From aefea95bd3c0be1c92cf282bdd31ffdcecdf4229 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 21 Sep 2023 15:37:19 +0200 Subject: [PATCH 01/27] Move code to radar-commons 1.1, Kotlin --- java-sdk/build.gradle.kts | 302 ++------- java-sdk/buildSrc/build.gradle.kts | 21 + java-sdk/buildSrc/src/main/kotlin/Versions.kt | 23 + java-sdk/config/checkstyle/checkstyle.xml | 212 ------- .../config/intellij-java-google-style.xml | 596 ------------------ java-sdk/config/pmd/ruleset.xml | 79 --- java-sdk/gradle.properties | 20 - java-sdk/gradle/wrapper/gradle-wrapper.jar | Bin 61574 -> 63721 bytes .../gradle/wrapper/gradle-wrapper.properties | 3 +- java-sdk/gradlew | 19 +- .../radar-catalog-server/build.gradle.kts | 16 +- .../service/SourceCatalogueJerseyEnhancer.kt | 8 +- .../schema/service/SourceCatalogueServer.kt | 24 +- .../schema/service/SourceCatalogueService.kt | 1 - .../service/SourceCatalogueServerTest.java | 72 --- .../service/SourceCatalogueServerTest.kt | 90 +++ .../radar-schemas-commons/build.gradle.kts | 24 +- java-sdk/radar-schemas-core/build.gradle.kts | 29 +- .../org/radarbase/schema/SchemaCatalogue.kt | 42 +- .../schema/specification/AppDataTopic.java | 40 -- .../schema/specification/AppDataTopic.kt | 29 + .../schema/specification/AppSource.java | 67 -- .../schema/specification/AppSource.kt | 48 ++ .../schema/specification/DataProducer.java | 99 --- .../schema/specification/DataProducer.kt | 79 +++ .../schema/specification/DataTopic.java | 167 ----- .../schema/specification/DataTopic.kt | 139 ++++ .../specification/SampleRateConfig.java | 42 -- .../schema/specification/SampleRateConfig.kt | 29 + .../schema/specification/SourceCatalogue.kt | 14 +- .../specification/active/ActiveSource.java | 85 --- .../specification/active/ActiveSource.kt | 68 ++ .../specification/active/AppActiveSource.java | 25 - .../specification/active/AppActiveSource.kt | 20 + .../questionnaire/QuestionnaireDataTopic.java | 43 -- .../questionnaire/QuestionnaireDataTopic.kt | 38 ++ .../questionnaire/QuestionnaireSource.java | 9 - .../questionnaire/QuestionnaireSource.kt | 10 + .../specification/config/PathMatcherConfig.kt | 10 +- .../specification/config/SchemaConfig.kt | 19 +- .../schema/specification/config/ToolConfig.kt | 10 +- .../connector/ConnectorSource.java | 55 -- .../connector/ConnectorSource.kt | 35 + .../specification/monitor/MonitorSource.java | 25 - .../specification/monitor/MonitorSource.kt | 18 + .../passive/PassiveDataTopic.java | 59 -- .../specification/passive/PassiveDataTopic.kt | 46 ++ .../specification/passive/PassiveSource.java | 50 -- .../specification/passive/PassiveSource.kt | 40 ++ .../schema/specification/push/PushSource.java | 48 -- .../schema/specification/push/PushSource.kt | 28 + .../specification/stream/StreamDataTopic.java | 148 ----- .../specification/stream/StreamDataTopic.kt | 110 ++++ .../specification/stream/StreamGroup.java | 49 -- .../specification/stream/StreamGroup.kt | 38 ++ .../radarbase/schema/util/SchemaUtils.java | 160 ----- .../org/radarbase/schema/util/SchemaUtils.kt | 149 +++++ .../schema/validation/SchemaValidator.kt | 71 +-- .../validation/SpecificationsValidator.java | 99 --- .../validation/SpecificationsValidator.kt | 108 ++++ .../validation/ValidationException.java | 52 -- .../schema/validation/ValidationException.kt | 24 + .../schema/validation/ValidationHelper.java | 143 ----- .../schema/validation/ValidationHelper.kt | 103 +++ .../schema/validation/config/ConfigItem.java | 83 --- .../rules/RadarSchemaFieldRules.java | 101 --- .../validation/rules/RadarSchemaFieldRules.kt | 119 ++++ .../rules/RadarSchemaMetadataRules.kt | 68 +- .../validation/rules/RadarSchemaRules.java | 253 -------- .../validation/rules/RadarSchemaRules.kt | 223 +++++++ .../schema/validation/rules/SchemaField.java | 21 - .../schema/validation/rules/SchemaField.kt | 6 + .../validation/rules/SchemaFieldRules.java | 53 -- .../validation/rules/SchemaFieldRules.kt | 53 ++ .../validation/rules/SchemaMetadata.java | 60 -- .../schema/validation/rules/SchemaMetadata.kt | 14 + .../validation/rules/SchemaMetadataRules.kt | 13 +- .../schema/validation/rules/SchemaRules.java | 139 ---- .../schema/validation/rules/SchemaRules.kt | 137 ++++ .../schema/validation/rules/Validator.java | 233 ------- .../schema/validation/rules/Validator.kt | 56 ++ .../specification/config/SchemaConfigTest.kt | 11 +- .../validation/SchemaValidatorTest.java | 165 ----- .../schema/validation/SchemaValidatorTest.kt | 173 +++++ .../SourceCatalogueValidationTest.java | 115 ---- .../SourceCatalogueValidationTest.kt | 124 ++++ .../SpecificationsValidatorTest.java | 62 -- .../validation/SpecificationsValidatorTest.kt | 95 +++ .../rules/RadarSchemaFieldRulesTest.java | 196 ------ .../rules/RadarSchemaFieldRulesTest.kt | 195 ++++++ .../rules/RadarSchemaMetadataRulesTest.java | 127 ---- .../rules/RadarSchemaMetadataRulesTest.kt | 122 ++++ .../rules/RadarSchemaRulesTest.java | 412 ------------ .../validation/rules/RadarSchemaRulesTest.kt | 337 ++++++++++ .../build.gradle.kts | 18 +- .../schema/registration/KafkaTopics.kt | 176 +++--- .../schema/registration/SchemaRegistry.kt | 215 ++++--- .../schema/registration/TopicRegistrar.java | 96 --- .../schema/registration/TopicRegistrar.kt | 97 +++ java-sdk/radar-schemas-tools/build.gradle.kts | 13 +- .../radarbase/schema/tools/CommandLineApp.kt | 11 +- .../schema/tools/KafkaTopicsCommand.kt | 26 +- .../org/radarbase/schema/tools/ListCommand.kt | 2 +- .../schema/tools/SchemaRegistryCommand.kt | 97 +-- .../schema/tools/ValidatorCommand.kt | 17 +- java-sdk/settings.gradle.kts | 19 +- 106 files changed, 3591 insertions(+), 5291 deletions(-) create mode 100644 java-sdk/buildSrc/build.gradle.kts create mode 100644 java-sdk/buildSrc/src/main/kotlin/Versions.kt delete mode 100644 java-sdk/config/checkstyle/checkstyle.xml delete mode 100644 java-sdk/config/intellij-java-google-style.xml delete mode 100644 java-sdk/config/pmd/ruleset.xml delete mode 100644 java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.java create mode 100644 java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/config/ConfigItem.java delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.java create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt delete mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.java create mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt delete mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.java create mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt delete mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.java create mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt delete mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.java create mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt delete mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.java create mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt delete mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.java create mode 100644 java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt delete mode 100644 java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.java create mode 100644 java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.kt diff --git a/java-sdk/build.gradle.kts b/java-sdk/build.gradle.kts index 91e43b33..3a9d76c7 100644 --- a/java-sdk/build.gradle.kts +++ b/java-sdk/build.gradle.kts @@ -1,16 +1,18 @@ -import com.github.benmanes.gradle.versions.updates.DependencyUpdatesTask -import org.jetbrains.kotlin.gradle.tasks.KotlinCompile +import org.radarbase.gradle.plugin.radarKotlin +import org.radarbase.gradle.plugin.radarPublishing plugins { - id("io.github.gradle-nexus.publish-plugin") - id("com.github.ben-manes.versions") - kotlin("jvm") apply false - id("org.jetbrains.dokka") apply false + id("org.radarbase.radar-root-project") version Versions.radarCommons + id("org.radarbase.radar-dependency-management") version Versions.radarCommons + id("org.radarbase.radar-kotlin") version Versions.radarCommons apply false + id("org.radarbase.radar-publishing") version Versions.radarCommons apply false + id("com.github.davidmc24.gradle.plugin.avro-base") version Versions.avroGenerator apply false + kotlin("plugin.allopen") version Versions.kotlin apply false } -allprojects { - version = "0.8.5-SNAPSHOT" - group = "org.radarbase" +radarRootProject { + projectVersion.set(Versions.project) + gradleVersion.set(Versions.gradle) } // Configuration @@ -19,33 +21,21 @@ val githubUrl = "https://github.com/${githubRepoName}.git" val githubIssueUrl = "https://github.com/$githubRepoName/issues" subprojects { - apply(plugin = "java") - apply(plugin = "org.jetbrains.kotlin.jvm") + apply(plugin = "org.radarbase.radar-kotlin") - repositories { - mavenCentral() - maven(url = "https://packages.confluent.io/maven/") - maven(url = "https://oss.sonatype.org/content/repositories/snapshots/") + radarKotlin { + javaVersion.set(Versions.java) + kotlinVersion.set(Versions.kotlin) + slf4jVersion.set(Versions.slf4j) + log4j2Version.set(Versions.log4j2) + junitVersion.set(Versions.junit) } afterEvaluate { configurations.all { - resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS) - resolutionStrategy.cacheDynamicVersionsFor(0, TimeUnit.SECONDS) exclude(group = "org.slf4j", module = "slf4j-log4j12") } } - - enableTesting() - - tasks.withType { - manifest { - attributes( - "Implementation-Title" to project.name, - "Implementation-Version" to project.version - ) - } - } } // Configure applications @@ -54,35 +44,6 @@ configure(listOf( project(":radar-catalog-server"), )) { apply(plugin = "application") - - extensions.configure(JavaApplication::class) { - applicationDefaultJvmArgs = listOf( - "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager", - ) - } - - setJavaVersion(17) - - tasks.withType { - compression = Compression.GZIP - archiveExtension.set("tar.gz") - } - - tasks.register("downloadDependencies") { - configurations.named("compileClasspath").map { it.files } - configurations.named("runtimeClasspath").map { it.files } - doLast { - println("Downloaded compile-time dependencies") - } - } - - tasks.register("copyDependencies") { - from(configurations.named("runtimeClasspath").map { it.files }) - into("$buildDir/third-party/") - doLast { - println("Copied third-party runtime dependencies") - } - } } // Configure libraries @@ -92,219 +53,28 @@ configure(listOf( project(":radar-schemas-registration") )) { apply(plugin = "java-library") + apply(plugin = "org.radarbase.radar-kotlin") + apply(plugin = "org.radarbase.radar-publishing") - setJavaVersion(11) - - enableDokka() - - enablePublishing() -} - -tasks.withType { - val stableVersionPattern = "(RELEASE|FINAL|GA|-ce|^[0-9,.v-]+)$".toRegex(RegexOption.IGNORE_CASE) - - rejectVersionIf { - !stableVersionPattern.containsMatchIn(candidate.version) + radarKotlin { + javaVersion.set(11) } -} - -nexusPublishing { - fun Project.propertyOrEnv(propertyName: String, envName: String): String? { - return if (hasProperty(propertyName)) { - property(propertyName)?.toString() - } else { - System.getenv(envName) - } - } - - repositories { - sonatype { - username.set(propertyOrEnv("ossrh.user", "OSSRH_USER")) - password.set(propertyOrEnv("ossrh.password", "OSSRH_PASSWORD")) - } - } -} - -tasks.wrapper { - gradleVersion = "7.6" -} -/** Set the given Java [version] for compiled Java and Kotlin code. */ -fun Project.setJavaVersion(version: Int) { - tasks.withType { - options.release.set(version) - } - tasks.withType { - kotlinOptions { - jvmTarget = version.toString() - languageVersion = "1.7" - apiVersion = "1.7" - } - } -} - -/** Add JUnit testing and logging, PMD, and Checkstyle to a project. */ -fun Project.enableTesting() { - dependencies { - val log4j2Version: String by project - val testRuntimeOnly by configurations - testRuntimeOnly("org.apache.logging.log4j:log4j-core:$log4j2Version") - testRuntimeOnly("org.apache.logging.log4j:log4j-slf4j2-impl:$log4j2Version") - testRuntimeOnly("org.apache.logging.log4j:log4j-jul:$log4j2Version") - - val junitVersion: String by project - val testImplementation by configurations - testImplementation("org.junit.jupiter:junit-jupiter:$junitVersion") - } - - tasks.withType { - systemProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager") - useJUnitPlatform() - inputs.dir("${project.rootDir}/../commons") - inputs.dir("${project.rootDir}/../specifications") - testLogging { - events("skipped", "failed") - setExceptionFormat("full") - showExceptions = true - showCauses = true - showStackTraces = true - showStandardStreams = true - } - } - - apply(plugin = "checkstyle") - - tasks.withType { - ignoreFailures = false - - configFile = file("$rootDir/config/checkstyle/checkstyle.xml") - - source = fileTree("$projectDir/src/main/java") { - include("**/*.java") - } - } - - apply(plugin = "pmd") - - tasks.withType { - ignoreFailures = false - - source = fileTree("$projectDir/src/main/java") { - include("**/*.java") - } - - isConsoleOutput = true - - ruleSets = listOf() - - ruleSetFiles = files("$rootDir/config/pmd/ruleset.xml") - } -} - -/** Enable Dokka documentation generation for a project. */ -fun Project.enableDokka() { - apply(plugin = "org.jetbrains.dokka") - - dependencies { - val dokkaVersion: String by project - val dokkaHtmlPlugin by configurations - dokkaHtmlPlugin("org.jetbrains.dokka:kotlin-as-java-plugin:$dokkaVersion") - - val jacksonVersion: String by project - val dokkaPlugin by configurations - dokkaPlugin(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")) - val dokkaRuntime by configurations - dokkaRuntime(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")) - - val jsoupVersion: String by project - dokkaPlugin("org.jsoup:jsoup:$jsoupVersion") - dokkaRuntime("org.jsoup:jsoup:$jsoupVersion") - } -} - -/** Enable publishing a project to a Maven repository. */ -fun Project.enablePublishing() { - val myProject = this - - val sourcesJar by tasks.registering(Jar::class) { - from(myProject.the()["main"].allSource) - archiveClassifier.set("sources") - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - val classes by tasks - dependsOn(classes) - } - - val dokkaJar by tasks.registering(Jar::class) { - from("$buildDir/dokka/javadoc") - archiveClassifier.set("javadoc") - val dokkaJavadoc by tasks - dependsOn(dokkaJavadoc) - } - - val assemble by tasks - assemble.dependsOn(sourcesJar) - assemble.dependsOn(dokkaJar) - - apply(plugin = "maven-publish") - - val mavenJar by extensions.getByType().publications.creating(MavenPublication::class) { - from(components["java"]) - - artifact(sourcesJar) - artifact(dokkaJar) - - afterEvaluate { - pom { - name.set(myProject.name) - description.set(myProject.description) - url.set(githubUrl) - licenses { - license { - name.set("The Apache Software License, Version 2.0") - url.set("https://www.apache.org/licenses/LICENSE-2.0.txt") - distribution.set("repo") - } - } - developers { - developer { - id.set("blootsvoets") - name.set("Joris Borgdorff") - email.set("joris@thehyve.nl") - organization.set("The Hyve") - } - developer { - id.set("nivemaham") - name.set("Nivethika Mahasivam") - email.set("nivethika@thehyve.nl") - organization.set("The Hyve") - } - } - issueManagement { - system.set("GitHub") - url.set(githubIssueUrl) - } - organization { - name.set("RADAR-base") - url.set("https://radar-base.org") - } - scm { - connection.set("scm:git:$githubUrl") - url.set(githubUrl) - } + radarPublishing { + githubUrl.set("https://github.com/$githubRepoName") + developers { + developer { + id.set("blootsvoets") + name.set("Joris Borgdorff") + email.set("joris@thehyve.nl") + organization.set("The Hyve") + } + developer { + id.set("nivemaham") + name.set("Nivethika Mahasivam") + email.set("nivethika@thehyve.nl") + organization.set("The Hyve") } } } - - apply(plugin = "signing") - - extensions.configure(SigningExtension::class) { - useGpgCmd() - isRequired = true - sign(tasks["sourcesJar"], tasks["dokkaJar"]) - sign(mavenJar) - } - - tasks.withType { - onlyIf { gradle.taskGraph.hasTask(myProject.tasks["publish"]) } - } } diff --git a/java-sdk/buildSrc/build.gradle.kts b/java-sdk/buildSrc/build.gradle.kts new file mode 100644 index 00000000..1854997d --- /dev/null +++ b/java-sdk/buildSrc/build.gradle.kts @@ -0,0 +1,21 @@ +import org.jetbrains.kotlin.gradle.dsl.JvmTarget +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +plugins { + kotlin("jvm") version "1.9.10" +} + +repositories { + mavenCentral() +} + +tasks.withType { + sourceCompatibility = "17" + targetCompatibility = "17" +} + +tasks.withType { + compilerOptions { + jvmTarget.set(JvmTarget.JVM_17) + } +} diff --git a/java-sdk/buildSrc/src/main/kotlin/Versions.kt b/java-sdk/buildSrc/src/main/kotlin/Versions.kt new file mode 100644 index 00000000..4c5ea0a5 --- /dev/null +++ b/java-sdk/buildSrc/src/main/kotlin/Versions.kt @@ -0,0 +1,23 @@ +object Versions { + const val project = "0.8.5-SNAPSHOT" + + const val kotlin = "1.9.10" + const val java = 17 + const val avroGenerator = "1.5.0" + + const val radarCommons = "1.1.1-SNAPSHOT" + const val avro = "1.11.1" + const val jackson = "2.15.2" + const val argparse = "0.9.0" + const val radarJersey = "0.11.0-SNAPSHOT" + const val junit = "5.10.0" + const val confluent = "7.5.0" + const val kafka = "$confluent-ce" + const val okHttp = "4.11.0" + const val ktor = "2.3.0" + const val slf4j = "2.0.9" + const val jakartaValidation = "3.0.2" + const val log4j2 = "2.20.0" + + const val gradle = "8.3" +} diff --git a/java-sdk/config/checkstyle/checkstyle.xml b/java-sdk/config/checkstyle/checkstyle.xml deleted file mode 100644 index 04b9832d..00000000 --- a/java-sdk/config/checkstyle/checkstyle.xml +++ /dev/null @@ -1,212 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/java-sdk/config/intellij-java-google-style.xml b/java-sdk/config/intellij-java-google-style.xml deleted file mode 100644 index 70c15157..00000000 --- a/java-sdk/config/intellij-java-google-style.xml +++ /dev/null @@ -1,596 +0,0 @@ - - - - - - diff --git a/java-sdk/config/pmd/ruleset.xml b/java-sdk/config/pmd/ruleset.xml deleted file mode 100644 index ff84bf9f..00000000 --- a/java-sdk/config/pmd/ruleset.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - - - This ruleset was parsed from the Codacy default codestyle. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/java-sdk/gradle.properties b/java-sdk/gradle.properties index c2f80d26..66929e69 100644 --- a/java-sdk/gradle.properties +++ b/java-sdk/gradle.properties @@ -1,21 +1 @@ org.gradle.jvmargs=-XX:MaxMetaspaceSize=512m - -kotlinVersion=1.7.22 -dokkaVersion=1.7.20 -nexusPluginVersion=1.1.0 -dependencyUpdateVersion=0.44.0 -jacksonVersion=2.14.1 -avroGeneratorVersion=1.5.0 - -avroVersion=1.11.1 -argparseVersion=0.9.0 -radarJerseyVersion=0.9.1 -junitVersion=5.9.1 -confluentVersion=7.3.0 -kafkaVersion=7.3.0-ce -okHttpVersion=4.10.0 -radarCommonsVersion=0.15.0 -slf4jVersion=2.0.5 -javaxValidationVersion=2.0.1.Final -jsoupVersion=1.15.3 -log4j2Version=2.19.0 diff --git a/java-sdk/gradle/wrapper/gradle-wrapper.jar b/java-sdk/gradle/wrapper/gradle-wrapper.jar index 943f0cbfa754578e88a3dae77fce6e3dea56edbf..7f93135c49b765f8051ef9d0a6055ff8e46073d8 100644 GIT binary patch delta 41154 zcmZ6yV|*sjvn`xVY}>YN+qUiGiTT8~ZQHhOPOOP0b~4GlbI-Z&x%YoRb#?9CzwQrJ zwRYF46@CbIaSsNeEC&XTo&pMwk%Wr|ik`&i0{UNfNZ=qKAWi@)CNPlyvttY6zZX-$ zK?y+7TS!42VgEUj;GF);E&ab2jo@+qEqcR8|M+(SM`{HB=MOl*X_-g!1N~?2{xi)n zB>$N$HJB2R|2+5jmx$;fAkfhNUMT`H8bxB3azUUBq`}|Bq^8EWjl{Ts@DTy0uM7kv zi7t`CeCti?Voft{IgV-F(fC2gvsaRj191zcu+M&DQl~eMCBB{MTmJHUoZHIUdVGA% zXaGU=qAh}0qQo^t)kR4|mKqKL-8sZQ>7-*HLFJa@zHy0_y*ua!he6^d1jMqjXEv;g z5|1we^OocE*{vq+yeYEhYL;aDUDejtRjbSCrzJ&LlFbFGZL7TtOu9F={y4$O^=evX zz%#OSQay8o6=^_YM(5N-H<35|l3C7QZUF@7aH=;k!R!Vzj=bMzl$**|Ne<1TYsn?T z@98M0#ZL9=Q&XFBoJ_Jf<0Fn;OcCl5x^koelbG4BbjMQ>*!nE0yT@6k7A+ebv`X1w zt|Xjn4FVXX9-Gr+Eak=408_Fui&@?foGz6qak-tHu>2o@ZVRQ-X;HZhb1Hw|ZAoxx z!)Cn4hxBI}ZbBCOTp3L63EU3Wv1dxk@J?)0_#oYR7HOP5Yx6W3jnagH;c}y$G^}eN z_gNT{1AanZ<}mw2ELMxx@ZzZ(2RvE4c)lH8c7Gi~3R2#hx}p9!hKPMW>ekYbK86>N zL&7Ky#*zv-P4iuIQ5RV(+vKjmwl+P}KH+$~xd=b5Dx1{hqqu0tbG{fYWstL&Kcz*d zOc@$}f?5vBmO8f3pj<+2PO7R}Jd6N{qRexKo>ElNYgVeYkyhIUY}X%clJ>unwsuOm z;;>SVKUJt$Kgz4Ax?PKY8F>##IJuP>EQ5R;Cq6}Xuvz;%La(_I4j$jv%s z_v}|apMsrN_%S~~HmEwu3RG@~x!CES{G~n#-()k{<4D?L%JT%I>3r{ML&;j7U#{u0 zJ?Wc+C3`^378b`@&yD4v8!cjFCp`ed7Vun)3h1Mkly&;(&fuUsq`8F2oWWnBfh9v! z%)WBwE2S9RJJIEHjIzyFh7TbyvbDRCqs zz`u%UBFGa1z6^Z;hSo~r?|SGTS_dE)60uPS35n|LB018jWS`wU7vFvrB4e$T&m zHc|hf8hn9fWZKeyH(lwiTQ1#0@gld4;-h@NX+Rzmyy}R9oxYJVHoXb zyV@nf36;c=c`b21vH@(g3?J$vx=?@!?R$yVrnPrplW!cQS})U%>{%lmdXH)bK|}WB zcslr*h|XiL-|~x4Ki6AvE3d+lTEd33pE)hY`fn@yv8^AoR52`*L^Kh!TF%3Zj&Vo) z=)bDG$a-IkN7fJsTT4x6FFNyqV+gZs@`P2OIF#{#7x)$_Cxj2bW2H2c)@w~>M9-`> z4Rw#yV$w+Qv?+!cb>ZXasldjG=R;#7T0@G-UcsiUBp%^VX-Dc8J_GSU8yDRiKwU|c zWvpbDr3EA4NPJjox0F|pxJqXQs*5zW32Z1yt8f{bm&ngF4za}c3?5YO)hu10?0t>G z?ULZt7!+Z}hMH(DP{TvGVkLv~GA_zNQf_1_ni6^ym;89EzQ5#iE4m6n-r2uEvoizl zq5cbd{wH>EyOaK;1d^KqLzrk_GD1tax$Dq$Q})b@IuYAblTIlc7NyShO4+UxQ!h@9 z`1~UTW%+i=c#J0?vlJ~q&h%e?Z+*S2@M z9)%F6JI5V&Z_>NgLbq|?usS;Lz#Hcsr^jx;DUTy_azC&RZ=O&Cop&s-TL-CH84KYl~J8>BsHHR%FFg^brE_t={xLMsXGwF zIyCKUONvr-f1;TKTPsMS*((XEUx+LCFvCe!sDD;lU=eO>tQ@>$nrs^M^q((M>TR#Q zOI>o=R+r!OkY1EKbUNuYY&$~TEk$WBzF19Z=DLh}j4c%g5#bz8au{mO(Tbi7uvF$Khaa+4M=?LiGQV#Lt>t>bsPrzJ1l+$MHNZAg*yv2Aj^GPdOj?yc~aVqIC*@K@(1i)SWh_{G{A zG1@USpgj^;P7~3AZ~V|GoHJ2?7%^R(%z)V*M!^T-q5otVw?hcavR3}JStYt4!&fXD z1+e)IzeoW7Z+C(-4G(4Cs?Tv2T4LY_Vi&j`Y32s=e7#vP1KE&fqM6+)W7s0H-(S1iQEl`JtY37ONAZL+Nu$hJdF28aC@KL1>?4iXE{ODGHT*$J!M(}w| z?iMo7ViHWSXq^tSRA9d49%mjWkK}6`jDOB=bRBJKkM^)P5DObI%N@QWmwBtA`U5as zY$MJ>tCT^Cl?=nqgIhYmmXxgSlTJp?*nuQde}DXE0r*uaEGzc|1QO)--|@1i^EYRU z-jUJ0(A^Onr66{}m%_N0m8V*Wgx!(Y+58UA>yEFY)xg)=ABaIlk4IPQu;Ff z^U0cjG$rBb6bPd4&~HD7 zuilr*e$ya*bYJ1slNQmcQRBfYGVv^7U*TP&1&j+6K!Gtya8k0ZVXlRaXonBQud{(- z8{H;11N->}EsfRH&PRJ+Zvv6nmNL5gZt^1ycQR+y^$-cE4ysf=aesOre{qVP8ZE-N z5b!{I@h=~}ezVU}r}w|kH1)|0eTt{uhLWwJF_ooj=394^#ps{7%#C64V{PAIM-QlV zWljWxJv?vy{cg$FR1<-R)1ooe&bh%H@q1B31dgl|L#Hi%;b1m+v3-Qi#xKFwtej6F zMD#OP7dy=d7x@>b$WbMbmRN5H4!ud^fkEiH^4c)#SM=rlV2(hQC})_B#wcQlF8lZe zG5d9j)R?jGyvJKno5h^QKFplNMt_2USAR%e+t$izw$>w&nxaUtQ<^8j*4Y`hJ=&70 zX!}IKNGDkF?b-aTbUt6IUAZ-_H)qqB}z z!Oxw~3$9y#kV1rG*7QSA92I_QlZsfNs`aV()|gms1UM2eQcsq<@USs>c&Gp?rddNQ zEV(xadXNq%+{o-xVl40Gp9^W}smgI{@XyRnBS|vC^n18J$sI&VO$Z4O<7O!Q^QmAM z=VJ|CUZTSd-k)5(U*-_`!=NxqE$3{g0d$9+KcYE)<3axb{$^F! zy^*(#FX8*az%oN7PXD!W!#xk;cyKXPlk#REJfCc@D3GUbxUdbf3 zgKAiY3UkwLeALOY#IYIP>YMzVjl!=0xvd{+phh(_O7tE9qy4gb>yre|RzH3^lT zWrRQ??y`cGvDufpSH>KBD+)tNgKaf$kj^Of{&pP#R7K8Q)1rNc)c#pAknYFKm6g5g zOW=*;dhTx-*{h7*GlF>Xh!oxu^ZvA7xfcsG7i<(iMKq?ht{pz!I?YZzNOki^74gx-@+C`zFrDH5GU4uDsNnfkcmY zQbAo?mp6?L4ni5+PG2%Zz&h=kLQn?S^b(Dt8DLm&ns$jXoaqk)El;XE@SK;iXX0wQ z;Olbo>zZ$ds`WKqciZ7*g0)utwY8VaYRl@26NmB|nw(xe&+Db*ldXdLA3d+d!5Pld z#$pjwmtrF~-?5pz)jXGt4sqBp0!26N_8b8iD|4ubbY3_O)aT;{K-ll#%wV!e8E)Ff zZt9=A;m691@9&~gi1oqV5Es86S%S0^+zH~VOTzgoDcz_X@d(}Xq%@uJsnC0)Q&1IY z-slwRxI@HX4M(nEzsE&vZxtyFLZ+F_)>Ne2^$IA3VfO}gAb?iJd!u^Zp!ak#LpeXGXMcSS#4&+DJBT91RSM<{qPz8@SJTKl;oJiy+6QQ@VK$5PjOa zD+x}7a3gCeP*X}*EGre%RbJ1fDeIQx!HOK|aONo)ukFgyfI!6{f)z*54Oco>&mI9i z;18~KEb$7_mh|HUv5!txYFdUQRaHc4J$-H^`SruU<8nJI(%i<(vp!&63A z!=>cO@-l5t{(3p5DoxawpiZul&;+*%46Q7W8tOty9cNCiNcm!@cTBA*_Sge^l>@eE0yb+7& z_G2$v0AnxOpW$Bfw?kEjDNw8x$j1q>M?gh4yM{&(@rM;tUsM8^hWY_z`J5riM7;CK zXlXQxK*Ska!rCWbb;(&bgG;Hb5qw>0eZ#Y?eVJDrz8L6*knEMm4+N7N(`k+2TB6u{ zP*lDK>Mi6JLU|r2J~*(|iBapcCaxQF(%pGfoCzq)y_CA_cws+oJ%9&=jAXjQtbN5k zAkClhvE(E$F&65^ij?_t*1kpm7|9VZEJ95(6bfqN%+8`g)#l5IQpmhG`ofn;5>7hk z2xnq?L2V}~_8;0Ll(dVlX(LSJO0x+1jr6Vw{Bo%vNJRugYT&*KUaL3&}YH4OWt#%tJVil>0MY&zxM zvAMLu22RDvj^Z_sa*ao26u32j#Gbhope{6`+4?eF)` zE3QBt`YUPT2C^v8Lt3;Or%uLTrW8xK5 zqLEc(9k<4`l{8L0=Vea0-xQYvFOQB(duQK#S=rMa^RK=p>fI!(^ef$BOyb)qUF|i~ zTl#JvRhkRlzl}D@lzj(;62K{qy$1rr=B~=Lb$%JgnRkS6>I{yw{h}QBka+IE&GX>% zAJ+|^G*Y#^rb6nMgMPQ3GkuC1B4U!BUk;Dd)rpy`_Yr1&E2!i z^7vz6B1W#bfEhpYDh3<@bGEu{6Jux__bwaZ2^g?PY_`Tg39vJlA>bfG>_pQj^Zq_6 zi#$Qa0DQ}Y6R}vkCm%Lt0&{NR63oo55%F%pOS?lg^XX1ghs3MiQf1Dt+2j*IGJMZa z#;0K^rLufIwaWc(uyfHqLcf`(@H^dMl)6c&#e6xWQ_(k zRz=x*OVFt#$cTpB?i@m*D8nm*lFVev555nBCQr+JihUaz;5fsw6-=qeW9iHz&hX|F zS&VP=r( zbO+X0bOM!y4TuJgS-&=u(*nR@cH5dzCPjGU>oS0CMPQMj^F@SYX(rvl+Y_76GURaR zp^G)7`Er$dE7Z-tH5)^X|2PfO8!}okjcZz8d-)|VT0R3v@@&4{g70e)0cTWq;*xOm z(e039+BRgcLB1nuoSwBO|5QIk3DjemLfsP#H=)+^8#8+J3)z15n?g%BFq#&yf_7EO zfboQ=qKNN1+=K$ZC!5;4mB7lqUt<5XQQP&I?f8PVp{Ss!{*_G;r@nDPQ&mY8R2sjM zxw4d?#_I?))gJ4O*V9&Rsx*U{fp-ncs_ng#Z?c5hplhQI$TVrp(5v3H%;YCL3+Ss1 z@~NQVv3~ibw5b*z1+1!z?twQOa?Q`OS#VheAa&;=;`&|UHmni$-h(qeO3wV5F;DBM z>Rzon?A7Hk;9}!a=XHn0klvPBC)cbM32aD#8!3$18Lf;z1s zG}(1&!y$ehWEo1unGS_G3z!!A`(GAjnMmxq6>>m{LCm?+e-_slha9vVFc1)#e+&xO z{}k7K4#<>CZWN%#E?`9x{d+x~OoDohJ4$Ssh&WVN)-)Gf);hNw=GQ`HPus_XphMt>}b*b=*@rzV<@1ijU?f6raCIlI+Jv) z_0^LwE%@~_m9Py3lW*#h3gZajMH(|r!5rbOj`l3l7#$X@_;ot*I=44BnR^WVW+{|f zt~onHYA&99JI6s+EY=zmEPc^){`=&kUD;P{at;X{_ARTe zb*LtuT`NFT6Gy-TS6^0$;50mdO<$$Z?t=u8bmqZ0RE46zk=w{TlhFPSwqLyMMt7K2 z%Xg6IA$cy(qYA|k zb)SKGwihPbq|>C0fY40>&8}gl98cThVt>8?(GfU{+og%;xM7#A#h_x_&-6#Y!tAf80_?y=XIxJt2Q&4q!8vC7 z?^~enOF_MOt1-6R5rje3P%fEa>l`txDAwOh$KS`=Bk+;j$DeuIoDi{%Hr*1dYJKUg z1@ddnOA9vBgGilNZyj|9f)XpAPPHx(go4{{KYs`#5%s~11b9v)@UYZt#g*C#j`9(# z*s!3d_`Ot_ek2y5cK*F{kXLdukiN@AE{O(0_zWb3m?Zb3p{gD|EM5}mrb)9VXKe|T z0?TD!ZawCi>si-w93t>jw&I?a!^WwqoIfVWxOt@cl6BJ z9Xl_11OE;aC;o4y$JGf7{3p2eau=Jc)qHMN*LA^w5D+YLtcBgj#G1UE-CP;fk|)dt zfy<;ibE&YHTwEe@3;iZ)lLrGyo!>mtWnd^#Z|@hdpzFf9!=yf}|C;j`PO>3gt3XC7 z#CF?=MEI1bm3~D<=R9(Qk9$m!)0RhFTHden(}ClhcnVr?j+EdoMt%-!sn{C#FT!3Mr`9asC7OOBkKx)@ZaE+XxKZ*xJ8L>uixI6iBh zKUc6oC)GTS)SciDQbhnvHur8HUtwTsFoRfVBx zND}|`cdIj36VJDmIW1haD0==ic!Q|+{Vrmd60J?2*7nU~Jw526CG7mpcM^D9Z@Vhk zK2Ntl6F|}%t4oMlc-^|JC+#vh3=Q(W}UY9Jo^1{B~gIY24 z0=mOyd=lVUu3W}us9s0D z{J*xZHKGUkBI?n~O}$@9gzpR#;(T0rtYDbPT{hlRan>z*%oZFuxGnU{ls$ECJm9UH z>BXmC*me*j;V>t%HpXHgBw)Au0BR!#tGk0vAw8@Mw0F5oo1sKKa#@+f;elcwo_p|i zf4zh1(PPF;vHKJm!Y}szf*YVt0CEmRp6t)d6`pxRBz!!1u_4dXst;7PqakTnr&yb# zy5R0SPn_YGvQuRQ1KHmt;Rg|7lPy&9=MNW@sgdll7K$pJ3agxoXmcJ1Bx`J6&_6PL z!oi)a7D|1iLw|mQJVW#d7Xziw&2yruRgPgk>;o&9C!vx~#WD|VPTrYi{lI7Z=t)~q zxvr6u_Y`)br5%qsy>llS%aIK2j=5Y@(nyb2w zsH`8K_@s+-Wt0x zEHp8g-ad7(dJ^(Jj-xbu1N);g{@8BcEE3FavmjOQn0uDn@%43f#smUoy(L{@OBP~_ zspPQQXkjuTnwRK(A;aV&A-#q-0p5ZJZ!m1Tk#ci5)_Gf z-!|L|W^Gt2u8&+SJ9Weu6C;9p(LXJLd;D^@G>K}79RO>Sj7Bx1*~i|xgr9GJVwFFM z*oST)uxtKzO`Ni}yjp?VJeLJsA(76F ze}2NOjg1)CrQ<^^Fk>zqr~~`bB;YN>fOYUs7DJ14AcvSzh~c99I7Qz zvf#)6h3UvIytr|wARx4~ARv_g`w>VWqnW*lt81Q)jj`TZ+IKv|#nb{*4jL7TIf_o? zwHHiK=BQ2{1oNokAjyypbo7@!ohCWi6nS`KsPGnzT#E@*GN@?!`;C7x{T3|eSCQv!&ugyhg20UDg1^u4<|7n{e8v~h+j^wp z@;=MwPeYUsKI@$pnj=2zJ@9SkR7HEVfuLbisk5Xl+ew5)i%A0A0*#FMycc;@T6_iJHNuhjtinw9&QSk0TF z)>0Yd#5Yq~&LP@b)&R{UR=%hBZEd({8IxVrp7~nov|wx5s#G)bI*ez&r$1=LGNk)x z=uSi%YSmL};Jc)a|B-hdZYtEsF5)=mO8&Mg~ndT{dj5?Ua_g^DK4wGAqwD^9n^0wTT%=+EHSoJ z!PP+cszWE*1f*+no9GPTd^rMC3;2uB69^nl9T!sd2U2DQVrQTHt$dgNZpG$MWNXwS7B`M_O7>WCgcfzU z4gLmu*mwix+Y@J#n^I^J+)TyENce+W#Hg#m>5i-05n6XzqOsLBc`gU|my@INVPL3t z7A8b$Q?{>eyRhcw^RQYGpPL+zh}mP{?5O-1)-DWV>UT>}@91Fj$nzs%)lPy>B|wSd z+*&gC;VzNwda2y4HAuwA$u8enHkQB0*|zjVMP>x5flRL>PLy2wN3CF579W!f)OL~* zxM0NSaF{#Z({GiM2&j$fOqndh&nst7cZs#aZ0{%pF$72TU1xG6Q$7D&gqgIo+Lq+3 zT$mOp`AbF$S3ois-io~}YrTgJ!+P)wy$nVd9VYCzBmu~lDKA`ZH_YAi_65~pGXfrs zxJV8#Keo(o*%#r1+_It?bs;?dm*r{hl0T+yrPV56t{QWazt$Igo<=1-tH58%77&>8 zF;0^=Ezh>NX+2?@Vkw_PnW?`j1dIO2KEK6U7vWld#P3g>>rWe58mS{2>WR3O8?s%S z;3kfzBS|ApxFx09m27tCxMOk1x#M`KxYh%NdPObrN#~|QwmW4F2WQx#cEG%uU?#r{9!X$A%NlnuM zbm@~&UwMu_;c76nrZwtmw*NZnx+>QNl)32w()1msIGX2@?JW3;N~{BFxkXqydPjlD zS0_FaPYiO7iFhyxK86Z4I(|@|O~x{@X?1i=COZ|NTFuCMsBx0T={u#Vglk+3!9|p5 zEW`f0^c~uOnjOoj>uKcu^y~B;5>H(~#*X#WZs$hw?W92ZPL25Ui(Y|t`$^A(z`C-I zvFh0P0^6T%QrqpPnuAtQO<@5pBn#kAg3G3rSP|UkUE^ky{xaca5rKK?7>`h<-_qQx7YR_N4!|zc`@m|)gjvL0QLZGvVMZvHuDbq_7kZGY)^I_sFCB?jm-T9Z2I>m z*U=wB(d0?W}1#g=l!qus4$Xk4k)Svul8k}pbG_&G;N0ANuif%WAR*S$K@ zw!*1wOaXPo_iA#5`mzQCY$$LfsZ(fiHFdLnL~aB;x&4WYm%W!$;`n=R$g2h@yOj!n z<2sNO%Wpry@m^09puOh>w}Yf!V(~L0$46SU3sUyABc8n$4~hF8*Yv4W;frKE)a}+0 zD*I!nHUh&Ymfun;N5fifef_7-Zo8opQRODhPPMQ3`ARmLVT78*<h-gwf(YuMTpacqNgSyG2=nR1QhH+2ax1bbjX~wwhYy z1ml%qPoUeL>g>Gu2o1RA-;buAcS*=X`x%$Z<^V<=^DzMZ0_+k{XwY2Lf=kyJN}ZFk zv}d}2a~H5f7`^<>;PN#U`kY5sYb1$|VMUi5;Rx&IsLXY1&F>9EPd}|1P_J14%XocI zv>HQv0fV~w#Im^G?;ld(Z&veQme0F|ilV2jp3-JcSQ^ah00*pTu|IU`qO|%lXXS3n zWNrR-V|4&|eK9Pck2UU`+AC(fV|1*N>}sL>T$e`>;YEOeYw7xxQ=eDBonm@cWmivC z$d-DZr11h1Ef{@2PF6MJp`y74)v@Wat|V}oqj-(cjG^l->d{HDS3QynIhhc8MS55Y z7GXPm!kJF}1pw-yx8`Ouyfj02FfLd@D#@`gFZI(_uG2^__&i&Pj%}rWr|_aA^$C-C zzg+MjVbvgp^+W1p5>j#{c5flgNE@B;MKy1j@~vYdPztrT)hNNTwb*+HO5U|@<>4kl zy~?jcrn2nN?pb>@e0LYw^y&wcJ^mX@u16!7*NVxH@d0*6e1e`lG2xjtQ#dNocjbr? zG_9WuEzNlGLqTC@N7;SUI+fa4&RRkU`E0I^naoC&w(5zFcYL7ROFUC_OD&RO`aO5^ zI<>OdpEPdp%D1#g*DFlpB~vPVA&E^|H=7Mr?xuFvRe|3ggf2~IewENZMD zWy^0umLP7`Xh;a>+}bgjmq}!ymHVLXkc6llH%XkT4TBCS;2QuL?>h$A zO=9^^U2w2H%mAox4>R=;Qv!nyJ;H;=1~{tgL7CF0E*U=n*0{R2Up`|j#gHay>3_x*zLks^As z4{DVs=>T5JMYNg`Ib2jVzwNf*LV)~K5sDP8PX1`LE?;j(qJf3AESX4GT`isjy1Ksd za#&Tgmo1j824DH~)uTs|Jru0p-ib#QEYMMN54gr?vb zI}Rf=5>6#9jT@`x%>(6!wQ+N;B-Q$XZLNiEt=XVatW+bRuQQAx>0cQ55<|j2AVMdPgs~Nx3C*w2;pZ$N z**f#|?k?x>^_-wjaPmEB>egW-h8}sW+N@({F)1c~6CBc;5wpIbt~Bh&q@zWINub zD>xfG{A&S=#VQJVlP5ZdAMQE7XdI&1o{8jf1~{POKNkLGj?@(I#bkg?bZ4h$sHqLs>BZFN zdbPV5EUkV=*0ZQ*u`Q-b|2*IDlt$s#$pw$O02x$Gy(`IsLtb3q`V|7o?<_4l=@?MiG(0dFeV(YETtlz{=rf*Tek(1 zSdx|f!?So9fYB)+)P!d~Fitjb_hbYVHg$Mx*?NorFgK z#us}*O<|*P)#LQJGO$9S?&rYrY6+>B9k1duYBp||BLo2BQ(5c6vX(mC!e8g78vRU~ z#LKbYTs;O)SL?x#4Y*3DNewhQ@MnY0#GD+B?44~{$C|`{zi9`gRv|a=50F}-#UoyS zG{?>}rSPdO;T5c2n5<5~BMVJ_{kHt|yALSe6_LpSg&je}d=s#+ zHxb*YRC!@i{F|khl+uu*zMoO>kLdUTf=-~(v}!NS%pINSmR>V~(~Q5D)ZS3f1L0oE z>pdR9Rfie#DbqL|>~rU(nOE8}LcK57zwxKoUkNNx)}Cx_f56S|;S@S@v-#(9@0D_6K8gA0{x*4tnbax7>#T zOY8m{M9CZ6HM%;&odxZKZpPk^xFDcN*5%vuBNr=gaP|Z!@=s;e^M~1z`iWzW>RP`^ncxsp-UY2&+-}%hSy=srh9knmjX2Ng)i?zLM3DGL*VU`Z zh#`Bkw3_ouYHo+`f>4O1MO`{$>y7*(xbKSo+0hozMU9IVPyM+U3(roD1HPPy;&@tB z_-NUuOEyLOsi;04(DqEHa{>k&g7%wUIc1wIZNNHesErepVq*!QJF6elioGY}|4cyj zk7ofURP-|csQXBDarH=?Cv%_1m(F8_Lams+ekz;pILR`_578nbmr@=AApl~d4FrBt z!@2|6*~qC7pO1v@3ZhcFgX;jftS&cbeK)Xd%k$P;-*R>Gzl07KbTVCijM$smfXVI_ zID^x%y?+%AvM|qa2DKK~!;q06Hyk?w1!JSZ3ZKXUm~;NOieeYZR&Aa5c0tZ}K=vu4 z#rYS&dH@PVBCTc%pf6Rchk6@(d&~aVo=;%YP|_u5%h6IIMyMYrjA`bpic)!Y|- zy_U+KdCg(p(bTt|7IJOhK=$=)KTwwRKpb!}^$Gm1eppJt8BWV@y+^2j!oLGEGO&Nb zKl*c=76Pm8|0M<7v|j#S;=q48#FRl>-2ZLe*^>QVJu#wrQu&^Lq*&CyaSOJTds}>< zvWc6uI>5xk0^n+5FJ^6FW@iET?;cs2x}FxE2Ksk6xFxh0lUfr5t)x$o{5Fn{h+I)? zrfOX|4X1FKgh7OJcCH62+Cpw1|NBt^F>o+Luo8(zF5}}S0noKTUS<=AL}`~dv-kP? zcDv*K>elElh%>~#`C`HhPV8|sFscT#J}YzXK+G>y1a{-uW_}oN- zzstd7YIx!!zr%UrA8FBpDL8eYwu3in^`>6~i+Phnjf<^~T%;TWsk+kT4tC+!I){MI z5SfUD*T%r8wWTSHT7jIV(>Pzc_!`e#S53-!fJLfvPnYZfwc|vM@)5@%_ zmu(-hm<{$z%P4T=aT<)@Qmc2D&?FN&tAJbBM0^Cp)clj2OjFL)T28Vj?SE6eNNognH=FibthG z`YBIiJIOjg$3Ab}fGrRQ6zh(NQ;xzl!fGN`l{3Mv8l~&Py`9Icfg8XM8LX9qx18maYTf%gsvQ|Q>NdR3+m&^`L(lyJE-=1)g+%Yo>mubEh7(QAz%E+m)j z%t*58Q5Eati6k^X{=5pQvqEo;g5uP?3kwghE(wi+gx?>p{$*?r{OO!Bf`DhI-Qgl~ z^~wK``tyk&FQJw5)H|p3BWm-}56lwX7k6nigOk&Febfw3N%*FJc%yXBKW$U)Z%x?V z!9F8-+rx_VdL}FLM#-!atP|8u&xlVuG(tGd(W$P%waUHOSZQ&(vIf|C&3uuM$H1&s z7X7^w9zXqK=@>mB(9v_xO>I90qX7rI+PRIigf|1X$RW|3B#YO!xxa1MWZRP_@-8tN zc8M{=8`D!kwL>9+`ySMv=A#Js#q8Fy#4Ey8;2|cro537VE=IIh;ZBSaPbOEh%Snut z(u#BhKkq^4G$`+eb_4qH;&RDV%9-o-;rZlLy0Z)lX*m1`xbhW6uNt*M)(XbsbBY=k zW3Wf%jCf{KAZs7D0xs6F81$YmZBwGt0Z|hLSI@R7S{@~{fg_7p66(Zt*g5YEC-uVO z7g+Miydp%J=i?G7D5(O?fQQN}hX^q;JX zitgBu$iEgk&OhCU;Qv-8Tcy0)q64)6CeF?l0C5{vH-L?)yPJ)ZqXxiU%*pXzRdD>ObjV$Sz&viz$nu=E?RJQCOUiW>Yarq%av_mmaT=&S17>$3(^=t2{380C(0551jmfkZgt*2hvF%{ zUyMu+YYw9bFFI3|`3fe{q20hy#S>9uj$JQB)yo?RkKB6VG6TGNCTcXs#pMBBod7OBz6_B>N|0NHdwf!rc(X z)|6`l3m7FRs7XHtqL%Bf)k{In+g-%icG=Mu<>g&-jdJ|#RZRYy6GGA=wY4o$h$C6g zy3GGmgz7<@sEe4$gX2}u@uAW4ZKuXeDYRU5dzf|0G1tZm8}qNrT{MYR=H3l81CoS6 zJ4I4G9fmcb8tbfnJ}pvN3r1yK{B1)-v+XgYJ>(}KX8hl5?=cE3FmSKRp1Ts;ZEf7F zmWBUo-<>7aAokJWSlEkwIBQ0svmo`?#MczFJmO|?m-SZqVtoe_qK!6M*+U_R!i(6B zvKK(f=hjOc0!vmagR@gu7ityBUBBByfjNQxi};sJV3tTSKIII_oODIT{9ym+9rRSu zCQpn?vIiFk(5zF2H->+lW||x*2`jTa=1T4nMcmZ|h+g%KEg3}yYE(?((cvko zG@s3_z&DQaN{?y^{-JqH8^(x6$&AyXGm7r0a!OzBlCuYXlgI`3f(8*&i_@$cx?gs? z)p_fidF5^h67c`7kEBC@%o`6J_mB>eN zORD8d)_f`fuH`VG@Y^)D1rnPMdh}rlcgKjewMBN-c}iMJRP#~{zh{`4Gkx0ypG{t~ zuaXZsaf-M??w})`U<#2%>En6Xyt)&n#WH+Jf6GsJ-|N@ZEL*z97p7F%SbQzozhp4r zUw*b|8l({I^JoC&=FR6MndV;NEA1|o{Eto|Q>Y#izgk$J{k-m_CBQa0sd+bK9*VUt zp${49PPx$ka2(RXXd~ZU*FHo z3JRnrfOF2cs(V}yq~!mmVoWHoi;8$Oaf>n(r?bxB+b8ZLiaybh|)ak{MX~F-lPH3nfTvzj2uSXN8rls|oB|{E#|HCdXYsAk80gvcS^Vlul|B&PX{_#+l5KUU(u*@?HiK3bI%U94%*{#yCeWSvm!d zNU4SX1VR%%l#8159s()ZVfz2a)j3Aj6}Q_yjT+mw+1S{zZQHhX(Ac(ZG>vUrjcsSA zaeDLKbH=#mo-x*!^?l)a{_{8I{K<-&tCe_1wCy-*??rdu` zV~ci=Fwte~L|<9mGHoBWVm&>Vg9~lQ-ZHhTn8h>W#8Qg;E>qbsQG0P-rI4gFF;(^2 zWMjSGNe1G(zT1x~>BwJbRCzU2y$ z)>w1eVh zC*|vy*ZXwI(W81S6|AUqkpM{R>!fLKb!==0-NShiaKC$<%oisn#ftHNz~LG~zLbnsvrI$NmtaIkvri72296&WoTLTaK)RO~ zEN@5qjFXSj>DDsZUCeGU%zGV#@ss8mBY&O;^CYOko~AN*)){CxfDP9(q>0v}af=9D z?L_ykdV%^u25N=t8H9k^Irzr04F7j&_h&HiE&1RryhDM*IzU^s6c9@&F=#y93`ggF z@#pmOv)W#|o?tmybEi}?`x3L3&}j-^_5p(nuiAd-rSjEfT9ZNbjX`z58)9!c*z>qO zdAo_wpu+LRss`A2@mD9WMNgH{L8+(l+^tH&XM!nF647yWm9cI?_;f6dVXxwKOB;J7 z8Sa+TGf5s=RS|@{x9;XsFIQG*vBa6FLH7H+f%hp##mCoV7SDQ1adAF!J_hlD$&s5i z_24cCT@`h{ueL=}h0FdrwqIDIiw%Jtq4U_XI@NLEy#ctTdxZt)v{;R4<;-<6`PJ5O zzJ+Te5+mTOK8#mJp}#|YMuZI%WMO@^A}p$h6u=dLAm1?RU66%0DEqyP8OADCy^l*0 zg(H9~!6Kv4ocRbS0v2HGh)kw7_Re?18&VxU{RmGqTNK z4~C@Rz3KKbeI63?rRC;kNrb$k_Sg+5x9r{a5P$~cNe1=KB0F^(3t(LWuHX5#)qO%b}j;A4t z{%6sGJpOm3Y-DPdAbHDINuE4k*dT>(<)%N{pN{ilr zwWa9jw)1h?{hBfRg7a!9+Tl;Lrra#rKm2SF;9wOi!qk1Z#nxZN=qV!%f-Kh-?P_P2 zwg9a9y?+rBmC_n`ElG~Ak2(&6ZdF|abBT0a46GKWWW*tjB6_SX zB2x6jgI~q3)jkj>F8MINA^pINir}9eyySb}oDRFAA36@)dctm8Nga>=41I(AXQDW{IQ~ll(;%defD&}PVx2tW$dN#GvblIL3bzJXe*@RIc_vx z_}!7J3#xNpdpQN>pix5s$>S=}o!DYaT46sj4Wjuwn^Sz$;hEHWth6K9~I%K;rNeLNK?j5L?!^DF2HT@(am z0j-<&5%?Fxtn?X{M|6pBEmC^-$5qUV4F&lF&R#v^pQxOishMA>6HIU_nf4=qTmw~1 z3j=l~jtFZMM%E<9-6YFh+QWK5)=J)ktt}?Sj4MRB3Hs1RE)T!_HykDEMS;Cf4_=BP z7tM*OkB^ZRG9xQ+Ydb?F`P@~H%%Z>KmHZX*q@)8m*J@P4ppYYQ*-fRCp+|Tl=9Q1k zcI%v|2-uUdtC|rupWyt>IB8y1`U=2&F-n2ohtVm87M5U+%`zHRno=#sBy-57CV{E# zQ!l?Spp0{veSfclkxWl2lUOvMROVpIq9cvHg@ULrTOuRnMQwse^k4%l- zX7Q@$NSO~!I?`9+S~Xbrzx!e>=sfH$9+n=xnYk|(9yhD$LLUgb3^LGh#_TeK+7SL; znw2L-UdT7}XAls?`&~h-F&Aw{B)}>#Wxbf)q%3C712`%-z1RYj{*t(O1ki3)5M&*_ zBk@IB;Q@LW6L71F>Hz^le3kxWB9G?JkJi0N8F8O>Y0tq%ePulAU8t{*ge*cxW!xAD z4bZlmMgdTqcR6&ss^&OjjNr)DKoeiZ_?vXgP|AfhNC&x|{kZv-jm`no2lDoq!|goc zJR^=K8uVi=S5e6IEY6R2Bhg%cHi0b1{RSUpZVZ;Z==9EUx7vIB7JE@!P5!}p@NK;gnMk}+A4_7&~DT_m=qsV^C0~I;A)F(;Du_!R9 zU+B2Q0KZ(>TGMb9daHKIXd=&t+sPO?B*p1}?oaaqT03YuJ$j0%-DDHy1$mrfQ} zdF&rp;jxtaeV*_az=7;r{zhqJRl07Kg0dazoK#UC*borX)4cBVzO#F@6r6}^dKB-A z{K8CP*}R=u7?H@N9Vv*=8V}m)k__P%Utw+x;!mG+m%OW%yT{<5VM(ZUo%uNoFdnco zKvr3e)SclCbM;+}h`gf<%CsWx8nV1FZY`d>W)Ie9W z$j`4bYO8zdFWgV$k3vxrEFf=)v5On}oFhomyU2BloHLrQRSI^q4<+{=3-^hbG_KTF zeLBo%hDin@%pr|ToaR=cpcS==Ra*oBA=hOyczs%c{{lxv2#`2%GAKe4_UYN0p<0B1 zAsZ24s+5R)svKG*u_X9vq}W==cUUP;DC!O|m+WxqpZlnA^~j5wumAqnio5_pGSB>$LTzez$NXs6Q22BV?{!%}=>gJmyRki1Wdk+WFP*0Nh( zkMj6sQW~w(+LFe!U_y_MLccDq+xf@8HCi{le&xD)`bp@i`%e<|Z5J=A?cT>ok}USGT$}eOdRq z`L-1ReEZDc<0eUTEYbSNiO(s$U*5>1TR>_!*4;~!OVG^Zk!$EwO^QV-yZi#XZI{jg zyui{J@Rz$o;%sz@cJYJGi`{a&yx@s%MbN7CX5E8NE_0f4czE8if;H#Z89vALLfZzw zwtW;}>y;dyhv_g2*J|ngi#=Ux@uKjAdv{OpI^80AMpvLYY85l_y^@4(PxB!#Ja5mQ z*YWAL)Gzb0P0xa9)hm3ae*RAiBO%@mM(y`fAa2q~l7&_lsv2u5+9yZ(pI%l}f-;r`17hVGGy0i~GZT#Sq zf%CXXy7MgwxY63IWo#?jgBD~MhS-15k;JD8r{~9{mZF9`f*aeQM5&m|{$A^5N5t#w zc{$C+NU~^e@BC`CTwKW`)Lr+5$j$Z^f-+)Er0=Ep;bXJ<=o5g%x5!;N!f z1;EOlgvdp&{H{0L*ja8ZF7I}{DBF(Z1HSThZg4$5U7cQEo}VK$x7wd;V;k+yh!(lh zWyt8ft=2oQf``tPE%17`%3=q zECeyFEWb5o3*IUTdfniYs~LZoMPBwdEGOe^Sc|_+<&w(k5#X`|bf>J8MrKOr1@V5C z!CU;mGIMy_ky)WF%H_m?y$N%M04_54E4ZhzvcXTwmU|b#u*6*tT6TW$P^X(DW;jbnRhyF{yr+Q+3Un~nAO9R_fRrbGkQYu) zkd+QLP|CQi4LT7MrW#%qgFnK3YFDXhaKI}UzHuh$nF1ZlbCaAfTBc@e+=dPgKDzZQ zn2mqJAwmB9BO~d`var@(>3>u3rW#x9r=5hv z5y1RI^i|jl(toUx&gK*&61YfKgB->{*=vD>7#e*s=yi^#|&T)8tZ%C`2(j;Yw+?j33JXCVOSesfKP)WND=39QQ zr%OS~ka2uWlV>`|#wHsyw#!6+t(HSDSOuq+s$r%|CYToi0h`7X20RKj;vS{ln<^S< zweiayX|;V9jJ=WKg9y;!#)MG)Xd$sAYhWheda{sJhYD%UYTVsbTVkBPs6LyBUgZxt zV|{0II7L8~42;ROn9>Od@byx{oSQ~tbMkE6wFQ+$Nn7#*j=%z zhXrR8&na5IG-iLQ10F5G?TQ^Utzp=66&DsLO^+8%w8WC>C5oSFu!x*A*ASkEt(9W! zR`Q{y(>R7iCg8TdE~atQ_vX7SYox(f)29o@0i4}~IJa{SFnTgAG*1Nj$z635Xb#V{ zO^|bZbs{`JtHJZ4TP)Wo9A)xR9 zGM*nZaBLUwZX6;sKy03sdU9@bJNjGhQH-7_jVd6;yL$C zPuhaS00f5&1c#ZDMCeGq{&5=OHdi2ds%&I~@zQ3jci+{vxcl~!EXDZ)e^PF6o6R}z za}LEKf8qICNW9BJf#Do8V&1MPH1WxIRDNbdM5Q0R>#KEa&ya(Ed&~X>FNy{GK(Rx# zqpZBK3)$UD2Mp~>4u8+zn=PAByS)$(7VD7>N7^@~19Ix3_a{Ws7yGTV#F_5BU2>1V;xmpzK#0g=P%T_B`)R*2;}{GFU?;dvBV2tt2kY{9|x_EQ8pZ%)XNW9p{hq=x%-#8<1*xR{XfU^eKjYwkSwvmXzOu z2D{43g)pXj>|H2G~Y0ThIgWY6i zfLzb5?_bZ{Wq0%f-^8Wp5_V%q-(IqQ9Q$W(fA5J$R1=+VSE8_oWt z1C;9CFX#QtUqYeQzL2vIam99^(AM`!X64Z%Y31A{3M znjfCmzj%I(=&fCV`UaB<+xL6}f+m7x49myC-J^Tf`}pEqHYBigoBEGhhRqCXYSDa% zHH7+6LOBApV!Sfjis@Bsb^079Mok0Wp+V3>D<7BHmescdAAUj)-s2oDk-fIf0Zk3X z9bSK`n-~0lvqY&bu1o}|^bF%bas`89>}fyvY-{Iv?CMQhuS}${O%*oNPWCZS zALXPCGrrN<_FnD6{uJha-1HD%{?%3C<6E84NhV48TP>tqbE3y?JXVkBw6m8XQ2Yk*7k~MVkYj8gj_j2&08}kS7K#V97WK6^` zGFESge(0cnWm&rPumDN1p4r503pLep%P4CKSN)`h5{vYLPC=Wvn9A?F&$J>!v#o>w ze%Tl0gIv|d~gn3GO^aHE!aZKN)jPn&vOd3}Fogcfs1rd*It6!Gw z*^VGZ#E)&EpPVRoEk??vQYBx~;Q9 zxtoVcf3kGys)Zz=Mk}0x^`5Hbi6t)jspntRB(Ucs=c*gW&x%2;kGhjCl+e|AFe(K; zWHN;&Zux^&KiQLZTs16MvktNfiYjX~RG?~AYGzuwO0?C1W!mar7jI1o^=rG+gz+o) zN?!_mBiX)#pvZL)>_Uf4QVDUnN!fMB!J%=6GY>DNTzta3sxB}`CNoJbOo3>$4FSk0z!U`ZcewC;{lZnzbHOZOd%#D<>3~OBqTN$}l`TninpOvvtaqdHAU>YR- ziXrHJUI6@_;uu$j4o6T$QE~Yj*~lK;*8b2ZvI~!J@${L3kuqHZd7V5Kflg`5KY1;s zQ^|^XcW0-;0%G^){Rp7N_*BPh(7v;~Zu{gOQ$0_0@41L&68mEJuScnDw0z#`Rd8!C zI~d#|SVIsQ4TDM+9@59wT>Tj8#iC42IALR6Ul)+--*SOPa2LmKNox)H59KWV16RUQ z9*&-(;vo*|3Y&r!hhPOh8CTomw)iCEp@$zy%!MY+*de~(eRAiFAg03%kCm}=0b6Rw z|8gX=Q#1%UTbnf|7jzh9ZGSV=E;oJM5Y(1XSGZc9wK7QdCO>=sBytb#8*nJp)_DMH zd;)?F*n7cfs@002Y(O}v`30d69Q-1d1mr-8+8>mn%+uw9Rb`Aae%X5}lJBrk6TvT( z86OD#E3iS6EY!h7bpjHWRA)8U!D$^7xgRi$HZCuE+r!d2DykO%lDrUQ4!L%A=>{&b zdrDY%>8j+i9&-^&|2?KEJ`qF+>I&3(H(=dU7X{;>as7Q>{7f)~{;qzULXw8u+(dG? zm3y+S#W|ImodmX5_Ej#~_<8aZ017!)6(O@vqZg`;6b~$?)%ZvyOFX^5IGw!sx`5XQ zF)3MEz8O7{3uXt|_=d&qC(S>^tM%2G-VMjWV_+IGdy9` z)6g0ypVQx;NuLvF8R$7->wCm-Qdl3F2cAxUNNbwI^?$ZQ0-P^&QZ-Nkwuc4QhHD=6+XOheXV=qnia5P`2xGLic0q!$Czj>tG<0}U_fS)3f1brp@5<&jcJ$u^)VW7<~N^#GU zqjm>Y_eFzUo2;~kC*@?_|&@}m|_l?yoxI06k4e^YL)Yxv3V<}xUqT5r#wHC z=`@{9um_yc3R%!G>8pNKQ;~M1r6aZGOP^-^lA1xYZHD^x{!URPDlQ0qf-E&BCpw;f zkcb)I@vhS+eXrR+161KYSDb74rpMjFmL+@ViW|T*I*at)Wf43@uAfBI9r8QrUajCQ zan|FQ;yvE@SdbSUio}}81PoNr zaJJpPNzK@hoj~G3f60ai_oj!(c0PZm8A*Fhwi|Vi$lwTG2e)oGmAH;^Y6=KA^e{D6)EssBzj^?Jw|C^-F!O%7MM}JEX;0ZE0{+{XI(kINw0X zkwNs-K}4E9GRbgdl@s@hKI0V4L6&4u;A`!Vm2b5I*)s1q1rw64l5A#jOO=hTxZ0uRP7Z zcpsL#@s_CKvxRQ_@wyYtO%4^U+*q{b7j44cUdE)9w;ia_ON%U>DdJ2ejCv&w6O4`@itcXXSSw1?zv)qZ()b;XeK$LPC#}lQ;~g!qt+3e@oXm zUm%l;g%TqpSzlL3vc$=pDq%yPZ}Hf98fMD*>)H#7)`!XQQFt3x{7Cj$&)eop77k7% zcXHY3eA@ch_S|`Y+_?dQaR;{hTn<}9vqD?q@DCbE0qDcjW2}^%HHLu|VLk|KE^(fw z?hy|@d9()zR5)@!+6s(ORPlVA6Z=bj_@hs}JhcZOyn?jdETpZZ$Vx@_;fk#VGc=5? z)J4$;Dq$ChIB~)9 z;!~_>JhKh8&ZBy0O(j5VLgMJeISC8d^%YF=TvxYa)j2^kzB8-!dDXI*8D1Yw`rK2q zhQH}eNq)6l_HFiCa2^_HQQCFo*;EgNYz%{Zg?+H~BU(hNlr^WX5N~UOg(ORk9Tzg9p7p?ePhI3t95VTo{Sl|P zi3u2Tql^4B>8h%$3xl#v>I3nu(wY*v$3kd&nVrj%|+x~o*ljX_wTsJ^L0B}Wp^Xkr@n6*cwRMC1LfLW80+ z-wB2Jt}1H_lLfH2B)=)C>}_{;iaJ zC1wx-k!FMapJi^2mQ=w^wy6|1$U0+}<^7+mn zzmA^sW<=Cr$+);uxvZ|)OEyXvl9%DsKK?hg{x{9=nUA-JVV4jVy+;7+!XSb5 z2_D(wjg8ZzwKO#wu>uRPL z?sqe=MeOe^AkuBBm~Me5{#?q{il|V^b(-IX48Gzc)2nI@(2zzE^zD@eq6ID1%o!#8 z8*r2pBZq*Lh1F=?W{R49q9i$)w$TeTqOaY!_lkJVriR~C2f<^O*kCnwi%DCd z^4+hs*OZ4MYp;@dB*twe2boSM_k8lLu?<6G&E1#h3(X9`vZD}`5D3W|#+I}G#M$Q# zfya>mCzm=P=(cp;EJ6UrJHJQ3zWRa2y6AfHK9hc@7^}eIH>?p*1BTBsPgKiJ_24F2rV&y}hm>kSJ{ab+zVU6U{7UC-*37MG}w zqc-^cgh%Ezh+pS&w6R(H(3j}#qP)Y$UK?(|QTEfg)U9h!q{@<*FAp6kV4QIo1hTGD zuqd_mL=+2{D}t;=Lf{PuMlzmEWr{{tS9#b7VlFu9rL1r* ze3INmX~hl^lRxIraL;v`pL)(eT+=m})h6u9W)K=3WjsdphB{G$Z2W{n>XDp;Nc9tO zVu3wQ<)!d`>Ra>u<+laHI2I_nZ^t60f-W_osDBkmsZDT4oDr3PY_OI#RN3yD@E)K+Ky9SPU>c<$cQ)VtZBSrU%-lvu<)EcIA#je*I8tEm9R*;pn8 z=vK<`Ax{=>Q8^1AVlALEs^?q8q9ytc-}+tLGoMO%qd-IF0u9N=Y>RMO3(k;%XGU}~cZ5(@yoGQL;1_+Cc?B$Jo^LQ)BjC>zT)H5bK`E2s% z6)l(f@zz}Qu$w3#Ki#J0bMoN~+fQ8ZBdI=RRGlcG*Uj*1&(`cZ0NF5mcJ=P@-Z_Nd z0d)Jl3q;%_eS+*$DgNvg>zJ0OTY{Os65i!U4_uQ)?U5gPjkt8~8*IJs3wH}xk|jQh z2TGsh67|S#d-}c*^{fsOrza}HK;)-H=HK6nFaxuM$nk+1CvRO#gZPIB0oso|na_dY z#7i#;GvNa7-pD`^iQdyv!2l^DfI;5OATM#^)1U#~F7p}xeyP7npyc641%HQoz|>^? z1Nyz!f^7QjFwtjIc>evp=5w|8JG&4$@SXo+uYUZE=g;8ZnWs2GIn5& zuRIN!OpQ5jCkV%dP&dib(s$m2%2L01(kyEUBPxRt!k^H>&K4!aB+tr{rAq(@e!O+- zOb!%gw4%-9*+TGb)0fZGg2i|xd>^)KnTK-CxZC*ZT4`38Ap=I7oFke67!M;}ElzC` zH8bU0CO#?;hvshlrd44o+|xQdAcxL)kIJUpUHcnV6>fmc#D9c87x?qKtZ_?jaz{NI zex!B)se?tCII5IWanhn<+B5X^2%k4ZDC48)OE5U)M9=O1Ltw`|U6#N&mC<;x!p(0a zI>g?&|5ypOr~k}0JQhU-Y(dsE#5u2ruBIjG2RfGpZ1{vk%(VmwwmEpBFa*XCv9U7I zuoN<)Uh?Iuzl z*^f-sX>gDYm@AEAte;M}q~!;Lgdr!CTP(A(7bR#{TFPOHtDRkeRD0I?7He`DQ8O!6 zz~uJPpUlHU*fOK4&Tf&ixREuH$!wR)kenj!HXaDbf2j}FgeUz$jOm5 z2`9AV)~_Gu#Om9D$RDJ_s;y*okNuApy3q#~C&COVI5iH?ZQ$A$0D-cF=we+ZhC!^v z&mc$-){w9CC|>Aq2K{0Qw8)3GTZxk+&dmWN7+Aph7i`{tD&<0=2fkBU6}~Ks)w;#= zKV41P_Nj);C>$#Hk4uz4{8dGU+=EwX4g;G(4TQhJKq z`0;NhsHSqTi?mzWxz78?|N78eCKj>f%!A3nf3wb@6%_9~+1 zO_1UVFZxXi#Jhl}LW9H2F{Y4_yS@PnHn*~rWuT+wKSR464=5|TL$^`sFZaPGC&9-* z4gdVHXB2GS(_v+3$O0bD$wG_wYfI}yvoKuAPm(6M30jU%2K(Eut$8n5rKwy?<4764 zgET+b1?uK2 zN1}euHFy5AAA#Gbif$Sfy&WoPcTQBP9Ke%E&QSFTo!WuTV9=FONo{E&yQ1(qg9S*a>EmNRgrVQ6^E*{|( z&VRXp>r_63=x`_S6Bcu)>9iHvKaPmyl*E6%V0O+Du_OMP>)G?&H}@aOjS${D_2;jC z;GR&i0&kdf8ccgH-aFSPpVu_T@GkIH=o_gd(9rI-*DFk6D;k2kPk0Q~@`!ZJ17_ppZ7uY;^xU9wUGOwG*g-PRYv5XnNm*d>fu5lT(F!&e)9s8(aC86P>2x5=vHvP6*WpM{T=IK>=?%93X+{!`zyNu>p z*67^*vwRqE+oV5P1YGOrwv@XshI}c~u?e0K{)HKsMRWDD#$_ zaC-5~bv1jPg}9caA1D)ZWwwHV?82|Q676+6{cKY!R}L0l#cbpUYiite@IN=3i>XiM zx<1CzeucgCHY2GK+@X}gg%LtHxN@w>Q+4-TYn6s2*Akrf*>4H|217n6tx2m3fVIuu zoSr%14gmUj15kC>)A%Qlv|5mR7ROrBmG-rAu(`bW0DCovyX_y3{4!l!-}Fd<_gIIX$~1 z@9yzuH!RZ;La3J)>0`Gyh?G8Gp*m!6dZzxLVva09;b(>!59}>-JH*i@#wK&fsLHfenDqt~v_jT(Zy`0grYU;3SD1=fGe69gv5+TN z^1{UBtf4)+bx~zY758-O(Lh4)lK;EwoS|GBV8I&{|>|2 z6w=I~slaGU9wcvnU_s+!msh5Knnft7hB@AmdtQN2?IwAmFJRY5P!e$2BWEZI1R+2ZYO zo?#Sl#m-e`AUIm*_t(zgfx0*(_{L3rPElT2>~Th8XbKqxb(?8LF|IP^rzlx`*Y9u& zw*o~*!eoE5)O9==%2xn)VLhKi1)IUumvsT3IFcSucRyw1Uo*N?;>OF5mzM4fzjGfH z!WU9}UlLN-OgVEk|NS^`1-^!M=_o>2w8ph&c16C;XK8XeUE>mef(U}+k$Odo|nX}fyq z;)8PXQxG1qWla*jEIFQjwdA=Gf$GeV$)xpnX@JZOPKENfZH%qxLwt-1h3iBf>Jy^8 z!$|boym3u^N0t@nQMMr6iSZocBgtV}uJN*iN#K3`CH}Ou@cyyYlpRdA{~Tq@1h!a< z(69QMC704^DV7?Wf?C!bc+3*d4-b0(i~HYEXQL{{I%xI zEN~ve3)}cQ#0_S4@Y#pCeJt`RxXIWhEjFRLdrn_?7Ag4?#d~6cxTvcsDtt^=;|1l2 zScA`xXcqTy#1&Jcu7K7J&Pz+)l}4Ca8PWe6xjB~nE17^;iOv9eb(&LYW!mkL@C^!L zv1G*#z&q+b>YnsR)?|;=iq`#i(V!ZOSg4}X zd?ALfDk;Xi4!>e?q#8WdYRHk#@Vbs|2!<{FDU1LDm0oj3j~ICYOCr_+Ifz>;8=Q?_ zL{T&Ymp!>BCM`N|0FU~Zd2p(JPLpxuh3#~5aBN!e1VtXUjevgZI+Zsg-zSiN7o5Ttkq{*7!=Y{GETe!wmpv& z;(_GsGH|ke!M{{crv@0KfLF+KMb6&ppYb005N0LV!dL0^4G*C9LylU=;IhXb)HJy3 z4sKtwU zH`)YtSRq^7l(JkEU!0M>lIYj4Zy?$Pa33y$5WE{q2nA#f0q{D~)^8T2;u?&y8w+TJ zd}^|Gdytl^^R7-V*fa(J!|wuIZCz14-y~PhvNPJV_;2PQbIGP&;ufD7fj_)bj*}$I zO>(2$UekO8>#0yK*e@7yGajM&*%kwt=b|+TZpqi=5V*J>As{|LM%Y%iFSE58vTV^V&B`O>K);cR7CJWxtmG%k(e2ZVc z=O=O+XnaUo(L*vxm9z@Q0e(5?Z`3o{6h!LVX1;1hh=a8(lVLAVKa0+|z@BL4@TPOR1&PMS zx|(Odg@iOl`r()z{LsXl%)tfvG{4XuN7Jzf5~_`BHDxSrDa#f!I)_+Hn)0aWm3?L7 z*7!OL?*5J?qoafHR4@k?71L^0q@1MF!P8EN?$&;5A#gc<;f+&|brE8D(jsh;JBAP8 z_Scyd6^}AelX5snpnN4+e6vKZ&Gt}I$>567X*h@+zpeM%k6@SVi9q;r4o!Z!-*Swp z$mn!;5Y1?@ywKf6cB56TTgOYy&HI&zd`NMEu3A^gVNad6UHBe7-xK|q?S}vqFgXpm< zFF}fIzIQ80-AHU9#k5YsQP@eO-H~Xlz~rVi^`S3_kqBqlhGb{@DiHF@Yy4`-kmEMo zTN3FKLInL|@am4|Bp0xkT-c0t!xbBlqi%^y=^_N#Zg>%L=1oh^yu=Q$B`yN`%C?-A z5!UX;kjE0Z9U<(TYh)aZLDtzmXF~A zoumoLY3~n5RpT_E8z`I(Ad#7j0D^PIa3-}liEI!|O(vGs!XjpBA33 z!)z;~Fpnh9KDu;6CGoW>bPa3zmmTTA(a5eSCmks1m&|u;<5+!b>~ui<)`F{ z=E&+kqIp}2yiDZqYy?yJAlfnjme5ZfL{gjnPpanDz+WYmn&ci7WNxW>$u?HMV+C=w zMJ$n@pB7a%PNh|K1*BEe6X=PTQ)ax2xmiy=1ctrAmvh49t!HcxO&a4yUY@@)lyIeg zC6Udm3O76q|Ap&?9|SwMfM$98-AP<3)mh8}$3=4)j^2mOWQAXrQDGag|1Eu%Lo5=a zxt}fvdi}_EmgP$Q_ae+yh{yNZb8Bhez6W;shqF*@9oB<~X2f~%G1K~}BxVO5sb36D z7jq0SBneD}MUy25-HfS<$wF+lz%FL}?^@aiEG4uO%5I3FvfHg+BZ%qsz+Ny(57M3h ze^8Vc8RmnT&IlC7uIOnyj1f!d%u%JApkndlnxtl9e%)TC8{=$I_FPY>wQolNG7(4aw**KwoHVV`gmq z=ynxt?lX-wkT#Qs^?79qF@NbmHfno#-)gc<$M?Rit(Il{u>;)Up2}C;e`LImXZ(bz z>2adO4&2}UgZ*Zvq{S|j%j_1;l3)Y8LgFpgaJ->86D#QHy53>*@4Wv{U0)p+)%L|p zcXvyJbPe4|3(_DUNK1D~3?U&y58W}M^cCqGx{+481%v?xP*6eM==FCm-1px6vCls1 zthHn9edcq{K6`z?tzNM;PF(!KH$+c(U+W$eeM-OIPBa%(o*D|QXz2U26qeyoAuzwq z5uAHMnrv89U1r`tt=C@TSQC&#Au&HuBj1^8Ty|As{Z&t#K_fSP!g?3B?X?Vih5GiZ zzWU9@YY!DBF5{=A8Unh?;1-(V#w-dr36<4--hm4Zi+r4S%2^Va!=o?PO^Q-eP+cVDu$}Ss#&RUI(PlziL-#O2aF}dH|I}nM!=twm3*$TVD74-Ek;f`2Yuf6 z$`07dv!+`WG+JoU?|XhtF1mygx2h-7xP7~1BvQ8Cv=6xPX7RyQ+*N|fUI?rp_P7)5 z*`*8Zix$d0yqEG(+#{KNeVvJyPMRQbHJaW%Q<3=*O{cU0uvz;uu!)Kic>Os)CUSKr zz*5_|qDeR;ZCn3-(uXP58n%12F@y740@Lyb0jPkJ{rVKKDL%InH#da`E(0&CqA*TY z2hH}F-IQO{Pa&)$FMQhpzEI9$QCV!=4Ml+ND4R|ht@-!{c!OV3PcKU|Fy-{@KwqP) zVDymHUnUdCE%&~ZyBTFbYb(E@Zlng=NgZe=0PLEWbDy~{xq{WjfQXnbW{jMVC0I-L z9&%2*z;LW^8LE1}6QeK18;TdQS;mI0P3FbBGYQN1nm!@*%v$+XDV2cFVE2?VJZYrky>gwh|#J3)t#|;@u!m0DS6(hsp%t17@ zVIb2~8c2t-+cr5}l*IVCEn)4pp`Q!jX?mWvkFTE>#NA-o3B=VOv6j>G`XkR?ETQJU zBqpQ|X_&^s=3s-T&FD13_EjFBzE`5$G$K&npyPgpg-(MTPP+%-pbR?dJg23=rEBxv zH#kRxR|Pu2&@}6i7bJ)4v|N6@{56zj*~DwYQZ#b&CVAZ}dNyE<|GJ-3roomX! zx1m@aQG;iKNWr;Bc<&gyynpRJsX3y4SKU2wo3_Wee6W=i5iKuI@C)Mq7k&)18~+c) zf4b4WCG7`tnaB)cYZGQ0si%M09nvsileKv+Q4Qjg^bIMj*S-3vexgRx_i;L22$azF z+PBF^YZ0QAE4q?<1W91ysE1$t)N&2&@V8G6i)H_I`l(aQU*e+u$5GHt=mpFlDX;rl zK-;F8-ZL%0gixvfi-5XV0OuJ{hqxGCHvz7Qb?DuL(jdT-vzbh;8hWud*!kBst(5v; z0?*;u0--p1<=veo-0NEGrQGzer zV@~Lee)18n*{H5L?4uL&N1vcl0I7O3ncC@kl1y#}nJtJoXU%*V`(d>^Q+{W0PLr($D3Cb9IV*&nu!s zpWHVAS16RaEmIIl*E&@IeHEZ@Kf1wI-lfvW$Z@1V$&UnBN;4$qm3Ugc&WqrW(oML^#X}wDg_yJw(bUB5tDUH7o*-V<|2*gzHp(eDb(v}N-NvC?L ze0gFGa&Ks@vDss(rVJe`ZL6E!;8g`7E94I~8Vp_SM zT!topq5#>jlvZ~p= z&+PZ6CnUZQX%?Cc*}hv6Pr52(-w{o&@_s~twZ4D7|FLN_s@bqPVd5U`h7o0brSbx^ zfB45a5ik+Y^QnlpRc&4Z8Lll);70UaEp^Rqdri#%$z{LE(J&}wdO{1pZvjOLKIOLf z4fgXnXND_1Fw-5iRwT{AwZ-KKzH4-TIg}o<$+IOclc7CBI-Vpe9siE#L@=j;N#QK% zI4g;{XC-MHOHBTI+<8(CUP98eOO#LWIV|x5Qy;1S6vd;}sAK%W%LtoKui=~tgOiDz zt(@l^j%=TEHUu9cCH7gNscy=Ctl187CpUpp3GKSC=A(JdiCMFcr};Uvs03Z zU-GJ$TC(X=eomT_H0$wsD%_Y@ z&4oP}6@VwK+DX6j{H5p-cAQY8cAa~Mvpb6^zbxzlsk#liF=r~}h1?#5gRG0Nz4?6# zknP7IM}c(OE%rPWu+B_`kKfNZ$wc3n|hiR;SO0bT~6EFn*3zaYdBs% z);gn6zIE!Jhag2^V7Y`my_r`9T< zHjmlHKt)^YRbx`G(!~W*r$%={$@-%i9ts&HTEB7R;Gu*Wq+o`q0Sw~k8tw0vK5_cp zs`;l7=agyb$gh83Xx>3b2+_&@-NGSnshnkpKx2E14Ln7#I7*n z=^6YksAc8mh`%ZG>ihK;2hu~R3f`swcdt2`g>o_dCp(i^C^PSwZN?DK=wFJV=?}xl zoQ0f)$m~oUiqh~0fei_fInE~Rk&T;gzv}91tr+?@;>_S}Ccx3hr>K1{B@D-_-mt}I zlgl0_d}0t(6Lm{ZtbaLNt_MpCzZw=lHL5&;<1cZy=5~3Rzz?y2<}iZ7 zv>}Uerg`m}G$73r%AJ}5m)9&B+V-`(aT-4cM?akS&zQUQ#!qWMmVj<>xoq~54 z8_kh;J_Nw3*K^}v*w7`5ajnp%Bleq3 z$*oNJ{q{;d$kY@wQC4iHAu#f1XY(zO8|v#&nu=7zEt-MV@+gvIYAMWTg<_O}{QM)I zy5ENG^)UWK-n=AyDNkzHXGq!+u?r4=gx)E2vJYhe?Gj^#pit%0E`|n6c85fVvR=@e z)NHBHL(Ek*>22NV;qz1G%%ruw<9P;{JC(*xNUryWp0&yM3<$c=4659B@uj83FQ)H% zvq>4#){F{lW{3__upyV}&+%R>`ZBs!!npL1SIRu#z%!sp1gr)5k2^%V8AqlDi}K! zDXlUNCQ7zN65>O1+)^mOQ7{lxqa_qd$jK&3Hb4TN>R^#N42k1Nw=QeUMJk*wGoqysEQJa66vzFru&x!} zz=Z?&kaPo*n-r5N#Y2!|dm`JF#(xkIeUH-JqJDIC`XAc9Og8~k7&hYR_9cP_i}Tmh zZR!ao*mi~ph#gGkKz{S6ZrCMSosl**mFjZ_hMFjo!U+B=EyZNsmNB;oY^S_K?bPt` z2|vFKqLiHM7s>P?IW(*l0myl?_ijYM)ac|L9Cw{JuK&SM~~CY}b|0+C|4j z=Z#e7#p(sj=8?=LQ5Zn6Jk~h2c_ztNgR`gdLA$9UHZW0*$b(Yff@QNIv|YRJfQ@c| zmNji7frMf`HdajCB$maFHBbz=I#yWP4rln;9_7Ery-^)NJKC|5<*bkA-f34Vwyr+q5ko5u6r zUO9L<2@|-mtREU25h6K07IW!s+DDC@d!o)R$74lOxG7VZae^hwvZ0%2XZc?Jl80ey zVV5Yk7e8hH3;aWxgy^8w?--?gMlhWq|A3&NdguB{Gi1GyN&N~dCnr<+ z1eoW*EQ#y2gk<0oL5>C`&EzHWMo3u`Jcm`o=DC-7F45#{H7%(tX*9{BH?A|$sU;z< zZ7?8$KxUXKu6$p8(Ew0G3vsBW5pHtE2=U!23TsICww|eYC|NWhRG$D1&VQcAWA?F{ zZEkgJHp}Tjx?m$O;21T2*z49~wf_7o zPwm3fSBsr#NEjy@os9W>>0`_9bD+q2>y-kbV&(;)o#=|bCTGW)Nz8;7Vf_hO780f@ zlj#9n2jB29n2sreqXV~3|;cUc~WpB1!G(nd*=-!F|-jGET#9}(d6P;{> z@y70OfpW74ih&&5MJ@0fbyVF3bBF-Uaankci60_BX~n!Td@$(?erTj8gY%=Bsto94 zg1Mm|j~%Cy9z!6c_>KvE)>4BBJ!A~^hB@%q5hrSaVyXO5%6nYS;0Y_;)7ocw?s>aR zquS~WhlW&krGuTL%5jxj8tk6MrE>v{QMo;7gI4aJ)Ml^*_klBZ3j&*tFA(7&V>*Pf zxSR{`qhl|*>?(`Pe0mS3rX5LgL8(BpDPfg|4Vpfl3dQw8?O}E|ZIB^C9@H3vQx`hHw2Pg$4T7Z8sveK+8ecHr>IBq zwW!%^3_Nm;H?zBn@84E3V_Shn02bXm6)+Axh4I6=73dqF>&T@HgG0BO?G-W*61qN>vu`f>(VMA=&tBF4u%)xZom<(1*Unz%bj775&+`UWbee5E3U~b|I%?8d(qaIqWitb7Lg3L3he zFk_f=5xf0+TvoKI&S&1hyr5k&^D5k#s3vzin@+qR`i35=-pIrNm^yFrPEHQLvwkMo z&?l!%lYLMf>ms1!m5Z<5V#i++qRqW;DbxG*$GPpfiT7H|AHLtcr0G5Mqj#k_t?{rf zs@c>g3^*X%kKj%WjzHAMiQ#H`sEM*A^~a$PS5U-fzqtA z_E(dTDd{AVb<$b8{Hgew&R@`V8=iZ6AYT(%gqiwSVOAGtRWetVe$oEWn(?!z=8K-` z+PW#GW5;NNTaj;*Gq59dUZ%!3cD|#=mm4LxV_F(2Rt1>O*R(;uenH*C%Mi~b~9Mgoc*OHs=vf_fuk z;)P?^i#+U4gBuM}N_jl=IIYmit~i*!KO+%`CfxAFRNV6_Kj6NT=W;*+8~5ytJRzM~ zxd)}|3Gq%aCkO|{LJbPiE)kmbA_>-a{ArLa1PGL67x)ok}))? zz+zvL-QuTsf7#OkyK_9P$scp*MwYzBhGb2cKS+rs0Wy3bO26l=t%5flcQKce84hiw zu9CyOPiB(YAH6dF*V~gT|LN=)UFD4T`M`Xem)<`OlP{;#CaX5xnBm{}Wumlted=?A za_+a9?Ew!2)jFhW>Pn1NXezuHYasQ=f%+<)hB|?0Z{#Z+t4iM%9NNPaMoV64e}%_) zI6(TTUIShH`-^k7pbA8S!=95GZ!-gAQ&&}Q&TG7w>!MY@4!alrXovzd4(>Fq!Oi4J zTHkezFIe{cz!ghs5zX8deeF?7T8SoM&~shXKi}#4&fU%%yNy`f?sQltINzv@cq;>&pPz07fbcjkOULeO&m_h zM+e8Mw{~7M*8wSW8&v!3n2wlC7p6Wr@KDk55 zZ_#vw4+8972#m4SaEYb6}_h2iLOGNB>y?H5VxS0GrQ3t zw!pb4%@RgF|5mROSqIGy8&FuL*{S~(r4EKBG71^$-$W0ND@@2_DSXsm$OlPC;hi9hjZ3qJj4TU061&Qgz#Uv%h`V0Yw*n$Im(g{c!QG#EV0eR{_o{( zKjNfcJpjknm*yxLf=_xO|)LmikxMT+&>`E3P^g5|Y^ebP-2L_=Y$_ zT>=c;!nmb=hfDsB`jj+yN=g#kwT*GBt-qP%!G$~IC=;^3D@PE>)BYV2sq^=^{ljVd zo0mKF6FJJT!XM4s&HPP*jObM}0uff|PQ9%Uemh}FiTGFDx0-r~B=?R9f$DD)0TqV- znA{=By+UHcPSOx33{K$VXHsB-bwoJn?^#N;Pk;h<1~cwc{Sllvq2c}A03sxq0+S2a zV*mCa-(fDf(@@i2s&vZ#UmlbHy8XXA2>&Y#5^nE-D2bNh|4oMgzF8x`N*%pIJ~Wo`c_h=DuZr z?>08@9eaf!ggpZSD)_egZ^Tc;91lg@O(J*H$AMta1L<2O|BEn)gv4}5wIdQyGCla@ z;8&}z4_Ht{v%njv!vBoC`5_Amdp0=yPzrIqJBRwu@cr@uZ_)2gX(MBRuMdAw(NMuy zP~auMg?g~te!LS!e5d-Q(%ap8t;go%p0X zb+J_aF~t9;cUDI%C{G4{i*t{D&FLD10DJhi0NTy=e-(b`ThyJxVIzNx@WE!szkCTD zx$Ub5)4wktj?jaor-Z<2D}x{F(!eI~^_3c2h;C4BN%%wM4{G&`hsv7%DZk5%b(wVxUR7hDh3;g)F$NKg)je>(rQ%S5ojB1!-c;Q4s_}T^0c^fB-6pnC+5QRsQ z!uw7+But7hfp-+wc1FuqXnkrPNLr4wXRsU3%t4nsSzZt8ZK1TwEUBW6X-v|Jh15x? z$g&kO&p3DG`rH~sw~D?*->F8-gA*)f;eZ<$oL%iStr@@IO@>VN)t#-KjG%jlDwLYH zXy@MLxId-^Ea~3dJ7f5Gtj^}i=h`vy$$2~ATHG|c3ix<&d z^@?p~tBiiMVQe}>^Ews-z;OI;i!%=^Q9v078P6urO^8>DT0<aBuUP zGRH$2`WVB%Xp@MDrZs|`YD>OrW*U2bQL({tf~tjLEy0EUs2r_|nt0}Er6-WJkx#`8 zuwura3^-xnbZ}Q!GCjWtCmY(&)b+aEG@g2F7Xo>9on$`pp!KbtKQBzltVAUE z|ITNhuFd@kT&S}?j*xR_o>xlpTx#}Oxju|dfpFI^25=!w#Ve-ioxfCVU{aO zHLoop_tPK5>ep2Pc1iJuJnk)vdy8?xB7ZH+!?X(xV7$fRn ze|_?UNnAzVGeD5CzK9{F!+&OI43Yd6)F8ppMA`hq3bqj|Mt0oORonuzZC(U(_?)X& zAJZ99WScZcF?lukuV`ZNDl2I9*)9cKHXM4u=veO#&ImdH*Hy{kovrwS_1iXwsirE)*ohdM-swhL$d|e)<{3 zyk|)t{}s>w9bWAo#`&N)QW3yG2}1;R?9-32$Ca_Qf<#CQGML^uD4J|k{FamgOJQD8 z#fafbMXAou(vKz(vM+|2LPdt-4&t>iwrQ;?r}?NqgQ|BX{fL&(jr55Z*RR zf!Xjk{Nf#oxHB4jY16@e3I-xIzA`*Eta`(fB3;+885Zq(^O-6cLl3~A`hahhoQc5G z!)47XkJMucEgpz5@#gp$P&1vV|5yb%M>}+H88DNU@RlW)R!mtxxWkqnzbIz52pn_Z zHnyz==n45B^5-d^C!@CNyZaQIfNZbg2&3>QNF$4W(_Z-J_8B#4`7`}3ODc3~e$47S zPMeaL(Y-4rw|y|HMg-uP?C3gLB_fEG#8LSyaecF0+36VH)rV|hTqCvbB$^vm-uz5H z@RS(*4wN`E``ENk)%E7>M09AsV>FOAYHbVm`gm7x`?UCaO{I5F?2Q82~GSeWL}BJvla2`h;Q zc67R-IwilL3-BNF?hfE?q{=c@(u>kUF%P7^q%|AL%LiH|(*{+An(NSu({UxX&Et}w z9IbF8SRb!(sS%~j5uzVgf_Or4<{7pwsp3NkpXnB3+ZgqPjo+)x?rN_=rWM;^() z)MTD+tPTigprFRf&rs%vd180}vljig@50dcQGkLdOo9IOFi6-AOqg;HQM<)#228Di zH1@{rBdmAWfEf2OqGYz7KX&Ce^HMhDeiSg5>m*AP^8boLo?zE*;AYdN@aNkZ4w#!a z#UaCDxwUo*YZ!-=W<(ez9-cmuDc%}SUCa#pSe0@Ysn{sr*bJDX%XXRz%-2cWerPF0 zN!)BgA0XZj@$d7Rq#)lAOIo$gvHFIp7oD$cHEv~#Zc9}bKkv};O{JzmTVqL&c}7If zw6oo!-d_(SsqUqs^xRF;#8q2-iDo zuq+>+fCs(PauI{<(AXlF=~ok2Tt-)=qljfc#WJ;_IFZ8rh`bdxP_JttVh}0#?#U^Y zFYStR4es#Zu%%ufkAjHOs+{Wd&cl?gPb7j{xaxURrIeE}rDD7uK<;x)G*|=j>$PGx z1v0RP2*sMy{SaLXSATF!;w2>x5%DdB_(7ep78&E7@LaP~C=FM(|M@n6EwultE`qj& zh{i00B`|D-7?YProY7@@Rk=aQKKIq5Wbex;WEC^sffT=X!(?2QXk_5R|bmj?rvH zG!^N}vPGacH`qKeJ3lVeT#M0-VgU9i0oe^-_xyrChUJ{U28xl8$4U2@tzzZBBku={ zkMKBzvMy@n-Ix{N3NikTWtRXNZ@&}@B7SrxIJ5GS^(^{cbYLU~y%Tpp>XBt=u}-G3 zj@FS5)R9kVRx;{)6&QJn5Pj2P4VDRE&{S)_CzTei#6>rE!{kmJ=HM))r3R#TcgJ0& z34@zRQG=Uilpjjy7p94>*Gawp4zhF|@Wgk}!Nl{n^q&xp5vCJ951+Hu@fcbCY-k92 zu`!eg>NSoSd?1y5Xs38TLOfVTo`Y2@1QXBihF*ZOa>gmjpWP#$h@7+e5KaF4uXgpn z6Tq#*ExOznFlexwf93WHZ@JN2AX*9wEa38w%(4m}w+`(T5|LgsWvXV| zcB_yGdKq10g3H(2mDrOc#N*6c@};Ym;k!(yr)7|HT;2Ct6+) z7l&Syn)Vm{3OhyC#n&X={L=te(GBtOlr>n-V2nXpL#blNlN}nkumT$1WgG z=o#aEY)6baU5*7_hBQCLzc<5%fAi-JLQGrr-y+mg1FfW$KFG)jTBd>twG(9WR5?sx z)>n`*$@<+_F|SQRuQApU?_L_PMq%2~W}OmuD9;-sy72S#Ug7?Cz0oW7!&!m`1EWF% z?Tb)@+Aj!!8SOJK3=PcB9isY||Fw63D?7-8Dv~^Dx61x=Zc=45WA>`H*6xWcEoa5w-rZ67xpT z7}(@U;}xR@e{AHOE!0g}LP+sg?LiGhUJo;ZY>xdsvED|IFYHIbY}?;qe0-z_hy4G- z8VT!0jNJx>jb%Q(}h1+HjOg`t(UI{4Ek87mbDcFrawqzasl|!IU%uf-Z*0Ze6m4CS?qj&aj zsPnt9#NYg$cC3{&rD%*zI@;!G3lZ$m#EYGE<$1QHuPR||a}{ebt9X9>ud8Y+X>Yv- zH0OAV2&K<1OVOE+Y1|mwl-kyyN{bM^_ow}Tl_)7Tht?$C=7gP4c9Yybd@C{M8MG(8~;68b2%8bFDeDi0t3KJ9lhxaNtkp zG3u$kz)NIYF`YF8UIG0>0 zmQ{?rzFLqpk8y4}rFlq=wnMY80ab6JEdsNN)g9jg|Fd5g&Iw*$eSRx9;!v3hczYA2 zf~hq&G~~JXgY9zFs3@fZJTnjRA1tuzoAFD_@rcZ{!Yrn4H)SF-iq zocXo?H#~CV)7QNr8N-(;i2+j2SE*O+LFzBIg$F9mxKfZzLNj?Z^8z!rdEy+@C^aUM z4k87CRGf^;f<2lN4&L39n+7|pfc{ijuu*#flx^9<=AT&+r1VvmkxrxH4?mkcnZ`mh zTag9l33ifS$JwIO0;Sn9ZF8U%MM25Y@_x<1 zyP$>cnWIiUk#CGXWg%?qKe`+kR9`}M(H#SUiq{Fes;BFAs<%s7@IGB<=*&z2(OP8EE zSgo!emqGChCgB1cG!&jK4y7=w1UZ&&R4zBGnZ~OWU`gw~eViG!&6MTvdZR{y%4axI zM1M1AMW7OR_%v=)9z5{@_n(SQvs*UEA?H!E677g4x;FmkNAMw+1z&|Jo3*4yR4^qH z3fo_3M9FEQMkP_>R};QPbH$RN=PLCtk`pIj>*9!h8P=mgb1Km9j`K0)t{NuJp?)@v z@`lkupM}q7uNzzm@-kHj_cAqz>Bg|ryUf+zMcrxJ&-&ISND^{K$r?sxCC(s zf498@*BvHpj!6(epbI$DLFoJezn1J+eMTs{;Dd114Sf~dN+5U@Oi7`HKax3=F_%0x z&Wk%*V^(o#!JO{_csHUqZTR2O(v4Utk9?=rzQcnG(ei5-r6dtF_+m5D%&tn2=t)D6 z%`!BSj)5!jMKa_oyim{lcpGU5k<~78v1LeAFZtZ;g)}zK$6!-ziKfS^Il`$RXS7@Q z(CB<`EW+#zq=m^xL2T06f{w7HCurTroyTa8pW!^3Gxi_^fcX44ZqgBuw0?wZ!UBD3BJN<6`A(*_PFzQXa3%;949&C{Q8`%}gr!rbu( zq66L^(aB1lsy{tdb7JnWeClG@QXd|?_lB?q5ac#WxTdgJEH9i(b(W z#W4`6yr25Bbe04u9juN<37j6gyh)=(55m9pqgV(i>HP|#47HH)nq6`WJZZVg@9PVM z$QVeD$Asrwq$$&(qxDdgg63Y?NJ*ZQk*8)Ao6lj~bu~wCgAHYdcuRE_TrvQj!ky4# ztyHtF8yN-W9$}j_#%j|q>MAxYeU@4$rxc4x&1-FC*dGamzvvv$crn_%y}&)Z8G?m# zikfazx(Go`I+t#&v+S&y4*dcxX;`VP+YPp)5ED`T@xm_We;m9QkXscyf@mYwXqgv` ziOgh?(^A;o zVLR%hM){dzmJ&k?<&czpnL0ZnB1tt<8`3F{g)uTY^sffvJu)V|_Rs}@0vpb)hY`)> zp5ia%bWV40Si|)di9Dehk4dpwT3g>YUzRJzEp-#RHw>p3;^v|amQtm04>!s9T|=9| z`a4ib*+6ncMiy#5V&ij43+|^?baSM3!z0EWk*1kFGFL)Vd8E*Np&EwOt340Lm`)n1 zG`!p&{>UWQPF#@|QyUy0%84;=^hIRLfv|RD!I>E63$l~Uu+QurFT^7bK++r2yLiOZ zd=u%M|3#mP!+!srJSW*b&6L4t)EcrEhcZLcc$U=%_q~I0%pD=i@i=2>1(x&cDO$9f z|3hCur)xR0^uijLRagWm6GGl#3+(-O#zT{#=WqOx$s4xSKYC1f4@C}B-HEtEqN%>j z39KQ;k`4@2_DfeVYeR`C@7BZD{SYbx+gKL<->w{K_{9Wt&8Y`lno@#O?y&d`q{8(L zkhYsoO6p>bBR<5ZVyPVXra6)Vjm1vqif@{sp`z@POKRwmrQo<0o#wz6i%q05w*pnq zIj!Gfa-8S7pVhJ=oIx4!{bkX0>5cdlS^sxI;;F?{Yd1e43U$c-z&*$U+G3?rr4jCI z-I|lW%zKm`=^ha?m(D4r<44IAKUO9aAa*{{YQ_6JiHy^$J8?)n^5n6_HDVjuRVULP z-p{bqlX+?YQut^!O{VM)KpdJEzrzA%+>jjC+$fc_Jp*j+dBrkfI_2Bxr5Rl=k;a5b zzJHZ@4rK1!i%lroP|7}EHS4|7lBVZnRN-7>uqo+OoRgLyf}`-r8O@0g%vp2+Fox)U zd2A1cL`x9KXOq(BvTSarqwHsDy+zlay_H3(OaSc7*@w{9}AT9GNOo%jb1A}>N z@_*$VG`1~pQSu%h?fv45)BNU-lL`jP>+W=~1_+dH*_@iE`{Xq{va95B@n@?wmOPf+ zd^$%m&9>!u8}vSpKZx77E(~k|>JxmVQCjlHj^#5k{K0}8Cz8w*45d64O#C8nmK@S* z>F+GHGUQzdmOtuY@zlCtf&00TS=Ahif(BNKb)Jav^dj8c!Y6_GpE7Z%E{6R-=DMX@ z#U)}dacI6GzmZ|WOnKglBm0oGf0x6wYL7+Bw}J_9P#<`WWl?!rnCvBWx<{@QtjJzW zMw=9PCyo*v<_b{zy&kAnSt$dZCY3Vdd5VqffzW%cu~`jkL0qZcjVie%V>HC8izG{~ zIM*cm-4wcx=Kj;nI180k_kqH-*dXTA>3{njVk5|~>t713`jiD=#jf?x3`! zj2U>nx}d^GSP$PDgt!AA%JvO48kT8+L8sq5VmQHqqp8GBW(y675DsGw1SgN$Z|WPX z$d5f~MN;IVWiptX3Yc}f7Cdw zjX#c>Oq0i7V^|H%j%*drmrEYldgR7ShO$Tyq2Y&t9;&UfA>gn5)w|!j@WObHsF~a8 zcy(4caWzi+dLy4e+U0kY9dF>7CDmE|JAR5p%YMswD(%__nl!B{eoL94F3=dyc85)4 zkjvwDP`OWSSKhv&(eXU)u&=R7kyf}J?4UFwaiRM6CAdfQA$9+LO8+Eac9X+r+4xCW zcvVBLjlg!sOQEYU+zxUAXA|4&1tqSt-orN~8kP}kY_+81E|20TGmHE#9k zIqMgw*?f#U{ZZ^zo7+0zImCpX%R=WRHu?&$)$09M!$PJX88oRGA~P=4Vb!S1A!6w# z@!vjVY|nU;cD)Ph=)wWZlcOqxYWxBu#rlsG)iOrOb(IcWg64`*rJ>qn z6jf%}XjGSZ&K{@ACe(%nHf)loa6gg$x0`1&^c|LigMk%8fPwM+x0|B~N<*Lq;5F$A zdV{0<&xntgf%>u%v@i4x_*DEig!p*&y-Gc_wnIaekYHFOW|Tq$Lab|8S;;aAaQ`^j zFdyIgu5A-ZejTB{HIjxzSMUe>I?2<;2-_@EC}U-1Y1R8?X}Ki~03XmSkyZh?L6xA4 z)g)1sj8Y0q_wArk6V1qoB2){~U&-zCrYD@+Yqvq9qoI6ao<7+C@GEVqr?UkcDqhRT zSo(El{7z66n`ka74Y%w**CjV^kS|pC&W=mWcjN9dHFva(sQWOxj*;$7C zQc_QG+_+{kgb|m`$S03TU7t9DYe#n~P`T|KyuPDV!-J^$fE$0iItty%2X#~A+FXb; zv1)4pf!eM^-Mj3Rk432 zRuRK6q1d&658~f$-IgI1t-SowN7;{%rPIy!-d{J68ox`)(^$5HS;$op^+iX&VTXze za3xAcWTeYNWB(Xuam9Xli$7ew?C-~5z-y0Ug_ayxV=3DQXvNM%yYbjQ@3gqvu58^W zJuo>J{30G#A58!F;@*+K3-c^s4=Jb5U<)mRPG-ukR=zv71tDv7S>c~~Gn3(#=Nm-_ zcEZPjEM;^xV>4#kACW?c>2wIpM^5FlBfn=Z{Z^hi5L(ZZCrY-V>sPN}h!;xx7Hxw| z_32HIMBWY$NGqh6sA7a;F4_=Wu@yE4OFY0V+y4QJa>D%$<#329WAdDozu*G)mtQi0 z>0Qa-QZ-QCRTh?^OpaI*p>5`nFMyKW_n=I45#fL-QVs5X7 z))E6+k=T_+yHl3(O zbBK0f>A31QSX$D=e+Xn%#&Y7R15I{g`(^SD;<0HMoJ+%XsLhiu7_7MA{XAxtX^G8?fb| z(KAH8A2a`e7n^mV8@PfdVvOckd;P}PbQ~i%#+DuLfc~u9?$@}%z7La@-`A-cE@CH% z#+ggIj56H7LbTsHjjMAugK6hJ8dH@@`J1?Z#_Alp6A1TA~=K%gq-%jP%vp?9;&|3Sn(sLdM7a-lYuZB zX9LUwPiB6+=i{8JHfS?MEL1O>W}JU2%~QHCv|(LK%|$qFn)k757kHJ2nj{)l!ZU@J zT7M@4F_33x2*LnT$YcLHjLah$^KoudusV|z2GG4b3J@XcV_fYHEfD~r5P+buk@1^T zSH+Eber#Ed!kgNQZU=*6c^0OEd7T%X5)h^4R~L`M-z}Ykeg%v0hUcv_%R(vYU!r*+ zWw?_Ux8`-+q@?H}Xva(19%Q`dNzxnSaa5Zjyl)rFlO;ZNUtBjgGWp@X!mh_LyK~J` z9i}y2W+j(jFtc!j2$9=oog7bVZZ5FF*eIFi1!bXykQE1x^9Q3nl6F(eS$Oc|j-I4( zsa?@C=bjdmRrF_Am=&~j7MU#WC-VPJ&^3sogixRvaaq8>UEyR5)g`n+WhZyQnrvY> z%wCSAYkdcSqejlh?x&~ZCy4u}^+C9B|5pnErzt7|3Te=@|0D05&#ceZE)e+f#W^S~ zj-!f;t-|tdSQ1Hz@lv}2gN<46OrjpP()TdT*fXS1PW#GtQLr2L#o$TgF<>^ObKu)6 zwWtqY9tALSI*kC=VOA)r^+bl|!uDw>tL5R37QxhK?*YN-Dk+N?u!i@Vw*)t8MjF{E zrmB@q=Y#=oc2|1iT2fS;kc1;WsN3VmBm_5x<9hT+c08fdq!-BKo?G;TKf*k!CDdJ~yj;e45OBIE z<`NPnU;cnm_~ADYj^BPG5O99Mo{^SEg@T|Hq5%+bJ?kkT%m_RyH~&yt*R$jzzS{MS ze?PeCb6(4$lp0`1QdC5B=2K9p!mP9k4PY{}H*s^z)$uV_U+MS^n3&$4n1K={>8F_*C))_6 zK^w%!=9K+{4>?4pn4E1s&Vpe{CAwV^;!!!WS{vd)TU*v@3&k0va&C8_YFFJVRNJd* zwDsaIr&a5KrSs1J4;8mej*0qfcFp^ebK`%a2b}PG847(ont-t%R}wZK0!rt1R!^@n z@xJ~A`(i|n{so8iuhJ*CyBvQEhJ;=3sDxGv4>5>%d5ZSch=yD^aK@YkaDF?7pafGD znMIoWQ0k4*xY!)5vT6z#E-ZIx3A$p2*A>gT_Lx>AnVhrg$Hm^t5?drC8BVO3r^MU5 z2Juf>E5N&B9zfK`n$VlA0A@M5`)Ubln4xM|yLacTA`>>X*%>iIY*8qLYs2-$s)|eI zjqw|75}?W~`6X5@#WfYmB`k|VDhWR9MR?VkV%-^a*(j<~zBIZBrwNMslm$A~I!n54 zHnA($2rbL1XL*C!WSuIl%888*ZQ7?4#Rpi}V^$40f*jL~Px zl+a2kZ5~0ksfA4uRBEVGrp_-lX5m_Q{?8cSqTs=!<6GK=D!qrHj)=)Ax=!U(xNDNu z`N|QU0uwea=l%~o19bRdQ%iXkO9?3fT)4Hd;sNK2tg)10u9e2Dc} zn9I;+td0(Ino@eTwCAcj=-kE6*76{akcHK8-T`X{!`%((4lbC%8S@|O_EF+9frnW& zN_Yjnb|kizmt7)>*{sgA-12Q{#>>&E)wY_;4mH(v2CXiD#Z#5W6jK$(YuZ0tnz-UL z7Dfz9rDWG&O6<(rs6}l~R2F?vG#X?Hw5bxt) zx}uVfuV@w6S~O;u$7Qn-pX)Z}!8IZoXJRmQZ%-4?QlJ#}s=Cx%M+BO_Ii$8j5XbtRyg_tBRfgq zFZ&zNv_zeBRasHc{|2Ew%m!ZmGuCY&Gn4!w;tMVQ^#Vl!St$Mpb3BypnUYbbR24rT z?z|^D8dIz^w0nHKc=n7no7JTXrM}d%eS>}?;e;H9qYq2j(TpQ^*LU=E7}4DAWrv7E zm8afmM+7!fw<^vWO$ok3I&30J^TE5Ib^Cq5ohrsHd8h<0kCY?aEz%mJbt4;M3sUWU zYk@Vn{8{ak!rU!)*?ABcTWNw8<|bUVS*(x&Q9=5xl}RUpbJw%PEhJ0Rm~-iX?_n+> zlN?i>(>s-uuSbt6m>1p7X2q8JF9Im<6qaSONJ?f_axt#6?cfBAt(ME4jORgf0>Qv< z+*x|fG@a9$q%6WB)qMU^waiMGtgLBm`|}z+cE}}o7^z?~;su>8%g5TBTFpG(lvIex zXo?u#=x{IM^o{Zk!>?ztTn${R-vune%3Vjs`fJxM_Ncb)_ok{|G`1zf&bJP_ztzNT zS)1Z7lGQ?Aj|P1CH2%RK1hX`n$OQ+u!fccfUxiGHBoK_9E7l_0G;SpR5Jx;qWQ-7( zAe9A9@X*s0@jL&KsHQBc3+(5KmJpO$(3*_s>GizM{vc6CxXYB2YK^Bx=u>JDxW?I2 zyJIrNVo}okO2;9%hw9KoGq4&*-jMNb{a^-u_@J$VJ#Xtd@) z=RESId|VHm@VdJmMOu z>f41LZX|7+B4Vf^G*%rMb;AoZz~WSR*Ki(1ohs_R2xGg-J$FebH3D^kWT}3zZ#!~P ze@W2c760{@nKj}vO%Sh2U8)!eW(uE1j8g3jHac(+yH3i6WI4KUtuI)qnipdQx@R6A zVDF@K|HL(Wb~E5UQ!DPees-j>1XjC8znSI6vAW4oqtJ7ruiGzbvH~Q(62im`KDPdei$s7E)C+{IhfUw?xp}4q`S#5l?xp7 zxWuO2qkjTh`V$>fr_FLyG(b-D?~yuwC0KgmGppaekJz4A(xuRq=!kDfJ}c#8XJ4bJ z_9-tTa(KOE`&32F{Born@k$Zvvl}O;JF7~A9Nd@Q{%Y;0Sm97#YVt)!?>*Z-OIrXn z?spsXp~J_S#nKegb`Sx;>9d?TcI1~OP>1dmW4wdPyBXmW9d-!r%ZPezjSyC;$gzIp zWLq4W#Q?KFxjkjerO5m_z5HWH9uyF9sTC5-tlxKA^+cql89DxP?onyDF^V)I5g|aT zv{(@;$r$rnA`~6TS0$gFCI+;#5t*zr%k4WbJqMuR!*Ayw+4Tc8g@w;S2@qlu@gMoH zkfr%2WE?4Gln)Q06BB1ds?2>TxHVIW$1f&$ShQ1)<>%a?(_fbMGbwG+)!z4#tPkOa zFFS1To|KEa>G8E&{c1vT>Bq3Jb88MBa!Y=ybANA`#c`i)Mkyr0{{zqS+VYHO*Y+n^ z^+y4GnY|E|SWgMSygX9Fh+56wHd4z>Q*)_ra$(_alBDL7)5bf$QgOY#;C=tDrF^RF zU8iA$-0FVMb1s$m2nlBkjeBaOWpv29yIl4sqjZOU$8ejzs*OS--OJD_GhWZ%*)yR);aLN(6SDy|8z?SvB2#D~Y`@A&m_otw=Q zXNpsSP)BWSLE|!%^dd7U*myTeUTNo9SbRF9H^W0;E_CP7bq=2iUoJq(zK!qHG?)TC z9v5~&wqFAAz%W^YKb7LYQ46oD=oKwk-aq?mVtUUePLhv*6>>%B?M$b>SEhm|TgY3L znym4wHH8u&Xg51FQzce7_YJ!&QV~kk>pcnY%nl$`dDgWZ%_$nqd?r>$>b4k&^r&Wt-AGFD6dr$l%nPZl@Cdjz zYYrx67uFKY<2jwnQ1dW zX$cu1+~a62CJrJfuc%7zVelY`+>_J%#QyMRZF_I4;&RIA6uS?=wei+FdGPFI{Kn;% z!z2WNX}K_OBf;$X!0Y_3N%gv|Y=W{J2;XQ22@aKw8NA|e+4(f=D!`A}3F%2>tLeL< z>Y{WflxKL?NK7$6o7opZLac0s`tf{vJR8{G9N#!@OAU;3{I*hOO}E$Y8MiqN|Ec_k zq7j)?%NcAG9I^Up*o5%ADH?UFbSi z$>;AYfHCtVug5VXSYCAT*l}f!3=rh8;8r9pol`ha^JD3qJbZXG{sLD=HXt5TuzfO{ zy&Ec%N}Syogc&GN{l@WiS3h0HPU*17t;4#p<&-04^;j`6xpwYKW;NWNuorLDO+VNz+X}aWtJvjI~o;B>9@+{NjPp~ z3`=;Fa-0UwSDc3qeDE3*SN#ws4rBgRHmp_F>(xUM`+iV6=4V8lH&Xh(R*64K+$&6$ z^OqSm#Ijh482NA2uM1QX>l9YS2IpwW9_tL*s7c7~<16cA+~jm9q*MvFe`)Q3RI7PP z<~?{HthMo*1_Pmfr{s^7JKN6e$lEfyf(hM1w!x!kC*9`%VgCq>P0T&ddLZSjH~=I@|mG2f6~`#M$BPm0hDi!xOv{vazLu zjKRjftd?hKSHHpD=9D^<^IXk+3YFXaGy~zFj*fQI(o8$h_K0c;cRp#4I=J2l39UhA zPIoo1u|xfn`8wu!;t|~%huMKGshvC2fil30)XWSipeLpo9nf|^Wte+J==KPdD_ zgjY?wGcSaO;fW0+Pm&qnRD7Q&_K6_3J)w2ZHKy_9Xtwj#95N9K|MiNimQb}2AjOgw z)2md;2r?I*yMi6M_hh!@P9f|kJaELi^LB*Tcl!xGT~99u{tq50VDxQBY}HdgrfZs3 zk+_w^J;GX%(;hZ!s?@Z{2E{=YhDB4#P{~c0*r>R?zl zPCUMpi}6HGcd}MGbsFzIMn(Gn{NsjlAaE^;W{bInlW2GM6N2&6AGzJ1SLiP))41j% zyMNuEkUz)I^u=PiTrIKyWu=rv1m&6dikx&6Ry)?X2BEk?X?8E0EC=O*m-i>3IMXmqt6&*0`z z=T8#36fWP`9mK+&*unY3!B7kvG4ll3AMLb53!UJ6)G0$CiRiU*J5=k~qnpWT@8n+Y z&tUYE0j}bQT-1uV;PpF8V88Wh`_0uWY2dJR;8(Rf0V}umUC3_Zh1wF#os`j`07L{< z&TCg%!{G!Cp{cD20h1exKVgNrCrC9Sbd?CxxMG{Q9!|RJ1mFA_d1LeSHV^aI4k4&(I+^#0cGfL;Rw7%A39B)e5 z3);`!jOa$pbn32!B;)K)2m?NMq%}(Q#Vm{}QW_lq^`(d+t(fglmZ}GrQI+ooLZ?#q z2=3;3q|Grr15F%`lgnHi?J0j)w$a0KVEXGb@!b`-cG#{dYoOHMPx^4D(i-I50ic3C z`sA&?{fp*}!Mm!-x@p~CCAkENqaXBtV;fI_DNPwxnE8dp7%8^sj0UPJxW#IMpN`CT6* z_$UQY5?LYHzmB>$fIwm0lonmT`BrKXj7z1arIDt?GDCH)it{%xu9Y-hmW;p@c?LP~ zsHx;BjjZ9Kmzr~$t;Rsg82>#b+&JL}_csIp;RVH}qpy+boj~{2OMj%};uh}8E6nGE zcHxAk_*r`rp?JYEm(6h&x#?aymn~AQe0qPFwkf7p{Vh#Et@-d)7iL05o#h)f(LhSV zzEH#$`YgO;wfpW@r?R>LRjnTy@oH4FfhD^JCYqagSEHX5xgD>-9e^Cu^lrqFgYyaW z9nP|a<>LADDH&v9>#zqg_w~S#5+l|*wm0VYr~H|LK@0rC2yvt6<34eS-Nj&*Q zJ@Iv6;2C~35N#xpOtZ;#XOP|?#Ub`eztyER*PfZWtJc1);mXu6Ty8AYzW(61cB@uX zPsEj{EAi|zf_=S(UoJfY|N1q>-zR&NK5NqBZ0`xsgVKKjoK3_Ah_L^b$hvRvn#3*O zA^vTfDOoT5$F^DEEnbs=ZO4Cz|JL0=2=@5@PvW;H$N3+KtGf%!I`H4r3im&Vo}gf7 zLO|EXSbD$!Qx7MVj0XtU6ODL4k%G;8wKx-FshQ6 zD0Ut(i;2|HlC!exkEmA^_n{(p6W2|(&YOhuGolShoN1Am%#MVPU9Ydjz{lT1%AkkW zx*&m&FY$*M82P$9P6wjkIIJ2$!E?alS#tQ4Z+Tf)#-?Br zr`s?*;_M8x?0yUrJ{B1-wpG@QimhtGaH@u-p8S(**gDX7kb2|v;2XJQlF20DWQO!K zsB6Jv7)3r}k@7P!s-i;sw{WUPojD;Z;`n1>EZ0FT2R-T=&ZG-@<@an=vyI$Gf1=G% zIDcz1LJak!HSM-RWpy8K#!7(8mP{2vSW~=*$lv|ao$uG=a-8Y0tCpK5knEvly;`TZ zQQp`S?{4)Mj+Mh@Yn1HTy1k3*dY=IT!G3$K-F_w_oygKY<*t9Ur1j+ub3*$efIx23 z_Vok5R;29tZ_q%jnh|-HcIzE#xxBxf8B{eFaj^bOt(J!K?%MgMiX(uNRd(r;v)pc) zalH^BRm$ylDF`K~C(Uj*>0YR*wY8k1ekZOnwM-TIRMoV6be;W>EkGOd6`!o?yFDZW znuGhbyV|Z|Y6UB1wdjt*L8ml_=oYD*fFCLcWPsdp&%z}e>V8qQ++CXVFP}}$s~sW1 z`{A+2?+^1v#PV~Pwd4Q=R5N?+lVOlxx!6~sq84E_>G zXihi*{J0+&qtT$Lzu|wP_fx$VKEM3h{poaMVy^PCfZXYDz@dVB5iKN|#onA?(R&Fw zFhA~P97v_QX)|dzU~9VQ2CYlO!FtX+a_O9MdqqV!hi7lHiwq5TH2^S{*IGxhR(_{r zH1zYQ*v{CI&hh5?u5hP)oEszNRWUNc3QaDlRNoSS$Z2Lj^Bf4sRq>zRzj$HvoU_zn zPL#_C?sJARN1t;FV;8f{B`aY2p(Y>5qT9pB+vO?B;4CFSV|GQx#p}EqXTIkdM=n6J zF7Aq)gc=aO5e+di5_-g=RY{r|T;&u$>93l8{vRq&D&Zx3c$u9u+6BP8qkF&}+MB+2fnV?UCMbMG@#lcT&a)}raSPV14q)0NI;li& z$FnQ`85=2RF=x(fR)%&GH9cFpVj-a5>sWvg4BV!&P7+%+(qImaxj6n-*`4H2TcDt}vXv=q}`9>9m&l_B;=}#&=DSfgams>eYF&p6?nR1878+6zipuT z0SMWgv4G4#h$(^sn}AZ;Qv;DHOxKqydTu7=&*Ddi8&S%f{>@#5RHOxT#vfiIo%Z{o zTs8t&uexkayzHybm#61VF2zAJ<60Q&beMZ;PXris_f@#_izg>5{=s;X4mw#`BtYY5>ARFYQASXqQ$;14Y8?J z!?q*(LwvQ7TkHKuzThf`BLn}3C}RW8Q2iAO42%Lq>cItUsbOg0eiFHD{9yRbN*5K< zASwff!Q7(xLo<$M9*Pm%p?Q;Og0#lb^USh%-u9-pN8ly6?E~(aa-FSZDc!RS03AN^ zcge_@cK1(jH0?gU7;t|I{r$@tRQxI<35LODI0OZUBbJfELTQG7GkGhL%0h0UDT$bF zu0QI>_A3#<8taZl#WvR;&~WMgDypbl7w&8@2!pQ5M%GShM#xR+zaC}iPa%R}qOv|2 zQ%r))WFj*f2u>1F6!wqwHObMD%i8RknT{7y?ylyg$o`5xL67(mtFec;rLl9=l)6aP zcG4mdN@KDzX*Y^BvpV7zcj}eq(U_#p*pEU zZ0$euh!rr@Awoo5oHJg!)Y5qcOy1c~F0W!=`BhVJ@5U(&mABuYzrPS1rq(f>4cZE6pVluo}otiRXZ4I zva+}!gdhGo+6=53eHGI`LvNKU1X2Y_*K*k-#cJTq+g5A5u=U>exwJryB{2Ka~D*0i4|$T_dVq6E&0$q zJsr!VP(5IC->Qvtd})&_sY@Z3X0T}8s6>^QyE1zBbgt>N@GI))ilkOeaF7|}v_jIZ zc%zGT`Dpe9p9V-hKS@3OE^Ts|pBzvK=B4Un1i88BhIZ~(#y(=%?sW~`eAg|h$Q9? zZD5KC$A_f^CWE*HGJ!0nkeIxLq!{ zrI4#i&3aY9cSI|i(rCCl6aaqHpRilSA9TJ&AGh&zgJ!|jB;BAAa$X~Et=8xrvjOWw zA8e`9fVq{5Pv*xuvG{jDEJY;fxvN0!w8V{BWG52&u`oZ7Czd=OQKDY;54Q^{Wpt0| z!Y(-y+I9)fH%1;8esRwXKee=fIp<*rwY~I<{@= zjcwbuZ9928>Daby+fK(;$F}X$-_+EpnW@@8Vb|XG^IYp%w?)D0iI*>G{f49GHN;an zsoF!e$uEM>^Ap^)p|1^yJvF>E>`z?1!r0GVL7Yc?zdr;W#pqUH$XJS;*{bDKOV)9M zrO;1IpftiUi5o;6PIM3!)V$8e5HJaZB|i{Ru|Rmip1?+-ILAykdN>tMf12)Hd#D6| z|F^5*`VKqM*L&=L47EfPXvxg~I&!O!|C9>0gF5DGm5_} z3@3i#SUbn_hD*f!2a?)%77|InoX_H(LU^KD15D(tYla8suoh3v}2o!512b zzK@J&Yqi`$a-{qxY*)XG(y^lgMfQX2md~?5>G6ioAAqKXg;Hu6JqZqP(Ql$tt`%Ja zvnljj>?@O1gP#}C?ZWon+LrT^(qd$WVcjR@VJUi{32$Oos-HO77p%W~5_L_ZP=sHn zquPOgRkKiZv$K`c)byS2DWFvs)-^8=LyqH*4>Q7d>^vu?Fw9G@QeG<@+7aqc@)7|Y zwr2^c1E%^k9zz~}3tRcv+l6h7MX4JQ!v}zV=8i-Q%^x@?BW^!0Iqma)W<$v5P4sEO zp;j8f6Jn={j0p-%V?AtiRU2*f!P8l4qU8W%Pc;$qrW}Q9{<=P_wTJBM(E6@WBc-zX z8t<1p4;CesS2lhE&LPV$9?9^x8|mDW09I9VR%?b5(*g9C$3usXg_3al*Q z!g}(VB){yl$b2Hc01F+$Cd`e{DcBo=L@Z4p6{u&GKH}*M{mCL_n8OlJy9XKY~ zl0^qcWk>mPo!M~ru5Tt15IS>Zn(RH1rkF|r;*P~~dy=Pzcre0Ytf-Qx6(k5vET-i^ z`x28NF~)IiDPc4&j6t+W8S3LO-KaknQe_c%-7DfZC3Ld>8v6PX#?BO?)XJ(@<0t6w z;z6!+@N^8l4NohwOCaf) zB~esXWb~_dP}B~%oRasx^>*lLuYW*%#m1LYGRG4*g#-S=hEyCy`~nep3E#c2`ztjs zACc|)oV9tJ#dpkQ@yzf2wJ`HTzR+}sO1zLYi+wiKBiWlG;`yd0fuHe)cp3+5unidF z0#N{iG%SIfrEwo@uR9bJBZG`alD6oyCWu&rCAnjAc!`Ry;ihYHa<@`#%{;@X=-TaE za}k~ZaKX(fQ!Pr(bmhQXM=y2)&1uA&5SU?1I}4L#U~F5eCOpJ3KXf0nRZGF7Lnph*TAPT-jEVf#T|L9&U79YssezB7?rx zoc@KI`_ZeJj0Uvb!aTxMDL^@>*jEXG&3-*k+qD#|b!VszQ3^qY%)^fIY6Tvp-$fLs z#a!B&sX^e??sASYJFzl*`_CG4qVvXTM?`T8GC*uvqs5KXegf;{;n)ZK4>JBExR@t|i2ZxyqSsVVaGX z3Q?3+A}qDfNHkb7O^(7~rv*kVvhyu%r~*RCbmCOA4kmW65T>T5HZ;vS9~=BL3&<)k zk0Y8E@zAT0A~??BHqQz!*qyEsdYY$VJq*Lo7-Ut{E63(|*({&_70uTDrdluXHW42O zsDec7#g%{N_d^tX zvgHrb0U7?ej_NR{^Bf?bJ%AJH&h&!#==4}dNa6w*wmB1z?- zMV#`I({HGzV@9_J+wZTRvA;R-tQi@lAc%)G%q?MU{}Ilu$incY(l{m7b%HoT>}A6A zb(8yYRM*568(3>rX!M`Mq-gqtB@9o!|3~{0TBrK9`48m!Mg8Lk@qY$_|K_-+w4r_f z!*TUYIy+Es5MvWDhB4B}JVB8df-0e)K$-T-fQnt_TB(-$;X@6F5Uyg&7&B{0oDzr8nqOtv5U9`hWprzXDq zz0UQC4@`XA@Vo;(vwjxaHX2cS>i$7SLZct>P|Ul@CU@Qp=asRrM}N}9xm z2I>--YlHCh2g6`OhyC=pV`V~%%ojDP4VCul(40gXloiTTJ=EG;v`zMdnYo?i>i8+4!fsgjjPq%%qy=+lI-py`*~r4wjRPw0_-p`)VT!04ambG*_epY!U@R1W1Yfi`IG zLIlq|DU$N2@=N%xgykz8QrzlQIyb&xgyHI-=~J%iF@&fy*&+>!Gc|?h>%rl}c+hl7 z9lG!>4g!RMzvz>H65t%!C;O;wRo!8L@F`d}t%!VfsU6x8ypiYD-hsmyPzkQ9^+=lO z<)@-;?e6klX{;f6z)|s*-NcOxC3{qK$sERw4+f6bHK+gVkKkIz_8%ID;1)nx&=vkLjUPp}* zlkUB`Q5CuuGM>r^&|KeWC@HP>@sV(OPMNNWUhmXd&*bsxXl-q6X>Nwd$ov;;L!`AW z2{p9=)5`PeON)!!ZG7_FY*+-%hE$u%D_!(7T!`wrt@vIXp=ep!MdJH~`fsG{;!g}O z4OOoZv73Dg-u{FkSKB3K8cOwdh!CRTzeDNloZDi)4a8f^S%SHnDh-P@# z3=tto?>p6T&>6XJgK2GWo5nYi-AzS%!-NvdR5L1;ZY2Z$F;Mz8C$xYa>y{ptMkWkN zAma@$DfUY0Q4fAaC@PUJn0SiwQ#2X6YVP@$td7Uks?|gKV^?I<&9eXf? z+6!|qgN3yKQ-H#1c}?29AW)Vp3u+iz9l6EtVdhUCw6#>7*I}-T0y7lB`-Dc)feU@0 z8xd!iDo1_bpQv*xR7vaJDLm=;%eg=Hz-B(Z>Zjp;Njn)vQ^`$QZ(uLd4{AnSbhCo( zu=M^qKM$oREijk+Eq)VG+dW681aq5J)&e8Gpr5yC$a*Z6tI04<-K6}CL=}QtE#gQB zqoJK}{iXrfOpl70$ONpR%=XN0i9}n!7Srh{(f zppi8<%Zk&}$=vugR%1pZGbh_uWwp1su{@5UV@*%DPUHrbT9EyV6}VM(Zpz6x zMK$5|TcVF0ch`Ro0YekEGnGclq1fj5C zM5ImG7H|FqqDN^rll6;r8(4J<@U0xl-Ca*BIi@ zJyCy)5^tSr#@Uo-H2XyR1IwQ;1v&WP70oZ>Vd=`TnK#$y3dPT6nnB7&O7aZjV(p8$ z9lb%8SI>&|mJ@MzaAbhgDY5Ir))U*Ccl-JAYXni4=K(a@`a$F^9yo${J1F@*hPOz? zQa+cPULFU-!s2?A(q@1a)?0R4_>m@lJ@13_8NweF4L-Ou&%pMDLOA;}@qr%QOA|8s z3LTw$P|Es+L#X;<6`i|VYx6<3oxz$e8Jb<5XL3g#{BS*I@|7pKIQJ4}^{@hU7K$cs z$vYm(#y>E_6hB~s0~DBXjKU0ZKL!d{h#$yDWsG<-l9_anNns0q3$ij_th|Wv(}biV z1%H9ybD!4UXD(-dG1uS2AI$A)w(neK{t-!Ms<~|qI`-3nV}M1+Lv7R;x}kx{dO_%N ziolC5N;PkRWrGc3VJ~f!4EZo05-5-`2{D*0GMKBK6O-|n2fna<(Jy=fY&y8Ld6r*& zjD;H>H#ajsnD^fsReA&hDYj}u^B>aYLLvHdr_TZ<8!FzM{`AUwm-G_$5WruB!Diun zm&675GQ;LSZ1^CSZw{ z?F;#v6PCft)jP(!{{VBgo6RRwgpDU~_ba2hqde2onN@3s^wfm3t*=SbM%08Xha1!M zO%I`a>@;>*LY#zX;TqDG@U8|396WMA*|af>$Ki?8JS}F$Z5yj$ zA1DSE+;)=Gn7x***92_yXEfunRaw;05H5lI z{1leLzz}@Ysz~6pW-QK|r+X;c6K&)!8s(1aF7c(d6@$f3iM((pTe52JKNtVWUz1T@ zrvlQr?ll^AlA)0z$Ip$OaQ_tzY>-GjF~VzGV6F*N2ro4{3%98+T%4*@X3V&dk;OnXszzVKxn+$vCN0YxRCjKzF8=OuJ zG3&M%(nsE)%R2pVq1Z@&jJgxfq2J%*)c@V``%329`6LnetQNTcc>(jvJ9 zta>ip5u24z|CkzAlzbYL>0)*S;PuJ&pzbbKxAD*zB-b?9MH_2np*J{JB0Vad@U~As z0byxK=&)?$JQ$(wbrFdqm0l&Gf( zUm0!k9R45`OZNNvk(p~Z%2CBW$zm>Jl%H(05l1+JlmL|c95s7t#O436xqNUxe7nm! zUW$93$ib4p&vd56q^IC-iww+&4yqIxuH^$_6I{eXp(nKEU=VprM9=6uGO% z2b_&|^XZd16nuXzfExr!$Z*#&7lEJC#^#IO{;$-BH&d{r2PeS$eKXUTE zrE03qYL79pJ#M$3>wO$KkxO5hQ@w{B9Rh(q` zp|--mi{f714WuCQ)2Ef3<;a=?Bdm(ankTvANYoPk>oHD5VIj^%1-A0a#u_4ut_9%t zUzIR9KgT4@tH%>PLLIU+Yv2>d?l^E!fXgz(FDq&J1;M1Fs0NL(mT27c@Cs%$|9F5x z=dqj!xYeLR+>7l50npK9=1Gi}dG-{~5XYp%{?HR9Q#K7sW_Fvc()4^kS%QwOTCuF#Okskg? zDTHEhm6Bq^{g(f8+_2`Y-S8MSwQTc?cF*Auz3`yAj8}Al7FdBe7X_1hk2FuWP%m9tWL`s>G=_3<|AL1la9@G-Wl=htDo_rcMt9puvD<_F%yRH zeDa}hL$_VpexK_mhgt~*Vl20hS}I**0j|Jz8z4ssp}iZV@x5 z)lbyW6}MdZ2nK=HiHQq-(_nydm*k|md+Tf(b(vJ3I}NmhTB?V3U+ci>6TO_=XCghv zv=s7_NuM&usFY`Kji~wY!eVfi`{#377-`tU*1CUR*le~e;PGdX(M03-w2xIx5H>B0 zki9!O(6AU>qHIfUiGPWSML?($A!<9c`gCgv+WMwFCjQKY{N&=yq^X))=ol~Widzf` zuTcCTjpF(*7#l3KPS-hOqp9Y#-7v2ZuJ1UN9j`VA;5j&=ZI{d zE=edsB^x}9a^5uW#8`VUBtlFO8l{y9G2r>*qvKb)pA*CsbsN68RHb7>^uV z@#0sT?LZ%zBZg2(P}ris*yBDj8;~piVLLF>{&I>_+}Gd2bawh^UxQZ+kTA=8pR01U zILzt%(Vi(EoVnqUTCGC-SovTQZUC>rWo$vr5&o@Ef{XQUo^vPgMiECUZ_Eb;vQD3u zg?QyWJ*fwlg4uPavSK5xBO$46-6zJo2889FDRs<7;FE>A@@G-1JpaslBwBn&;`9nK z&kuW3vehV@E%8YXQN^*5EUt%mqsAjcEY+BwG4;mu718P~1l4pGm(HCCc;Nrsn5PWA zXz+qvY~d5aY}W941iQR>;?mCpet}`*t8IbQ6Sx|0p5GvV|EO|NXLRnz%I)~zP8*anwVW#0VE0pX} zXxu~5O`rgx@_gXPA+D7!m%833gNLlLGqI{=>@dex=&1xOe!_3eEL)|9L8@BM+0V-Q z@=0I4N?M|SBucQhFBkhHxD(ualV2lG#AO`P&on~uw(+UB;yBdhJL84 zgP}j(lP9Xu1BUeIx+@5ZY=>6I;PM|<$uEZaf%~|PZ#C;89PpFvB60Nv^EZz*y5>Uk zcf?8EHVdv#O7kn@mmf0OEEfLO_1M4T0P7_kWclVG<1=uoG3&mj5AQy$}CLvh>UUn8}GAu^VOb zD*hui_j$xlL@b>B9~m0}qDo)ypR+vS-?bXae_<$fPmqalPf);Y4KH6b4gBw(3#JKE zP%El@TuWT)78xQf;dT@RSm+9#OocC)H#GjhMz-b{^ zIs#?ht~AKZP*xyfOq9AQ8D2c{KwVHFv#F`4Q&g`#{4P0fZ<2@p$l!z*=k%Z*bAo4Z zlzmr1==$*vLRUnuK471%Em7(!!HGj3ci=qj;gPs2z^q$oB=g36%S(p*s$VDl#;(iP zhuqjxf_%@D5|Na6V`BYz%oB49P?%dSxyQ%br7`d>>;KZgfRmPK7z zkngmIot&w)?H1TSNogNtr%0Eb3cotNA}X0kqccwreH0y0PnS2;Dm9}lfYyn|)plJ~ zSD~JYc>;R7=nzxihRQ5Gz?D@lS6z;gfm`#Mg;-M*@QE5ygb|;>otzfaf@7pE%o~;9 zFEOUs7uugqn+NcaupnbV=5NqkpOf>jZJwF=fPdJjXBF_*s&F*d$Tk=>dQ^U9g=Wah zPv|c#du6XbG^T<|hgv0gipZ6+Y3sB7rj*GXvjPf870{_kq$E>l%<(m}fB@LZb`xtZ z%CjTM01TaRNkL=s%xvML23fl#CpP^>^D)`$F(XZ?^rOzo&7rL8q_xa!fE@eQ5V=9+ z{h_wpx)gOpo6BNN6+GIda{M{CfqyCZrBta4>0hZPvbGjZ*HPuyo2ArQdpd+s^S$ab zC*b1hECSnMgXBw6kO~R`kqxDsf($3X(ThLLUfkme(VA(QuT(-Its=2rYBS{&3>-qy zZHT$0I$gdgI`Gfg_$ukMCN@urm(cieN%$BP>-uDxQ8BbM535CAgv?7*AS6?TA+Eh@ z9dag`?a{7Rn@~T7t=epeJNSA$xmIum8ZgQzwQoQ<%zo-bT(=QDYCkG4)G_7muPHqB z3tVmMej^OQNKoL+?Ty#gJ7`bYj?7C#VC_yNN>9noIcMn(?n_CK1ez~wEX@l$1(~lz zzsD_KfS`(WGd}qCDB_u zy%(I%SdOZ_D{^4Uw(2cWYkCX{A1Kc|JToMEVockjgS6^gTvgABxzilT5_u zFAj(pa>_NOe5lRAYO;`7L09yQ!+}SMIW~HWEs+xe8Pkj6p@mBO4Hv#ix{)visvb_o zL4VV#%VTspK>=nwc54J&)dH_18)TIz8I3Y#wcXrtmYc;C&GR%_9q^hhQg%g5$|aZb z>*M`dSRyfpcF?-0)GJRi?;gcD_A@`H*izrr6%~Bkr%8EQA4&nF`<9)m7QkPig(Jq= z8T1v&u4?4we&FbxOp()BD(jQh8k0pWX^Guq_cg{_ckOgDIvEq7rq6gXJUDuX{8KHk z3}Jc)=^|fN)(w4U=UVqAS~m0{Lm$lCP*T+T*g?nOhjGHt2qW$Jip-eFN&;>MQob6O z^==)}q{E0oX2lMLT{@MyY{0xjmkld&6nf3bzU)pzb5M_$f&nVjkmNLGJNV2Nr>7FP zgP#q%B!2cZhWRfC$4lhMv=9iC#G=g#k3H)Jsb*4I7{acXjhT7_yq~v7^OU#-c11V5 z+eqQ!IEo8;f0fSc{1>wCAgWA$;#ipUecfGl>Uuo9N$vWS?MQLp>jTf%QZniLQ-p^M zq$eh5Esel&zpgEsXe03c3NiU znrcrc5WSLhAn_=%i04`84EHB}I|1_KC~V+AG!vr7S@yITy;6w|MkuMWvc?RoAU(wn zY$G8=k<20?#E`5qLI6)e3aa{GE#XiEBo|<apV@dKp=(WY!L~cVC2vWO6;znh!)s0A7J4vTzBRPjf}N*YD6s^Z|3&IL#h)Y` zJ>*f3GUhL?xD8`1%L#`4CZSe-DBQX>KDVdKcY-Ae+QtxepxkW@Wo(XB zTvxD^o}FfN-{IUncNboj+6Y=W87UNf#ByIiZNpSB$^~kM@z)c)53A!Ltvg0?hKZ0s zN3)g>i1)S;GC3hQ-9qtI&zK1EBIxmk3*9ongp{FRJAAQQ^ zKgfVwPxBgaMfgP7?@;?eH@f(uL#QW(Dgj1LgiC86dwld){Zp#h{Ji96j<5&`N%xwU z(g-_ZL%?FBZV&gcg8X@TmIK&TN@BA7cdN{R)u+ikA` zni8tBVN(i$2pSIgNaS58qKg12MLvuAFV!_^b>6@%$V6jHJjYWvxQ=@yxMw=(#JeAh zv-+@Iz4m`E13vx_8ykQv@Br=e&o_qs4`&Db|EWax2^E-}r1u|FO~|%Yt6s;ZaP{_! zTCt=Kj1Ei&(lC}(7=;*vb;izS9h{kw)tG(d7pX(rtZ2d9kPO3z(g;iAI-)c@tK7`w zTJ|&7^;-JoKi|>^sA9j#PbW09GzCSfu{xp!F?=&FdThzO0+aN9OHbcXbOxQ!K%<6^ znQfN|E1(u`3~kPAu-;W0zJe{x_A$J6JhXaS*IJ#m-Ha8xWimmHF{1(7C%ebyBcBmr zSrc`aTk3ny4fgV##Zs{KgpFnZ^!e}6*q9)T;G_+r=rhanYJ3`IHMfjCw_`^gPlwl* zO5JALtL66NKfGOKgzj?Z{-Z6_Mw(N(SB_a{x4=^Q1uFrQH0;1A%L;k(bsq#Zywbqk z7q?xm&X`rBup`)OEahhQRz&t!rPZWL@tZK>&F3d`AA?o9kkfCIMT({PVh$Q127NH>(_iM;R-q$chX)9*!qhqe;w*hH z;V&4p{=dJp%IQ_2CU|TyG%W2QDjn6B;MNYPgyW-$(BzRpIgr;3C07gRr1#O1bE&3t z1!+^{kv``KP0PSEp>cEs0|CYfS}^JGrogljfZ!BaR(ZNDyRd-wol!a_>$p$(o`z*_ zzD;hPSnR2=YoCo-pA8Mzb}7OXBO6#3_>`e;ZaoD zB%DCq+#a5}L|7pvs=&pir@;q){QVF$dCs}rQ(js&CBhR|tucsI^ZUaX36$NU0Pg?h zNi1Z{wpY=9`~U)e{2==8_Y@vjt@WQcdP#t?v1{yR6%-_7N(tJO{9*zupBM}}rz3c>`M(j%-Obg_faT2=*ZS%*GQ$eX<~AK0kLp!z8=doIH9NJQ?{)`s zW+?*DFMi1@ziWDn&s>+8POp#OKG&25Tpu;L?suA2FvZxEQ=V272ZcarNfsr2%GdHB zN*mr`v5!c|kAvfKW{(6`A%WbU<-9n)_snEP7O+c+F9YVb+?f)N7RAb--nmfb zUhfCxCfXjlM6^aq^Z0I%`?W&328{50E)uI>?kFG>w!W&&>z3V?exe=Dyu6ZZktnPo~(Nv6)j? zlZZ20QKW;?S_3uI9jdk+rz1K>;Lp!f95xhNTXwVDmT=XX&=+$pY&mF1#jCm-=A?s6 zF4bfQZT5s2%m@fV%`(oK>g>h5_1j!=)Fs$Nqu3>a?x^K)mk(?DOMv0#8ZumR&_z%4s zI?DSqgMc(!DKWz>yZ{Ai0?FbbYHzGPXZ*UQETFXws^*cM-t@s4V<#g)h{HPCU>8-O zX@ne+RZ9_yQDvty{4BI(4ODBU6l{~O9q8`?x|I%1i*ZoOg)r$RHjlyg5HrTZ-(3KZ zg&<;OfJQ=CAugcMY#paiWmk6JhV?5@+=umxWTEZSoMdA;+ou2VmNU+6gs3`{<}xgm z6)0bNctjD#$d3s}J83FIvSIbUx``t%xUR(RHz`4LJ~$5NpZQSXi57y(!D!gQ!y<%= z*W@%NSf8qH^-=84wy;|{Bpu;$jESgrkrqMP&|G%O%GqyOuPiPqL@t1>K&~x|jx;z~ zBi}g8tg{rwU~Ga?0g-*1RM{ah8(2)*3PjMYm@_e)$Fh&bR;K|uYK#%G31j<_RAjZ} z4&sx7<(Qow7Lq6x{$psz&{~mIp^lT7$Nu-Yt)2r;wL7z%cJ*yi@^GbbH*-!HPD(LG z8>ys!-ojdxXpYIGTX%F-$WRZ_kO*Dm~Liv3gqj^VX7tRn9xQ>npnHIrn&xU@aox|1)WR-clfqs6? zGIEtO&wwP)`({_c>>y~v*tN4HO5Ij%9?Q_6abp?# zp)d*ZwJupw0BW9$6O$md%yAu5$%xlj}q)jcbI!s?*V5!3C;6sLF$w{UVp_^ zB-Rii3m;KiLZ`}d{h|QErjvNEQ-%qFbleeqwM=jCUY~vNeML-r+&stEWldm31M}_? zU*)r(@ASC0r=BGn6^;pdgpT#S<4-phlDqmhlu0=Ysklg2aX1qDftnjI1$)g=j&zno z9rbhp`&{%~j^=gje(I3bo^|k)LbDeG^#rE~)6+HQIpH+L25rq#$3~jRpT2^}KJ5y} zgtvOK7Wismtk-0kL%-)@DFUXSZ$EJe7hggA?l0 zHTQ!bsM(19MH}@}!0{5t$nm{?&&-8o!%UFlmKSwB6p0-?&my4kdT?3ir?mcZ?L`5Fr<=+!JW-_yqf=#XjU>#_k4^M_VBm}{JRiZT1z?{0s_VfZx##TXuf*x4# z$ON%^Sz&l-HyV{E$K_PODK|?|7btVTwHh5l`s^!m56GjEK+kfiu{x)<`8gJJhlG;a zgT?ll=)g$|TGhSDL&0?HJu0^VwSq!_fUBnQz&2h?@xCS$Z#2!GCHm?QSBoARUI6|a zr!>XN!0Wvm@Qd0QUNvX#)#9USS^%>z)};n69TXw7m9O-k;U!i0L}UKL`y1L6uHkE62<*vw=ewOynCau?ETQAv?ge2o4s<$$_Tuqv9NyG zY`lvLhNRzix1v;3ug7B1z}kDlAtr5JE|!o8rTtV_;1_9c)ore;fADwJy}?Uu+LqBu zDtGTVaN(8BAD_J11iN@a=Obj=A%b*!uh)W(x5$0|l`x_6yZpZQ#Vq?9aFxTmifsdl z_8ZlC!RadLWmjM?rPa+l^{eup=R+^=_9XPjLezd$sG`3`33-{4!!*}z>|4}z@DBz`Sb;AjKS0vDxo_Cx%YISfR2bx&o(*`vYY zOFg}hoXeO5{VIg^Wm~dK4OzGNn@sKB+f2D#AZRIFUl^ibWdp&)|%N*G&~6K zWQckUOV0IOoPqH+lS)46* zfu?p%3j-^)t#a9LXRy{#_0=MP3o8E>_gp~pyH$)a{J7N$U5rXjQmjA>-zN`10vio=>HOB@6ktb?4c;{gq*xoY)Rg zXWjbWfJ5P-9?1Ge6*m`V0x7MMq4zuhu>IE9eGOZ{fWBA#uZDIZNI$W4P$Dfe8~ue< z6R1)t!TT37ow!gM4k<=#zIgQNd=#yFo2SxT9GXmITYJn|ui3mimJ#eDB*IVhOs;fa zU->{BPh6h<*9760=+j*3?Ub=Hg@P7+y>I{*F&SdJW+GZUg$r!DO6RVsO1P0HkQ>=- z3z7i#$ufDv&kTm*k?xJOwtuLP#z&-cmFuO>&tAX9HP{ z49&csH%>l}bH(Z>!;-i=))?C@Fh~oRsqh9t6FxFsPyNlUF0;W)^!L&Q?%0Nl?wkff z6gDz7JT9rT83&D3Q+m4a+}2^VeWSzQpPzMj&cD^uZOT9}-6RHkEg+{U66H&>RfRNyORw5 zN_>?B8Wo<->CZpU-OM-m6(y%!lR8)p^W)TOIKA-x4L=a$!R4W~qMHhUbFreCDpaxJ z*Zci%JrjjrK#ycmjwpq?0*P9VH%hxM)+dTyByzmDz^jxi~+UM8EHkahI*sK#1m4?tVIcvCl zp^9aCf=lugfc9AVjad&e-1Jl9q%vXVA09HE>o&3UmDd=2>3T;YjC4=}EDHrs$X%WNh-mDbzxj9^|w-wVnUG39wyir9k?Jk$I1Aa1m|Pv-=_ z?YcMGJi3Gw#3SG1D){|0DGp=S`$Y8_z)H7+%~ly&Vu#>qT4s4quD|7SU}ePdDG$Yk zGNJ_mlB(2vrZ;Yz2`odpbT4cEN!tOE4ZE|K?TX@;U_M(~mPJ3*qsMqOyxJUTJy#oO zg1hr~%F%`eZymbL?WZFqh_!~{1LM}xzp!uTfX{KwQJsi5j?S1t`{H zeJx&j&h(oyesyc7bsw>e=OyoqwX_rC8=q;Y>(OK@K4^<4J44TwI8_oP&%aMO{qUlO z_}I`x%Q~96QjdZ*&{5J{YD?GpPP(SR@7apgwM-Imf5HNBGI`OX<@zehuYbXsveQpu$Tev%NKj?x6c7Y08d<&d2& znl{(whpHJTQ9{r2K0ctLPkNLuj(z{^*YzMQkeJ6I5Mqe$?1;r6T^3G=s&HRGah1Wz zd)$C=+WiE=ULesuY=~3JyiC=LTsLhNr9w9SY+{YQh~v@aUoD0d{MJvyriP12`@u08 za*Mej^v9KO;B>c2#$v$|+#jCyc&_jP3+< zG7~|ALeT`Hh=Zak;9E5pU>E}V2`7m!ei1n>CO;s64>WUx14i|>9PQ6fb1x+gfw7Ou zk2v-xg@q`q~RJ z@WiI~&2^95tF!AO7VN>kp_aaaw3-eQl6@tnHLnid%mj*gxC2L z*sCN!0+_v^**8mCD*u9=DgtgD!?2pr>C=3dXW>lYqN)k2!n0bjvpS=;;E{Ykv}J#| zCm+z`!$i$nR-zXbm^qfZlbO1I)b0d>CGICm>2+=WHzU<2#_1K8=OJSmU##LMzVHM1 z&Y9Scx94AHGUJ8Wzs}7wpsi=h;%Gs;E&8j0ze{FkK;#KIt06B9Kmo+uzx~1A2o#^@ z9s2s=JGgUlR1ASskpZEqjD-;5Z$A0~-ALuP9$IxP2{RsGzJAz*`=#QAhdAANbJp-y zL0g68Sg1s8E|~Fjf5XT#Lm`Ep7#HhV_OLw@({NM_FG`O(%Zfv_&nI9e39LIweL&xV zO9=u(M;b{_`xk(JHvf}PAM_%xyOmVdUC6vjvW5y~bSanF=2~h2QSzE%m2s!#Y~2b+ z4Sq9(Un?^9TpYEOvF9-|x?xS2^}bv9zZ=FY4Ien_e4@qY&Y@a36nVR$DZ9iPZ+d=h zO+)BFtAEAohj_s%7X0Bl80b+hyNN&ncGKA0(Y!S>x2JhcY=Ot~zE&!C zKzJfu`_r{|#Bv`zAad-`2`I$*{Aw9PHXc_A%6C^q*?%cGRc|o}1x>Iw!NMR-c8W4cK zN_N=qbVVD-(L#ty=f?i@$9&T(TXZ*bS@J%>EBH2g@;ji(Dv`tx%aVXy6Ps9-Yre== zyC?))l(H}9&pAfqE977AAvYyT%$0hvUJ3)b-YVM!)4ll|^+lr{$8G9ngp&wsE4i&< z&;D@i;J|N?IL>UNPuEW4d|JN&CEg@EGrr0lb7J@Q4IUiYs1qMAvcLP~o#fc8knnGd z=hm+mNTLh+9+4Sm;K9?IELgom6-ghuL|(s(dDX~FW5c2ipXon2{6sgxm;$WkEIFjx zLzq|Hofkl#SFA+6f@yZ>S6vR=emT*gC}?~$m0By)lQAgd&Ee&Z@nRnWN6|7yR9p~l z3#vxM>b?+@M#P}zIF^X*G3DrS0$g7G)Fd~HU1xk0BC`Fim585V`Nky(`&`^@myA=h z2(+!RW9$roN4yD++0bWCpqWJM zKC9<-Yd*f2Y<*K(c+wU?qS}hFERS{Q(afb`n{!a^*I!jX@t^T2B`vUa>`9IB;piP7 z``cddoG#spOo4c>4N1OvrYqF^nkzLbzEFfxo7j|FF#M#RrWrGWlt)~Rua1;3nRr1r zR6`}dnO4)El%j;Z!tAS#nO+2uiaKxl>re^d{FB8FwjX)nOO3xE|C>f=G#;t^4|IV1 zjNM2%HV*PXsDr%S79OyFtzKG$|LAJ|BflV+zyJbqm(cl40HhQ9;KNdN^{B(p^>kwx zQqW19MWQX?=yW`f;@Ll@C2=MaC0UG9hEC<7ZD0ZK%F@p%S0&`i5wO13+lzPS-7F?! z?9KrU4p%-d+kD449>>$~&!vR^a66p*F#A~Q+Q<^|p-YUax(JDG!J#OKX2>`U=e*@% zo zp}UO_Tl2T7Uw;l%Wxt-7^jH8|jTnas)les#0z5AE#|dl_Tc9o33OJFj?=bV1d?ReX&{F7!92WY`*8016b+gjexZc` zq987rX@jwG^ja!z_~I%9HHu|AF&6x_&y2zp!80iA`PoV^jX@9wRJnR(SYTti9+{yR zW|VUG7DqR=XYjb-kPheo8i!oy5#FD^D^Ru`bb(wZyQ~E}(%Ksm^tYAaHb+QkgIhr)~~ueF}r#wJBuy`}2{ci~c* zxZ+gZHMxSl!Ww-&qQi}4pk5%aX{;^0A_&J`>V>jY&!p|-xL7XR9%$i^d_&A$rrA{@ z#WsHE$Sh5csWw$stvB~z6T|&ounB*}!eyVfo^sT+_cYu_i{RgXohm#O;*N2WgCEUs zxv1+=WtD&JdXz+s7%~q78qzy!ajk~Q!6dUidEmIP)eD#{c!yWQb>Z?xO&!FSW5uH6 zlIf5g5>?Sx!hloj0|p#zyNDoKooDTR<@z5yMMT}pOrqwSbe8toh`>5&IQnD=un*}0 zEHz*n#CAdJBdz4QHE(7E>!UgwI8v;*>UZo>Vs+5S7ARR?|6_6nP1rS7~2o3@Y2BcbO(xi8hq7*@@ zNG}2ckuC^-;QM^!y~$cLH|Oqsa!+O^nRUxP5ejn?uA@~SO{&F`T;jT7CxnRSye&na zC`?x|D$1j7Fr{g-W}<`o#H(ZX`tHxD(@%x|KJT?Q%KcU7(0++&%nYgXf;Vy(Xl3(t1^nAlA(6uF_UJL z18TQf?ASl#kOJgx@T7Z_0knOE7zN1mPEhx|R zOlou2Yg2p?1**3XPf zh;Y-Ezi_2GZTZ62Mmb*oER$-CvNqS-UeR+-9Y6ZMO6;Y>p0%|rs8bN}$3^(q6L$cI z{1^!DN3;GWIa>7QM0361_K?>;3eJ>#Sd3^;mA$M#DN3L;-y29y{y5oaNM`-W^juteTN-66In;gFY3{66|JWD)H~#(9pXVcd1f*lT0Wboi(!02 zz^_BZRZ3}e?~Y+x1}kA=x(4*N;T77>=iU}|z9e=23=e59;RU@}VE{BZn0&AotSmSW zm}qH}Q_fl|D9ZOBnIWsjTxt=IJUJPT@I;pHw|j-mTP`3sIt3)u9A*cMT{nCxmjSj2 zF}h(m(IG!upB>63nYDS}DouV0**-*lPYzYqzaP$kOrxluPUQPVBJgp>`4`8@P$ST_ zb3CPwzSQ^A4AH57P^PeCa|kFMWdP3I`~L`e{VpjGGsK9tM)s>P-0e|w8KqnF+hd?* z7|qa7HNcPVg%p^0Su{dDQxedvFLq+x=`zM|I@MlqDqW~u zQy%Hx?3uXUfNw**twwW~k3f;Cu@n~5d>1bje>k&iLCW4`7&1xxQ9)|uQ#?IPOeHMT ztY3I+;FSp4QW8awmE$v4%C-DTgyV5^DiPPSE$eqi$9Ar~=^^7* zcq{1C**O}R^O+kK+$U;~-7t75!wQ=*c3~CDy_8<3mmZR9P9(vDZprWRMZT0a-uq-MDr$`UYi&O5 zIeK781o=F4_v}=)>q1Q-{84u?6J5n{u5)!wR7o3DXeWf;^!B(P(_8~t!;@=aWWae%n3@M%3(+nfE|G!{-Kevdw7 zRy(@dK`53aEa5fV&!BP*Z568s1}Zyag?D}Pv52}G{L-{-H`F4?G&Hq8&A&T3DCb=I z=tcc%Vd*86-+B$@jFw0vo^R`Cyd1w<&0RWTd9{k@d56|URuZ)$iWw>8tKR80^+Lah zxZaVBO#-%uGGyKIllgayP~xBNPOZgtiNpYt?^EzNFtln%Wb%1lek>tQ#az`GuXB## z^x2q=Za}<$G=7t?)pO|XUrs)`>WU~4j27H9?cFN=sEA0pYHbtwg&Ln1PyASoZYD3N zm~Os8FE;Jigb_~LsX{7Y(ed+PyJj@x(ufe1#r(x4sOY`MaA6-+Ze{S8pW#?et=&v$ zJ{=xY3|qE}-w%$bh1e5Hr-rCJVs9rD)ea@@xTTu32g8ThGf7t|!vv@`=EI51>JW5stuKXtPIzYX zsq{}8_P1tf0b}HNc=?QYc>E{Np!K_q=*Ol3HVku&M|vfxBb0!R3^xr~wQZt8t+eQ9ZT_bumwcP4Js-*iN_D+&f6mhPO-b9Jd76x6Nl*RSp8 zmam0Ku5{ldK^zV5#qHEj)J*Jr2aW?fLQxfv8<$=*fbu(SN};Mn=s zfSef9$aPm}k)apUwbvOCQpOKW_|PKcc_vcV`y0xKwU~=hyOX~T0tk^4%+0C(!IEpW zkxrgeQyz-Sjp=Wr*3Ys}bj*?l+?I>*!|u|czp^z4`71W3N2&XZJzTTw~8+q-65tn&7Pjf_bu4o`rY$tK2zMlPP`=KSJYT4Cwd z<2-`hPV|6|V9V*2WS)%F7C3zOu*l6on3<6jd#k_dzhiJ^W$b+MXLT!sOPA3ebk3*#kJN@CAP z-hi_?pKRApE1R2KpB}ZfZ!ZHS9nX&i=&?$>NGdqSfEkz^EM~sXB*{GO8{09ni0dx* z^lQ@#&$Dre zD7m;y4S^mK8RJ#Ww9~@`fBTX;=u=2aupOZq=c#a!J{6p!(k~oRnvj=v4lQuLJ;On} z5?1|zHvaSX?WRL+Qf^O3K1#rt=Yf$iOl$^epPA#9MevEWOacY7bIl6Dl#xNrJH4$_ zg-2B{@s(DEe|Il>jCFygCt%HQ9aY{8`V1d;&J$jzo9;W}L)csUhh83mj$f!*J&1o~ z74;-W%p=QXG&?FQsjN95|tBwH@%33;F=AIU}-zN0XyLCWyb+gfx$QK)eoesp& z@x`-fCdiY;0(cdVU<2sBoBq7KPk_E@gw+hb&+=Et{Gr;Q4DXgL zo33`Hyr%ACg18ZH$>yhnPwg*sy7FDLHtUQngkI$8n$^L-v>69OuitRFp3{xOg&^9dudj@iV zIM97p6tI0R#vu8e$Z{hTf=+!<<2^QFe``!+yz}dqA5wY{tR-VZ%i7h0FiDO0`9OQw>GJK?W}k`C*%T@4*t9jzt?$7ce5LN|%XNRfRs8n&ZKSV9 zz2{T&yPcWqLGR_z&w%=>A%jtbqen1G{*aHBrJnSH)XD+a~ZXQq!{ zGh&8yt6F%dQGEI}`8tm_Mb*ohMY=!!Aw{JJdefIV*;XOs>H-ahC&NwL zn20q>*!li2gW8T=6QyfZv+kt$2k5J~oGU8NE{%h0d4-YSu6LR6?71+^SjHnQICm%~y^HU?Te=@w z9?$+fJ6-*Bx}GB0PI_)onjuCyrPX5Py*z7cd4#vp^L+ICmdd!pJDv>PF|?g6&TXYw z)~O{VbDIyoxE(3Nw(SX$$7&qx&LGQv8ZMZno7=E0#p|5RA+b3ZZy8~(X?N~Id0n(( z$R2%~ej?(1uUw>%vlGwU07Ic7i&&(*FxQ~YJxl&pOMxs;8WtG^v%+htRuNK{bi@XE ziM~-LwcDdhue`?l9!I<5ZwREevTHauPsIsXZ4o7!RobzbdZstM;uN=5iggo|EKD|D zREINa3(Sb#zUn1e*at^Wy-gF^Q06M(iFPV>yr)=Pp@2zgmj7uT#H>sGuuIdxO~(Xj zxEh&Ql3^|qS%9&R5O|jAY^7}{6jsVk`+Uq+b|V1Y7Iq^$;JjCC!+`97u6ph_3M;H>UBVSUKm2XyNGn8m^vQ1ZPjGX51?b_B<# zoBA|PaO}#S)N8sFzseL@I@yh(chd!Y#bGS%6K-`JT<Pjb31^_dz`X294HV#6Q2LGW zU*H2v84K6I{n3ImxOZMWE+ZJBz~tgFiv1fB++$gxK=e1oQ)%rMcU>BQ&)+OhnJRs4 z(gEcq_~s{#P9!g>mhlQyc69Jl@XDj;0*6!aw7BnYhpB`76uJg2Fjd^-obbPI4;+>Gzgq(3!ph*)3r@EFvt@@3dHp8Cz#!LtzzI}upBNqn()=d?97WMT zK8^$6_WMmD2ZDll{zE(w4g|Yf{sK>~6b`XG=ns)Jl=(kI9LX{q;!E^zA{iA<$lxMQ zIKjI|e^&yYDG<`X&rbtbRSuj~?QQac@xr#?1hr334A(MP{@{W@bLcZ5V@vSV6^FCb z6Q?Pz7+@U-db|MR&-x1-C}|#`vMqheE6x;n9O%cFpjXy^1=qR_Czz=}Y4G7T9T)-a z4xHd&(}{t-iwZE?VL#;@XPgMfxlhY)3jLp;yFgDhoxu8z+$j$@{bV>Ex_SVwA+i6Q jFr01y9D;B^02>p87$Atj-#$D%S?~p?$HUVbKYsT=?Xdg@ diff --git a/java-sdk/gradle/wrapper/gradle-wrapper.properties b/java-sdk/gradle/wrapper/gradle-wrapper.properties index f398c33c..ac72c34e 100644 --- a/java-sdk/gradle/wrapper/gradle-wrapper.properties +++ b/java-sdk/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/java-sdk/gradlew b/java-sdk/gradlew index 65dcd68d..0adc8e1a 100755 --- a/java-sdk/gradlew +++ b/java-sdk/gradlew @@ -83,10 +83,8 @@ done # This is normally unused # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -133,10 +131,13 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. @@ -144,7 +145,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac @@ -152,7 +153,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then '' | soft) :;; #( *) # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -197,6 +198,10 @@ if "$cygwin" || "$msys" ; then done fi + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + # Collect all arguments for the java command; # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of # shell script including quotes and variable substitutions, so put them in diff --git a/java-sdk/radar-catalog-server/build.gradle.kts b/java-sdk/radar-catalog-server/build.gradle.kts index 4a5d54de..77bcf30c 100644 --- a/java-sdk/radar-catalog-server/build.gradle.kts +++ b/java-sdk/radar-catalog-server/build.gradle.kts @@ -1,20 +1,14 @@ description = "RADAR Schemas specification and validation tools." dependencies { - val radarJerseyVersion: String by project - implementation("org.radarbase:radar-jersey:$radarJerseyVersion") + implementation("org.radarbase:radar-jersey:${Versions.radarJersey}") implementation(project(":radar-schemas-core")) - val argparseVersion: String by project - implementation("net.sourceforge.argparse4j:argparse4j:$argparseVersion") + implementation("net.sourceforge.argparse4j:argparse4j:${Versions.argparse}") - val log4j2Version: String by project - runtimeOnly("org.apache.logging.log4j:log4j-core:$log4j2Version") - runtimeOnly("org.apache.logging.log4j:log4j-slf4j2-impl:$log4j2Version") - runtimeOnly("org.apache.logging.log4j:log4j-jul:$log4j2Version") - - val okHttpVersion: String by project - testImplementation("com.squareup.okhttp3:okhttp:$okHttpVersion") + testImplementation(platform("io.ktor:ktor-bom:${Versions.ktor}")) + testImplementation("io.ktor:ktor-client-content-negotiation") + testImplementation("io.ktor:ktor-serialization-kotlinx-json") } application { diff --git a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueJerseyEnhancer.kt b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueJerseyEnhancer.kt index dd92ac1f..2150428b 100644 --- a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueJerseyEnhancer.kt +++ b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueJerseyEnhancer.kt @@ -2,16 +2,16 @@ package org.radarbase.schema.service import jakarta.inject.Singleton import org.glassfish.jersey.internal.inject.AbstractBinder -import org.glassfish.jersey.server.ResourceConfig import org.radarbase.jersey.enhancer.JerseyResourceEnhancer import org.radarbase.jersey.filter.Filters.logResponse import org.radarbase.schema.specification.SourceCatalogue -class SourceCatalogueJerseyEnhancer(private val sourceCatalogue: SourceCatalogue) : - JerseyResourceEnhancer { +class SourceCatalogueJerseyEnhancer( + private val sourceCatalogue: SourceCatalogue, +) : JerseyResourceEnhancer { override val classes: Array> = arrayOf( logResponse, - SourceCatalogueService::class.java + SourceCatalogueService::class.java, ) override val packages: Array = emptyArray() diff --git a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueServer.kt b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueServer.kt index b3754d02..4f56eebc 100644 --- a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueServer.kt +++ b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueServer.kt @@ -9,7 +9,6 @@ import org.radarbase.jersey.config.ConfigLoader.createResourceConfig import org.radarbase.jersey.enhancer.Enhancers.exception import org.radarbase.jersey.enhancer.Enhancers.health import org.radarbase.jersey.enhancer.Enhancers.mapper -import org.radarbase.jersey.enhancer.Enhancers.okhttp import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.SourceCatalogue.Companion.load import org.radarbase.schema.specification.config.ToolConfig @@ -25,18 +24,19 @@ import kotlin.system.exitProcess * This server provides a webservice to share the SourceType Catalogues provided in *.yml files as * [org.radarbase.schema.service.SourceCatalogueService.SourceTypeResponse] */ -class SourceCatalogueServer(private val serverPort: Int) : Closeable { +class SourceCatalogueServer( + private val serverPort: Int, +) : Closeable { private lateinit var server: GrizzlyServer fun start(sourceCatalogue: SourceCatalogue) { val config = createResourceConfig( listOf( mapper, - okhttp, exception, health, - SourceCatalogueJerseyEnhancer(sourceCatalogue) - ) + SourceCatalogueJerseyEnhancer(sourceCatalogue), + ), ) server = GrizzlyServer(URI.create("http://0.0.0.0:$serverPort/"), config, false) server.listen() @@ -49,13 +49,6 @@ class SourceCatalogueServer(private val serverPort: Int) : Closeable { companion object { private val logger = LoggerFactory.getLogger(SourceCatalogueServer::class.java) - init { - System.setProperty( - "java.util.logging.manager", - "org.apache.logging.log4j.jul.LogManager" - ) - } - @JvmStatic fun main(args: Array) { val logger = LoggerFactory.getLogger(SourceCatalogueServer::class.java) @@ -108,8 +101,11 @@ class SourceCatalogueServer(private val serverPort: Int) : Closeable { private fun loadConfig(fileName: String): ToolConfig = try { loadToolConfig(fileName) } catch (ex: IOException) { - logger.error("Cannot configure radar-catalog-server from config file {}: {}", - fileName, ex.message) + logger.error( + "Cannot configure radar-catalog-server from config file {}: {}", + fileName, + ex.message, + ) exitProcess(1) } } diff --git a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueService.kt b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueService.kt index 03eedc62..fedcf5d6 100644 --- a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueService.kt +++ b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueService.kt @@ -49,5 +49,4 @@ class SourceCatalogueService( monitorSources = sourceCatalogue.monitorSources, connectorSources = sourceCatalogue.connectorSources, ) - } diff --git a/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.java b/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.java deleted file mode 100644 index 09172062..00000000 --- a/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.java +++ /dev/null @@ -1,72 +0,0 @@ -package org.radarbase.schema.service; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.IOException; -import java.nio.file.Paths; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.Response; -import okhttp3.ResponseBody; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.radarbase.schema.specification.SourceCatalogue; -import org.radarbase.schema.specification.config.SchemaConfig; -import org.radarbase.schema.specification.config.SourceConfig; - -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class SourceCatalogueServerTest { - private SourceCatalogueServer server; - private Thread serverThread; - private Exception exception; - - @BeforeEach - public void setUp() { - exception = null; - server = new SourceCatalogueServer(9876); - serverThread = new Thread(() -> { - try { - SourceCatalogue sourceCatalog = SourceCatalogue.Companion.load(Paths.get("../.."), new SchemaConfig(), new SourceConfig()); - server.start(sourceCatalog); - } catch (IllegalStateException e) { - // this is acceptable - } catch (Exception e) { - exception = e; - } - }); - serverThread.start(); - } - - @AfterEach - public void tearDown() throws Exception { - serverThread.interrupt(); - server.close(); - serverThread.join(); - if (exception != null) { - throw exception; - } - } - - @Test - public void sourceTypesTest() throws IOException, InterruptedException { - Thread.sleep(5000L); - - OkHttpClient client = new OkHttpClient(); - Request request = new Request.Builder() - .url("http://localhost:9876/source-types") - .build(); - - try (Response response = client.newCall(request).execute()) { - assertTrue(response.isSuccessful()); - ResponseBody body = response.body(); - assertNotNull(body); - JsonNode node = new ObjectMapper().readTree(body.byteStream()); - assertTrue(node.isObject()); - assertTrue(node.has("passive-source-types")); - assertTrue(node.get("passive-source-types").isArray()); - } - } -} diff --git a/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.kt b/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.kt new file mode 100644 index 00000000..520924b6 --- /dev/null +++ b/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.kt @@ -0,0 +1,90 @@ +package org.radarbase.schema.service + +import io.ktor.client.HttpClient +import io.ktor.client.call.body +import io.ktor.client.engine.cio.CIO +import io.ktor.client.plugins.contentnegotiation.ContentNegotiation +import io.ktor.client.request.get +import io.ktor.http.isSuccess +import io.ktor.serialization.kotlinx.json.json +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.asFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.mapNotNull +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.radarbase.schema.specification.SourceCatalogue.Companion.load +import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.specification.config.SourceConfig +import java.nio.file.Paths +import kotlin.time.Duration.Companion.milliseconds + +internal class SourceCatalogueServerTest { + private lateinit var server: SourceCatalogueServer + private lateinit var serverThread: Thread + private var exception: Exception? = null + + @BeforeEach + fun setUp() { + exception = null + server = SourceCatalogueServer(9876) + serverThread = Thread { + try { + val sourceCatalog = load(Paths.get("../.."), SchemaConfig(), SourceConfig()) + server.start(sourceCatalog) + } catch (e: IllegalStateException) { + // this is acceptable + } catch (e: Exception) { + exception = e + } + } + serverThread.start() + } + + @AfterEach + @Throws(Exception::class) + fun tearDown() { + serverThread.interrupt() + server.close() + serverThread.join() + exception?.let { throw it } + } + + @Test + fun sourceTypesTest(): Unit = runBlocking { + val client = HttpClient(CIO) { + install(ContentNegotiation) { + json( + Json { + ignoreUnknownKeys = true + }, + ) + } + } + + val response = (0 until 5000).asFlow() + .mapNotNull { + try { + client.get("http://localhost:9876/source-types") + .takeIf { it.status.isSuccess() } + } catch (ex: Exception) { + null + }.also { + if (it == null) delay(10.milliseconds) + } + } + .first() + + val body = response.body() + val obj = body.jsonObject + assertTrue(obj.containsKey("passive-source-types")) + obj["passive-source-types"]!!.jsonArray + } +} diff --git a/java-sdk/radar-schemas-commons/build.gradle.kts b/java-sdk/radar-schemas-commons/build.gradle.kts index f9f5c3e2..16aa04d5 100644 --- a/java-sdk/radar-schemas-commons/build.gradle.kts +++ b/java-sdk/radar-schemas-commons/build.gradle.kts @@ -16,11 +16,9 @@ sourceSets { } dependencies { - val avroVersion: String by project - val jacksonVersion: String by project - api("org.apache.avro:avro:$avroVersion") { - api("com.fasterxml.jackson.core:jackson-core:$jacksonVersion") - api("com.fasterxml.jackson.core:jackson-databind:$jacksonVersion") + api("org.apache.avro:avro:${Versions.avro}") { + api("com.fasterxml.jackson.core:jackson-core:${Versions.jackson}") + api("com.fasterxml.jackson.core:jackson-databind:${Versions.jackson}") exclude(group = "org.xerial.snappy", module = "snappy-java") exclude(group = "com.thoughtworks.paranamer", module = "paranamer") exclude(group = "org.apache.commons", module = "commons-compress") @@ -28,20 +26,22 @@ dependencies { } } -//---------------------------------------------------------------------------// +// ---------------------------------------------------------------------------// // Clean settings // -//---------------------------------------------------------------------------// +// ---------------------------------------------------------------------------// tasks.clean { delete(avroOutputDir) } -//---------------------------------------------------------------------------// +// ---------------------------------------------------------------------------// // AVRO file manipulation // -//---------------------------------------------------------------------------// +// ---------------------------------------------------------------------------// val generateAvro by tasks.registering(GenerateAvroJavaTask::class) { - source(rootProject.fileTree("../commons") { - include("**/*.avsc") - }) + source( + rootProject.fileTree("../commons") { + include("**/*.avsc") + }, + ) setOutputDir(avroOutputDir) } diff --git a/java-sdk/radar-schemas-core/build.gradle.kts b/java-sdk/radar-schemas-core/build.gradle.kts index 7ca28010..c934184b 100644 --- a/java-sdk/radar-schemas-core/build.gradle.kts +++ b/java-sdk/radar-schemas-core/build.gradle.kts @@ -1,25 +1,25 @@ +plugins { + kotlin("plugin.allopen") +} + description = "RADAR Schemas core specification and validation tools." dependencies { - val avroVersion: String by project - api("org.apache.avro:avro:$avroVersion") { + api("org.apache.avro:avro:${Versions.avro}") { exclude(group = "org.xerial.snappy", module = "snappy-java") exclude(group = "com.thoughtworks.paranamer", module = "paranamer") exclude(group = "org.apache.commons", module = "commons-compress") exclude(group = "org.tukaani", module = "xz") } - val javaxValidationVersion: String by project - api("javax.validation:validation-api:$javaxValidationVersion") + api("jakarta.validation:jakarta.validation-api:${Versions.jakartaValidation}") api(project(":radar-schemas-commons")) - val jacksonVersion: String by project - api(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")) + api(platform("com.fasterxml.jackson:jackson-bom:${Versions.jackson}")) api("com.fasterxml.jackson.core:jackson-databind") implementation("com.fasterxml.jackson.module:jackson-module-kotlin") implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml") - val confluentVersion: String by project - implementation("io.confluent:kafka-connect-avro-data:$confluentVersion") { + implementation("io.confluent:kafka-connect-avro-data:${Versions.confluent}") { exclude(group = "org.glassfish.jersey.core", module = "jersey-common") exclude(group = "jakarta.ws.rs", module = "jakarta.ws.rs-api") exclude(group = "io.swagger", module = "swagger-annotations") @@ -27,14 +27,15 @@ dependencies { exclude(group = "io.confluent", module = "kafka-schema-serializer") } - val kafkaVersion: String by project - implementation("org.apache.kafka:connect-api:$kafkaVersion") { + implementation("org.apache.kafka:connect-api:${Versions.kafka}") { exclude(group = "org.apache.kafka", module = "kafka-clients") exclude(group = "javax.ws.rs", module = "javax.ws.rs-api") } - val okHttpVersion: String by project - val radarCommonsVersion: String by project - api("com.squareup.okhttp3:okhttp:$okHttpVersion") - api("org.radarbase:radar-commons-server:$radarCommonsVersion") + api("com.squareup.okhttp3:okhttp:${Versions.okHttp}") + api("org.radarbase:radar-commons-server:${Versions.radarCommons}") +} + +allOpen { + annotation("org.radarbase.config.OpenConfig") } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt index 508646a9..a06858d1 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt @@ -13,7 +13,6 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.PathMatcher import java.util.* -import java.util.stream.Stream import kotlin.collections.HashMap import kotlin.collections.HashSet import kotlin.io.path.exists @@ -22,7 +21,7 @@ import kotlin.io.path.inputStream class SchemaCatalogue @JvmOverloads constructor( private val schemaRoot: Path, config: SchemaConfig, - scope: Scope? = null + scope: Scope? = null, ) { val schemas: Map val unmappedAvroFiles: List @@ -34,7 +33,7 @@ class SchemaCatalogue @JvmOverloads constructor( if (scope != null) { loadSchemas(schemaTemp, unmappedTemp, scope, matcher, config) } else { - for (useScope in Scope.values()) { + for (useScope in Scope.entries) { loadSchemas(schemaTemp, unmappedTemp, useScope, matcher, config) } } @@ -53,11 +52,11 @@ class SchemaCatalogue @JvmOverloads constructor( fun getGenericAvroTopic(config: AvroTopicConfig): AvroTopic { val (keySchema, valueSchema) = getSchemaMetadata(config) return AvroTopic( - config.topic, - keySchema.schema, - valueSchema.schema, + requireNotNull(config.topic) { "Missing Avro topic in configuration" }, + requireNotNull(keySchema.schema) { "Missing Avro key schema" }, + requireNotNull(valueSchema.schema) { "Missing Avro value schema" }, + GenericRecord::class.java, GenericRecord::class.java, - GenericRecord::class.java ) } @@ -67,12 +66,12 @@ class SchemaCatalogue @JvmOverloads constructor( unmappedFiles: MutableList, scope: Scope, matcher: PathMatcher, - config: SchemaConfig + config: SchemaConfig, ) { val walkRoot = schemaRoot.resolve(scope.lower) val avroFiles = buildMap { if (walkRoot.exists()) { - Files.walk(walkRoot).use, Unit> { walker -> + Files.walk(walkRoot).use { walker -> walker .filter { p -> matcher.matches(p) && SchemaValidator.isAvscFile(p) @@ -84,10 +83,9 @@ class SchemaCatalogue @JvmOverloads constructor( } } } - config.schemas(scope) - .forEach { (key, value) -> - put(walkRoot.resolve(key), value) - } + config.schemas(scope).forEach { (key, value) -> + put(walkRoot.resolve(key), value) + } } var prevSize = -1 @@ -97,8 +95,12 @@ class SchemaCatalogue @JvmOverloads constructor( // at all. while (prevSize != schemas.size) { prevSize = schemas.size - val useTypes = schemas.mapValues { (_, value) -> value.schema } - val ignoreFiles = schemas.values.mapTo(HashSet()) { it.path } + val useTypes = schemas + .mapNotNull { (k, v) -> v.schema?.let { k to it } } + .toMap() + val ignoreFiles = schemas.values.asSequence() + .map { it.path } + .filterNotNullTo(HashSet()) schemas.putParsedSchemas(avroFiles, ignoreFiles, useTypes, scope) } @@ -114,7 +116,7 @@ class SchemaCatalogue @JvmOverloads constructor( customSchemas: Map, ignoreFiles: Set, useTypes: Map, - scope: Scope + scope: Scope, ): Unit = customSchemas.asSequence() .filter { (p, _) -> p !in ignoreFiles } .forEach { (p, schema) -> @@ -139,13 +141,13 @@ class SchemaCatalogue @JvmOverloads constructor( fun getSchemaMetadata(config: AvroTopicConfig): Pair { val parsedKeySchema = schemas[config.keySchema] ?: throw NoSuchElementException( - "Key schema " + config.keySchema - + " for topic " + config.topic + " not found." + "Key schema " + config.keySchema + + " for topic " + config.topic + " not found.", ) val parsedValueSchema = schemas[config.valueSchema] ?: throw NoSuchElementException( - "Value schema " + config.valueSchema - + " for topic " + config.topic + " not found." + "Value schema " + config.valueSchema + + " for topic " + config.topic + " not found.", ) return Pair(parsedKeySchema, parsedValueSchema) } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.java deleted file mode 100644 index 65668eb2..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.radarbase.schema.specification; - -import static org.radarbase.schema.util.SchemaUtils.expandClass; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; -import java.util.Map; - -@JsonInclude(Include.NON_NULL) -public class AppDataTopic extends DataTopic { - @JsonProperty("app_provider") - private String appProvider; - - @JsonSetter - @SuppressWarnings("PMD.UnusedPrivateMethod") - private void setAppProvider(String provider) { - this.appProvider = expandClass(provider); - } - - public String getAppProvider() { - return appProvider; - } - - @Override - protected void propertiesMap(Map map, boolean reduced) { - map.put("app_provider", appProvider); - super.propertiesMap(map, reduced); - } - - public static class DataField { - @JsonProperty - private String name; - - public String getName() { - return name; - } - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.kt new file mode 100644 index 00000000..72e5468c --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppDataTopic.kt @@ -0,0 +1,29 @@ +package org.radarbase.schema.specification + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonSetter +import org.radarbase.config.OpenConfig +import org.radarbase.schema.util.SchemaUtils + +@JsonInclude(NON_NULL) +@OpenConfig +class AppDataTopic : DataTopic() { + @JsonProperty("app_provider") + @set:JsonSetter + var appProvider: String? = null + set(value) { + field = SchemaUtils.expandClass(value) + } + + override fun propertiesMap(map: MutableMap, reduced: Boolean) { + map["app_provider"] = appProvider + super.propertiesMap(map, reduced) + } + + class DataField { + @JsonProperty + var name: String? = null + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.java deleted file mode 100644 index 42d118f6..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.java +++ /dev/null @@ -1,67 +0,0 @@ -package org.radarbase.schema.specification; - -import static org.radarbase.schema.util.SchemaUtils.expandClass; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; -import java.util.Objects; - -@JsonInclude(Include.NON_NULL) -public abstract class AppSource extends DataProducer { - @JsonProperty("app_provider") - private String appProvider; - - @JsonProperty - private String vendor; - - @JsonProperty - private String model; - - @JsonProperty - private String version; - - @JsonSetter - @SuppressWarnings("PMD.UnusedPrivateMethod") - private void setAppProvider(String provider) { - this.appProvider = expandClass(provider); - } - - public String getAppProvider() { - return appProvider; - } - - public String getVersion() { - return version; - } - - public String getVendor() { - return vendor; - } - - public String getModel() { - return model; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - AppSource provider = (AppSource) o; - return Objects.equals(appProvider, provider.appProvider) - && Objects.equals(version, provider.version) - && Objects.equals(model, provider.model) - && Objects.equals(vendor, provider.vendor) - && Objects.equals(getData(), provider.getData()); - } - - @Override - public int hashCode() { - return Objects.hash(appProvider, vendor, model, version, getData()); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.kt new file mode 100644 index 00000000..8a5cd957 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/AppSource.kt @@ -0,0 +1,48 @@ +package org.radarbase.schema.specification + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonSetter +import org.radarbase.config.OpenConfig +import org.radarbase.schema.util.SchemaUtils +import java.util.Objects + +@JsonInclude(NON_NULL) +@OpenConfig +abstract class AppSource : DataProducer() { + @JsonProperty("app_provider") + @set:JsonSetter + var appProvider: String? = null + set(value) { + field = SchemaUtils.expandClass(value) + } + + @JsonProperty + var vendor: String? = null + + @JsonProperty + var model: String? = null + + @JsonProperty + var version: String? = null + + override fun equals(other: Any?): Boolean { + if (this === other) { + return true + } + if (javaClass != other?.javaClass) { + return false + } + other as AppSource<*> + return appProvider == other.appProvider && + version == other.version && + model == other.model && + vendor == other.vendor && + data == other.data + } + + override fun hashCode(): Int { + return Objects.hash(appProvider, vendor, model, version, data) + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.java deleted file mode 100644 index a8b2e5a8..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.java +++ /dev/null @@ -1,99 +0,0 @@ -package org.radarbase.schema.specification; - -import static org.radarbase.schema.util.SchemaUtils.applyOrEmpty; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Stream; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; -import org.radarbase.schema.SchemaCatalogue; -import org.radarbase.schema.Scope; -import org.radarbase.topic.AvroTopic; - -/** - * A producer of data to Kafka, generally mapping to a source. - * @param type of data that is produced. - */ -@JsonInclude(Include.NON_NULL) -public abstract class DataProducer { - @JsonProperty @NotBlank - private String name; - - @JsonProperty @NotBlank - private String doc; - - @JsonProperty - private Map properties; - - @JsonProperty - private Map labels; - - /** - * If true, register the schema during kafka initialization, otherwise, the producer should do - * that itself. The default is true, set in the constructor of subclasses to use a different - * default. - */ - @JsonProperty("register_schema") - protected boolean registerSchema = true; - - public String getName() { - return name; - } - - public String getDoc() { - return doc; - } - - @NotNull - public abstract List getData(); - - @NotNull - public abstract Scope getScope(); - - public Map getLabels() { - return labels; - } - - public Map getProperties() { - return properties; - } - - @JsonIgnore - public Stream getTopicNames() { - return getData().stream().flatMap(DataTopic::getTopicNames); - } - - @JsonIgnore - public Stream> getTopics(SchemaCatalogue schemaCatalogue) { - return getData().stream().flatMap(applyOrEmpty(t -> t.getTopics(schemaCatalogue))); - } - - public boolean doRegisterSchema() { - return registerSchema; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - DataProducer producer = (DataProducer) o; - return Objects.equals(name, producer.name) - && Objects.equals(doc, producer.doc) - && Objects.equals(getData(), producer.getData()); - } - - @Override - public int hashCode() { - return Objects.hash(name, doc, getData()); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.kt new file mode 100644 index 00000000..0a25302c --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataProducer.kt @@ -0,0 +1,79 @@ +package org.radarbase.schema.specification + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import jakarta.validation.constraints.NotBlank +import jakarta.validation.constraints.NotNull +import org.radarbase.config.OpenConfig +import org.radarbase.schema.SchemaCatalogue +import org.radarbase.schema.Scope +import org.radarbase.schema.util.SchemaUtils.applyOrEmpty +import org.radarbase.topic.AvroTopic +import java.util.Objects +import java.util.stream.Stream + +/** + * A producer of data to Kafka, generally mapping to a source. + * @param type of data that is produced. + */ +@JsonInclude(NON_NULL) +@OpenConfig +abstract class DataProducer { + @JsonProperty + var name: @NotBlank String? = null + + @JsonProperty + var doc: @NotBlank String? = null + + @JsonProperty + var properties: Map? = null + + @JsonProperty + var labels: Map? = null + + /** + * If true, register the schema during kafka initialization, otherwise, the producer should do + * that itself. The default is true, set in the constructor of subclasses to use a different + * default. + */ + @JsonProperty("register_schema") + var registerSchema = true + + abstract val data: @NotNull MutableList + abstract val scope: @NotNull Scope? + + @get:JsonIgnore + val topicNames: Stream + get() = data.stream().flatMap(DataTopic::topicNames) + + @JsonIgnore + fun topics(schemaCatalogue: SchemaCatalogue): Stream> = + data.stream().flatMap( + applyOrEmpty { t -> + t.topics(schemaCatalogue) + }, + ) + + fun doRegisterSchema(): Boolean { + return registerSchema + } + + override fun equals(other: Any?): Boolean { + if (this === other) { + return true + } + if (javaClass != other?.javaClass) { + return false + } + other as DataProducer<*> + return name == other.name && + doc == other.doc && + data == other.data + } + + override fun hashCode(): Int { + return Objects.hash(name, doc, data) + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.java deleted file mode 100644 index c29d2556..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.java +++ /dev/null @@ -1,167 +0,0 @@ -package org.radarbase.schema.specification; - -import static com.fasterxml.jackson.dataformat.yaml.YAMLGenerator.Feature.MINIMIZE_QUOTES; -import static com.fasterxml.jackson.dataformat.yaml.YAMLGenerator.Feature.WRITE_DOC_START_MARKER; -import static org.radarbase.schema.util.SchemaUtils.expandClass; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import java.io.IOException; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; -import org.radarbase.config.AvroTopicConfig; -import org.radarbase.schema.SchemaCatalogue; -import org.radarbase.topic.AvroTopic; -import org.radarcns.catalogue.Unit; -import org.radarcns.kafka.ObservationKey; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** DataTopic topic from a data producer. */ -@JsonInclude(Include.NON_NULL) -public class DataTopic extends AvroTopicConfig { - private static final Logger logger = LoggerFactory.getLogger(DataTopic.class); - - /** Type of topic. Its meaning is class-specific.*/ - @JsonProperty - private String type; - - /** Documentation string for this topic. */ - @JsonProperty - private String doc; - - /** Sampling rate, how frequently messages are expected to be sent on average. */ - @JsonProperty("sample_rate") - private SampleRateConfig sampleRate; - - /** Output unit. */ - @JsonProperty - private Unit unit; - - /** Record fields that the given unit applies to. */ - @JsonProperty - private List fields; - - /** - * DataTopic using ObservationKey as the default key. - */ - public DataTopic() { - // default value - setKeySchema(ObservationKey.class.getName()); - } - - /** Get all topic names that are provided by the data. */ - @JsonIgnore - public Stream getTopicNames() { - return Stream.of(getTopic()); - } - - /** Get all Avro topics that are provided by the data. */ - @JsonIgnore - public Stream> getTopics(SchemaCatalogue schemaCatalogue) throws IOException { - return Stream.of(schemaCatalogue.getGenericAvroTopic(this)); - } - - public String getType() { - return type; - } - - public String getDoc() { - return doc; - } - - public SampleRateConfig getSampleRate() { - return sampleRate; - } - - public Unit getUnit() { - return unit; - } - - public List getFields() { - return fields; - } - - @Override - @JsonSetter - public void setKeySchema(String schema) { - super.setKeySchema(expandClass(schema)); - } - - @Override - @JsonSetter - public void setValueSchema(String schema) { - super.setValueSchema(expandClass(schema)); - } - - @Override - public String toString() { - return toString(false); - } - - /** - * Convert the topic to String, either as dense string or as verbose YAML string. - * @param prettyString Whether the result should be a verbose pretty-printed string. - * @return topic as a string. - */ - public String toString(boolean prettyString) { - String name = getClass().getSimpleName(); - // preserves insertion order - Map properties = new LinkedHashMap<>(); - propertiesMap(properties, !prettyString); - - if (prettyString) { - YAMLFactory factory = new YAMLFactory(); - factory.configure(WRITE_DOC_START_MARKER, false); - factory.configure(MINIMIZE_QUOTES, true); - ObjectMapper mapper = new ObjectMapper(factory); - try { - return mapper.writeValueAsString(Map.of(name, properties)); - } catch (JsonProcessingException ex) { - logger.error("Failed to convert data to YAML", ex); - return name + properties; - } - } else { - return name + properties; - } - } - - /** - * Turns this topic into an descriptive properties map. - * @param map properties to add to. - * @param reduced whether to set a reduced set of properties, to decrease verbosity. - */ - protected void propertiesMap(Map map, boolean reduced) { - map.put("type", type); - if (!reduced && doc != null) { - map.put("doc", doc); - } - - String topic = getTopic(); - if (topic != null) { - map.put("topic", topic); - } - map.put("key_schema", getKeySchema()); - map.put("value_schema", getValueSchema()); - - if (!reduced) { - if (sampleRate != null) { - map.put("sample_rate", sampleRate); - } - if (unit != null) { - map.put("unit", unit); - } - if (fields != null) { - map.put("fields", fields); - } - } - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.kt new file mode 100644 index 00000000..c77f69f5 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.kt @@ -0,0 +1,139 @@ +package org.radarbase.schema.specification + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonSetter +import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator.Feature.MINIMIZE_QUOTES +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator.Feature.WRITE_DOC_START_MARKER +import org.radarbase.config.AvroTopicConfig +import org.radarbase.config.OpenConfig +import org.radarbase.schema.SchemaCatalogue +import org.radarbase.schema.specification.AppDataTopic.DataField +import org.radarbase.schema.util.SchemaUtils +import org.radarbase.topic.AvroTopic +import org.radarcns.catalogue.Unit +import org.radarcns.kafka.ObservationKey +import org.slf4j.LoggerFactory +import java.io.IOException +import java.util.stream.Stream + +/** DataTopic topic from a data producer. */ +@JsonInclude(NON_NULL) +@OpenConfig +class DataTopic : AvroTopicConfig() { + /** Type of topic. Its meaning is class-specific. */ + @JsonProperty + val type: String? = null + + /** Documentation string for this topic. */ + @JsonProperty + val doc: String? = null + + /** Sampling rate, how frequently messages are expected to be sent on average. */ + @JsonProperty("sample_rate") + val sampleRate: SampleRateConfig? = null + + /** Output unit. */ + @JsonProperty + val unit: Unit? = null + + /** Record fields that the given unit applies to. */ + @JsonProperty + val fields: List? = null + + @get:JsonIgnore + val topicNames: Stream + /** Get all topic names that are provided by the data. */ + get() = Stream.of(topic) + + /** Get all Avro topics that are provided by the data. */ + @JsonIgnore + @Throws(IOException::class) + fun topics(schemaCatalogue: SchemaCatalogue): Stream> { + return Stream.of(schemaCatalogue.getGenericAvroTopic(this)) + } + + @JsonProperty("key_schema") + @set:JsonSetter + override var keySchema: String? = ObservationKey::class.java.getName() + set(schema) { + field = SchemaUtils.expandClass(schema) + } + + @JsonProperty("value_schema") + @set:JsonSetter + override var valueSchema: String? = null + set(schema) { + field = SchemaUtils.expandClass(schema) + } + + override fun toString(): String { + return toString(false) + } + + /** + * Convert the topic to String, either as dense string or as verbose YAML string. + * @param prettyString Whether the result should be a verbose pretty-printed string. + * @return topic as a string. + */ + fun toString(prettyString: Boolean): String { + val name = javaClass.getSimpleName() + // preserves insertion order + val properties: MutableMap = LinkedHashMap() + propertiesMap(properties, !prettyString) + return if (prettyString) { + val mapper = ObjectMapper( + YAMLFactory().apply { + disable(WRITE_DOC_START_MARKER) + enable(MINIMIZE_QUOTES) + }, + ) + try { + mapper.writeValueAsString(mapOf(name to properties)) + } catch (ex: JsonProcessingException) { + logger.error("Failed to convert data to YAML", ex) + name + properties + } + } else { + name + properties + } + } + + /** + * Turns this topic into an descriptive properties map. + * @param map properties to add to. + * @param reduced whether to set a reduced set of properties, to decrease verbosity. + */ + protected fun propertiesMap(map: MutableMap, reduced: Boolean) { + map["type"] = type + if (!reduced && doc != null) { + map["doc"] = doc + } + val topic: String? = topic + if (topic != null) { + map["topic"] = topic + } + map["key_schema"] = keySchema + map["value_schema"] = valueSchema + if (!reduced) { + if (sampleRate != null) { + map["sample_rate"] = sampleRate + } + if (unit != null) { + map["unit"] = unit + } + if (fields != null) { + map["fields"] = fields + } + } + } + + companion object { + private val logger = LoggerFactory.getLogger(DataTopic::class.java) + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.java deleted file mode 100644 index 7711b910..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.radarbase.schema.specification; - -import com.fasterxml.jackson.annotation.JsonProperty; - -public class SampleRateConfig { - @JsonProperty - private Double interval; - - @JsonProperty - private Double frequency; - - @JsonProperty - private boolean dynamic; - - @JsonProperty - private boolean configurable; - - public Double getInterval() { - return interval; - } - - public Double getFrequency() { - return frequency; - } - - public boolean isDynamic() { - return dynamic; - } - - public boolean isConfigurable() { - return configurable; - } - - @Override - public String toString() { - return "SampleRateConfig{interval=" + interval - + ", frequency=" + frequency - + ", dynamic=" + dynamic - + ", configurable=" + configurable - + '}'; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.kt new file mode 100644 index 00000000..1eabcdeb --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SampleRateConfig.kt @@ -0,0 +1,29 @@ +package org.radarbase.schema.specification + +import com.fasterxml.jackson.annotation.JsonProperty +import org.radarbase.config.OpenConfig + +@OpenConfig +class SampleRateConfig { + @JsonProperty + var interval: Double? = null + + @JsonProperty + var frequency: Double? = null + + @JsonProperty("dynamic") + var isDynamic = false + + @JsonProperty("configurable") + var isConfigurable = false + + override fun toString(): String { + return ( + "SampleRateConfig{interval=" + interval + + ", frequency=" + frequency + + ", dynamic=" + isDynamic + + ", configurable=" + isConfigurable + + '}' + ) + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt index 60f57506..360e2d1e 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt @@ -34,8 +34,10 @@ import org.radarbase.schema.validation.ValidationHelper.SPECIFICATIONS_PATH import org.radarbase.topic.AvroTopic import org.slf4j.LoggerFactory import java.io.IOException -import java.nio.file.* -import java.util.* +import java.nio.file.Files +import java.nio.file.InvalidPathException +import java.nio.file.Path +import java.nio.file.PathMatcher import java.util.stream.Stream import kotlin.io.path.exists import kotlin.streams.asSequence @@ -47,7 +49,7 @@ class SourceCatalogue internal constructor( val passiveSources: List, val streamGroups: List, val connectorSources: List, - val pushSources: List + val pushSources: List, ) { val sources: Set> = buildSet { @@ -66,7 +68,7 @@ class SourceCatalogue internal constructor( /** Get all topics in the catalogue. */ val topics: Stream> get() = sources.stream() - .flatMap { it.getTopics(schemaCatalogue) } + .flatMap { it.topics(schemaCatalogue) } companion object { private val logger = LoggerFactory.getLogger(SourceCatalogue::class.java) @@ -94,7 +96,7 @@ class SourceCatalogue internal constructor( .withGetterVisibility(JsonAutoDetect.Visibility.NONE) .withIsGetterVisibility(JsonAutoDetect.Visibility.NONE) .withSetterVisibility(JsonAutoDetect.Visibility.NONE) - .withCreatorVisibility(JsonAutoDetect.Visibility.NONE) + .withCreatorVisibility(JsonAutoDetect.Visibility.NONE), ) } val schemaCatalogue = SchemaCatalogue( @@ -109,7 +111,7 @@ class SourceCatalogue internal constructor( initSources(mapper, specRoot, Scope.PASSIVE, pathMatcher, sourceConfig.passive), initSources(mapper, specRoot, Scope.STREAM, pathMatcher, sourceConfig.stream), initSources(mapper, specRoot, Scope.CONNECTOR, pathMatcher, sourceConfig.connector), - initSources(mapper, specRoot, Scope.PUSH, pathMatcher, sourceConfig.push) + initSources(mapper, specRoot, Scope.PUSH, pathMatcher, sourceConfig.push), ) } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.java deleted file mode 100644 index f988b350..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.specification.active; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import java.util.List; -import javax.validation.constraints.NotBlank; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.DataProducer; -import org.radarbase.schema.specification.DataTopic; -import org.radarbase.schema.specification.active.questionnaire.QuestionnaireSource; - -/** - * TODO. - */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "assessment_type") -@JsonSubTypes(value = { - @JsonSubTypes.Type(name = "QUESTIONNAIRE", value = QuestionnaireSource.class), - @JsonSubTypes.Type(name = "APP", value = AppActiveSource.class)}) -@JsonInclude(Include.NON_NULL) -public class ActiveSource extends DataProducer { - public enum RadarSourceTypes { - QUESTIONNAIRE - } - - @JsonProperty("assessment_type") - @NotBlank - private String assessmentType; - - @JsonProperty - private List data; - - @JsonProperty - private String vendor; - - @JsonProperty - private String model; - - @JsonProperty - private String version; - - public String getAssessmentType() { - return assessmentType; - } - - @Override - public List getData() { - return data; - } - - @Override - public Scope getScope() { - return Scope.ACTIVE; - } - - public String getVendor() { - return vendor; - } - - public String getModel() { - return model; - } - - public String getVersion() { - return version; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.kt new file mode 100644 index 00000000..8ead1648 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.kt @@ -0,0 +1,68 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.specification.active + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonSubTypes.Type +import com.fasterxml.jackson.annotation.JsonTypeInfo +import com.fasterxml.jackson.annotation.JsonTypeInfo.Id.NAME +import jakarta.validation.constraints.NotBlank +import org.radarbase.schema.Scope +import org.radarbase.schema.Scope.ACTIVE +import org.radarbase.schema.specification.DataProducer +import org.radarbase.schema.specification.DataTopic +import org.radarbase.schema.specification.active.questionnaire.QuestionnaireSource + +/** + * TODO. + */ +@JsonTypeInfo(use = NAME, property = "assessment_type") +@JsonSubTypes( + value = [ + Type( + name = "QUESTIONNAIRE", + value = QuestionnaireSource::class, + ), Type(name = "APP", value = AppActiveSource::class), + ], +) +@JsonInclude( + NON_NULL, +) +open class ActiveSource : DataProducer() { + enum class RadarSourceTypes { + QUESTIONNAIRE, + } + + @JsonProperty("assessment_type") + val assessmentType: @NotBlank String? = null + + @JsonProperty + override val data: MutableList = mutableListOf() + + @JsonProperty + val vendor: String? = null + + @JsonProperty + val model: String? = null + + @JsonProperty + val version: String? = null + override val scope: Scope + get() = ACTIVE +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.java deleted file mode 100644 index f5debc79..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.radarbase.schema.specification.active; - -import static org.radarbase.schema.util.SchemaUtils.expandClass; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; -import org.radarbase.schema.specification.AppDataTopic; - -@JsonInclude(Include.NON_NULL) -public class AppActiveSource extends ActiveSource { - @JsonProperty("app_provider") - private String appProvider; - - @JsonSetter - @SuppressWarnings("PMD.UnusedPrivateMethod") - private void setAppProvider(String provider) { - this.appProvider = expandClass(provider); - } - - public String getAppProvider() { - return appProvider; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.kt new file mode 100644 index 00000000..f9c60f57 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/AppActiveSource.kt @@ -0,0 +1,20 @@ +package org.radarbase.schema.specification.active + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonSetter +import org.radarbase.config.OpenConfig +import org.radarbase.schema.specification.AppDataTopic +import org.radarbase.schema.util.SchemaUtils + +@JsonInclude(NON_NULL) +@OpenConfig +class AppActiveSource : ActiveSource() { + @JsonProperty("app_provider") + @set:JsonSetter + var appProvider: String? = null + set(value) { + field = SchemaUtils.expandClass(value) + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.java deleted file mode 100644 index 9f111724..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.specification.active.questionnaire; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.net.URL; -import java.util.Map; -import org.radarbase.schema.specification.DataTopic; - -/** - * TODO. - */ -@JsonInclude(Include.NON_NULL) -public class QuestionnaireDataTopic extends DataTopic { - @JsonProperty("questionnaire_definition_url") - private URL questionnaireDefinitionUrl; - - public URL getQuestionnaireDefinitionUrl() { - return questionnaireDefinitionUrl; - } - - @Override - protected void propertiesMap(Map props, boolean reduced) { - super.propertiesMap(props, reduced); - props.put("questionnaire_definition_url", questionnaireDefinitionUrl); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.kt new file mode 100644 index 00000000..01e83135 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.kt @@ -0,0 +1,38 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.specification.active.questionnaire + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import org.radarbase.config.OpenConfig +import org.radarbase.schema.specification.DataTopic +import java.net.URL + +/** + * TODO. + */ +@JsonInclude(NON_NULL) +@OpenConfig +class QuestionnaireDataTopic : DataTopic() { + @JsonProperty("questionnaire_definition_url") + var questionnaireDefinitionUrl: URL? = null + + override fun propertiesMap(map: MutableMap, reduced: Boolean) { + super.propertiesMap(map, reduced) + map["questionnaire_definition_url"] = questionnaireDefinitionUrl + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.java deleted file mode 100644 index 39ea808c..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.radarbase.schema.specification.active.questionnaire; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import org.radarbase.schema.specification.active.ActiveSource; - -@JsonInclude(Include.NON_NULL) -public class QuestionnaireSource extends ActiveSource { -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.kt new file mode 100644 index 00000000..c1728be9 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireSource.kt @@ -0,0 +1,10 @@ +package org.radarbase.schema.specification.active.questionnaire + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import org.radarbase.config.OpenConfig +import org.radarbase.schema.specification.active.ActiveSource + +@JsonInclude(NON_NULL) +@OpenConfig +class QuestionnaireSource : ActiveSource() diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/PathMatcherConfig.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/PathMatcherConfig.kt index fbaeec42..6eb8d690 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/PathMatcherConfig.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/PathMatcherConfig.kt @@ -1,6 +1,10 @@ package org.radarbase.schema.specification.config -import java.nio.file.* +import java.nio.file.FileSystem +import java.nio.file.FileSystems +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.PathMatcher import kotlin.io.path.relativeTo interface PathMatcherConfig { @@ -48,6 +52,8 @@ interface PathMatcherConfig { companion object { fun Path.relativeToAbsolutePath(absoluteBase: Path) = if (isAbsolute) { relativeTo(absoluteBase) - } else this + } else { + this + } } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/SchemaConfig.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/SchemaConfig.kt index 3882396a..ce3358c6 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/SchemaConfig.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/SchemaConfig.kt @@ -1,7 +1,6 @@ package org.radarbase.schema.specification.config import org.radarbase.schema.Scope -import org.radarbase.schema.Scope.* data class SchemaConfig( override val include: List = listOf(), @@ -15,14 +14,14 @@ data class SchemaConfig( val push: Map = emptyMap(), val stream: Map = emptyMap(), ) : PathMatcherConfig { - fun schemas(scope: Scope): Map = when(scope) { - ACTIVE -> active - KAFKA -> kafka - CATALOGUE -> catalogue - MONITOR -> monitor - PASSIVE -> passive - STREAM -> stream - CONNECTOR -> connector - PUSH -> push + fun schemas(scope: Scope): Map = when (scope) { + Scope.ACTIVE -> active + Scope.KAFKA -> kafka + Scope.CATALOGUE -> catalogue + Scope.MONITOR -> monitor + Scope.PASSIVE -> passive + Scope.STREAM -> stream + Scope.CONNECTOR -> connector + Scope.PUSH -> push } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/ToolConfig.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/ToolConfig.kt index de24a370..16000440 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/ToolConfig.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/config/ToolConfig.kt @@ -6,9 +6,7 @@ import com.fasterxml.jackson.module.kotlin.KotlinFeature import com.fasterxml.jackson.module.kotlin.kotlinModule import java.io.IOException import java.nio.file.Paths -import kotlin.io.path.bufferedReader import kotlin.io.path.inputStream -import kotlin.io.path.reader data class ToolConfig( val kafka: Map = emptyMap(), @@ -24,9 +22,11 @@ fun loadToolConfig(fileName: String?): ToolConfig { } val mapper = ObjectMapper(YAMLFactory.builder().build()) - .registerModule(kotlinModule { - enable(KotlinFeature.NullIsSameAsDefault) - }) + .registerModule( + kotlinModule { + enable(KotlinFeature.NullIsSameAsDefault) + }, + ) return Paths.get(fileName).inputStream().use { stream -> mapper.readValue(stream, ToolConfig::class.java) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.java deleted file mode 100644 index 346008d4..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.radarbase.schema.specification.connector; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.DataProducer; -import org.radarbase.schema.specification.DataTopic; - -/** - * Data producer for third-party connectors. This data topic does not register schemas to the schema - * registry by default, since Kafka Connect will do that itself. To enable auto-registration, set - * the {@code register_schema} property to {@code true}. - */ -@JsonInclude(Include.NON_NULL) -public class ConnectorSource extends DataProducer { - @JsonProperty - private List data; - - @JsonProperty - private String vendor; - - @JsonProperty - private String model; - - @JsonProperty - private String version; - - public ConnectorSource() { - registerSchema = false; - } - - @Override - public List getData() { - return data; - } - - @Override - public Scope getScope() { - return Scope.CONNECTOR; - } - - public String getVendor() { - return vendor; - } - - public String getModel() { - return model; - } - - public String getVersion() { - return version; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.kt new file mode 100644 index 00000000..c8b7214a --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/connector/ConnectorSource.kt @@ -0,0 +1,35 @@ +package org.radarbase.schema.specification.connector + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import org.radarbase.config.OpenConfig +import org.radarbase.schema.Scope +import org.radarbase.schema.Scope.CONNECTOR +import org.radarbase.schema.specification.DataProducer +import org.radarbase.schema.specification.DataTopic + +/** + * Data producer for third-party connectors. This data topic does not register schemas to the schema + * registry by default, since Kafka Connect will do that itself. To enable auto-registration, set + * the `register_schema` property to `true`. + */ +@JsonInclude(NON_NULL) +@OpenConfig +class ConnectorSource : DataProducer() { + @JsonProperty + override var data: MutableList = mutableListOf() + + @JsonProperty + var vendor: String? = null + + @JsonProperty + var model: String? = null + + @JsonProperty + var version: String? = null + + override var registerSchema = false + + override val scope: Scope = CONNECTOR +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.java deleted file mode 100644 index 36c2e5a3..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.radarbase.schema.specification.monitor; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.AppDataTopic; -import org.radarbase.schema.specification.AppSource; - -@JsonInclude(Include.NON_NULL) -public class MonitorSource extends AppSource { - @JsonProperty - private List data; - - @Override - public List getData() { - return data; - } - - @Override - public Scope getScope() { - return Scope.MONITOR; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.kt new file mode 100644 index 00000000..e08479ad --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/monitor/MonitorSource.kt @@ -0,0 +1,18 @@ +package org.radarbase.schema.specification.monitor + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import org.radarbase.config.OpenConfig +import org.radarbase.schema.Scope +import org.radarbase.schema.Scope.MONITOR +import org.radarbase.schema.specification.AppDataTopic +import org.radarbase.schema.specification.AppSource + +@JsonInclude(NON_NULL) +@OpenConfig +class MonitorSource : AppSource() { + @JsonProperty + override val data: MutableList = mutableListOf() + override val scope: Scope = MONITOR +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.java deleted file mode 100644 index 9ef4c322..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.specification.passive; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.Objects; -import org.radarbase.schema.specification.AppDataTopic; -import org.radarcns.catalogue.ProcessingState; - -/** - * TODO. - */ -@JsonInclude(Include.NON_NULL) -public class PassiveDataTopic extends AppDataTopic { - - @JsonProperty("processing_state") - private ProcessingState processingState; - - public ProcessingState getProcessingState() { - return processingState; - } - - public void setProcessingState(ProcessingState processingState) { - this.processingState = processingState; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!super.equals(o)) { - return false; - } - PassiveDataTopic passiveData = (PassiveDataTopic) o; - return Objects.equals(processingState, passiveData.processingState); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), processingState); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.kt new file mode 100644 index 00000000..46743be8 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.kt @@ -0,0 +1,46 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.specification.passive + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import org.radarbase.schema.specification.AppDataTopic +import org.radarcns.catalogue.ProcessingState +import java.util.Objects + +/** + * TODO. + */ +@JsonInclude(NON_NULL) +class PassiveDataTopic : AppDataTopic() { + @JsonProperty("processing_state") + var processingState: ProcessingState? = null + override fun equals(other: Any?): Boolean { + if (this === other) { + return true + } + if (!super.equals(other)) { + return false + } + other as PassiveDataTopic + return processingState == other.processingState + } + + override fun hashCode(): Int { + return Objects.hash(super.hashCode(), processingState) + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.java deleted file mode 100644 index 89c94ade..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.specification.passive; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.AppSource; - -import javax.validation.constraints.NotEmpty; -import java.util.List; - -/** - * TODO. - */ -@JsonInclude(Include.NON_NULL) -public class PassiveSource extends AppSource { - @JsonProperty @NotEmpty - private List data; - - @Override - public List getData() { - return data; - } - - @Override - public Scope getScope() { - return Scope.PASSIVE; - } - - @Override - public String getName() { - return super.getVendor() + '_' + super.getModel(); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.kt new file mode 100644 index 00000000..9e57225d --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.specification.passive + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import jakarta.validation.constraints.NotEmpty +import org.radarbase.config.OpenConfig +import org.radarbase.schema.Scope +import org.radarbase.schema.Scope.PASSIVE +import org.radarbase.schema.specification.AppSource + +/** + * TODO. + */ +@JsonInclude(NON_NULL) +@OpenConfig +class PassiveSource : AppSource() { + @JsonProperty + @NotEmpty + override var data: MutableList = mutableListOf() + override var scope: Scope = PASSIVE + + override var name: String? = null + get() = field ?: "${vendor}_$model" +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.java deleted file mode 100644 index 67b16434..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.radarbase.schema.specification.push; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; -import javax.validation.constraints.NotNull; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.DataProducer; -import org.radarbase.schema.specification.DataTopic; - -@JsonInclude(Include.NON_NULL) -public class PushSource extends DataProducer { - - @JsonProperty - private List data; - - @JsonProperty - private String vendor; - - @JsonProperty - private String model; - - @JsonProperty - private String version; - - @Override - public @NotNull List getData() { - return data; - } - - @Override - public @NotNull Scope getScope() { - return Scope.PUSH; - } - - public String getVendor() { - return vendor; - } - - public String getModel() { - return model; - } - - public String getVersion() { - return version; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.kt new file mode 100644 index 00000000..662470bf --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/push/PushSource.kt @@ -0,0 +1,28 @@ +package org.radarbase.schema.specification.push + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import org.radarbase.config.OpenConfig +import org.radarbase.schema.Scope +import org.radarbase.schema.Scope.PUSH +import org.radarbase.schema.specification.DataProducer +import org.radarbase.schema.specification.DataTopic + +@JsonInclude(NON_NULL) +@OpenConfig +class PushSource : DataProducer() { + @JsonProperty + override var data: MutableList = mutableListOf() + + @JsonProperty + var vendor: String? = null + + @JsonProperty + var model: String? = null + + @JsonProperty + var version: String? = null + + override val scope: Scope = PUSH +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.java deleted file mode 100644 index dd7e4eaf..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.java +++ /dev/null @@ -1,148 +0,0 @@ -package org.radarbase.schema.specification.stream; - -import static org.radarbase.schema.util.SchemaUtils.applyOrEmpty; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; -import org.radarbase.config.AvroTopicConfig; -import org.radarbase.schema.SchemaCatalogue; -import org.radarbase.schema.specification.DataTopic; -import org.radarbase.stream.TimeWindowMetadata; -import org.radarbase.topic.AvroTopic; -import org.radarcns.kafka.AggregateKey; -import org.radarcns.kafka.ObservationKey; - -/** - * Topic used for Kafka Streams. - */ -@JsonInclude(Include.NON_NULL) -public class StreamDataTopic extends DataTopic { - /** Whether the stream is a windowed stream with standard TimeWindow windows. */ - @JsonProperty - private boolean windowed = false; - - /** Input topic for the stream. */ - @JsonProperty("input_topics") - private final List inputTopics = new ArrayList<>(); - - /** - * Base topic name for output topics. If windowed, output topics would become - * {@code [topicBase]_[time-frame]}, otherwise it becomes {@code [topicBase]_output}. - * If a fixed topic is set, this will override the topic base for non-windowed topics. - */ - @JsonProperty("topic_base") - private String topicBase; - - @JsonSetter - @SuppressWarnings("PMD.UnusedPrivateMethod") - private void setWindowed(boolean windowed) { - this.windowed = windowed; - if (windowed && (this.getKeySchema() == null - || this.getKeySchema().equals(ObservationKey.class.getName()))) { - this.setKeySchema(AggregateKey.class.getName()); - } - } - - @JsonSetter("input_topic") - @SuppressWarnings("PMD.UnusedPrivateMethod") - private void setInputTopic(String inputTopic) { - if (topicBase == null) { - topicBase = inputTopic; - } - if (!this.inputTopics.isEmpty()) { - throw new IllegalStateException("Input topics already set"); - } - this.inputTopics.add(inputTopic); - } - - /** Get human readable output topic. */ - @Override - public String getTopic() { - if (windowed) { - return topicBase + "_"; - } else if (super.getTopic() == null) { - return topicBase + "_output"; - } else { - return super.getTopic(); - } - } - - public boolean isWindowed() { - return windowed; - } - - /** Get the input topics. */ - public List getInputTopics() { - return inputTopics; - } - - @JsonSetter - @SuppressWarnings("PMD.UnusedPrivateMethod") - private void setInputTopics(Collection topics) { - if (!this.inputTopics.isEmpty()) { - throw new IllegalStateException("Input topics already set"); - } - this.inputTopics.addAll(topics); - } - - public String getTopicBase() { - return topicBase; - } - - @JsonIgnore - @Override - public Stream getTopicNames() { - if (windowed) { - return Arrays.stream(TimeWindowMetadata.values()) - .map(label -> label.getTopicLabel(topicBase)); - } else { - String currentTopic = getTopic(); - if (currentTopic == null) { - currentTopic = topicBase + "_output"; - setTopic(currentTopic); - } - return Stream.of(currentTopic); - } - } - - @JsonIgnore - @Override - public Stream> getTopics(SchemaCatalogue schemaCatalogue) { - return getTopicNames() - .flatMap(applyOrEmpty(topic -> { - AvroTopicConfig config = new AvroTopicConfig(); - config.setTopic(topic); - config.setKeySchema(getKeySchema()); - config.setValueSchema(getValueSchema()); - return Stream.of(schemaCatalogue.getGenericAvroTopic(config)); - })); - } - - /** Get only topic names that are used with the fixed time windows. */ - @JsonIgnore - public Stream getTimedTopicNames() { - if (windowed) { - return getTopicNames(); - } else { - return Stream.empty(); - } - } - - @Override - protected void propertiesMap(Map properties, boolean reduce) { - properties.put("input_topics", inputTopics); - properties.put("windowed", windowed); - if (!reduce) { - properties.put("topic_base", topicBase); - } - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt new file mode 100644 index 00000000..9a07c9c7 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt @@ -0,0 +1,110 @@ +package org.radarbase.schema.specification.stream + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonSetter +import org.radarbase.config.AvroTopicConfig +import org.radarbase.config.OpenConfig +import org.radarbase.schema.SchemaCatalogue +import org.radarbase.schema.specification.DataTopic +import org.radarbase.schema.util.SchemaUtils +import org.radarbase.stream.TimeWindowMetadata +import org.radarbase.topic.AvroTopic +import org.radarcns.kafka.AggregateKey +import org.radarcns.kafka.ObservationKey +import java.util.Arrays +import java.util.stream.Stream + +/** + * Topic used for Kafka Streams. + */ +@JsonInclude(NON_NULL) +@OpenConfig +class StreamDataTopic : DataTopic() { + /** Whether the stream is a windowed stream with standard TimeWindow windows. */ + @JsonProperty + @set:JsonSetter + var windowed = false + set(value) { + field = value + if (value && (keySchema == null || keySchema == ObservationKey::class.java.getName())) { + keySchema = AggregateKey::class.java.getName() + } + } + + /** Input topic for the stream. */ + @JsonProperty("input_topics") + var inputTopics: MutableList = mutableListOf() + + /** + * Base topic name for output topics. If windowed, output topics would become + * `[topicBase]_[time-frame]`, otherwise it becomes `[topicBase]_output`. + * If a fixed topic is set, this will override the topic base for non-windowed topics. + */ + @JsonProperty("topic_base") + var topicBase: String? = null + + @JsonSetter("input_topic") + private fun setInputTopic(inputTopic: String) { + if (topicBase == null) { + topicBase = inputTopic + } + check(inputTopics.isEmpty()) { "Input topics already set" } + inputTopics.add(inputTopic) + } + + override var topic: String? = null + /** Get human readable output topic. */ + get() = if (windowed) { + "${topicBase}_" + } else { + field ?: "${topicBase}_output" + } + + @get:JsonIgnore + override val topicNames: Stream + get() = if (windowed) { + Arrays.stream(TimeWindowMetadata.entries.toTypedArray()) + .map { label: TimeWindowMetadata -> label.getTopicLabel(topicBase) } + } else { + var currentTopic = topic + if (currentTopic == null) { + currentTopic = topicBase + "_output" + topic = currentTopic + } + Stream.of(currentTopic) + } + + @JsonIgnore + override fun topics(schemaCatalogue: SchemaCatalogue): Stream> { + return topicNames + .flatMap( + SchemaUtils.applyOrEmpty { topic -> + val config = AvroTopicConfig() + config.topic = topic + config.keySchema = keySchema + config.valueSchema = valueSchema + Stream.of(schemaCatalogue.getGenericAvroTopic(config)) + }, + ) + } + + @get:JsonIgnore + val timedTopicNames: Stream + /** Get only topic names that are used with the fixed time windows. */ + get() = if (windowed) { + topicNames + } else { + Stream.empty() + } + + override fun propertiesMap(map: MutableMap, reduced: Boolean) { + map["input_topics"] = inputTopics + map["windowed"] = windowed + if (!reduced) { + map["topic_base"] = topicBase + } + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.java deleted file mode 100644 index d4d62b41..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.radarbase.schema.specification.stream; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; -import java.util.stream.Stream; -import javax.validation.constraints.NotEmpty; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.DataProducer; - -/** - * Data producer for Kafka Streams. This data topic does not register schemas to the schema registry - * by default, since Kafka Streams will do that itself. To disable this, set the - * {@code register_schema} property to {@code true}. - */ -@JsonInclude(Include.NON_NULL) -public class StreamGroup extends DataProducer { - @JsonProperty @NotEmpty - private List data; - - @JsonProperty - private String master; - - public StreamGroup() { - registerSchema = false; - } - - @Override - public List getData() { - return data; - } - - @Override - public Scope getScope() { - return Scope.STREAM; - } - - /** Get only the topic names that are the output of a timed stream. */ - @JsonIgnore - public Stream getTimedTopicNames() { - return data.stream().flatMap(StreamDataTopic::getTimedTopicNames); - } - - public String getMaster() { - return master; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.kt new file mode 100644 index 00000000..fe1a2786 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamGroup.kt @@ -0,0 +1,38 @@ +package org.radarbase.schema.specification.stream + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL +import com.fasterxml.jackson.annotation.JsonProperty +import jakarta.validation.constraints.NotEmpty +import org.radarbase.config.OpenConfig +import org.radarbase.schema.Scope +import org.radarbase.schema.Scope.STREAM +import org.radarbase.schema.specification.DataProducer +import java.util.stream.Stream + +/** + * Data producer for Kafka Streams. This data topic does not register schemas to the schema registry + * by default, since Kafka Streams will do that itself. To disable this, set the + * `register_schema` property to `true`. + */ +@JsonInclude(NON_NULL) +@OpenConfig +class StreamGroup : DataProducer() { + @JsonProperty + @NotEmpty + override val data: MutableList = mutableListOf() + + @JsonProperty + val master: String? = null + + override var registerSchema: Boolean = false + + override val scope: Scope = STREAM + + @get:JsonIgnore + val timedTopicNames: Stream + /** Get only the topic names that are the output of a timed stream. */ + get() = data.stream() + .flatMap { it.timedTopicNames } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.java deleted file mode 100644 index 68f4ad9a..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.util; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Locale; -import java.util.Properties; -import java.util.function.Function; -import java.util.stream.Stream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * TODO. - */ -public final class SchemaUtils { - private static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class); - private static final String GRADLE_PROPERTIES = "exchange.properties"; - private static final String GROUP_PROPERTY = "project.group"; - private static String projectGroup; - - private SchemaUtils() { - //Static class - } - - /** - * TODO. - * @return TODO - */ - public static synchronized String getProjectGroup() { - if (projectGroup == null) { - Properties prop = new Properties(); - ClassLoader loader = ClassLoader.getSystemClassLoader(); - try (InputStream in = loader.getResourceAsStream(GRADLE_PROPERTIES)) { - if (in == null) { - projectGroup = "org.radarcns"; - logger.debug("Project group not specified. Using \"{}\".", projectGroup); - } else { - prop.load(in); - projectGroup = prop.getProperty(GROUP_PROPERTY); - if (projectGroup == null) { - projectGroup = "org.radarcns"; - logger.debug("Project group not specified. Using \"{}\".", projectGroup); - } - } - } catch (IOException exc) { - throw new IllegalStateException(GROUP_PROPERTY - + " cannot be extracted from " + GRADLE_PROPERTIES, exc); - } - } - - return projectGroup; - } - - /** - * Expand a class name with the group name if it starts with a dot. - * @param classShorthand class name, possibly starting with a dot as a shorthand. - * @return class name or {@code null} if null or empty. - */ - public static String expandClass(String classShorthand) { - if (classShorthand == null || classShorthand.isEmpty()) { - return null; - } else if (classShorthand.charAt(0) == '.') { - return getProjectGroup() + classShorthand; - } else { - return classShorthand; - } - } - - /** - * Converts given file name from snake_case to CamelCase. This will cause underscores to be - * removed, and the next character to be uppercase. This only converts the value up to the - * first dot encountered. - * @param value file name in snake_case - * @return main part of file name in CamelCase. - */ - public static String snakeToCamelCase(String value) { - char[] fileName = value.toCharArray(); - - StringBuilder builder = new StringBuilder(fileName.length); - - boolean nextIsUpperCase = true; - for (char c : fileName) { - switch (c) { - case '_': - nextIsUpperCase = true; - break; - case '.': - return builder.toString(); - default: - if (nextIsUpperCase) { - builder.append(String.valueOf(c).toUpperCase(Locale.ENGLISH)); - nextIsUpperCase = false; - } else { - builder.append(c); - } - break; - } - } - - return builder.toString(); - } - - /** Apply a throwing function, and if it throws, log it and let it return an empty Stream. */ - public static Function> applyOrEmpty(ThrowingFunction> func) { - return t -> { - try { - return func.apply(t); - } catch (Exception ex) { - logger.error("Failed to apply function, returning empty.", ex); - return Stream.empty(); - } - }; - } - - - /** Apply a throwing function, and if it throws, log it and let it return an empty Stream. */ - public static Function> applyOrIllegalException( - ThrowingFunction> func) { - return t -> { - try { - return func.apply(t); - } catch (Exception ex) { - throw new IllegalStateException(ex.getMessage(), ex); - } - }; - } - - /** - * Function that may throw an exception. - * @param type of value taken. - * @param type of value returned. - */ - @FunctionalInterface - @SuppressWarnings("PMD.SignatureDeclareThrowsException") - public interface ThrowingFunction { - /** - * Apply containing function. - * @param value value to apply function to. - * @return result of the function. - * @throws Exception if the function fails. - */ - R apply(T value) throws Exception; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt new file mode 100644 index 00000000..d014bb7e --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt @@ -0,0 +1,149 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.util + +import org.slf4j.LoggerFactory +import java.io.IOException +import java.util.Properties +import java.util.function.Function +import java.util.stream.Stream + +/** + * TODO. + */ +object SchemaUtils { + private val logger = LoggerFactory.getLogger(SchemaUtils::class.java) + private const val GRADLE_PROPERTIES = "exchange.properties" + private const val GROUP_PROPERTY = "project.group" + + @JvmStatic + @get:Synchronized + var projectGroup: String? = null + /** + * TODO. + * @return TODO + */ + get() { + if (field == null) { + val prop = Properties() + val loader = ClassLoader.getSystemClassLoader() + try { + loader.getResourceAsStream(GRADLE_PROPERTIES).use { `in` -> + if (`in` == null) { + field = "org.radarcns" + logger.debug("Project group not specified. Using \"{}\".", field) + } else { + prop.load(`in`) + field = prop.getProperty(GROUP_PROPERTY) + if (field == null) { + field = "org.radarcns" + logger.debug("Project group not specified. Using \"{}\".", field) + } + } + } + } catch (exc: IOException) { + throw IllegalStateException( + GROUP_PROPERTY + + " cannot be extracted from " + GRADLE_PROPERTIES, + exc, + ) + } + } + return field + } + private set + + /** + * Expand a class name with the group name if it starts with a dot. + * @param classShorthand class name, possibly starting with a dot as a shorthand. + * @return class name or `null` if null or empty. + */ + fun expandClass(classShorthand: String?): String? { + return when { + classShorthand.isNullOrEmpty() -> null + classShorthand[0] == '.' -> projectGroup + classShorthand + else -> classShorthand + } + } + + /** + * Converts given file name from snake_case to CamelCase. This will cause underscores to be + * removed, and the next character to be uppercase. This only converts the value up to the + * first dot encountered. + * @param value file name in snake_case + * @return main part of file name in CamelCase. + */ + @JvmStatic + fun snakeToCamelCase(value: String): String { + val fileName = value.toCharArray() + val builder = StringBuilder(fileName.size) + var nextIsUpperCase = true + for (c in fileName) { + when (c) { + '_' -> nextIsUpperCase = true + '.' -> return builder.toString() + else -> if (nextIsUpperCase) { + builder.append(c.toString().uppercase()) + nextIsUpperCase = false + } else { + builder.append(c) + } + } + } + return builder.toString() + } + + /** Apply a throwing function, and if it throws, log it and let it return an empty Stream. */ + fun applyOrEmpty(func: ThrowingFunction?>): Function?> { + return Function { t: T -> + try { + return@Function func.apply(t) + } catch (ex: Exception) { + logger.error("Failed to apply function, returning empty.", ex) + return@Function Stream.empty() + } + } + } + + /** Apply a throwing function, and if it throws, log it and let it return an empty Stream. */ + fun applyOrIllegalException( + func: ThrowingFunction?>, + ): Function?> { + return Function { t: T -> + try { + return@Function func.apply(t) + } catch (ex: Exception) { + throw IllegalStateException(ex.message, ex) + } + } + } + + /** + * Function that may throw an exception. + * @param type of value taken. + * @param type of value returned. + */ + fun interface ThrowingFunction { + /** + * Apply containing function. + * @param value value to apply function to. + * @return result of the function. + * @throws Exception if the function fails. + */ + @Throws(Exception::class) + fun apply(value: T): R + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt index d9e30aa2..d799406d 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt @@ -21,34 +21,33 @@ import org.radarbase.schema.Scope import org.radarbase.schema.specification.DataProducer import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.config.SchemaConfig -import org.radarbase.schema.validation.rules.* +import org.radarbase.schema.validation.ValidationHelper.matchesExtension +import org.radarbase.schema.validation.rules.RadarSchemaMetadataRules +import org.radarbase.schema.validation.rules.RadarSchemaRules +import org.radarbase.schema.validation.rules.SchemaMetadata +import org.radarbase.schema.validation.rules.SchemaMetadataRules +import org.radarbase.schema.validation.rules.Validator import java.nio.file.Path import java.nio.file.PathMatcher -import java.util.* +import java.util.Arrays +import java.util.Objects import java.util.stream.Collectors import java.util.stream.Stream /** * Validator for a set of RADAR-Schemas. + * + * @param schemaRoot RADAR-Schemas commons directory. + * @param config configuration to exclude certain schemas or fields from validation. */ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { - val rules: SchemaMetadataRules - private val pathMatcher: PathMatcher - private var validator: Validator - - /** - * Schema validator for given RADAR-Schemas directory. - * @param schemaRoot RADAR-Schemas commons directory. - * @param config configuration to exclude certain schemas or fields from validation. - */ - init { - pathMatcher = config.pathMatcher(schemaRoot) - rules = RadarSchemaMetadataRules(schemaRoot, config) - validator = rules.getValidator(false) - } + val rules: SchemaMetadataRules = RadarSchemaMetadataRules(schemaRoot, config) + private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) + private var validator: Validator = rules.getValidator(false) fun analyseSourceCatalogue( - scope: Scope?, catalogue: SourceCatalogue + scope: Scope?, + catalogue: SourceCatalogue, ): Stream { validator = rules.getValidator(true) val producers: Stream> = if (scope != null) { @@ -62,9 +61,9 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { .flatMap { it.data.stream() } .flatMap { topic -> val (keySchema, valueSchema) = catalogue.schemaCatalogue.getSchemaMetadata(topic) - Stream.of(keySchema, valueSchema) + Stream.of(keySchema, valueSchema).filter { it.schema != null } } - .sorted(Comparator.comparing { it.schema.fullName }) + .sorted(Comparator.comparing { it.schema!!.fullName }) .distinct() .flatMap(this::validate) .distinct() @@ -73,13 +72,9 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { } } - /** - * TODO. - * @param scope TODO. - */ fun analyseFiles( scope: Scope?, - schemaCatalogue: SchemaCatalogue + schemaCatalogue: SchemaCatalogue, ): Stream { if (scope == null) { return analyseFiles(schemaCatalogue) @@ -100,7 +95,7 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { val parser = Schema.Parser() parser.addTypes(useTypes) try { - parser.parse(p.path.toFile()) + parser.parse(p.path?.toFile()) return@map null } catch (ex: Exception) { return@map ValidationException("Cannot parse schema", ex) @@ -109,15 +104,11 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { .filter(Objects::nonNull) .map { obj -> requireNotNull(obj) }, schemaCatalogue.schemas.values.stream() - .flatMap { this.validate(it) } + .flatMap { this.validate(it) }, ).distinct() } - - /** - * TODO. - */ private fun analyseFiles(schemaCatalogue: SchemaCatalogue): Stream = - Arrays.stream(Scope.values()) + Arrays.stream(Scope.entries.toTypedArray()) .flatMap { scope -> analyseFiles(scope, schemaCatalogue) } /** Validate a single schema in given path. */ @@ -127,8 +118,10 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { /** Validate a single schema in given path. */ private fun validate(schemaMetadata: SchemaMetadata): Stream = if (pathMatcher.matches(schemaMetadata.path)) { - validator.apply(schemaMetadata) - } else Stream.empty() + validator.validate(schemaMetadata) + } else { + Stream.empty() + } val validatedSchemas: Map get() = (rules.schemaRules as RadarSchemaRules).schemaStore @@ -146,17 +139,13 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { |${ex.message} | | - |""".trimMargin() + | + """.trimMargin() } .collect(Collectors.joining()) } - /** - * TODO. - * @param file TODO - * @return TODO - */ - fun isAvscFile(file: Path?): Boolean = - ValidationHelper.matchesExtension(file, AVRO_EXTENSION) + fun isAvscFile(file: Path): Boolean = + matchesExtension(file, AVRO_EXTENSION) } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.java deleted file mode 100644 index b6ee64ca..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.config.SchemaConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.PathMatcher; -import java.util.stream.Stream; - -import static org.radarbase.schema.validation.ValidationHelper.SPECIFICATIONS_PATH; - -/** - * Validates RADAR-Schemas specifications. - */ -public class SpecificationsValidator { - private static final Logger logger = LoggerFactory.getLogger(SpecificationsValidator.class); - - public static final String YML_EXTENSION = "yml"; - private final Path specificationsRoot; - private final ObjectMapper mapper; - private final PathMatcher pathMatcher; - - /** - * Specifications validator for given RADAR-Schemas directory. - * @param root RADAR-Schemas directory. - * @param config configuration to exclude certain schemas or fields from validation. - */ - public SpecificationsValidator(Path root, SchemaConfig config) { - this.specificationsRoot = root.resolve(SPECIFICATIONS_PATH); - this.pathMatcher = config.pathMatcher(specificationsRoot); - this.mapper = new ObjectMapper(new YAMLFactory()); - } - - /** Check that all files in the specifications directory are YAML files. */ - public boolean specificationsAreYmlFiles(Scope scope) throws IOException { - Path baseFolder = scope.getPath(specificationsRoot); - if (baseFolder == null) { - logger.info("{} sources folder not present at {}", scope, - specificationsRoot.resolve(scope.getLower())); - return false; - } - - try (Stream walker = Files.walk(baseFolder)) { - return walker - .filter(pathMatcher::matches) - .allMatch(SpecificationsValidator::isYmlFile); - } - } - - public boolean checkSpecificationParsing(Scope scope, Class clazz) throws IOException { - Path baseFolder = scope.getPath(specificationsRoot); - if (baseFolder == null) { - logger.info("{} sources folder not present at {}", scope, - specificationsRoot.resolve(scope.getLower())); - return false; - } - - try (Stream walker = Files.walk(baseFolder)) { - return walker - .filter(pathMatcher::matches) - .allMatch(f -> { - try { - mapper.readerFor(clazz).readValue(f.toFile()); - return true; - } catch (IOException ex) { - logger.error("Failed to load configuration {}: {}", f, - ex.toString()); - return false; - } - }); - } - } - - private static boolean isYmlFile(Path path) { - return ValidationHelper.matchesExtension(path, YML_EXTENSION); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt new file mode 100644 index 00000000..25bd2f5c --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt @@ -0,0 +1,108 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import org.radarbase.schema.Scope +import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.validation.ValidationHelper.SPECIFICATIONS_PATH +import org.radarbase.schema.validation.ValidationHelper.matchesExtension +import org.slf4j.LoggerFactory +import java.io.IOException +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.PathMatcher +import kotlin.io.path.walk + +/** + * Validates RADAR-Schemas specifications. + */ +class SpecificationsValidator(root: Path, config: SchemaConfig) { + private val specificationsRoot: Path + private val mapper: ObjectMapper + private val pathMatcher: PathMatcher + + /** + * Specifications validator for given RADAR-Schemas directory. + * @param root RADAR-Schemas directory. + * @param config configuration to exclude certain schemas or fields from validation. + */ + init { + specificationsRoot = root.resolve(SPECIFICATIONS_PATH) + pathMatcher = config.pathMatcher(specificationsRoot) + mapper = ObjectMapper(YAMLFactory()) + } + + /** Check that all files in the specifications directory are YAML files. */ + @Throws(IOException::class) + fun specificationsAreYmlFiles(scope: Scope): Boolean { + val baseFolder = scope.getPath(specificationsRoot) + if (baseFolder == null) { + logger.info( + "{} sources folder not present at {}", + scope, + specificationsRoot.resolve(scope.lower), + ) + return false + } + Files.walk(baseFolder).use { walker -> + return walker + .filter { path: Path? -> pathMatcher.matches(path) } + .allMatch { path: Path -> isYmlFile(path) } + } + } + + @Throws(IOException::class) + fun checkSpecificationParsing(scope: Scope, clazz: Class?): Boolean { + val baseFolder = scope.getPath(specificationsRoot) + if (baseFolder == null) { + logger.info( + "{} sources folder not present at {}", + scope, + specificationsRoot.resolve(scope.lower), + ) + return false + } + Files.walk(baseFolder).use { walker -> + return walker + .filter { path: Path? -> pathMatcher.matches(path) } + .allMatch { f: Path -> + try { + mapper.readerFor(clazz).readValue(f.toFile()) + return@allMatch true + } catch (ex: IOException) { + logger.error( + "Failed to load configuration {}: {}", + f, + ex.toString(), + ) + return@allMatch false + } + } + } + } + + companion object { + private val logger = LoggerFactory.getLogger( + SpecificationsValidator::class.java, + ) + const val YML_EXTENSION = "yml" + private fun isYmlFile(path: Path): Boolean { + return matchesExtension(path, YML_EXTENSION) + } + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.java deleted file mode 100644 index 935b5a2b..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation; - -import java.util.Objects; - -/** - * TODO. - */ -public class ValidationException extends RuntimeException { - private static final long serialVersionUID = 1; - - public ValidationException(String message) { - super(message); - } - - public ValidationException(String message, Throwable exception) { - super(message, exception); - } - - @Override - public boolean equals(Object obj) { - if (obj == this) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - ValidationException ex = (ValidationException) obj; - return Objects.equals(getMessage(), ex.getMessage()) - && Objects.equals(getCause(), ex.getCause()); - } - - @Override - public int hashCode() { - return getMessage().hashCode(); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt new file mode 100644 index 00000000..579a2dc2 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation + +/** + * TODO. + */ +class ValidationException : RuntimeException { + constructor(message: String?) : super(message) + constructor(message: String?, exception: Throwable?) : super(message, exception) +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.java deleted file mode 100644 index 27d0f16c..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation; - -import static org.radarbase.schema.util.SchemaUtils.getProjectGroup; -import static org.radarbase.schema.util.SchemaUtils.snakeToCamelCase; - -import java.nio.file.Path; -import java.util.Locale; -import java.util.Objects; -import java.util.function.Predicate; -import java.util.regex.Pattern; -import org.radarbase.schema.Scope; - -/** - * TODO. - */ -public final class ValidationHelper { - public static final String COMMONS_PATH = "commons"; - public static final String SPECIFICATIONS_PATH = "specifications"; - - /** Package names. */ - public enum Package { - AGGREGATOR(".kafka.aggregator"), - BIOVOTION(".passive.biovotion"), - EMPATICA(".passive.empatica"), - KAFKA_KEY(".kafka.key"), - MONITOR(".monitor"), - PEBBLE(".passive.pebble"), - QUESTIONNAIRE(".active.questionnaire"); - - private final String name; - - Package(String name) { - this.name = name; - } - - public String getName() { - return name; - } - } - - // snake case - private static final Pattern TOPIC_PATTERN = Pattern.compile( - "[A-Za-z][a-z0-9-]*(_[A-Za-z0-9-]+)*"); - - private ValidationHelper() { - //Static class - } - - /** - * TODO. - * @param scope TODO - * @return TODO - */ - public static String getNamespace(Path schemaRoot, Path schemaPath, Scope scope) { - // add subfolder of root to namespace - Path rootPath = scope.getPath(schemaRoot); - if (rootPath == null) { - throw new IllegalArgumentException("Scope " + scope + " does not have a commons path"); - } - Path relativePath = rootPath.relativize(schemaPath); - - StringBuilder builder = new StringBuilder(50); - builder.append(getProjectGroup()).append('.').append(scope.getLower()); - for (int i = 0; i < relativePath.getNameCount() - 1; i++) { - builder.append('.').append(relativePath.getName(i)); - } - return builder.toString(); - } - - /** - * TODO. - * @param path TODO - * @return TODO - */ - public static String getRecordName(Path path) { - Objects.requireNonNull(path); - - return snakeToCamelCase(path.getFileName().toString()); - } - - /** - * TODO. - * @param topicName TODO - * @return TODO - */ - public static boolean isValidTopic(String topicName) { - return topicName != null && TOPIC_PATTERN.matcher(topicName).matches(); - } - - /** - * TODO. - * @param file TODO. - * @return TODO. - */ - public static boolean matchesExtension(Path file, String extension) { - return file.toString().toLowerCase(Locale.ENGLISH) - .endsWith("." + extension.toLowerCase(Locale.ENGLISH)); - } - - /** - * TODO. - * @param file TODO - * @param extension TODO - * @return TODO - */ - public static boolean equalsFileName(String str, Path file, String extension) { - return equalsFileName(file, extension).test(str); - } - - - /** - * TODO. - * @param file TODO - * @param extension TODO - * @return TODO - */ - public static Predicate equalsFileName(Path file, String extension) { - return str -> { - String fileName = file.getFileName().toString(); - if (fileName.endsWith(extension)) { - fileName = fileName.substring(0, fileName.length() - extension.length()); - } - - return str.equalsIgnoreCase(fileName); - }; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt new file mode 100644 index 00000000..4addb5f8 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt @@ -0,0 +1,103 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation + +import org.radarbase.schema.Scope +import org.radarbase.schema.util.SchemaUtils.projectGroup +import org.radarbase.schema.util.SchemaUtils.snakeToCamelCase +import java.nio.file.Path +import java.util.Objects +import java.util.function.Predicate +import java.util.regex.Pattern + +/** + * TODO. + */ +object ValidationHelper { + const val COMMONS_PATH = "commons" + const val SPECIFICATIONS_PATH = "specifications" + + // snake case + private val TOPIC_PATTERN = Pattern.compile( + "[A-Za-z][a-z0-9-]*(_[A-Za-z0-9-]+)*", + ) + + /** + * TODO. + * @param scope TODO + * @return TODO + */ + fun getNamespace(schemaRoot: Path?, schemaPath: Path?, scope: Scope): String { + // add subfolder of root to namespace + val rootPath = scope.getPath(schemaRoot) + ?: throw IllegalArgumentException("Scope $scope does not have a commons path") + val relativePath = rootPath.relativize(schemaPath) + val builder = StringBuilder(50) + builder.append(projectGroup).append('.').append(scope.lower) + for (i in 0 until relativePath.nameCount - 1) { + builder.append('.').append(relativePath.getName(i)) + } + return builder.toString() + } + + /** + * TODO. + * @param path TODO + * @return TODO + */ + @JvmStatic + fun getRecordName(path: Path): String { + Objects.requireNonNull(path) + return snakeToCamelCase(path.fileName.toString()) + } + + /** + * TODO. + * @param topicName TODO + * @return TODO + */ + @JvmStatic + fun isValidTopic(topicName: String?): Boolean { + return topicName != null && TOPIC_PATTERN.matcher(topicName).matches() + } + + /** + * TODO. + * @param file TODO. + * @return TODO. + */ + @JvmStatic + fun matchesExtension(file: Path, extension: String): Boolean { + return file.toString().lowercase() + .endsWith("." + extension.lowercase()) + } + + /** + * TODO. + * @param file TODO + * @param extension TODO + * @return TODO + */ + fun equalsFileName(file: Path, extension: String): Predicate { + return Predicate { str: String -> + var fileName = file.fileName.toString() + if (fileName.endsWith(extension)) { + fileName = fileName.substring(0, fileName.length - extension.length) + } + str.equals(fileName, ignoreCase = true) + } + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/config/ConfigItem.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/config/ConfigItem.java deleted file mode 100644 index a85f297a..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/config/ConfigItem.java +++ /dev/null @@ -1,83 +0,0 @@ -package org.radarbase.schema.validation.config; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; -import java.util.Collection; -import java.util.HashSet; -import java.util.Locale; -import java.util.Set; - -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * TODO. - */ -public class ConfigItem { - - /** Possible check status. */ - private enum CheckStatus { - DISABLE, ENABLE; - - private final String name; - - CheckStatus() { - this.name = this.name().toLowerCase(Locale.ENGLISH); - } - - public String getName() { - return name; - } - } - - @JsonProperty("record_name_check") - @SuppressWarnings("PMD.ImmutableField") - private CheckStatus nameRecordCheck = CheckStatus.ENABLE; - - private final Set fields = new HashSet<>(); - - public ConfigItem() { - // POJO initializer - } - - /** - * TODO. - */ - @JsonSetter("fields") - public void setFields(Collection fields) { - if (!this.fields.isEmpty()) { - this.fields.clear(); - } - this.fields.addAll(fields); - } - - /** - * TODO. - * - * @return TODO - */ - public Set getFields() { - return fields; - } - - @Override - public String toString() { - return "ConfigItem{" - + "nameRecordCheck=" + nameRecordCheck - + ", fields=" + fields - + '}'; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.java deleted file mode 100644 index 8d91f0b7..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.java +++ /dev/null @@ -1,101 +0,0 @@ -package org.radarbase.schema.validation.rules; - -import static org.radarbase.schema.validation.rules.RadarSchemaRules.validateDocumentation; -import static org.radarbase.schema.validation.rules.Validator.check; -import static org.radarbase.schema.validation.rules.Validator.matches; -import static org.radarbase.schema.validation.rules.Validator.valid; -import static org.radarbase.schema.validation.rules.Validator.validateNonNull; - -import java.util.EnumMap; -import java.util.Map; -import java.util.regex.Pattern; -import java.util.stream.Stream; -import org.apache.avro.JsonProperties; -import org.apache.avro.Schema; -import org.radarbase.schema.validation.ValidationException; - -/** - * Rules for RADAR-Schemas schema fields. - */ -public class RadarSchemaFieldRules implements SchemaFieldRules { - private static final String UNKNOWN = "UNKNOWN"; - // lowerCamelCase - public static final Pattern FIELD_NAME_PATTERN = Pattern.compile( - "^[a-z][a-z0-9]*([a-z0-9][A-Z][a-z0-9]+)?([A-Z][a-z0-9]+)*[A-Z]?$"); - - private final Map> defaultsValidator; - - /** - * Rules for RADAR-Schemas schema fields. - */ - public RadarSchemaFieldRules() { - defaultsValidator = new EnumMap<>(Schema.Type.class); - defaultsValidator.put(Schema.Type.ENUM, this::validateDefaultEnum); - defaultsValidator.put(Schema.Type.UNION, this::validateDefaultUnion); - } - - @Override - public Validator validateFieldTypes(SchemaRules schemaRules) { - return field -> { - Schema schema = field.getField().schema(); - Schema.Type subType = schema.getType(); - if (subType == Schema.Type.UNION) { - return validateInternalUnion(schemaRules).apply(field); - } else if (subType == Schema.Type.RECORD) { - return schemaRules.validateRecord().apply(schema); - } else if (subType == Schema.Type.ENUM) { - return schemaRules.validateEnum().apply(schema); - } else { - return valid(); - } - }; - } - - @Override - public Validator validateDefault() { - return input -> defaultsValidator - .getOrDefault(input.getField().schema().getType(), - this::validateDefaultOther) - .apply(input); - } - - - @Override - public Validator validateFieldName() { - return validateNonNull(f -> f.getField().name(), matches(FIELD_NAME_PATTERN), message( - "Field name does not respect lowerCamelCase name convention." - + " Please avoid abbreviations and write out the field name instead.")); - } - - @Override - public Validator validateFieldDocumentation() { - return field -> validateDocumentation(field.getField().doc(), - (m, f) -> message(m).apply(f), field); - } - - - private Stream validateDefaultEnum(SchemaField field) { - return check(!field.getField().schema().getEnumSymbols().contains(UNKNOWN) - || field.getField().defaultVal() != null - && field.getField().defaultVal().toString().equals(UNKNOWN), - message("Default is \"" + field.getField().defaultVal() - + "\". Any Avro enum type that has an \"UNKNOWN\" symbol must set its" - + " default value to \"UNKNOWN\".").apply(field)); - } - - private Stream validateDefaultUnion(SchemaField field) { - return check( - !field.getField().schema().getTypes().contains(Schema.create(Schema.Type.NULL)) - || field.getField().defaultVal() != null - && field.getField().defaultVal().equals(JsonProperties.NULL_VALUE), - message("Default is not null. Any nullable Avro field must" - + " specify have its default value set to null.").apply(field)); - } - - private Stream validateDefaultOther(SchemaField field) { - return check(field.getField().defaultVal() == null, message( - "Default of type " + field.getField().schema().getType() + " is set to " - + field.getField().defaultVal() + ". The only acceptable default values are the" - + " \"UNKNOWN\" enum symbol and null.").apply(field)); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt new file mode 100644 index 00000000..a69b4b4a --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt @@ -0,0 +1,119 @@ +package org.radarbase.schema.validation.rules + +import org.apache.avro.JsonProperties +import org.apache.avro.Schema +import org.apache.avro.Schema.Type +import org.apache.avro.Schema.Type.ENUM +import org.apache.avro.Schema.Type.NULL +import org.apache.avro.Schema.Type.RECORD +import org.apache.avro.Schema.Type.UNION +import org.radarbase.schema.validation.ValidationException +import org.radarbase.schema.validation.rules.RadarSchemaRules.Companion.validateDocumentation +import org.radarbase.schema.validation.rules.SchemaFieldRules.Companion.message +import org.radarbase.schema.validation.rules.Validator.Companion.check +import org.radarbase.schema.validation.rules.Validator.Companion.validate +import java.util.EnumMap +import java.util.stream.Stream + +/** + * Rules for RADAR-Schemas schema fields. + */ +class RadarSchemaFieldRules : SchemaFieldRules { + private val defaultsValidator: MutableMap> + + /** + * Rules for RADAR-Schemas schema fields. + */ + init { + defaultsValidator = EnumMap(Type::class.java) + defaultsValidator[ENUM] = Validator { validateDefaultEnum(it) } + defaultsValidator[UNION] = Validator { validateDefaultUnion(it) } + } + + override fun validateFieldTypes(schemaRules: SchemaRules): Validator { + return Validator { field -> + val schema = field.field.schema() + val subType = schema.type + return@Validator when (subType) { + UNION -> validateInternalUnion(schemaRules).validate(field) + RECORD -> schemaRules.validateRecord().validate(schema) + ENUM -> schemaRules.validateEnum().validate(schema) + else -> Validator.valid() + } + } + } + + override fun validateDefault(): Validator { + return Validator { input: SchemaField -> + defaultsValidator + .getOrDefault( + input.field.schema().type, + Validator { validateDefaultOther(it) }, + ) + .validate(input) + } + } + + override fun validateFieldName(): Validator { + return validate( + { f -> f.field.name()?.matches(FIELD_NAME_PATTERN) == true }, + "Field name does not respect lowerCamelCase name convention." + + " Please avoid abbreviations and write out the field name instead.", + ) + } + + override fun validateFieldDocumentation(): Validator { + return Validator { field: SchemaField -> + validateDocumentation( + field.field.doc(), + { m, f -> message(f, m) }, + field, + ) + } + } + + private fun validateDefaultEnum(field: SchemaField): Stream { + return check( + !field.field.schema().enumSymbols.contains(UNKNOWN) || + field.field.defaultVal() != null && field.field.defaultVal() + .toString() == UNKNOWN, + message( + field, + "Default is \"" + field.field.defaultVal() + + "\". Any Avro enum type that has an \"UNKNOWN\" symbol must set its" + + " default value to \"UNKNOWN\".", + ), + ) + } + + private fun validateDefaultUnion(field: SchemaField): Stream { + return check( + !field.field.schema().types.contains(Schema.create(NULL)) || + field.field.defaultVal() != null && field.field.defaultVal() == JsonProperties.NULL_VALUE, + message( + field, + "Default is not null. Any nullable Avro field must" + + " specify have its default value set to null.", + ), + ) + } + + private fun validateDefaultOther(field: SchemaField): Stream { + return check( + field.field.defaultVal() == null, + message( + field, + "Default of type " + field.field.schema().type + " is set to " + + field.field.defaultVal() + ". The only acceptable default values are the" + + " \"UNKNOWN\" enum symbol and null.", + ), + ) + } + + companion object { + private const val UNKNOWN = "UNKNOWN" + + // lowerCamelCase + internal val FIELD_NAME_PATTERN = "[a-z][a-z0-9]*([a-z0-9][A-Z][a-z0-9]+)?([A-Z][a-z0-9]+)*[A-Z]?".toRegex() + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt index 2b248de3..e193d91b 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt @@ -3,22 +3,22 @@ package org.radarbase.schema.validation.rules import org.apache.avro.Schema import org.radarbase.schema.specification.config.SchemaConfig import org.radarbase.schema.validation.ValidationHelper -import org.radarbase.schema.validation.rules.RadarSchemaRules +import org.radarbase.schema.validation.rules.Validator.Companion.check +import org.radarbase.schema.validation.rules.Validator.Companion.raise +import org.radarbase.schema.validation.rules.Validator.Companion.valid import java.nio.file.Path import java.nio.file.PathMatcher -/** Rules for schemas with metadata in RADAR-Schemas. */ -class RadarSchemaMetadataRules -/** - * Rules for schemas with metadata in RADAR-Schemas. +/** Rules for schemas with metadata in RADAR-Schemas. + * * @param schemaRoot directory of RADAR-Schemas commons * @param config configuration for excluding schemas from validation. * @param schemaRules schema rules implementation. */ -@JvmOverloads constructor( +class RadarSchemaMetadataRules( private val schemaRoot: Path, config: SchemaConfig, - override val schemaRules: SchemaRules = RadarSchemaRules() + override val schemaRules: SchemaRules = RadarSchemaRules(), ) : SchemaMetadataRules { private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) @@ -27,43 +27,49 @@ class RadarSchemaMetadataRules .and(validateNameSchemaLocation()) private fun validateNamespaceSchemaLocation(): Validator = - Validator { metadata: SchemaMetadata -> + Validator { metadata -> try { val expected = ValidationHelper.getNamespace( - schemaRoot, metadata.path, metadata.scope + schemaRoot, + metadata.path, + metadata.scope, ) - val namespace = metadata.schema.namespace - return@Validator Validator.check( - expected.equals(namespace, ignoreCase = true), message( - "Namespace cannot be null and must fully lowercase dot separated without numeric. In this case the expected value is \"$expected\"." - ).invoke(metadata) + val namespace = metadata.schema?.namespace + return@Validator check( + expected.equals(namespace, ignoreCase = true), + message( + metadata, + "Namespace cannot be null and must fully lowercase dot separated without numeric. In this case the expected value is \"$expected\".", + ), ) } catch (ex: IllegalArgumentException) { - return@Validator Validator.raise( - "Path " + metadata.path - + " is not part of root " + schemaRoot, ex + return@Validator raise( + "Path " + metadata.path + + " is not part of root " + schemaRoot, + ex, ) } } private fun validateNameSchemaLocation(): Validator = - Validator { metadata: SchemaMetadata -> + Validator { metadata -> + if (metadata.path == null) { + return@Validator raise(message(metadata, "Missing metadata path")) + } val expected = ValidationHelper.getRecordName(metadata.path) - if (expected.equals( - metadata.schema.name, - ignoreCase = true - ) - ) Validator.valid() else Validator.raise( - message( - "Record name should match file name. Expected record name is \"$expected\"." - ).invoke(metadata) - ) + if (expected.equals(metadata.schema?.name, ignoreCase = true)) { + valid() + } else { + raise(message(metadata, "Record name should match file name. Expected record name is \"$expected\".")) + } } override fun schema(validator: Validator): Validator = - Validator { metadata: SchemaMetadata -> - if (pathMatcher.matches(metadata.path)) validator.apply( - metadata.schema - ) else Validator.valid() + Validator { metadata -> + when { + metadata.schema == null -> raise("Missing schema") + pathMatcher.matches(metadata.path) -> validator.validate(metadata.schema) + else -> valid() + } } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.java deleted file mode 100644 index 5d67ee67..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.java +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation.rules; - -import io.confluent.connect.avro.AvroData; -import io.confluent.connect.avro.AvroDataConfig; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Type; -import org.radarbase.schema.validation.ValidationException; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.function.BiFunction; -import java.util.regex.Pattern; -import java.util.stream.Stream; - -import static io.confluent.connect.avro.AvroDataConfig.CONNECT_META_DATA_CONFIG; -import static io.confluent.connect.avro.AvroDataConfig.ENHANCED_AVRO_SCHEMA_SUPPORT_CONFIG; -import static io.confluent.connect.avro.AvroDataConfig.SCHEMAS_CACHE_SIZE_CONFIG; -import static java.util.function.Predicate.not; -import static org.radarbase.schema.validation.rules.Validator.check; -import static org.radarbase.schema.validation.rules.Validator.matches; -import static org.radarbase.schema.validation.rules.Validator.raise; -import static org.radarbase.schema.validation.rules.Validator.valid; -import static org.radarbase.schema.validation.rules.Validator.validate; -import static org.radarbase.schema.validation.rules.Validator.validateNonEmpty; -import static org.radarbase.schema.validation.rules.Validator.validateNonNull; - -/** - * Schema validation rules enforced for the RADAR-Schemas repository. - */ -public class RadarSchemaRules implements SchemaRules { - // used in testing - static final String TIME = "time"; - private static final String TIME_RECEIVED = "timeReceived"; - private static final String TIME_COMPLETED = "timeCompleted"; - - static final Pattern NAMESPACE_PATTERN = Pattern.compile("^[a-z]+(\\.[a-z]+)*$"); - private final Map schemaStore; - - // CamelCase - // see SchemaValidatorRolesTest#recordNameRegex() for valid and invalid values - static final Pattern RECORD_NAME_PATTERN = Pattern.compile( - "^([A-Z]([a-z]*[0-9]*))+[A-Z]?$"); - - // used in testing - static final Pattern ENUM_SYMBOL_PATTERN = Pattern.compile("^[A-Z][A-Z0-9_]*$"); - - private static final String WITH_TYPE_DOUBLE = "\" field with type \"double\"."; - - private final RadarSchemaFieldRules fieldRules; - - /** - * RADAR-Schema validation rules. - */ - public RadarSchemaRules(RadarSchemaFieldRules fieldRules) { - this.fieldRules = fieldRules; - this.schemaStore = new HashMap<>(); - } - - public RadarSchemaRules() { - this(new RadarSchemaFieldRules()); - } - - @Override - public SchemaFieldRules getFieldRules() { - return fieldRules; - } - - @Override - public Validator validateUniqueness() { - return schema -> { - String key = schema.getFullName(); - Schema oldSchema = schemaStore.putIfAbsent(key, schema); - return check(oldSchema == null || oldSchema.equals(schema), messageSchema( - "Schema is already defined elsewhere with a different definition.") - .apply(schema)); - }; - } - - @Override - public Validator validateNameSpace() { - return validateNonNull(Schema::getNamespace, matches(NAMESPACE_PATTERN), - messageSchema("Namespace cannot be null and must fully lowercase, period-" - + "separated, without numeric characters.")); - } - - @Override - public Validator validateName() { - return validateNonNull(Schema::getName, matches(RECORD_NAME_PATTERN), - messageSchema("Record names must be camel case.")); - } - - @Override - public Validator validateSchemaDocumentation() { - return schema -> validateDocumentation(schema.getDoc(), (m, t) -> messageSchema(m).apply(t), - schema); - } - - static Stream validateDocumentation(String doc, - BiFunction message, T schema) { - if (doc == null || doc.isEmpty()) { - return raise(message.apply("Property \"doc\" is missing. Documentation is" - + " mandatory for all fields. The documentation should report what is being" - + " measured, how, and what units or ranges are applicable. Abbreviations" - + " and acronyms in the documentation should be written out. The sentence" - + " must end with a period '.'. Please add \"doc\" property.", schema)); - } - - Stream result = valid(); - if (doc.charAt(doc.length() - 1) != '.') { - result = raise(message.apply("Documentation is not terminated with a period. The" - + " documentation should report what is being measured, how, and what units" - + " or ranges are applicable. Abbreviations and acronyms in the" - + " documentation should be written out. Please end the sentence with a" - + " period '.'.", schema)); - } - if (!Character.isUpperCase(doc.charAt(0))) { - result = Stream.concat(result, raise( - message.apply("Documentation does not start with a capital letter. The" - + " documentation should report what is being measured, how, and what" - + " units or ranges are applicable. Abbreviations and acronyms in the" - + " documentation should be written out. Please end the sentence with a" - + " period '.'.", schema))); - } - return result; - } - - - @Override - public Validator validateSymbols() { - return validateNonEmpty(Schema::getEnumSymbols, messageSchema( - "Avro Enumerator must have symbol list.")) - .and(schema -> schema.getEnumSymbols().stream() - .filter(not(matches(ENUM_SYMBOL_PATTERN))) - .map(s -> new ValidationException(messageSchema( - "Symbol " + s + " does not use valid syntax. " - + "Enumerator items should be written in" - + " uppercase characters separated by underscores.") - .apply(schema)))); - } - - /** - * TODO. - * @return TODO - */ - @Override - public Validator validateTime() { - return validateNonNull(s -> s.getField(TIME), - time -> time.schema().getType().equals(Type.DOUBLE), messageSchema( - "Any schema representing collected data must have a \"" - + TIME + WITH_TYPE_DOUBLE)); - } - - /** - * TODO. - * @return TODO - */ - @Override - public Validator validateTimeCompleted() { - return validateNonNull(s -> s.getField(TIME_COMPLETED), - time -> time.schema().getType().equals(Type.DOUBLE), - messageSchema("Any ACTIVE schema must have a \"" - + TIME_COMPLETED + WITH_TYPE_DOUBLE)); - } - - /** - * TODO. - * @return TODO - */ - @Override - public Validator validateNotTimeCompleted() { - return validate(s -> s.getField(TIME_COMPLETED), Objects::isNull, - messageSchema("\"" + TIME_COMPLETED - + "\" is allow only in ACTIVE schemas.")); - } - - @Override - public Validator validateTimeReceived() { - return validateNonNull(s -> s.getField(TIME_RECEIVED), - time -> time.schema().getType().equals(Type.DOUBLE), - messageSchema("Any PASSIVE schema must have a \"" - + TIME_RECEIVED + WITH_TYPE_DOUBLE)); - } - - @Override - public Validator validateNotTimeReceived() { - return validate(s -> s.getField(TIME_RECEIVED), Objects::isNull, - messageSchema("\"" + TIME_RECEIVED + "\" is allow only in PASSIVE schemas.")); - } - - @Override - public Validator validateAvroData() { - return schema -> { - AvroDataConfig avroConfig = new AvroDataConfig.Builder() - .with(CONNECT_META_DATA_CONFIG, false) - .with(SCHEMAS_CACHE_SIZE_CONFIG, 10) - .with(ENHANCED_AVRO_SCHEMA_SUPPORT_CONFIG, true) - .build(); - AvroData encoder = new AvroData(10); - AvroData decoder = new AvroData(avroConfig); - try { - org.apache.kafka.connect.data.Schema connectSchema = encoder - .toConnectSchema(schema); - Schema originalSchema = decoder.fromConnectSchema(connectSchema); - return check(schema.equals(originalSchema), - () -> "Schema changed by validation: " - + schema.toString(true) + " is not equal to " - + originalSchema.toString(true)); - } catch (Exception ex) { - return raise("Failed to convert schema back to itself"); - } - }; - } - - @Override - public Validator fields(Validator validator) { - return schema -> { - if (!schema.getType().equals(Schema.Type.RECORD)) { - return raise("Default validation can be applied only to an Avro RECORD, not to " - + schema.getType() + " of schema " + schema.getFullName() + '.'); - } - if (schema.getFields().isEmpty()) { - return raise("Schema " + schema.getFullName() + " does not contain any fields."); - } - return schema.getFields().stream() - .flatMap(field -> { - SchemaField schemaField = new SchemaField(schema, field); - return validator.apply(schemaField); - }); - }; - } - - public Map getSchemaStore() { - return Collections.unmodifiableMap(schemaStore); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt new file mode 100644 index 00000000..cdfbdab8 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt @@ -0,0 +1,223 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation.rules + +import io.confluent.connect.avro.AvroData +import io.confluent.connect.avro.AvroDataConfig +import io.confluent.connect.avro.AvroDataConfig.Builder +import io.confluent.connect.schema.AbstractDataConfig +import org.apache.avro.Schema +import org.apache.avro.Schema.Type.DOUBLE +import org.apache.avro.Schema.Type.RECORD +import org.radarbase.schema.validation.ValidationException +import org.radarbase.schema.validation.rules.Validator.Companion.check +import org.radarbase.schema.validation.rules.Validator.Companion.raise +import org.radarbase.schema.validation.rules.Validator.Companion.valid +import org.radarbase.schema.validation.rules.Validator.Companion.validate +import java.util.stream.Stream + +/** + * Schema validation rules enforced for the RADAR-Schemas repository. + */ +class RadarSchemaRules( + override val fieldRules: RadarSchemaFieldRules = RadarSchemaFieldRules(), +) : SchemaRules { + val schemaStore: MutableMap = HashMap() + + override fun validateUniqueness() = Validator { schema: Schema -> + val key = schema.fullName + val oldSchema = schemaStore.putIfAbsent(key, schema) + check( + oldSchema == null || oldSchema == schema, + messageSchema(schema, "Schema is already defined elsewhere with a different definition."), + ) + } + + override fun validateNameSpace() = validate( + { it.namespace?.matches(NAMESPACE_PATTERN) == true }, + messageSchema("Namespace cannot be null and must fully lowercase, period-separated, without numeric characters."), + ) + + override fun validateName() = validate( + { it.name?.matches(RECORD_NAME_PATTERN) == true }, + messageSchema("Record names must be camel case."), + ) + + override fun validateSchemaDocumentation() = Validator { schema -> + validateDocumentation( + schema.doc, + { m, t -> messageSchema(t, m) }, + schema, + ) + } + + override fun validateSymbols() = validate( + { !it.enumSymbols.isNullOrEmpty() }, + messageSchema("Avro Enumerator must have symbol list."), + ).and(validateSymbolNames()) + + private fun validateSymbolNames() = Validator { schema -> + schema.enumSymbols.stream() + .filter { !it.matches(ENUM_SYMBOL_PATTERN) } + .map { s -> + ValidationException( + messageSchema( + schema, + "Symbol $s does not use valid syntax. " + + "Enumerator items should be written in uppercase characters separated by underscores.", + ), + ) + } + } + + /** + * TODO. + * @return TODO + */ + override fun validateTime(): Validator = validate( + { it.getField(TIME)?.schema()?.type == DOUBLE }, + messageSchema("Any schema representing collected data must have a \"$TIME$WITH_TYPE_DOUBLE"), + ) + + /** + * TODO. + * @return TODO + */ + override fun validateTimeCompleted(): Validator = validate( + { it.getField(TIME_COMPLETED)?.schema()?.type == DOUBLE }, + messageSchema("Any ACTIVE schema must have a \"$TIME_COMPLETED$WITH_TYPE_DOUBLE"), + ) + + /** + * TODO. + * @return TODO + */ + override fun validateNotTimeCompleted(): Validator = validate( + { it.getField(TIME_COMPLETED) == null }, + messageSchema("\"$TIME_COMPLETED\" is allow only in ACTIVE schemas."), + ) + + override fun validateTimeReceived(): Validator = validate( + { it.getField(TIME_RECEIVED)?.schema()?.type == DOUBLE }, + messageSchema("Any PASSIVE schema must have a \"$TIME_RECEIVED$WITH_TYPE_DOUBLE"), + ) + + override fun validateNotTimeReceived(): Validator = validate( + { it.getField(TIME_RECEIVED) == null }, + messageSchema("\"$TIME_RECEIVED\" is allow only in PASSIVE schemas."), + ) + + override fun validateAvroData(): Validator { + val avroConfig = Builder() + .with(AvroDataConfig.CONNECT_META_DATA_CONFIG, false) + .with(AbstractDataConfig.SCHEMAS_CACHE_SIZE_CONFIG, 10) + .with(AvroDataConfig.ENHANCED_AVRO_SCHEMA_SUPPORT_CONFIG, true) + .build() + + return Validator { schema: Schema -> + val encoder = AvroData(10) + val decoder = AvroData(avroConfig) + try { + val connectSchema = encoder.toConnectSchema(schema) + val originalSchema = decoder.fromConnectSchema(connectSchema) + check(schema == originalSchema) { + "Schema changed by validation: " + + schema.toString(true) + " is not equal to " + + originalSchema.toString(true) + } + } catch (ex: Exception) { + raise("Failed to convert schema back to itself") + } + } + } + + override fun fields(validator: Validator): Validator = + Validator { schema: Schema -> + when { + schema.type != RECORD -> raise( + "Default validation can be applied only to an Avro RECORD, not to ${schema.type} of schema ${schema.fullName}.", + ) + schema.fields.isEmpty() -> raise("Schema ${schema.fullName} does not contain any fields.") + else -> schema.fields.stream() + .flatMap { field -> validator.validate(SchemaField(schema, field)) } + } + } + + companion object { + // used in testing + const val TIME = "time" + private const val TIME_RECEIVED = "timeReceived" + private const val TIME_COMPLETED = "timeCompleted" + + val NAMESPACE_PATTERN = "[a-z]+(\\.[a-z]+)*".toRegex() + + // CamelCase + // see SchemaValidatorRolesTest#recordNameRegex() for valid and invalid values + val RECORD_NAME_PATTERN = "([A-Z]([a-z]*[0-9]*))+[A-Z]?".toRegex() + + // used in testing + val ENUM_SYMBOL_PATTERN = "[A-Z][A-Z0-9_]*".toRegex() + private const val WITH_TYPE_DOUBLE = "\" field with type \"double\"." + + fun validateDocumentation( + doc: String?, + message: (String, T) -> String, + schema: T, + ): Stream { + if (doc.isNullOrEmpty()) { + return raise( + message( + "Property \"doc\" is missing. Documentation is" + + " mandatory for all fields. The documentation should report what is being" + + " measured, how, and what units or ranges are applicable. Abbreviations" + + " and acronyms in the documentation should be written out. The sentence" + + " must end with a period '.'. Please add \"doc\" property.", + schema, + ), + ) + } + var result: Stream = valid() + if (doc[doc.length - 1] != '.') { + result = raise( + message( + "Documentation is not terminated with a period. The" + + " documentation should report what is being measured, how, and what units" + + " or ranges are applicable. Abbreviations and acronyms in the" + + " documentation should be written out. Please end the sentence with a" + + " period '.'.", + schema, + ), + ) + } + if (!Character.isUpperCase(doc[0])) { + result = Stream.concat( + result, + raise( + message( + "Documentation does not start with a capital letter. The" + + " documentation should report what is being measured, how, and what" + + " units or ranges are applicable. Abbreviations and acronyms in the" + + " documentation should be written out. Please end the sentence with a" + + " period '.'.", + schema, + ), + ), + ) + } + return result + } + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.java deleted file mode 100644 index a33c4df8..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.radarbase.schema.validation.rules; - -import org.apache.avro.Schema; - -public class SchemaField { - private final Schema schema; - private final Schema.Field field; - - public SchemaField(Schema schema, Schema.Field field) { - this.schema = schema; - this.field = field; - } - - public Schema getSchema() { - return schema; - } - - public Schema.Field getField() { - return field; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt new file mode 100644 index 00000000..af1027f1 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt @@ -0,0 +1,6 @@ +package org.radarbase.schema.validation.rules + +import org.apache.avro.Schema +import org.apache.avro.Schema.Field + +data class SchemaField(@JvmField val schema: Schema, @JvmField val field: Field) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.java deleted file mode 100644 index c1756328..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.radarbase.schema.validation.rules; - -import static org.radarbase.schema.validation.rules.Validator.raise; -import static org.radarbase.schema.validation.rules.Validator.valid; - -import java.util.function.Function; -import org.apache.avro.Schema; - -public interface SchemaFieldRules { - /** Recursively checks field types. */ - Validator validateFieldTypes(SchemaRules schemaRules); - - /** Checks field name format. */ - Validator validateFieldName(); - - /** Checks field documentation presence and format. */ - Validator validateFieldDocumentation(); - - /** Checks field default values. */ - Validator validateDefault(); - - /** Get a validator for a field. */ - default Validator getValidator(SchemaRules schemaRules) { - return validateFieldTypes(schemaRules) - .and(validateFieldName()) - .and(validateDefault()) - .and(validateFieldDocumentation()); - } - - /** Get a validator for a union inside a record. */ - default Validator validateInternalUnion(SchemaRules schemaRules) { - return field -> field.getField().schema().getTypes().stream() - .flatMap(schema -> { - Schema.Type type = schema.getType(); - if (type == Schema.Type.RECORD) { - return schemaRules.validateRecord().apply(schema); - } else if (type == Schema.Type.ENUM) { - return schemaRules.validateEnum().apply(schema); - } else if (type == Schema.Type.UNION) { - return raise(message("Cannot have a nested union.") - .apply(field)); - } else { - return valid(); - } - }); - } - - /** A message function for a field, ending with given text. */ - default Function message(String text) { - return schema -> "Field " + schema.getField().name() + " in schema " - + schema.getSchema().getFullName() + " is invalid. " + text; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt new file mode 100644 index 00000000..fee9c975 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt @@ -0,0 +1,53 @@ +package org.radarbase.schema.validation.rules + +import org.apache.avro.Schema +import org.apache.avro.Schema.Type.ENUM +import org.apache.avro.Schema.Type.RECORD +import org.apache.avro.Schema.Type.UNION + +interface SchemaFieldRules { + /** Recursively checks field types. */ + fun validateFieldTypes(schemaRules: SchemaRules): Validator + + /** Checks field name format. */ + fun validateFieldName(): Validator + + /** Checks field documentation presence and format. */ + fun validateFieldDocumentation(): Validator + + /** Checks field default values. */ + fun validateDefault(): Validator + + /** Get a validator for a field. */ + fun getValidator(schemaRules: SchemaRules): Validator { + return validateFieldTypes(schemaRules) + .and(validateFieldName()) + .and(validateDefault()) + .and(validateFieldDocumentation()) + } + + /** Get a validator for a union inside a record. */ + fun validateInternalUnion(schemaRules: SchemaRules): Validator { + return Validator { field: SchemaField -> + field.field.schema().types.stream() + .flatMap { schema: Schema -> + val type = schema.type + return@flatMap when (type) { + RECORD -> schemaRules.validateRecord().validate(schema) + ENUM -> schemaRules.validateEnum().validate(schema) + UNION -> Validator.raise( + message(field, "Cannot have a nested union."), + ) + else -> Validator.valid() + } + } + } + } + + companion object { + /** A message function for a field, ending with given text. */ + fun message(field: SchemaField, text: String): String { + return "Field ${field.field.name()} in schema ${field.schema.fullName} is invalid. $text" + } + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.java deleted file mode 100644 index add41dfe..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.radarbase.schema.validation.rules; - -import java.nio.file.Path; -import java.util.Objects; - -import org.apache.avro.Schema; -import org.radarbase.schema.Scope; - -/** - * Schema with metadata. - */ -public class SchemaMetadata { - private final Schema schema; - private final Scope scope; - private final Path path; - - /** - * Schema with metadata. - */ - public SchemaMetadata(Schema schema, Scope scope, Path path) { - this.schema = schema; - this.scope = scope; - this.path = path; - } - - public Path getPath() { - return path; - } - - public Scope getScope() { - return scope; - } - - public Schema getSchema() { - return schema; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - SchemaMetadata that = (SchemaMetadata) o; - - return scope == that.scope - && Objects.equals(path, that.path) - && Objects.equals(schema, that.schema); - } - - @Override - public int hashCode() { - int result = scope != null ? scope.hashCode() : 0; - result = 31 * result + (path != null ? path.hashCode() : 0); - return result; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.kt new file mode 100644 index 00000000..a4dc7d24 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.kt @@ -0,0 +1,14 @@ +package org.radarbase.schema.validation.rules + +import org.apache.avro.Schema +import org.radarbase.schema.Scope +import java.nio.file.Path + +/** + * Schema with metadata. + */ +data class SchemaMetadata( + val schema: Schema?, + val scope: Scope, + val path: Path?, +) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt index f8bd0a1f..6bd94bf4 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt @@ -14,7 +14,10 @@ interface SchemaMetadataRules { * and type of the schema. */ fun getValidator(validateScopeSpecific: Boolean): Validator = - Validator { metadata: SchemaMetadata -> + Validator { metadata -> + if (metadata.schema == null) { + return@Validator Validator.raise("Missing schema") + } val schemaRules = schemaRules var validator = validateSchemaLocation() @@ -30,14 +33,14 @@ interface SchemaMetadataRules { } else { validator.and(schema(schemaRules.validateRecord())) } - validator.apply(metadata) + validator.validate(metadata) } /** Validates schemas without their metadata. */ fun schema(validator: Validator): Validator = - Validator { metadata: SchemaMetadata -> validator.apply(metadata.schema) } + Validator { metadata -> validator.validate(metadata.schema!!) } - fun message(text: String): (SchemaMetadata) -> String = { metadata -> - "Schema ${metadata.schema.fullName} at ${metadata.path} is invalid. $text" + fun message(metadata: SchemaMetadata, text: String): String { + return "Schema ${metadata.schema!!.fullName} at ${metadata.path} is invalid. $text" } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.java deleted file mode 100644 index 7f9257bf..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.java +++ /dev/null @@ -1,139 +0,0 @@ -package org.radarbase.schema.validation.rules; - -import static org.radarbase.schema.validation.rules.Validator.raise; - -import java.util.function.Function; -import org.apache.avro.Schema; - -public interface SchemaRules { - SchemaFieldRules getFieldRules(); - - /** - * Checks that schemas are unique compared to already validated schemas. - */ - Validator validateUniqueness(); - - /** - * Checks schema namespace format. - */ - Validator validateNameSpace(); - - /** - * Checks schema name format. - */ - Validator validateName(); - - /** - * Checks schema documentation presence and format. - */ - Validator validateSchemaDocumentation(); - - /** - * Checks that the symbols of enums have the required format. - */ - Validator validateSymbols(); - - /** - * Checks that schemas should have a {@code time} field. - */ - Validator validateTime(); - - /** - * Checks that schemas should have a {@code timeCompleted} field. - */ - Validator validateTimeCompleted(); - - /** - * Checks that schemas should not have a {@code timeCompleted} field. - */ - Validator validateNotTimeCompleted(); - - /** - * Checks that schemas should have a {@code timeReceived} field. - */ - Validator validateTimeReceived(); - - /** - * Checks that schemas should not have a {@code timeReceived} field. - */ - Validator validateNotTimeReceived(); - - /** - * Validate an enum. - */ - default Validator validateEnum() { - return validateUniqueness() - .and(validateNameSpace()) - .and(validateSymbols()) - .and(validateSchemaDocumentation()) - .and(validateName()); - } - - /** - * Validate a record that is defined inline. - */ - default Validator validateRecord() { - return validateUniqueness() - .and(validateAvroData()) - .and(validateNameSpace()) - .and(validateName()) - .and(validateSchemaDocumentation()) - .and(fields(getFieldRules().getValidator(this))); - } - - Validator validateAvroData(); - - /** - * Validates record schemas of an active source. - * - * @return TODO - */ - default Validator validateActiveSource() { - return validateRecord() - .and(validateTime() - .and(validateTimeCompleted()) - .and(validateNotTimeReceived())); - } - - /** - * Validates schemas of monitor sources. - * - * @return TODO - */ - default Validator validateMonitor() { - return validateRecord() - .and(validateTime()); - } - - /** - * Validates schemas of passive sources. - */ - default Validator validatePassive() { - return validateRecord() - .and(validateTime()) - .and(validateTimeReceived()) - .and(validateNotTimeCompleted()); - } - - default Function messageSchema(String text) { - return schema -> "Schema " + schema.getFullName() + " is invalid. " + text; - } - - /** - * Validates all fields of records. - * Validation will fail on non-record types or records with no fields. - */ - default Validator fields(Validator validator) { - return schema -> { - if (!schema.getType().equals(Schema.Type.RECORD)) { - return raise("Default validation can be applied only to an Avro RECORD, not to " - + schema.getType() + " of schema " + schema.getFullName() + '.'); - } - if (schema.getFields().isEmpty()) { - return raise("Schema " + schema.getFullName() + " does not contain any fields."); - } - return schema.getFields().stream() - .flatMap(field -> validator.apply(new SchemaField(schema, field))); - }; - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt new file mode 100644 index 00000000..3948db0f --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt @@ -0,0 +1,137 @@ +package org.radarbase.schema.validation.rules + +import org.apache.avro.Schema +import org.apache.avro.Schema.Type.RECORD +import org.radarbase.schema.validation.rules.Validator.Companion.raise + +interface SchemaRules { + val fieldRules: SchemaFieldRules + + /** + * Checks that schemas are unique compared to already validated schemas. + */ + fun validateUniqueness(): Validator + + /** + * Checks schema namespace format. + */ + fun validateNameSpace(): Validator + + /** + * Checks schema name format. + */ + fun validateName(): Validator + + /** + * Checks schema documentation presence and format. + */ + fun validateSchemaDocumentation(): Validator + + /** + * Checks that the symbols of enums have the required format. + */ + fun validateSymbols(): Validator + + /** + * Checks that schemas should have a `time` field. + */ + fun validateTime(): Validator + + /** + * Checks that schemas should have a `timeCompleted` field. + */ + fun validateTimeCompleted(): Validator + + /** + * Checks that schemas should not have a `timeCompleted` field. + */ + fun validateNotTimeCompleted(): Validator + + /** + * Checks that schemas should have a `timeReceived` field. + */ + fun validateTimeReceived(): Validator + + /** + * Checks that schemas should not have a `timeReceived` field. + */ + fun validateNotTimeReceived(): Validator + + /** + * Validate an enum. + */ + fun validateEnum(): Validator = validateUniqueness() + .and(validateNameSpace()) + .and(validateSymbols()) + .and(validateSchemaDocumentation()) + .and(validateName()) + + /** + * Validate a record that is defined inline. + */ + fun validateRecord(): Validator = validateUniqueness() + .and(validateAvroData()) + .and(validateNameSpace()) + .and(validateName()) + .and(validateSchemaDocumentation()) + .and(fields(fieldRules.getValidator(this))) + + fun validateAvroData(): Validator + + /** + * Validates record schemas of an active source. + */ + fun validateActiveSource(): Validator = validateRecord() + .and( + validateTime() + .and(validateTimeCompleted()) + .and(validateNotTimeReceived()), + ) + + /** + * Validates schemas of monitor sources. + */ + fun validateMonitor(): Validator = validateRecord() + .and(validateTime()) + + /** + * Validates schemas of passive sources. + */ + fun validatePassive(): Validator = validateRecord() + .and(validateTime()) + .and(validateTimeReceived()) + .and(validateNotTimeCompleted()) + + fun messageSchema(text: String): (Schema) -> String { + return { schema -> "Schema ${schema.fullName} is invalid. $text" } + } + + fun messageSchema(schema: Schema, text: String): String { + return "Schema ${schema.fullName} is invalid. $text" + } + + /** + * Validates all fields of records. + * Validation will fail on non-record types or records with no fields. + */ + fun fields(validator: Validator) = Validator { schema: Schema -> + if (schema.type != RECORD) { + return@Validator raise( + "Default validation can be applied only to an Avro RECORD, not to " + + schema.type + " of schema " + schema.fullName + '.', + ) + } + if (schema.fields.isEmpty()) { + return@Validator raise("Schema " + schema.fullName + " does not contain any fields.") + } + schema.fields.stream() + .flatMap { field -> + validator.validate( + SchemaField( + schema, + field, + ), + ) + } + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.java b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.java deleted file mode 100644 index 6c6fb72e..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.java +++ /dev/null @@ -1,233 +0,0 @@ - -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation.rules; - -import java.util.Collection; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.function.Supplier; -import java.util.regex.Pattern; -import java.util.stream.Stream; -import org.radarbase.schema.validation.ValidationException; - -/** - * TODO. - */ -public interface Validator extends Function> { - static Stream check(boolean test, String message) { - return test ? valid() : raise(message); - } - - static Stream check(boolean test, Supplier message) { - return test ? valid() : raise(message.get()); - } - - /** - * TODO. - * @param predicate TODO - * @param message TODO - * @return TODO - */ - static Validator validate(Predicate predicate, String message) { - return object -> check(predicate.test(object), message); - } - - /** - * TODO. - * @param predicate TODO - * @param message TODO - * @return TODO - */ - static Validator validate(Predicate predicate, Function message) { - return object -> check(predicate.test(object), message.apply(object)); - } - - static Validator validate(Function property, Predicate predicate, - Function message) { - return object -> check(predicate.test(property.apply(object)), message.apply(object)); - } - - /** - * TODO. - * @param predicate TODO - * @param message TODO - * @return TODO - */ - static Validator validateNonNull(Predicate predicate, String message) { - return validate(o -> o != null && predicate.test(o), message); - } - - /** - * TODO. - * @param predicate TODO - * @param message TODO - * @return TODO - */ - static Validator validateNonNull(Function property, Predicate predicate, - Function message) { - return validate(o -> { - V val = property.apply(o); - return val != null && predicate.test(val); - }, message); - } - - /** - * TODO. - * @param predicate TODO - * @param message TODO - * @return TODO - */ - static Validator validateNonNull(Function property, Predicate predicate, - String message) { - return validate(o -> { - V val = property.apply(o); - return val != null && predicate.test(val); - }, message); - } - - /** - * TODO. - * @param message TODO - * @return TODO - */ - static Validator validateNonNull(Function property, String message) { - return validate(o -> property.apply(o) != null, message); - } - - /** - * TODO. - * @param message TODO - * @return TODO - */ - static Validator validateNonEmpty(Function property, - Function message, Validator validator) { - return o -> { - String val = property.apply(o); - if (val == null || val.isEmpty()) { - return raise(message.apply(o)); - } - return validator.apply(val); - }; - } - - /** - * TODO. - * @param message TODO - * @return TODO - */ - static Validator validateNonEmpty(Function property, String message, - Validator validator) { - return o -> { - String val = property.apply(o); - if (val == null || val.isEmpty()) { - return raise(message); - } - return validator.apply(val); - }; - } - - /** - * TODO. - * @param message TODO - * @return TODO - */ - static > Validator validateNonEmpty(Function property, - String message) { - return validate(o -> { - V val = property.apply(o); - return val != null && !val.isEmpty(); - }, message); - } - - - /** - * TODO. - * @param message TODO - * @return TODO - */ - static > Validator validateNonEmpty(Function property, - Function message) { - return validate(o -> { - V val = property.apply(o); - return val != null && !val.isEmpty(); - }, message); - } - - - /** - * TODO. - * @param predicate TODO - * @param message TODO - * @return TODO - */ - static Validator validateOrNull(Predicate predicate, String message) { - return validate(o -> o == null || predicate.test(o), message); - } - - /** - * TODO. - * @param predicate TODO - * @param message TODO - * @return TODO - */ - static Validator validateOrNull(Function property, Predicate predicate, - String message) { - return validate(o -> { - V val = property.apply(o); - return val == null || predicate.test(val); - }, message); - } - - /** - * TODO. - * @param other TODO - * @return TODO - */ - default Validator and(Validator other) { - return object -> Stream.concat(this.apply(object), other.apply(object)); - } - - /** - * TODO. - * @param other TODO - * @return TODO - */ - default Validator and(Validator other, Function toOther) { - return object -> Stream.concat(this.apply(object), other.apply(toOther.apply(object))); - } - - static boolean matches(String str, Pattern pattern) { - return pattern.matcher(str).matches(); - } - - static Predicate matches(Pattern pattern) { - return str -> pattern.matcher(str).matches(); - } - - static Stream raise(String message) { - return Stream.of(new ValidationException(message)); - } - - static Stream raise(String message, Exception ex) { - return Stream.of(new ValidationException(message, ex)); - } - - static Stream valid() { - return Stream.empty(); - } -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt new file mode 100644 index 00000000..e8817d89 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt @@ -0,0 +1,56 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation.rules + +import org.radarbase.schema.validation.ValidationException +import java.util.stream.Stream + +class Validator( + private val validation: (T) -> Stream, +) { + fun and(other: Validator): Validator = Validator { obj -> + Stream.concat( + this.validate(obj), + other.validate(obj), + ) + } + + fun validate(value: T): Stream = this.validation.invoke(value) + + companion object { + fun check(test: Boolean, message: String): Stream = + if (test) valid() else raise(message) + + inline fun check(test: Boolean, message: () -> String): Stream { + return if (test) valid() else raise(message()) + } + + fun validate(predicate: (T) -> Boolean, message: String): Validator = + Validator { obj -> + check(predicate(obj), message) + } + + fun validate(predicate: (T) -> Boolean, message: (T) -> String): Validator = + Validator { obj: T -> + check(predicate(obj), message(obj)) + } + + fun raise(message: String, ex: Exception? = null): Stream = + Stream.of(ValidationException(message, ex)) + + fun valid(): Stream = Stream.empty() + } +} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt index 9fdc6ea2..c8fd1dcf 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt @@ -1,8 +1,7 @@ package org.radarbase.schema.specification.config +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test - -import org.junit.jupiter.api.Assertions.* import org.radarbase.schema.SchemaCatalogue import org.radarbase.schema.Scope import org.radarbase.schema.validation.ValidationHelper.COMMONS_PATH @@ -15,7 +14,8 @@ internal class SchemaConfigTest { fun getMonitor() { val config = SchemaConfig( exclude = listOf("**"), - monitor = mapOf("application/test.avsc" to """ + monitor = mapOf( + "application/test.avsc" to """ { "namespace": "org.radarcns.monitor.application", "type": "record", @@ -26,7 +26,8 @@ internal class SchemaConfigTest { { "name": "uptime", "type": "double", "doc": "Time since last app start (s)." } ] } - """.trimIndent()), + """.trimIndent(), + ), ) val commonsRoot = Paths.get("../..").resolve(COMMONS_PATH) .absolute() @@ -35,7 +36,7 @@ internal class SchemaConfigTest { assertEquals(1, schemaCatalogue.schemas.size) val (fullName, schemaMetadata) = schemaCatalogue.schemas.entries.first() assertEquals("org.radarcns.monitor.application.ApplicationUptime2", fullName) - assertEquals("org.radarcns.monitor.application.ApplicationUptime2", schemaMetadata.schema.fullName) + assertEquals("org.radarcns.monitor.application.ApplicationUptime2", schemaMetadata.schema!!.fullName) assertEquals(commonsRoot.resolve("monitor/application/test.avsc"), schemaMetadata.path) assertEquals(Scope.MONITOR, schemaMetadata.scope) } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.java b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.java deleted file mode 100644 index eb0ece1e..00000000 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation; - -import org.apache.avro.Schema; -import org.apache.avro.SchemaBuilder; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.radarbase.schema.SchemaCatalogue; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.SourceCatalogue; -import org.radarbase.schema.specification.config.SchemaConfig; -import org.radarbase.schema.specification.config.SourceConfig; - -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; -import static org.radarbase.schema.Scope.ACTIVE; -import static org.radarbase.schema.Scope.CATALOGUE; -import static org.radarbase.schema.Scope.CONNECTOR; -import static org.radarbase.schema.Scope.KAFKA; -import static org.radarbase.schema.Scope.MONITOR; -import static org.radarbase.schema.Scope.PASSIVE; -import static org.radarbase.schema.validation.ValidationHelper.COMMONS_PATH; - -/** - * TODO. - */ -public class SchemaValidatorTest { - private SchemaValidator validator; - private static final Path ROOT = Paths.get("../..").toAbsolutePath().normalize(); - private static final Path COMMONS_ROOT = ROOT.resolve(COMMONS_PATH); - - @BeforeEach - public void setUp() { - SchemaConfig config = new SchemaConfig(); - validator = new SchemaValidator(COMMONS_ROOT, config); - } - - @Test - public void active() throws IOException { - testScope(ACTIVE); - } - - @Test - public void activeSpecifications() throws IOException { - testFromSpecification(ACTIVE); - } - - @Test - public void monitor() throws IOException { - testScope(MONITOR); - } - - @Test - public void monitorSpecifications() throws IOException { - testFromSpecification(MONITOR); - } - - @Test - public void passive() throws IOException { - testScope(PASSIVE); - } - - @Test - public void passiveSpecifications() throws IOException { - testFromSpecification(PASSIVE); - } - - @Test - public void kafka() throws IOException { - testScope(KAFKA); - } - - @Test - public void kafkaSpecifications() throws IOException { - testFromSpecification(KAFKA); - } - - @Test - public void catalogue() throws IOException { - testScope(CATALOGUE); - } - - @Test - public void catalogueSpecifications() throws IOException { - testFromSpecification(CATALOGUE); - } - - @Test - public void connectorSchemas() throws IOException { - testScope(CONNECTOR); - } - - @Test - public void connectorSpecifications() throws IOException { - testFromSpecification(CONNECTOR); - } - - private void testFromSpecification(Scope scope) throws IOException { - SourceCatalogue sourceCatalogue = SourceCatalogue.Companion.load(ROOT, new SchemaConfig(), new SourceConfig()); - String result = SchemaValidator.format( - validator.analyseSourceCatalogue(scope, sourceCatalogue)); - - if (!result.isEmpty()) { - fail(result); - } - } - - private void testScope(Scope scope) throws IOException { - SchemaCatalogue schemaCatalogue = new SchemaCatalogue(COMMONS_ROOT, new SchemaConfig(), - scope); - String result = SchemaValidator.format( - validator.analyseFiles(scope, schemaCatalogue)); - - if (!result.isEmpty()) { - fail(result); - } - } - - @Test - public void testEnumerator() { - Path schemaPath = COMMONS_ROOT.resolve( - "monitor/application/application_server_status.avsc"); - - String name = "org.radarcns.monitor.application.ApplicationServerStatus"; - String documentation = "Mock documentation."; - - Schema schema = SchemaBuilder - .enumeration(name) - .doc(documentation) - .symbols("CONNECTED", "DISCONNECTED", "UNKNOWN"); - - Stream result = validator.validate(schema, schemaPath, MONITOR); - - assertEquals(0, result.count()); - - schema = SchemaBuilder - .enumeration(name) - .doc(documentation) - .symbols("CONNECTED", "DISCONNECTED", "un_known"); - - result = validator.validate(schema, schemaPath, MONITOR); - - assertEquals(2, result.count()); - } -} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt new file mode 100644 index 00000000..e1232687 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt @@ -0,0 +1,173 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation + +import org.apache.avro.SchemaBuilder +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.fail +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.radarbase.schema.SchemaCatalogue +import org.radarbase.schema.Scope +import org.radarbase.schema.Scope.ACTIVE +import org.radarbase.schema.Scope.CATALOGUE +import org.radarbase.schema.Scope.CONNECTOR +import org.radarbase.schema.Scope.KAFKA +import org.radarbase.schema.Scope.MONITOR +import org.radarbase.schema.Scope.PASSIVE +import org.radarbase.schema.specification.SourceCatalogue.Companion.load +import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.specification.config.SourceConfig +import org.radarbase.schema.validation.SchemaValidator.Companion.format +import org.radarbase.schema.validation.ValidationHelper.COMMONS_PATH +import java.io.IOException +import java.nio.file.Path +import java.nio.file.Paths +import java.util.stream.Stream + +class SchemaValidatorTest { + private lateinit var validator: SchemaValidator + + @BeforeEach + fun setUp() { + val config = SchemaConfig() + validator = SchemaValidator(COMMONS_ROOT, config) + } + + @Test + @Throws(IOException::class) + fun active() { + testScope(ACTIVE) + } + + @Test + @Throws(IOException::class) + fun activeSpecifications() { + testFromSpecification(ACTIVE) + } + + @Test + @Throws(IOException::class) + fun monitor() { + testScope(MONITOR) + } + + @Test + @Throws(IOException::class) + fun monitorSpecifications() { + testFromSpecification(MONITOR) + } + + @Test + @Throws(IOException::class) + fun passive() { + testScope(PASSIVE) + } + + @Test + @Throws(IOException::class) + fun passiveSpecifications() { + testFromSpecification(PASSIVE) + } + + @Test + @Throws(IOException::class) + fun kafka() { + testScope(KAFKA) + } + + @Test + @Throws(IOException::class) + fun kafkaSpecifications() { + testFromSpecification(KAFKA) + } + + @Test + @Throws(IOException::class) + fun catalogue() { + testScope(CATALOGUE) + } + + @Test + @Throws(IOException::class) + fun catalogueSpecifications() { + testFromSpecification(CATALOGUE) + } + + @Test + @Throws(IOException::class) + fun connectorSchemas() { + testScope(CONNECTOR) + } + + @Test + @Throws(IOException::class) + fun connectorSpecifications() { + testFromSpecification(CONNECTOR) + } + + @Throws(IOException::class) + private fun testFromSpecification(scope: Scope) { + val sourceCatalogue = load(ROOT, SchemaConfig(), SourceConfig()) + val result = format( + validator.analyseSourceCatalogue(scope, sourceCatalogue), + ) + if (result.isNotEmpty()) { + fail(result) + } + } + + @Throws(IOException::class) + private fun testScope(scope: Scope) { + val schemaCatalogue = SchemaCatalogue( + COMMONS_ROOT, + SchemaConfig(), + scope, + ) + val result = format( + validator.analyseFiles(scope, schemaCatalogue), + ) + if (result.isNotEmpty()) { + fail(result) + } + } + + @Test + fun testEnumerator() { + val schemaPath = COMMONS_ROOT.resolve( + "monitor/application/application_server_status.avsc", + ) + val name = "org.radarcns.monitor.application.ApplicationServerStatus" + val documentation = "Mock documentation." + var schema = SchemaBuilder + .enumeration(name) + .doc(documentation) + .symbols("CONNECTED", "DISCONNECTED", "UNKNOWN") + var result: Stream = validator.validate(schema, schemaPath, MONITOR) + assertEquals(0, result.count()) + schema = SchemaBuilder + .enumeration(name) + .doc(documentation) + .symbols("CONNECTED", "DISCONNECTED", "un_known") + result = validator.validate(schema, schemaPath, MONITOR) + assertEquals(2, result.count()) + } + + companion object { + private val ROOT = Paths.get("../..").toAbsolutePath().normalize() + private val COMMONS_ROOT: Path = ROOT.resolve(COMMONS_PATH) + } +} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.java b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.java deleted file mode 100644 index 9ac5fb93..00000000 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.opentest4j.MultipleFailuresError; -import org.radarbase.schema.specification.DataProducer; -import org.radarbase.schema.specification.SourceCatalogue; -import org.radarbase.schema.specification.config.SchemaConfig; -import org.radarbase.schema.specification.config.SourceConfig; - -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Collection; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.radarbase.schema.validation.ValidationHelper.isValidTopic; - -/** - * TODO. - */ -public class SourceCatalogueValidationTest { - private static SourceCatalogue catalogue; - public static Path BASE_PATH = Paths.get("../..").toAbsolutePath().normalize(); - - - @BeforeAll - public static void setUp() throws IOException { - catalogue = SourceCatalogue.Companion.load(BASE_PATH, new SchemaConfig(), new SourceConfig()); - } - - @Test - public void validateTopicNames() { - catalogue.getTopicNames().forEach(topic -> - assertTrue(isValidTopic(topic), topic + " is invalid")); - } - - @Test - public void validateTopics() { - List expected = Stream.of( - catalogue.getActiveSources(), - catalogue.getMonitorSources(), - catalogue.getPassiveSources(), - catalogue.getStreamGroups(), - catalogue.getConnectorSources(), - catalogue.getPushSources()) - .flatMap(Collection::stream) - .flatMap(DataProducer::getTopicNames) - .sorted() - .collect(Collectors.toList()); - - assertEquals(expected, catalogue.getTopicNames().sorted().collect(Collectors.toList())); - } - - @Test - public void validateTopicSchemas() { - catalogue.getSources().stream() - .flatMap(source -> source.getData().stream()) - .forEach(data -> { - try { - assertTrue(data.getTopics(catalogue.getSchemaCatalogue()) - .findAny().isPresent()); - } catch (IOException ex) { - fail("Cannot create topic from specification: " + ex); - } - }); - } - - @Test - public void validateSerialization() { - ObjectMapper mapper = new ObjectMapper(); - - List failures = catalogue.getSources() - .stream() - .map(source -> { - try { - String json = mapper.writeValueAsString(source); - assertFalse(json.contains("\"parallel\":false")); - return null; - } catch (Exception ex) { - return new IllegalArgumentException("Source " + source + " is not valid", ex); - } - }) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - - if (!failures.isEmpty()) { - throw new MultipleFailuresError("One or more sources were not valid", failures); - } - } -} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt new file mode 100644 index 00000000..de06815d --- /dev/null +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt @@ -0,0 +1,124 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation + +import com.fasterxml.jackson.databind.ObjectMapper +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Assertions.fail +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.opentest4j.MultipleFailuresError +import org.radarbase.schema.specification.DataProducer +import org.radarbase.schema.specification.DataTopic +import org.radarbase.schema.specification.SourceCatalogue +import org.radarbase.schema.specification.SourceCatalogue.Companion.load +import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.specification.config.SourceConfig +import org.radarbase.schema.validation.ValidationHelper.isValidTopic +import java.io.IOException +import java.nio.file.Paths +import java.util.Objects +import java.util.stream.Collectors +import java.util.stream.Stream + +/** + * TODO. + */ +class SourceCatalogueValidationTest { + @Test + fun validateTopicNames() { + catalogue.topicNames.forEach { topic: String -> + assertTrue( + isValidTopic(topic), + "$topic is invalid", + ) + } + } + + @Test + fun validateTopics() { + val expected = Stream.of>>( + catalogue.activeSources, + catalogue.monitorSources, + catalogue.passiveSources, + catalogue.streamGroups, + catalogue.connectorSources, + catalogue.pushSources, + ) + .flatMap { it.stream() } + .flatMap(DataProducer<*>::topicNames) + .sorted() + .collect(Collectors.toList()) + Assertions.assertEquals( + expected, + catalogue.topicNames.sorted().collect(Collectors.toList()), + ) + } + + @Test + fun validateTopicSchemas() { + catalogue.sources.stream() + .flatMap { source: DataProducer<*> -> source.data.stream() } + .forEach { data -> + try { + assertTrue( + data.topics(catalogue.schemaCatalogue) + .findAny() + .isPresent, + ) + } catch (ex: IOException) { + fail("Cannot create topic from specification: $ex") + } + } + } + + @Test + fun validateSerialization() { + val mapper = ObjectMapper() + val failures = catalogue.sources + .stream() + .map { source: DataProducer<*> -> + try { + val json = mapper.writeValueAsString(source) + assertFalse(json.contains("\"parallel\":false")) + return@map null + } catch (ex: Exception) { + return@map IllegalArgumentException("Source $source is not valid", ex) + } + } + .filter(Objects::nonNull) + .collect(Collectors.toList()) + + if (failures.isNotEmpty()) { + throw MultipleFailuresError("One or more sources were not valid", failures) + } + } + + companion object { + private lateinit var catalogue: SourceCatalogue + + val BASE_PATH = Paths.get("../..").toAbsolutePath().normalize() + + @BeforeAll + @JvmStatic + @Throws(IOException::class) + fun setUp() { + catalogue = load(BASE_PATH, SchemaConfig(), SourceConfig()) + } + } +} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.java b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.java deleted file mode 100644 index ba071001..00000000 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.radarbase.schema.validation; - -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.radarbase.schema.validation.SourceCatalogueValidationTest.BASE_PATH; - -import java.io.IOException; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.radarbase.schema.Scope; -import org.radarbase.schema.specification.active.ActiveSource; -import org.radarbase.schema.specification.config.SchemaConfig; -import org.radarbase.schema.specification.connector.ConnectorSource; -import org.radarbase.schema.specification.monitor.MonitorSource; -import org.radarbase.schema.specification.passive.PassiveSource; -import org.radarbase.schema.specification.push.PushSource; -import org.radarbase.schema.specification.stream.StreamGroup; - -public class SpecificationsValidatorTest { - private SpecificationsValidator validator; - - @BeforeEach - public void setUp() { - this.validator = new SpecificationsValidator(BASE_PATH, new SchemaConfig()); - } - - @Test - public void activeIsYml() throws IOException { - assertTrue(validator.specificationsAreYmlFiles(Scope.ACTIVE)); - assertTrue(validator.checkSpecificationParsing(Scope.ACTIVE, ActiveSource.class)); - } - - @Test - public void monitorIsYml() throws IOException { - assertTrue(validator.specificationsAreYmlFiles(Scope.MONITOR)); - assertTrue(validator.checkSpecificationParsing(Scope.MONITOR, MonitorSource.class)); - } - - @Test - public void passiveIsYml() throws IOException { - assertTrue(validator.specificationsAreYmlFiles(Scope.PASSIVE)); - assertTrue(validator.checkSpecificationParsing(Scope.PASSIVE, PassiveSource.class)); - } - - @Test - public void connectorIsYml() throws IOException { - assertTrue(validator.specificationsAreYmlFiles(Scope.CONNECTOR)); - assertTrue(validator.checkSpecificationParsing(Scope.CONNECTOR, ConnectorSource.class)); - } - - @Test - public void pushIsYml() throws IOException { - assertTrue(validator.specificationsAreYmlFiles(Scope.PUSH)); - assertTrue(validator.checkSpecificationParsing(Scope.PUSH, PushSource.class)); - } - - @Test - public void streamIsYml() throws IOException { - assertTrue(validator.specificationsAreYmlFiles(Scope.STREAM)); - assertTrue(validator.checkSpecificationParsing(Scope.STREAM, StreamGroup.class)); - } -} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt new file mode 100644 index 00000000..414c4b7c --- /dev/null +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt @@ -0,0 +1,95 @@ +package org.radarbase.schema.validation + +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.radarbase.schema.Scope.ACTIVE +import org.radarbase.schema.Scope.CONNECTOR +import org.radarbase.schema.Scope.MONITOR +import org.radarbase.schema.Scope.PASSIVE +import org.radarbase.schema.Scope.PUSH +import org.radarbase.schema.Scope.STREAM +import org.radarbase.schema.specification.active.ActiveSource +import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.specification.connector.ConnectorSource +import org.radarbase.schema.specification.monitor.MonitorSource +import org.radarbase.schema.specification.passive.PassiveSource +import org.radarbase.schema.specification.push.PushSource +import org.radarbase.schema.specification.stream.StreamGroup +import java.io.IOException + +class SpecificationsValidatorTest { + private lateinit var validator: SpecificationsValidator + + @BeforeEach + fun setUp() { + validator = SpecificationsValidator(SourceCatalogueValidationTest.BASE_PATH, SchemaConfig()) + } + + @Test + @Throws(IOException::class) + fun activeIsYml() { + Assertions.assertTrue(validator.specificationsAreYmlFiles(ACTIVE)) + Assertions.assertTrue( + validator.checkSpecificationParsing( + ACTIVE, + ActiveSource::class.java, + ), + ) + } + + @Test + @Throws(IOException::class) + fun monitorIsYml() { + Assertions.assertTrue(validator.specificationsAreYmlFiles(MONITOR)) + Assertions.assertTrue( + validator.checkSpecificationParsing( + MONITOR, + MonitorSource::class.java, + ), + ) + } + + @Test + @Throws(IOException::class) + fun passiveIsYml() { + Assertions.assertTrue(validator.specificationsAreYmlFiles(PASSIVE)) + Assertions.assertTrue( + validator.checkSpecificationParsing( + PASSIVE, + PassiveSource::class.java, + ), + ) + } + + @Test + @Throws(IOException::class) + fun connectorIsYml() { + Assertions.assertTrue(validator.specificationsAreYmlFiles(CONNECTOR)) + Assertions.assertTrue( + validator.checkSpecificationParsing( + CONNECTOR, + ConnectorSource::class.java, + ), + ) + } + + @Test + @Throws(IOException::class) + fun pushIsYml() { + Assertions.assertTrue(validator.specificationsAreYmlFiles(PUSH)) + Assertions.assertTrue(validator.checkSpecificationParsing(PUSH, PushSource::class.java)) + } + + @Test + @Throws(IOException::class) + fun streamIsYml() { + Assertions.assertTrue(validator.specificationsAreYmlFiles(STREAM)) + Assertions.assertTrue( + validator.checkSpecificationParsing( + STREAM, + StreamGroup::class.java, + ), + ) + } +} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.java b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.java deleted file mode 100644 index a13382a7..00000000 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation.rules; - -import org.apache.avro.Schema; -import org.apache.avro.Schema.Parser; -import org.apache.avro.SchemaBuilder; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.radarbase.schema.validation.ValidationException; -import org.radarbase.schema.validation.ValidationHelper; - -import java.nio.file.Paths; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.radarbase.schema.validation.rules.RadarSchemaFieldRules.FIELD_NAME_PATTERN; -import static org.radarbase.schema.validation.rules.Validator.matches; - -/** - * TODO. - */ -public class RadarSchemaFieldRulesTest { - - private static final String MONITOR_NAME_SPACE_MOCK = "org.radarcns.monitor.test"; - private static final String ENUMERATOR_NAME_SPACE_MOCK = "org.radarcns.test.EnumeratorTest"; - private static final String UNKNOWN_MOCK = "UNKNOWN"; - - private static final String RECORD_NAME_MOCK = "RecordName"; - private static final String FIELD_NUMBER_MOCK = "Field1"; - private RadarSchemaFieldRules validator; - private RadarSchemaRules schemaValidator; - - @BeforeEach - public void setUp() { - validator = new RadarSchemaFieldRules(); - schemaValidator = new RadarSchemaRules(validator); - } - - @Test - public void fileNameTest() { - assertEquals("Questionnaire", - ValidationHelper.getRecordName(Paths.get("/path/to/questionnaire.avsc"))); - assertEquals("ApplicationExternalTime", - ValidationHelper.getRecordName( - Paths.get("/path/to/application_external_time.avsc"))); - } - - @Test - public void fieldNameRegex() { - assertTrue(matches("interBeatInterval", FIELD_NAME_PATTERN)); - assertTrue(matches("x", FIELD_NAME_PATTERN)); - assertTrue(matches(RadarSchemaRules.TIME, FIELD_NAME_PATTERN)); - assertTrue(matches("subjectId", FIELD_NAME_PATTERN)); - assertTrue(matches("listOfSeveralThings", FIELD_NAME_PATTERN)); - assertFalse(matches("Time", FIELD_NAME_PATTERN)); - assertFalse(matches("E4Heart", FIELD_NAME_PATTERN)); - } - - @Test - public void fieldsTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .endRecord(); - - result = schemaValidator.fields(validator.validateFieldTypes(schemaValidator)) - .apply(schema); - - assertEquals(1, result.count()); - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .optionalBoolean("optional") - .endRecord(); - - result = schemaValidator.fields(validator.validateFieldTypes(schemaValidator)) - .apply(schema); - - assertEquals(0, result.count()); - } - - @Test - public void fieldNameTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredString(FIELD_NUMBER_MOCK) - .endRecord(); - - result = schemaValidator.fields(validator.validateFieldName()).apply(schema); - assertEquals(1, result.count()); - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredDouble("timeReceived") - .endRecord(); - - result = schemaValidator.fields(validator.validateFieldName()).apply(schema); - assertEquals(0, result.count()); - } - - @Test - public void fieldDocumentationTest() { - Schema schema; - Stream result; - - schema = new Parser().parse("{\"namespace\": \"org.radarcns.kafka.key\", " - + "\"type\": \"record\"," - + " \"name\": \"key\", \"type\": \"record\", \"fields\": [" - + "{\"name\": \"userId\", \"type\": \"string\" , \"doc\": \"Documentation\"}," - + "{\"name\": \"sourceId\", \"type\": \"string\"} ]}"); - - result = schemaValidator.fields(validator.validateFieldDocumentation()).apply(schema); - - assertEquals(2, result.count()); - - schema = new Parser().parse("{\"namespace\": \"org.radarcns.kafka.key\", " - + "\"type\": \"record\", \"name\": \"key\", \"type\": \"record\", \"fields\": [" - + "{\"name\": \"userId\", \"type\": \"string\" , \"doc\": \"Documentation.\"}]}"); - - result = schemaValidator.fields(validator.validateFieldDocumentation()).apply(schema); - assertEquals(0, result.count()); - } - - @Test - public void defaultValueExceptionTest() { - Stream result = schemaValidator.fields(validator.validateDefault()) - .apply(SchemaBuilder.record(RECORD_NAME_MOCK) - .fields() - .name(FIELD_NUMBER_MOCK) - .type(SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK) - .symbols("VAL", UNKNOWN_MOCK)) - .noDefault() - .endRecord()); - - assertEquals(1, result.count()); - } - - @Test - @SuppressWarnings("PMD.ExcessiveMethodLength") - // TODO improve test after having define the default guideline - public void defaultValueTest() { - Schema schema; - Stream result; - - String schemaTxtInit = "{\"namespace\": \"org.radarcns.test\", " - + "\"type\": \"record\", \"name\": \"TestRecord\", \"fields\": "; - - schema = new Parser().parse(schemaTxtInit - + "[ {\"name\": \"serverStatus\", \"type\": {\"name\": \"ServerStatus\", \"type\": " - + "\"enum\", \"symbols\": [\"Connected\", \"NotConnected\", \"UNKNOWN\"] }, " - + "\"default\": \"UNKNOWN\" } ] }"); - - result = schemaValidator.fields(validator.validateDefault()).apply(schema); - - assertEquals(0, result.count()); - - schema = new Parser().parse(schemaTxtInit - + "[ {\"name\": \"serverStatus\", \"type\": {\"name\": \"ServerStatus\", \"type\": " - + "\"enum\", \"symbols\": [\"Connected\", \"NotConnected\", \"UNKNOWN\"] }, " - + "\"default\": \"null\" } ] }"); - - result = schemaValidator.fields(validator.validateDefault()).apply(schema); - - assertEquals(1, result.count()); - } -} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt new file mode 100644 index 00000000..f983202d --- /dev/null +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt @@ -0,0 +1,195 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation.rules + +import org.apache.avro.Schema +import org.apache.avro.Schema.Parser +import org.apache.avro.SchemaBuilder +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.radarbase.schema.validation.ValidationException +import org.radarbase.schema.validation.ValidationHelper.getRecordName +import java.nio.file.Paths +import java.util.stream.Stream + +/** + * TODO. + */ +class RadarSchemaFieldRulesTest { + private lateinit var validator: RadarSchemaFieldRules + private lateinit var schemaValidator: RadarSchemaRules + + @BeforeEach + fun setUp() { + validator = RadarSchemaFieldRules() + schemaValidator = RadarSchemaRules(validator) + } + + @Test + fun fileNameTest() { + Assertions.assertEquals( + "Questionnaire", + getRecordName(Paths.get("/path/to/questionnaire.avsc")), + ) + Assertions.assertEquals( + "ApplicationExternalTime", + getRecordName( + Paths.get("/path/to/application_external_time.avsc"), + ), + ) + } + + @Test + fun fieldNameRegex() { + assertTrue("interBeatInterval".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue("x".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue(RadarSchemaRules.TIME.matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue("subjectId".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue("listOfSeveralThings".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) + assertFalse("Time".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) + assertFalse("E4Heart".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) + } + + @Test + fun fieldsTest() { + var result: Stream + var schema: Schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .endRecord() + result = schemaValidator.fields(validator.validateFieldTypes(schemaValidator)) + .validate(schema) + Assertions.assertEquals(1, result.count()) + schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .optionalBoolean("optional") + .endRecord() + result = schemaValidator.fields(validator.validateFieldTypes(schemaValidator)) + .validate(schema) + Assertions.assertEquals(0, result.count()) + } + + @Test + fun fieldNameTest() { + var result: Stream + var schema: Schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .requiredString(FIELD_NUMBER_MOCK) + .endRecord() + result = schemaValidator.fields(validator.validateFieldName()).validate(schema) + Assertions.assertEquals(1, result.count()) + schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .requiredDouble("timeReceived") + .endRecord() + result = schemaValidator.fields(validator.validateFieldName()).validate(schema) + Assertions.assertEquals(0, result.count()) + } + + @Test + fun fieldDocumentationTest() { + var result: Stream + var schema: Schema = Parser().parse( + """{ + |"namespace": "org.radarcns.kafka.key", + |"type": "record", + |"name": "key", "type": + |"record", + |"fields": [ + |{"name": "userId", "type": "string" , "doc": "Documentation"}, + |{"name": "sourceId", "type": "string"} ] + |} + """.trimMargin(), + ) + result = schemaValidator.fields(validator.validateFieldDocumentation()).validate(schema) + Assertions.assertEquals(2, result.count()) + schema = Parser().parse( + """{ + |"namespace": "org.radarcns.kafka.key", + |"type": "record", + |"name": "key", + |"type": "record", + |"fields": [ + |{"name": "userId", "type": "string" , "doc": "Documentation."}] + |} + """.trimMargin(), + ) + result = schemaValidator.fields(validator.validateFieldDocumentation()).validate(schema) + Assertions.assertEquals(0, result.count()) + } + + @Test + fun defaultValueExceptionTest() { + val result: Stream = schemaValidator.fields( + validator.validateDefault(), + ) + .validate( + SchemaBuilder.record(RECORD_NAME_MOCK) + .fields() + .name(FIELD_NUMBER_MOCK) + .type( + SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK) + .symbols("VAL", UNKNOWN_MOCK), + ) + .noDefault() + .endRecord(), + ) + Assertions.assertEquals(1, result.count()) + } + + @Test // TODO improve test after having define the default guideline + fun defaultValueTest() { + val schemaTxtInit = ( + "{\"namespace\": \"org.radarcns.test\", " + + "\"type\": \"record\", \"name\": \"TestRecord\", \"fields\": " + ) + var schema: Schema = Parser().parse( + schemaTxtInit + + "[ {\"name\": \"serverStatus\", \"type\": {\"name\": \"ServerStatus\", \"type\": " + + "\"enum\", \"symbols\": [\"Connected\", \"NotConnected\", \"UNKNOWN\"] }, " + + "\"default\": \"UNKNOWN\" } ] }", + ) + var result: Stream = + schemaValidator.fields(validator.validateDefault()).validate(schema) + Assertions.assertEquals(0, result.count()) + schema = Parser().parse( + schemaTxtInit + + "[ {\"name\": \"serverStatus\", \"type\": {\"name\": \"ServerStatus\", \"type\": " + + "\"enum\", \"symbols\": [\"Connected\", \"NotConnected\", \"UNKNOWN\"] }, " + + "\"default\": \"null\" } ] }", + ) + result = schemaValidator.fields(validator.validateDefault()).validate(schema) + Assertions.assertEquals(1, result.count()) + } + + companion object { + private const val MONITOR_NAME_SPACE_MOCK = "org.radarcns.monitor.test" + private const val ENUMERATOR_NAME_SPACE_MOCK = "org.radarcns.test.EnumeratorTest" + private const val UNKNOWN_MOCK = "UNKNOWN" + private const val RECORD_NAME_MOCK = "RecordName" + private const val FIELD_NUMBER_MOCK = "Field1" + } +} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.java b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.java deleted file mode 100644 index a4850e5d..00000000 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation.rules; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.radarbase.schema.validation.SourceCatalogueValidationTest.BASE_PATH; -import static org.radarbase.schema.validation.ValidationHelper.COMMONS_PATH; -import static org.radarbase.schema.Scope.MONITOR; -import static org.radarbase.schema.Scope.PASSIVE; - -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.stream.Stream; -import org.apache.avro.Schema; -import org.apache.avro.SchemaBuilder; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.radarbase.schema.specification.config.SchemaConfig; -import org.radarbase.schema.validation.SchemaValidator; -import org.radarbase.schema.validation.ValidationException; -import org.radarbase.schema.validation.ValidationHelper; - -/** - * TODO. - */ -public class RadarSchemaMetadataRulesTest { - - private static final String RECORD_NAME_MOCK = "RecordName"; - private RadarSchemaMetadataRules validator; - - @BeforeEach - public void setUp() { - SchemaConfig config = new SchemaConfig(); - validator = new RadarSchemaMetadataRules(BASE_PATH.resolve(COMMONS_PATH), config); - } - - @Test - public void fileNameTest() { - assertEquals("Questionnaire", - ValidationHelper.getRecordName(Paths.get("/path/to/questionnaire.avsc"))); - assertEquals("ApplicationExternalTime", - ValidationHelper.getRecordName( - Paths.get("/path/to/application_external_time.avsc"))); - } - - @Test - public void nameSpaceInvalidPlural() { - Schema schema = SchemaBuilder - .builder("org.radarcns.monitors.test") - .record(RECORD_NAME_MOCK) - .fields() - .endRecord(); - - Path root = MONITOR.getPath(BASE_PATH.resolve(COMMONS_PATH)); - assertNotNull(root); - Path path = root.resolve("test/record_name.avsc"); - Stream result = validator.validateSchemaLocation() - .apply(new SchemaMetadata(schema, MONITOR, path)); - - assertEquals(1, result.count()); - } - - @Test - public void nameSpaceInvalidLastPartPlural() { - - Schema schema = SchemaBuilder - .builder("org.radarcns.monitor.tests") - .record(RECORD_NAME_MOCK) - .fields() - .endRecord(); - - Path root = MONITOR.getPath(BASE_PATH.resolve(COMMONS_PATH)); - assertNotNull(root); - Path path = root.resolve("test/record_name.avsc"); - Stream result = validator.validateSchemaLocation() - .apply(new SchemaMetadata(schema, MONITOR, path)); - - assertEquals(1, result.count()); - } - - @Test - public void recordNameTest() { - // misspell aceleration - String fieldName = "EmpaticaE4Aceleration"; - Path filePath = Paths.get("/path/to/empatica_e4_acceleration.avsc"); - - Schema schema = SchemaBuilder - .builder("org.radarcns.passive.empatica") - .record(fieldName) - .fields() - .endRecord(); - - Stream result = validator.validateSchemaLocation() - .apply(new SchemaMetadata(schema, PASSIVE, filePath)); - - assertEquals(2, result.count()); - - fieldName = "EmpaticaE4Acceleration"; - filePath = BASE_PATH.resolve("commons/passive/empatica/empatica_e4_acceleration.avsc"); - - schema = SchemaBuilder - .builder("org.radarcns.passive.empatica") - .record(fieldName) - .fields() - .endRecord(); - - result = validator.validateSchemaLocation() - .apply(new SchemaMetadata(schema, PASSIVE, filePath)); - - assertEquals("", SchemaValidator.format(result)); - } -} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt new file mode 100644 index 00000000..595020f4 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt @@ -0,0 +1,122 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation.rules + +import org.apache.avro.SchemaBuilder +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.radarbase.schema.Scope.MONITOR +import org.radarbase.schema.Scope.PASSIVE +import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.validation.SchemaValidator.Companion.format +import org.radarbase.schema.validation.SourceCatalogueValidationTest +import org.radarbase.schema.validation.ValidationException +import org.radarbase.schema.validation.ValidationHelper +import java.nio.file.Paths +import java.util.stream.Stream + +/** + * TODO. + */ +class RadarSchemaMetadataRulesTest { + private lateinit var validator: RadarSchemaMetadataRules + + @BeforeEach + fun setUp() { + val config = SchemaConfig() + validator = RadarSchemaMetadataRules( + SourceCatalogueValidationTest.BASE_PATH.resolve(ValidationHelper.COMMONS_PATH), config, + ) + } + + @Test + fun fileNameTest() { + assertEquals( + "Questionnaire", + ValidationHelper.getRecordName(Paths.get("/path/to/questionnaire.avsc")), + ) + assertEquals( + "ApplicationExternalTime", + ValidationHelper.getRecordName( + Paths.get("/path/to/application_external_time.avsc"), + ), + ) + } + + @Test + fun nameSpaceInvalidPlural() { + val schema = SchemaBuilder + .builder("org.radarcns.monitors.test") + .record(RECORD_NAME_MOCK) + .fields() + .endRecord() + val root = + MONITOR.getPath(SourceCatalogueValidationTest.BASE_PATH.resolve(ValidationHelper.COMMONS_PATH)) + assertNotNull(root) + val path = root.resolve("test/record_name.avsc") + val result = validator.validateSchemaLocation() + .validate(SchemaMetadata(schema, MONITOR, path)) + assertEquals(1, result.count()) + } + + @Test + fun nameSpaceInvalidLastPartPlural() { + val schema = SchemaBuilder + .builder("org.radarcns.monitor.tests") + .record(RECORD_NAME_MOCK) + .fields() + .endRecord() + val root = + MONITOR.getPath(SourceCatalogueValidationTest.BASE_PATH.resolve(ValidationHelper.COMMONS_PATH)) + assertNotNull(root) + val path = root.resolve("test/record_name.avsc") + val result = validator.validateSchemaLocation() + .validate(SchemaMetadata(schema, MONITOR, path)) + assertEquals(1, result.count()) + } + + @Test + fun recordNameTest() { + // misspell aceleration + var fieldName = "EmpaticaE4Aceleration" + var filePath = Paths.get("/path/to/empatica_e4_acceleration.avsc") + var schema = SchemaBuilder + .builder("org.radarcns.passive.empatica") + .record(fieldName) + .fields() + .endRecord() + var result: Stream = validator.validateSchemaLocation() + .validate(SchemaMetadata(schema, PASSIVE, filePath)) + assertEquals(2, result.count()) + fieldName = "EmpaticaE4Acceleration" + filePath = + SourceCatalogueValidationTest.BASE_PATH.resolve("commons/passive/empatica/empatica_e4_acceleration.avsc") + schema = SchemaBuilder + .builder("org.radarcns.passive.empatica") + .record(fieldName) + .fields() + .endRecord() + result = validator.validateSchemaLocation() + .validate(SchemaMetadata(schema, PASSIVE, filePath)) + assertEquals("", format(result)) + } + + companion object { + private const val RECORD_NAME_MOCK = "RecordName" + } +} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.java b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.java deleted file mode 100644 index 8b2f49ea..00000000 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.java +++ /dev/null @@ -1,412 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.schema.validation.rules; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.radarbase.schema.validation.rules.RadarSchemaRules.ENUM_SYMBOL_PATTERN; -import static org.radarbase.schema.validation.rules.RadarSchemaRules.NAMESPACE_PATTERN; -import static org.radarbase.schema.validation.rules.RadarSchemaRules.RECORD_NAME_PATTERN; -import static org.radarbase.schema.validation.rules.Validator.matches; - -import java.util.stream.Stream; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Parser; -import org.apache.avro.SchemaBuilder; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.radarbase.schema.specification.config.SchemaConfig; -import org.radarbase.schema.validation.ValidationException; - -/** - * TODO. - */ -public class RadarSchemaRulesTest { - - private static final String ACTIVE_NAME_SPACE_MOCK = "org.radarcns.active.test"; - private static final String MONITOR_NAME_SPACE_MOCK = "org.radarcns.monitor.test"; - private static final String ENUMERATOR_NAME_SPACE_MOCK = "org.radarcns.test.EnumeratorTest"; - private static final String UNKNOWN_MOCK = "UNKNOWN"; - - private static final String RECORD_NAME_MOCK = "RecordName"; - private RadarSchemaRules validator; - - @BeforeEach - public void setUp() { - SchemaConfig config = new SchemaConfig(); - validator = new RadarSchemaRules(); - } - - @Test - public void nameSpaceRegex() { - assertTrue(matches("org.radarcns", NAMESPACE_PATTERN)); - assertFalse(matches("Org.radarcns", NAMESPACE_PATTERN)); - assertFalse(matches("org.radarCns", NAMESPACE_PATTERN)); - assertFalse(matches(".org.radarcns", NAMESPACE_PATTERN)); - assertFalse(matches("org.radar-cns", NAMESPACE_PATTERN)); - assertFalse(matches("org.radarcns.empaticaE4", NAMESPACE_PATTERN)); - } - - @Test - public void recordNameRegex() { - assertTrue(matches("Questionnaire", RECORD_NAME_PATTERN)); - assertTrue(matches("EmpaticaE4Acceleration", RECORD_NAME_PATTERN)); - assertTrue(matches("Heart4Me", RECORD_NAME_PATTERN)); - assertTrue(matches("Heart4M", RECORD_NAME_PATTERN)); - assertTrue(matches("Heart4", RECORD_NAME_PATTERN)); - assertFalse(matches("Heart4me", RECORD_NAME_PATTERN)); - assertTrue(matches("Heart4ME", RECORD_NAME_PATTERN)); - assertFalse(matches("4Me", RECORD_NAME_PATTERN)); - assertTrue(matches("TTest", RECORD_NAME_PATTERN)); - assertFalse(matches("questionnaire", RECORD_NAME_PATTERN)); - assertFalse(matches("questionnaire4", RECORD_NAME_PATTERN)); - assertFalse(matches("questionnaire4Me", RECORD_NAME_PATTERN)); - assertFalse(matches("questionnaire4me", RECORD_NAME_PATTERN)); - assertTrue(matches("A4MM", RECORD_NAME_PATTERN)); - assertTrue(matches("Aaaa4MMaa", RECORD_NAME_PATTERN)); - } - - @Test - public void enumerationRegex() { - assertTrue(matches("PHQ8", ENUM_SYMBOL_PATTERN)); - assertTrue(matches("HELLO", ENUM_SYMBOL_PATTERN)); - assertTrue(matches("HELLOTHERE", ENUM_SYMBOL_PATTERN)); - assertTrue(matches("HELLO_THERE", ENUM_SYMBOL_PATTERN)); - assertFalse(matches("Hello", ENUM_SYMBOL_PATTERN)); - assertFalse(matches("hello", ENUM_SYMBOL_PATTERN)); - assertFalse(matches("HelloThere", ENUM_SYMBOL_PATTERN)); - assertFalse(matches("Hello_There", ENUM_SYMBOL_PATTERN)); - assertFalse(matches("HELLO.THERE", ENUM_SYMBOL_PATTERN)); - } - - @Test - public void nameSpaceTest() { - Schema schema = SchemaBuilder - .builder("org.radarcns.active.questionnaire") - .record("Questionnaire") - .fields() - .endRecord(); - - Stream result = validator.validateNameSpace() - .apply(schema); - - assertEquals(0, result.count()); - } - - @Test - public void nameSpaceInvalidDashTest() { - Schema schema = SchemaBuilder - .builder("org.radar-cns.monitors.test") - .record(RECORD_NAME_MOCK) - .fields() - .endRecord(); - - Stream result = validator.validateNameSpace() - .apply(schema); - - assertEquals(1, result.count()); - - } - - @Test - public void recordNameTest() { - Schema schema = SchemaBuilder - .builder("org.radarcns.active.testactive") - .record("Schema") - .fields() - .endRecord(); - - Stream result = validator.validateName() - .apply(schema); - - assertEquals(0, result.count()); - } - - @Test - public void fieldsTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .endRecord(); - - result = validator.fields(validator.getFieldRules().validateFieldTypes(validator)) - .apply(schema); - - assertEquals(1, result.count()); - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .optionalBoolean("optional") - .endRecord(); - - result = validator.fields(validator.getFieldRules().validateFieldTypes(validator)) - .apply(schema); - - assertEquals(0, result.count()); - } - - @Test - public void timeTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder - .builder("org.radarcns.time.test") - .record(RECORD_NAME_MOCK) - .fields() - .requiredString("string") - .endRecord(); - - result = validator.validateTime().apply(schema); - - assertEquals(1, result.count()); - - schema = SchemaBuilder - .builder("org.radarcns.time.test") - .record(RECORD_NAME_MOCK) - .fields() - .requiredDouble(RadarSchemaRules.TIME) - .endRecord(); - - result = validator.validateTime().apply(schema); - - assertEquals(0, result.count()); - } - - @Test - public void timeCompletedTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder - .builder(ACTIVE_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredString("field") - .endRecord(); - - result = validator.validateTimeCompleted().apply(schema); - assertEquals(1, result.count()); - - result = validator.validateNotTimeCompleted().apply(schema); - assertEquals(0, result.count()); - - schema = SchemaBuilder - .builder(ACTIVE_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredDouble("timeCompleted") - .endRecord(); - - result = validator.validateTimeCompleted().apply(schema); - assertEquals(0, result.count()); - - result = validator.validateNotTimeCompleted().apply(schema); - assertEquals(1, result.count()); - } - - @Test - public void timeReceivedTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredString("field") - .endRecord(); - - result = validator.validateTimeReceived().apply(schema); - assertEquals(1, result.count()); - - result = validator.validateNotTimeReceived().apply(schema); - assertEquals(0, result.count()); - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredDouble("timeReceived") - .endRecord(); - - result = validator.validateTimeReceived().apply(schema); - assertEquals(0, result.count()); - - result = validator.validateNotTimeReceived().apply(schema); - assertEquals(1, result.count()); - } - - @Test - public void schemaDocumentationTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .endRecord(); - - result = validator.validateSchemaDocumentation().apply(schema); - - assertEquals(1, result.count()); - - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .doc("Documentation.") - .fields() - .endRecord(); - - result = validator.validateSchemaDocumentation().apply(schema); - - assertEquals(0, result.count()); - } - - @Test - public void enumerationSymbolsTest() { - Schema schema; - Stream result; - - schema = SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK) - .symbols("TEST", UNKNOWN_MOCK); - - result = validator.validateSymbols().apply(schema); - - assertEquals(0, result.count()); - - schema = SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK).symbols(); - - result = validator.validateSymbols().apply(schema); - - assertEquals(1, result.count()); - } - - @Test - public void enumerationSymbolTest() { - Schema schema; - Stream result; - - String enumName = "org.radarcns.monitor.application.ApplicationServerStatus"; - String connected = "CONNECTED"; - - schema = SchemaBuilder - .enumeration(enumName) - .symbols(connected, "DISCONNECTED", UNKNOWN_MOCK); - - result = validator.validateSymbols().apply(schema); - - assertEquals(0, result.count()); - - String schemaTxtInit = "{\"namespace\": \"org.radarcns.monitor.application\", " - + "\"name\": \"ServerStatus\", \"type\": " - + "\"enum\", \"symbols\": ["; - - String schemaTxtEnd = "] }"; - - schema = new Parser().parse(schemaTxtInit - + "\"CONNECTED\", \"NOT_CONNECTED\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd); - - result = validator.validateSymbols().apply(schema); - - assertEquals(0, result.count()); - - schema = SchemaBuilder - .enumeration(enumName) - .symbols(connected, "disconnected", UNKNOWN_MOCK); - - result = validator.validateSymbols().apply(schema); - - assertEquals(1, result.count()); - - schema = SchemaBuilder - .enumeration(enumName) - .symbols(connected, "Not_Connected", UNKNOWN_MOCK); - - result = validator.validateSymbols().apply(schema); - - assertEquals(1, result.count()); - - schema = SchemaBuilder - .enumeration(enumName) - .symbols(connected, "NotConnected", UNKNOWN_MOCK); - - result = validator.validateSymbols().apply(schema); - - assertEquals(1, result.count()); - - schema = new Parser().parse(schemaTxtInit - + "\"CONNECTED\", \"Not_Connected\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd); - - result = validator.validateSymbols().apply(schema); - - assertEquals(1, result.count()); - - schema = new Parser().parse(schemaTxtInit - + "\"Connected\", \"NotConnected\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd); - - result = validator.validateSymbols().apply(schema); - - assertEquals(2, result.count()); - } - - - @Test - public void testUniqueness() { - final String prefix = "{\"namespace\": \"org.radarcns.monitor.application\", " - + "\"name\": \""; - final String infix = "\", \"type\": \"enum\", \"symbols\": "; - final char suffix = '}'; - - Schema schema = new Parser().parse(prefix + "ServerStatus" - + infix + "[\"A\", \"B\"]" + suffix); - Stream result = validator.validateUniqueness().apply(schema); - assertEquals(0, result.count()); - result = validator.validateUniqueness().apply(schema); - assertEquals(0, result.count()); - - Schema schemaAlt = new Parser().parse(prefix + "ServerStatus" - + infix + "[\"A\", \"B\", \"C\"]" + suffix); - result = validator.validateUniqueness().apply(schemaAlt); - assertEquals(1, result.count()); - result = validator.validateUniqueness().apply(schemaAlt); - assertEquals(1, result.count()); - - Schema schema2 = new Parser().parse(prefix + "ServerStatus2" - + infix + "[\"A\", \"B\"]" + suffix); - result = validator.validateUniqueness().apply(schema2); - assertEquals(0, result.count()); - - Schema schema3 = new Parser().parse(prefix + "ServerStatus" - + infix + "[\"A\", \"B\"]" + suffix); - result = validator.validateUniqueness().apply(schema3); - assertEquals(0, result.count()); - result = validator.validateUniqueness().apply(schema3); - assertEquals(0, result.count()); - - result = validator.validateUniqueness().apply(schemaAlt); - assertEquals(1, result.count()); - } -} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt new file mode 100644 index 00000000..d99bfb40 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt @@ -0,0 +1,337 @@ +/* + * Copyright 2017 King's College London and The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.schema.validation.rules + +import org.apache.avro.Schema +import org.apache.avro.Schema.Parser +import org.apache.avro.SchemaBuilder +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.radarbase.schema.validation.ValidationException +import java.util.stream.Stream + +/** + * TODO. + */ +class RadarSchemaRulesTest { + private lateinit var validator: RadarSchemaRules + + @BeforeEach + fun setUp() { + validator = RadarSchemaRules() + } + + @Test + fun nameSpaceRegex() { + assertTrue("org.radarcns".matches(RadarSchemaRules.NAMESPACE_PATTERN)) + assertFalse("Org.radarcns".matches(RadarSchemaRules.NAMESPACE_PATTERN)) + assertFalse("org.radarCns".matches(RadarSchemaRules.NAMESPACE_PATTERN)) + assertFalse(".org.radarcns".matches(RadarSchemaRules.NAMESPACE_PATTERN)) + assertFalse("org.radar-cns".matches(RadarSchemaRules.NAMESPACE_PATTERN)) + assertFalse("org.radarcns.empaticaE4".matches(RadarSchemaRules.NAMESPACE_PATTERN)) + } + + @Test + fun recordNameRegex() { + assertTrue("Questionnaire".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertTrue("EmpaticaE4Acceleration".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Heart4Me".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Heart4M".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Heart4".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertFalse("Heart4me".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Heart4ME".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertFalse("4Me".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertTrue("TTest".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertFalse("questionnaire".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertFalse("questionnaire4".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertFalse("questionnaire4Me".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertFalse("questionnaire4me".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertTrue("A4MM".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Aaaa4MMaa".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + } + + @Test + fun enumerationRegex() { + assertTrue("PHQ8".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) + assertTrue("HELLO".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) + assertTrue("HELLOTHERE".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) + assertTrue("HELLO_THERE".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("Hello".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("hello".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("HelloThere".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("Hello_There".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("HELLO.THERE".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) + } + + @Test + fun nameSpaceTest() { + val schema = SchemaBuilder + .builder("org.radarcns.active.questionnaire") + .record("Questionnaire") + .fields() + .endRecord() + val result: Stream = validator.validateNameSpace() + .validate(schema) + Assertions.assertEquals(0, result.count()) + } + + @Test + fun nameSpaceInvalidDashTest() { + val schema = SchemaBuilder + .builder("org.radar-cns.monitors.test") + .record(RECORD_NAME_MOCK) + .fields() + .endRecord() + val result: Stream = validator.validateNameSpace() + .validate(schema) + Assertions.assertEquals(1, result.count()) + } + + @Test + fun recordNameTest() { + val schema = SchemaBuilder + .builder("org.radarcns.active.testactive") + .record("Schema") + .fields() + .endRecord() + val result: Stream = validator.validateName() + .validate(schema) + Assertions.assertEquals(0, result.count()) + } + + @Test + fun fieldsTest() { + var schema: Schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .endRecord() + var result: Stream = validator.fields( + validator.fieldRules.validateFieldTypes(validator), + ).validate(schema) + Assertions.assertEquals(1, result.count()) + schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .optionalBoolean("optional") + .endRecord() + result = validator.fields(validator.fieldRules.validateFieldTypes(validator)) + .validate(schema) + Assertions.assertEquals(0, result.count()) + } + + @Test + fun timeTest() { + var schema: Schema = SchemaBuilder + .builder("org.radarcns.time.test") + .record(RECORD_NAME_MOCK) + .fields() + .requiredString("string") + .endRecord() + var result: Stream = validator.validateTime().validate(schema) + Assertions.assertEquals(1, result.count()) + schema = SchemaBuilder + .builder("org.radarcns.time.test") + .record(RECORD_NAME_MOCK) + .fields() + .requiredDouble(RadarSchemaRules.TIME) + .endRecord() + result = validator.validateTime().validate(schema) + Assertions.assertEquals(0, result.count()) + } + + @Test + fun timeCompletedTest() { + var schema: Schema = SchemaBuilder + .builder(ACTIVE_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .requiredString("field") + .endRecord() + var result: Stream = validator.validateTimeCompleted().validate(schema) + Assertions.assertEquals(1, result.count()) + result = validator.validateNotTimeCompleted().validate(schema) + Assertions.assertEquals(0, result.count()) + schema = SchemaBuilder + .builder(ACTIVE_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .requiredDouble("timeCompleted") + .endRecord() + result = validator.validateTimeCompleted().validate(schema) + Assertions.assertEquals(0, result.count()) + result = validator.validateNotTimeCompleted().validate(schema) + Assertions.assertEquals(1, result.count()) + } + + @Test + fun timeReceivedTest() { + var schema: Schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .requiredString("field") + .endRecord() + var result: Stream = validator.validateTimeReceived().validate(schema) + Assertions.assertEquals(1, result.count()) + result = validator.validateNotTimeReceived().validate(schema) + Assertions.assertEquals(0, result.count()) + schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .requiredDouble("timeReceived") + .endRecord() + result = validator.validateTimeReceived().validate(schema) + Assertions.assertEquals(0, result.count()) + result = validator.validateNotTimeReceived().validate(schema) + Assertions.assertEquals(1, result.count()) + } + + @Test + fun schemaDocumentationTest() { + var schema: Schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .fields() + .endRecord() + var result: Stream = validator.validateSchemaDocumentation().validate(schema) + Assertions.assertEquals(1, result.count()) + schema = SchemaBuilder + .builder(MONITOR_NAME_SPACE_MOCK) + .record(RECORD_NAME_MOCK) + .doc("Documentation.") + .fields() + .endRecord() + result = validator.validateSchemaDocumentation().validate(schema) + Assertions.assertEquals(0, result.count()) + } + + @Test + fun enumerationSymbolsTest() { + var schema: Schema = SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK) + .symbols("TEST", UNKNOWN_MOCK) + var result: Stream = validator.validateSymbols().validate(schema) + Assertions.assertEquals(0, result.count()) + schema = SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK).symbols() + result = validator.validateSymbols().validate(schema) + Assertions.assertEquals(1, result.count()) + } + + @Test + fun enumerationSymbolTest() { + val enumName = "org.radarcns.monitor.application.ApplicationServerStatus" + val connected = "CONNECTED" + var schema: Schema = SchemaBuilder + .enumeration(enumName) + .symbols(connected, "DISCONNECTED", UNKNOWN_MOCK) + var result: Stream = validator.validateSymbols().validate(schema) + Assertions.assertEquals(0, result.count()) + val schemaTxtInit = ( + "{\"namespace\": \"org.radarcns.monitor.application\", " + + "\"name\": \"ServerStatus\", \"type\": " + + "\"enum\", \"symbols\": [" + ) + val schemaTxtEnd = "] }" + schema = Parser().parse( + schemaTxtInit + + "\"CONNECTED\", \"NOT_CONNECTED\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd, + ) + result = validator.validateSymbols().validate(schema) + Assertions.assertEquals(0, result.count()) + schema = SchemaBuilder + .enumeration(enumName) + .symbols(connected, "disconnected", UNKNOWN_MOCK) + result = validator.validateSymbols().validate(schema) + Assertions.assertEquals(1, result.count()) + schema = SchemaBuilder + .enumeration(enumName) + .symbols(connected, "Not_Connected", UNKNOWN_MOCK) + result = validator.validateSymbols().validate(schema) + Assertions.assertEquals(1, result.count()) + schema = SchemaBuilder + .enumeration(enumName) + .symbols(connected, "NotConnected", UNKNOWN_MOCK) + result = validator.validateSymbols().validate(schema) + Assertions.assertEquals(1, result.count()) + schema = Parser().parse( + schemaTxtInit + + "\"CONNECTED\", \"Not_Connected\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd, + ) + result = validator.validateSymbols().validate(schema) + Assertions.assertEquals(1, result.count()) + schema = Parser().parse( + schemaTxtInit + + "\"Connected\", \"NotConnected\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd, + ) + result = validator.validateSymbols().validate(schema) + Assertions.assertEquals(2, result.count()) + } + + @Test + fun testUniqueness() { + val prefix = ( + "{\"namespace\": \"org.radarcns.monitor.application\", " + + "\"name\": \"" + ) + val infix = "\", \"type\": \"enum\", \"symbols\": " + val suffix = '}' + val schema = Parser().parse( + prefix + "ServerStatus" + + infix + "[\"A\", \"B\"]" + suffix, + ) + var result: Stream = validator.validateUniqueness().validate(schema) + Assertions.assertEquals(0, result.count()) + result = validator.validateUniqueness().validate(schema) + Assertions.assertEquals(0, result.count()) + val schemaAlt = Parser().parse( + prefix + "ServerStatus" + + infix + "[\"A\", \"B\", \"C\"]" + suffix, + ) + result = validator.validateUniqueness().validate(schemaAlt) + Assertions.assertEquals(1, result.count()) + result = validator.validateUniqueness().validate(schemaAlt) + Assertions.assertEquals(1, result.count()) + val schema2 = Parser().parse( + prefix + "ServerStatus2" + + infix + "[\"A\", \"B\"]" + suffix, + ) + result = validator.validateUniqueness().validate(schema2) + Assertions.assertEquals(0, result.count()) + val schema3 = Parser().parse( + prefix + "ServerStatus" + + infix + "[\"A\", \"B\"]" + suffix, + ) + result = validator.validateUniqueness().validate(schema3) + Assertions.assertEquals(0, result.count()) + result = validator.validateUniqueness().validate(schema3) + Assertions.assertEquals(0, result.count()) + result = validator.validateUniqueness().validate(schemaAlt) + Assertions.assertEquals(1, result.count()) + } + + companion object { + private const val ACTIVE_NAME_SPACE_MOCK = "org.radarcns.active.test" + private const val MONITOR_NAME_SPACE_MOCK = "org.radarcns.monitor.test" + private const val ENUMERATOR_NAME_SPACE_MOCK = "org.radarcns.test.EnumeratorTest" + private const val UNKNOWN_MOCK = "UNKNOWN" + private const val RECORD_NAME_MOCK = "RecordName" + } +} diff --git a/java-sdk/radar-schemas-registration/build.gradle.kts b/java-sdk/radar-schemas-registration/build.gradle.kts index ceba6259..2d44b8f5 100644 --- a/java-sdk/radar-schemas-registration/build.gradle.kts +++ b/java-sdk/radar-schemas-registration/build.gradle.kts @@ -7,18 +7,14 @@ repositories { dependencies { api(project(":radar-schemas-commons")) api(project(":radar-schemas-core")) - val okHttpVersion: String by project - api("com.squareup.okhttp3:okhttp:$okHttpVersion") - val radarCommonsVersion: String by project - api("org.radarbase:radar-commons-server:$radarCommonsVersion") - val confluentVersion: String by project - implementation("io.confluent:kafka-connect-avro-converter:$confluentVersion") - implementation("io.confluent:kafka-schema-registry-client:$confluentVersion") + implementation("org.radarbase:radar-commons:${Versions.radarCommons}") + api("org.radarbase:radar-commons-server:${Versions.radarCommons}") + implementation("org.radarbase:radar-commons-kotlin:${Versions.radarCommons}") - val kafkaVersion: String by project - implementation("org.apache.kafka:connect-json:$kafkaVersion") + implementation("io.confluent:kafka-connect-avro-converter:${Versions.confluent}") + implementation("io.confluent:kafka-schema-registry-client:${Versions.confluent}") - val slf4jVersion: String by project - implementation("org.slf4j:slf4j-api:$slf4jVersion") + implementation("org.apache.kafka:connect-json:${Versions.kafka}") + implementation("io.ktor:ktor-client-auth:2.3.4") } diff --git a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/KafkaTopics.kt b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/KafkaTopics.kt index d41eea96..ee177ac7 100644 --- a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/KafkaTopics.kt +++ b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/KafkaTopics.kt @@ -1,22 +1,30 @@ package org.radarbase.schema.registration +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.firstOrNull +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.take +import kotlinx.coroutines.withContext +import org.apache.kafka.clients.admin.Admin import org.apache.kafka.clients.admin.AdminClient import org.apache.kafka.clients.admin.AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG import org.apache.kafka.clients.admin.ListTopicsOptions import org.apache.kafka.clients.admin.NewTopic import org.apache.kafka.common.config.SaslConfigs.SASL_JAAS_CONFIG +import org.radarbase.kotlin.coroutines.suspendGet +import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.config.ToolConfig import org.radarbase.schema.specification.config.TopicConfig -import org.radarbase.schema.specification.SourceCatalogue import org.slf4j.LoggerFactory -import java.time.Duration -import java.time.Instant -import java.util.* -import java.util.concurrent.ExecutionException -import java.util.concurrent.TimeUnit import java.util.concurrent.TimeoutException import java.util.stream.Collectors import java.util.stream.Stream +import kotlin.time.Duration.Companion.seconds +import kotlin.time.TimeMark +import kotlin.time.TimeSource /** * Registers Kafka topics with Zookeeper. @@ -25,7 +33,8 @@ class KafkaTopics( private val toolConfig: ToolConfig, ) : TopicRegistrar { private var initialized = false - private var topics: Set? = null + override lateinit var topics: Set + private val adminClient: AdminClient = AdminClient.create(toolConfig.kafka) /** @@ -35,8 +44,7 @@ class KafkaTopics( * @param brokers number of brokers to wait for * @throws InterruptedException when waiting for the brokers is interrupted. */ - @Throws(InterruptedException::class) - override fun initialize(brokers: Int) { + override suspend fun initialize(brokers: Int) { initialize(brokers, 20) } @@ -47,28 +55,34 @@ class KafkaTopics( * * @param brokers number of brokers to wait for. * @param numTries Number of times to retry in case of failure. - * @throws InterruptedException when waiting for the brokers is interrupted. */ - @Throws(InterruptedException::class) - override fun initialize(brokers: Int, numTries: Int) { - val numBrokers = retrySequence(Duration.ofSeconds(2), MAX_SLEEP) + override suspend fun initialize(brokers: Int, numTries: Int) { + val numBrokers = retryFlow(2.seconds, MAX_SLEEP) .take(numTries) .map { sleep -> try { - adminClient.describeCluster() - .nodes() - .get(sleep.toSeconds(), TimeUnit.SECONDS) - .size + withContext(Dispatchers.IO) { + adminClient.describeCluster() + .nodes() + .suspendGet(sleep) + .size + } } catch (ex: InterruptedException) { logger.error("Refreshing topics interrupted") throw ex } catch (ex: TimeoutException) { - logger.error("Failed to connect to bootstrap server {} within {} seconds", - kafkaProperties[BOOTSTRAP_SERVERS_CONFIG], sleep) + logger.error( + "Failed to connect to bootstrap server {} within {} seconds", + kafkaProperties[BOOTSTRAP_SERVERS_CONFIG], + sleep, + ) 0 } catch (ex: Throwable) { - logger.error("Failed to connect to bootstrap server {}", - kafkaProperties[BOOTSTRAP_SERVERS_CONFIG], ex.cause) + logger.error( + "Failed to connect to bootstrap server {}", + kafkaProperties[BOOTSTRAP_SERVERS_CONFIG], + ex.cause, + ) 0 } } @@ -90,7 +104,7 @@ class KafkaTopics( check(initialized) { "Manager is not initialized yet" } } - override fun createTopics( + override suspend fun createTopics( catalogue: SourceCatalogue, partitions: Int, replication: Short, @@ -105,9 +119,12 @@ class KafkaTopics( .filter { s -> pattern.matcher(s).find() } .collect(Collectors.toList()) if (topicNames.isEmpty()) { - logger.error("Topic {} does not match a known topic." - + " Find the list of acceptable topics" - + " with the `radar-schemas-tools list` command. Aborting.", pattern) + logger.error( + "Topic {} does not match a known topic." + + " Find the list of acceptable topics" + + " with the `radar-schemas-tools list` command. Aborting.", + pattern, + ) return 1 } if (createTopics(topicNames.stream(), partitions, replication)) 0 else 1 @@ -117,7 +134,7 @@ class KafkaTopics( private fun topicNames(catalogue: SourceCatalogue): Stream { return Stream.concat( catalogue.topicNames, - toolConfig.topics.keys.stream() + toolConfig.topics.keys.stream(), ).filter { t -> toolConfig.topics[t]?.enabled != false } } @@ -129,29 +146,29 @@ class KafkaTopics( * @param replication number of replicas for a topic * @return whether the whole catalogue was registered */ - private fun createTopics( + private suspend fun createTopics( catalogue: SourceCatalogue, partitions: Int, - replication: Short + replication: Short, ): Boolean { ensureInitialized() return createTopics(topicNames(catalogue), partitions, replication) } - override fun createTopics( - topicsToCreate: Stream, + override suspend fun createTopics( + topics: Stream, partitions: Int, - replication: Short + replication: Short, ): Boolean { ensureInitialized() return try { refreshTopics() logger.info("Creating topics. Topics marked with [*] already exist.") - val newTopics = topicsToCreate + val newTopics = topics .sorted() .distinct() .filter { t: String -> - if (topics?.contains(t) == true) { + if (this.topics.contains(t)) { logger.info("[*] {}", t) return@filter false } else { @@ -174,7 +191,7 @@ class KafkaTopics( kafkaClient .createTopics(newTopics) .all() - .get() + .suspendGet() logger.info("Created {} topics. Requesting to refresh topics", newTopics.size) refreshTopics() } else { @@ -188,23 +205,23 @@ class KafkaTopics( } @Throws(InterruptedException::class) - override fun refreshTopics(): Boolean { + override suspend fun refreshTopics(): Boolean { ensureInitialized() logger.info("Waiting for topics to become available.") - topics = null + topics = emptySet() val opts = ListTopicsOptions().apply { listInternal(true) } - topics = retrySequence(Duration.ofSeconds(2), MAX_SLEEP) + topics = retryFlow(2.seconds, MAX_SLEEP) .take(10) .map { sleep -> try { kafkaClient .listTopics(opts) .names() - .get(sleep.toSeconds(), TimeUnit.SECONDS) + .suspendGet(sleep) } catch (ex: TimeoutException) { logger.error("Failed to list topics within {} seconds", sleep) emptySet() @@ -217,15 +234,9 @@ class KafkaTopics( } } .firstOrNull { it.isNotEmpty() } + ?: emptySet() - return topics != null - } - - override fun getTopics(): Set { - ensureInitialized() - return Collections.unmodifiableSet(checkNotNull(topics) { - "Topics were not properly initialized" - }) + return topics.isNotEmpty() } override fun close() { @@ -236,71 +247,72 @@ class KafkaTopics( * Get current number of Kafka brokers according to Zookeeper. * * @return number of Kafka brokers - * @throws ExecutionException if kafka cannot connect - * @throws InterruptedException if the query is interrupted. */ - @get:Throws(ExecutionException::class, - InterruptedException::class) - val numberOfBrokers: Int - get() = adminClient.describeCluster() + suspend fun numberOfBrokers(): Int { + return adminClient.describeCluster() .nodes() - .get() + .suspendGet() .size - - override fun getKafkaClient(): AdminClient { - ensureInitialized() - return adminClient } - override fun getKafkaProperties(): Map = toolConfig.kafka + override val kafkaClient: Admin + get() { + ensureInitialized() + return adminClient + } + + override val kafkaProperties: Map + get() = toolConfig.kafka companion object { private val logger = LoggerFactory.getLogger(KafkaTopics::class.java) - private val MAX_SLEEP = Duration.ofSeconds(32) + private val MAX_SLEEP = 32.seconds + @JvmStatic fun ToolConfig.configureKafka( - bootstrapServers: String? + bootstrapServers: String?, ): ToolConfig = if (bootstrapServers.isNullOrEmpty()) { check(BOOTSTRAP_SERVERS_CONFIG in kafka) { "Cannot configure Kafka without $BOOTSTRAP_SERVERS_CONFIG property" } this } else { - copy(kafka = buildMap { - putAll(kafka) - put(BOOTSTRAP_SERVERS_CONFIG, bootstrapServers) - System.getenv("KAFKA_SASL_JAAS_CONFIG")?.let { - put(SASL_JAAS_CONFIG, it) - } - }) + copy( + kafka = buildMap { + putAll(kafka) + put(BOOTSTRAP_SERVERS_CONFIG, bootstrapServers) + System.getenv("KAFKA_SASL_JAAS_CONFIG")?.let { + put(SASL_JAAS_CONFIG, it) + } + }, + ) } - fun retrySequence( - startSleep: Duration, - maxSleep: Duration, - ): Sequence = sequence { + fun retryFlow( + startSleep: kotlin.time.Duration, + maxSleep: kotlin.time.Duration, + ): Flow = flow { var sleep = startSleep while (true) { // All computation for the sequence will be done in yield. It should be excluded // from sleep. - val endTime = Instant.now() + sleep - yield(sleep) - sleepUntil(endTime) { sleepMillis -> - logger.info("Waiting {} seconds to retry", (sleepMillis / 100) / 10.0) + val endTime = TimeSource.Monotonic.markNow() + sleep + emit(sleep) + sleepUntil(endTime) { timeUntil -> + logger.info("Waiting {} seconds to retry", timeUntil) } if (sleep < maxSleep) { - sleep = sleep.multipliedBy(2L).coerceAtMost(maxSleep) + sleep = (sleep * 2).coerceAtMost(maxSleep) } } } - private inline fun sleepUntil(time: Instant, beforeSleep: (Long) -> Unit) { - val timeToSleep = Duration.between(time, Instant.now()) - if (!timeToSleep.isNegative) { - val sleepMillis = timeToSleep.toMillis() - beforeSleep(sleepMillis) - Thread.sleep(sleepMillis) + private suspend fun sleepUntil(time: TimeMark, beforeSleep: (kotlin.time.Duration) -> Unit) { + val timeUntil = -time.elapsedNow() + if (timeUntil.isPositive()) { + beforeSleep(timeUntil) + delay(timeUntil) } } } diff --git a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/SchemaRegistry.kt b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/SchemaRegistry.kt index 4f3efa0d..4f8574c1 100644 --- a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/SchemaRegistry.kt +++ b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/SchemaRegistry.kt @@ -15,29 +15,38 @@ */ package org.radarbase.schema.registration -import okhttp3.Credentials.basic -import okhttp3.Headers.Companion.headersOf -import okhttp3.MediaType -import okhttp3.MediaType.Companion.toMediaTypeOrNull -import okhttp3.RequestBody -import org.radarbase.producer.rest.SchemaRetriever -import org.radarbase.producer.rest.RestClient -import org.radarcns.kafka.ObservationKey -import kotlin.Throws -import org.radarbase.schema.specification.SourceCatalogue -import org.radarbase.topic.AvroTopic -import okio.BufferedSink +import io.ktor.client.plugins.auth.Auth +import io.ktor.client.plugins.auth.providers.BasicAuthCredentials +import io.ktor.client.plugins.auth.providers.basic +import io.ktor.client.request.setBody +import io.ktor.client.request.url +import io.ktor.http.ContentType +import io.ktor.http.HttpMethod +import io.ktor.http.contentType +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.firstOrNull +import kotlinx.coroutines.flow.mapNotNull +import kotlinx.coroutines.flow.take import org.apache.avro.specific.SpecificRecord -import org.radarbase.config.ServerConfig -import org.radarbase.schema.registration.KafkaTopics.Companion.retrySequence +import org.radarbase.kotlin.coroutines.forkJoin +import org.radarbase.producer.io.timeout +import org.radarbase.producer.rest.RestException +import org.radarbase.producer.schema.SchemaRetriever +import org.radarbase.producer.schema.SchemaRetriever.Companion.schemaRetriever +import org.radarbase.schema.registration.KafkaTopics.Companion.retryFlow +import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.config.TopicConfig +import org.radarbase.topic.AvroTopic +import org.radarcns.kafka.ObservationKey import org.slf4j.LoggerFactory import java.io.IOException -import java.lang.IllegalStateException import java.net.MalformedURLException import java.time.Duration -import java.util.concurrent.TimeUnit import kotlin.streams.asSequence +import kotlin.time.Duration.Companion.seconds +import kotlin.time.toKotlinDuration /** * Schema registry interface. @@ -45,22 +54,28 @@ import kotlin.streams.asSequence * @param baseUrl URL of the schema registry * @throws MalformedURLException if given URL is invalid. */ -class SchemaRegistry @JvmOverloads constructor( - baseUrl: String, +class SchemaRegistry( + private val baseUrl: String, apiKey: String? = null, apiSecret: String? = null, private val topicConfiguration: Map = emptyMap(), ) { - private val httpClient: RestClient = RestClient.global().apply { - timeout(10, TimeUnit.SECONDS) - server(ServerConfig(baseUrl).apply { - isUnsafe = false - }) - if (apiKey != null && apiSecret != null) { - headers(headersOf("Authorization", basic(apiKey, apiSecret))) + private val schemaClient: SchemaRetriever = schemaRetriever(baseUrl) { + httpClient { + timeout(10.seconds) + if (apiKey != null && apiSecret != null) { + install(Auth) { + basic { + credentials { + BasicAuthCredentials(username = apiKey, password = apiSecret) + } + realm = "Access to the '/' path" + } + } + } } - }.build() - private val schemaClient: SchemaRetriever = SchemaRetriever(httpClient) + } + private val httpClient = schemaClient.restClient /** * Wait for schema registry to become available. This uses a polling mechanism, waiting for at @@ -70,29 +85,30 @@ class SchemaRegistry @JvmOverloads constructor( * @throws IllegalStateException if the schema registry is not ready after wait is finished. */ @Throws(InterruptedException::class) - fun initialize() { - check( - retrySequence(startSleep = Duration.ofSeconds(2), maxSleep = MAX_SLEEP) + suspend fun initialize() { + checkNotNull( + retryFlow(startSleep = 2.seconds, maxSleep = MAX_SLEEP.toKotlinDuration()) .take(20) - .any { + .mapNotNull { try { - httpClient.request("subjects").use { response -> - if (response.isSuccessful) { - true - } else { - logger.error("Schema registry {} not ready, responded with HTTP {}: {}", - httpClient.server, response.code, - RestClient.responseBody(response)) - false - } + httpClient.request> { + url("subjects") } + } catch (ex: RestException) { + logger.error( + "Schema registry {} not ready, responded with HTTP {}: {}", + baseUrl, + ex.status, + ex.message, + ) + null } catch (e: IOException) { - logger.error("Failed to connect to schema registry {}", - httpClient.server) - false + logger.error("Failed to connect to schema registry {}", e.toString()) + null } } - ) { "Schema registry ${httpClient.server} not available" } + .firstOrNull(), + ) { "Schema registry $baseUrl not available" } } /** @@ -101,10 +117,10 @@ class SchemaRegistry @JvmOverloads constructor( * @param catalogue schema catalogue to read schemas from * @return whether all schemas were successfully registered. */ - fun registerSchemas(catalogue: SourceCatalogue): Boolean { + suspend fun registerSchemas(catalogue: SourceCatalogue): Boolean { val sourceTopics = catalogue.sources.asSequence() - .filter { it.doRegisterSchema() } - .flatMap { it.getTopics(catalogue.schemaCatalogue).asSequence() } + .filter { it.registerSchema } + .flatMap { it.topics(catalogue.schemaCatalogue).asSequence() } .distinctBy { it.name } .mapNotNull { topic -> val topicConfig = topicConfiguration[topic.name] ?: return@mapNotNull topic @@ -118,17 +134,18 @@ class SchemaRegistry @JvmOverloads constructor( val configuredTopics = remainingTopics .mapNotNull { (name, topicConfig) -> loadAvroTopic(name, topicConfig) } - return (sourceTopics.asSequence() + configuredTopics.asSequence()) - .sortedBy(AvroTopic<*, *>::getName) - .onEach { t -> logger.info( - "Registering topic {} schemas: {} - {}", - t.name, - t.keySchema.fullName, - t.valueSchema.fullName, - ) } - .map(::registerSchema) - .reduceOrNull { a, b -> a && b } - ?: true + return (sourceTopics + configuredTopics) + .sortedBy(AvroTopic<*, *>::name) + .forkJoin { topic -> + logger.info( + "Registering topic {} schemas: {} - {}", + topic.name, + topic.keySchema.fullName, + topic.valueSchema.fullName, + ) + registerSchema(topic) + } + .all { it } } private fun loadAvroTopic( @@ -137,24 +154,30 @@ class SchemaRegistry @JvmOverloads constructor( defaultTopic: AvroTopic<*, *>? = null, ): AvroTopic<*, *>? { if (!topicConfig.enabled || !topicConfig.registerSchema) return null - if (topicConfig.keySchema == null && topicConfig.valueSchema == null) return defaultTopic + val topicKeySchema = topicConfig.keySchema + val topicValueSchema = topicConfig.valueSchema + + if (topicKeySchema == null && topicValueSchema == null) return defaultTopic val (keyClass, keySchema) = when { - topicConfig.keySchema != null -> { - val record: SpecificRecord = AvroTopic.parseSpecificRecord(topicConfig.keySchema) + topicKeySchema != null -> { + val record: SpecificRecord = AvroTopic.parseSpecificRecord(topicKeySchema) record.javaClass to record.schema } + defaultTopic != null -> defaultTopic.keyClass to defaultTopic.keySchema else -> ObservationKey::class.java to ObservationKey.`SCHEMA$` } val (valueClass, valueSchema) = when { - topicConfig.valueSchema != null -> { - val record: SpecificRecord = AvroTopic.parseSpecificRecord(topicConfig.valueSchema) + topicValueSchema != null -> { + val record: SpecificRecord = AvroTopic.parseSpecificRecord(topicValueSchema) record.javaClass to record.schema } defaultTopic != null -> defaultTopic.valueClass to defaultTopic.valueSchema else -> { - logger.warn("For topic {} the key schema is specified but the value schema is not", - name) + logger.warn( + "For topic {} the key schema is specified but the value schema is not", + name, + ) return null } } @@ -165,10 +188,16 @@ class SchemaRegistry @JvmOverloads constructor( /** * Register the schema of a single topic. */ - fun registerSchema(topic: AvroTopic<*, *>): Boolean { - return try { - schemaClient.addSchema(topic.name, false, topic.keySchema) - schemaClient.addSchema(topic.name, true, topic.valueSchema) + suspend fun registerSchema(topic: AvroTopic<*, *>): Boolean = coroutineScope { + try { + listOf( + async { + schemaClient.addSchema(topic.name, false, topic.keySchema) + }, + async { + schemaClient.addSchema(topic.name, true, topic.valueSchema) + }, + ).awaitAll() true } catch (ex: IOException) { logger.error("Failed to register schemas for topic {}", topic.name, ex) @@ -182,42 +211,24 @@ class SchemaRegistry @JvmOverloads constructor( * @param compatibility target compatibility level. * @return whether the request was successful. */ - fun putCompatibility(compatibility: Compatibility): Boolean { + suspend fun putCompatibility(compatibility: Compatibility): Boolean { logger.info("Setting compatibility to {}", compatibility) - val request = try { - httpClient.requestBuilder("config") - .put(object : RequestBody() { - override fun contentType(): MediaType? = - "application/vnd.schemaregistry.v1+json; charset=utf-8" - .toMediaTypeOrNull() - - @Throws(IOException::class) - override fun writeTo(sink: BufferedSink) { - sink.writeUtf8("{\"compatibility\": \"") - sink.writeUtf8(compatibility.name) - sink.writeUtf8("\"}") - } - }) - .build() - } catch (ex: MalformedURLException) { - // should not occur with valid base URL - return false - } return try { - httpClient.request(request).use { response -> - response.body.use { body -> - if (response.isSuccessful) { - logger.info("Compatibility set to {}", compatibility) - true - } else { - val bodyString = body?.string() - logger.info("Failed to set compatibility set to {}: {}", - compatibility, - bodyString) - false - } - } + httpClient.requestEmpty { + url("config") + method = HttpMethod.Put + contentType(ContentType("application", "vnd.schemaregistry.v1+json")) + setBody("{\"compatibility\": \"${compatibility.name}\"}") } + logger.info("Compatibility set to {}", compatibility) + true + } catch (ex: RestException) { + logger.info( + "Failed to set compatibility set to {}: {}", + compatibility, + ex.message, + ) + false } catch (ex: IOException) { logger.error("Error changing compatibility level to {}", compatibility, ex) false diff --git a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.java b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.java deleted file mode 100644 index e741e952..00000000 --- a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.java +++ /dev/null @@ -1,96 +0,0 @@ -package org.radarbase.schema.registration; - -import java.io.Closeable; -import java.util.Map; -import java.util.Set; -import java.util.regex.Pattern; -import java.util.stream.Stream; -import javax.validation.constraints.NotNull; -import org.apache.kafka.clients.admin.Admin; -import org.radarbase.schema.specification.SourceCatalogue; - -/** - * Registers topic on configured Kafka environment. - */ -public interface TopicRegistrar extends Closeable { - - /** - * Create a pattern to match given topic. If the exact match is non-null, it is returned as an - * exact match, otherwise if regex is non-null, it is used, and otherwise {@code null} is - * returned. - * - * @param exact string that should be exactly matched. - * @param regex string that should be matched as a regex. - * @return pattern or {@code null} if both exact and regex are {@code null}. - */ - static Pattern matchTopic(String exact, String regex) { - if (exact != null) { - return Pattern.compile("^" + Pattern.quote(exact) + "$"); - } else if (regex != null) { - return Pattern.compile(regex); - } else { - return null; - } - } - - /** - * Create all topics in a catalogue based on pattern provided. - * - * @param catalogue source catalogue to extract topic names from. - * @param partitions number of partitions per topic. - * @param replication number of replicas for a topic. - * @param topic Topic name if registering the schemas only for topic. - * @param match Regex string to register schemas only for topics that match the pattern. - * @return 0 if execution was successful. 1 otherwise. - */ - int createTopics(@NotNull SourceCatalogue catalogue, int partitions, short replication, - String topic, String match); - - /** - * Create a single topic. - * - * @param topics names of the topic to create. - * @param partitions number of partitions per topic. - * @param replication number of replicas for a topic. - * @return whether the topic was registered. - */ - boolean createTopics(Stream topics, int partitions, short replication); - - /** - * Wait for brokers to become available. This uses a polling mechanism, waiting for at most 200 - * seconds. - * - * @param brokers number of brokers to wait for - * @throws InterruptedException when waiting for the brokers is interrupted. - * @throws IllegalStateException when the brokers are not ready. - */ - void initialize(int brokers) throws InterruptedException; - - void initialize(int brokers, int numTries) throws InterruptedException; - - /** - * Ensures this topicRegistrar instance is initialized for use. - */ - void ensureInitialized(); - - /** - * Updates the list of topics from Kafka. - * - * @return {@code true} if the update succeeded, {@code false} otherwise. - * @throws InterruptedException if the request was interrupted. - */ - boolean refreshTopics() throws InterruptedException; - - /** - * Returns the list of topics from Kafka. - * - * @return {@code List} list of topics. - */ - Set getTopics(); - - /** Kafka Admin client. */ - Admin getKafkaClient(); - - /** Kafka Admin properties. */ - Map getKafkaProperties(); -} diff --git a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.kt b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.kt new file mode 100644 index 00000000..90b066d3 --- /dev/null +++ b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/TopicRegistrar.kt @@ -0,0 +1,97 @@ +package org.radarbase.schema.registration + +import org.apache.kafka.clients.admin.Admin +import org.radarbase.schema.specification.SourceCatalogue +import java.io.Closeable +import java.util.regex.Pattern +import java.util.stream.Stream + +/** + * Registers topic on configured Kafka environment. + */ +interface TopicRegistrar : Closeable { + /** + * list of topics from Kafka. + */ + val topics: Set + + /** Kafka Admin client. */ + val kafkaClient: Admin + + /** Kafka Admin properties. */ + val kafkaProperties: Map + + /** + * Create all topics in a catalogue based on pattern provided. + * + * @param catalogue source catalogue to extract topic names from. + * @param partitions number of partitions per topic. + * @param replication number of replicas for a topic. + * @param topic Topic name if registering the schemas only for topic. + * @param match Regex string to register schemas only for topics that match the pattern. + * @return 0 if execution was successful. 1 otherwise. + */ + suspend fun createTopics( + catalogue: SourceCatalogue, + partitions: Int, + replication: Short, + topic: String?, + match: String?, + ): Int + + /** + * Create a single topic. + * + * @param topics names of the topic to create. + * @param partitions number of partitions per topic. + * @param replication number of replicas for a topic. + * @return whether the topic was registered. + */ + suspend fun createTopics(topics: Stream, partitions: Int, replication: Short): Boolean + + /** + * Wait for brokers to become available. This uses a polling mechanism, waiting for at most 200 + * seconds. + * + * @param brokers number of brokers to wait for + * @throws IllegalStateException when the brokers are not ready. + */ + suspend fun initialize(brokers: Int) + + suspend fun initialize(brokers: Int, numTries: Int) + + /** + * Ensures this topicRegistrar instance is initialized for use. + */ + fun ensureInitialized() + + /** + * Updates the list of topics from Kafka. + * + * @return `true` if the update succeeded, `false` otherwise. + * @throws InterruptedException if the request was interrupted. + */ + @Throws(InterruptedException::class) + suspend fun refreshTopics(): Boolean + + companion object { + /** + * Create a pattern to match given topic. If the exact match is non-null, it is returned as an + * exact match, otherwise if regex is non-null, it is used, and otherwise `null` is + * returned. + * + * @param exact string that should be exactly matched. + * @param regex string that should be matched as a regex. + * @return pattern or `null` if both exact and regex are `null`. + */ + fun matchTopic(exact: String?, regex: String?): Pattern? { + return if (exact != null) { + Pattern.compile("^" + Pattern.quote(exact) + "$") + } else if (regex != null) { + Pattern.compile(regex) + } else { + null + } + } + } +} diff --git a/java-sdk/radar-schemas-tools/build.gradle.kts b/java-sdk/radar-schemas-tools/build.gradle.kts index fc539261..e48fd8e1 100644 --- a/java-sdk/radar-schemas-tools/build.gradle.kts +++ b/java-sdk/radar-schemas-tools/build.gradle.kts @@ -6,17 +6,14 @@ repositories { dependencies { implementation(project(":radar-schemas-registration")) - val jacksonVersion: String by project - implementation(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")) + implementation(platform("com.fasterxml.jackson:jackson-bom:${Versions.jackson}")) implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml") - val argparseVersion: String by project - implementation("net.sourceforge.argparse4j:argparse4j:$argparseVersion") + implementation("org.radarbase:radar-commons-kotlin:${Versions.radarCommons}") - val log4j2Version: String by project - implementation("org.apache.logging.log4j:log4j-core:$log4j2Version") - runtimeOnly("org.apache.logging.log4j:log4j-slf4j2-impl:$log4j2Version") - runtimeOnly("org.apache.logging.log4j:log4j-jul:$log4j2Version") + implementation("org.apache.logging.log4j:log4j-core:${Versions.log4j2}") + + implementation("net.sourceforge.argparse4j:argparse4j:${Versions.argparse}") } application { diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/CommandLineApp.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/CommandLineApp.kt index 10dc867a..024d9179 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/CommandLineApp.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/CommandLineApp.kt @@ -23,11 +23,11 @@ import net.sourceforge.argparse4j.inf.Namespace import org.apache.logging.log4j.Level import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.core.config.Configurator -import org.radarbase.schema.specification.config.ToolConfig -import org.radarbase.schema.specification.config.loadToolConfig import org.radarbase.schema.specification.DataProducer import org.radarbase.schema.specification.DataTopic import org.radarbase.schema.specification.SourceCatalogue +import org.radarbase.schema.specification.config.ToolConfig +import org.radarbase.schema.specification.config.loadToolConfig import org.slf4j.LoggerFactory import java.io.IOException import java.nio.file.Path @@ -152,8 +152,11 @@ class CommandLineApp( private fun loadConfig(fileName: String): ToolConfig = try { loadToolConfig(fileName) } catch (ex: IOException) { - logger.error("Cannot configure radar-schemas-tools client from config file {}: {}", - fileName, ex.message) + logger.error( + "Cannot configure radar-schemas-tools client from config file {}: {}", + fileName, + ex.message, + ) exitProcess(1) } diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/KafkaTopicsCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/KafkaTopicsCommand.kt index 0066edc7..a97be2a0 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/KafkaTopicsCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/KafkaTopicsCommand.kt @@ -1,5 +1,6 @@ package org.radarbase.schema.tools +import kotlinx.coroutines.runBlocking import net.sourceforge.argparse4j.inf.ArgumentParser import net.sourceforge.argparse4j.inf.Namespace import org.radarbase.schema.registration.KafkaTopics @@ -18,22 +19,27 @@ class KafkaTopicsCommand : SubCommand { val brokers = options.getInt("brokers") val replication = options.getShort("replication") ?: 3 if (brokers < replication) { - logger.error("Cannot assign a replication factor {}" - + " higher than number of brokers {}", replication, brokers) + logger.error( + "Cannot assign a replication factor {}" + + " higher than number of brokers {}", + replication, + brokers, + ) return 1 } val toolConfig: ToolConfig = app.config .configureKafka(bootstrapServers = options.getString("bootstrap_servers")) - try { + + return runBlocking { KafkaTopics(toolConfig).use { topics -> try { val numTries = options.getInt("num_tries") topics.initialize(brokers, numTries) } catch (ex: IllegalStateException) { logger.error("Kafka brokers not yet available. Aborting.") - return 1 + return@use 1 } - return topics.createTopics( + topics.createTopics( app.catalogue, options.getInt("partitions") ?: 3, replication, @@ -41,10 +47,6 @@ class KafkaTopicsCommand : SubCommand { options.getString("match"), ) } - } catch (e: InterruptedException) { - logger.error("Cannot retrieve number of addActive Kafka brokers." - + " Please check that Zookeeper is running.") - return 1 } } @@ -69,8 +71,10 @@ class KafkaTopicsCommand : SubCommand { .help("register the schemas of one topic") .type(String::class.java) addArgument("-m", "--match") - .help("register the schemas of all topics matching the given regex" - + "; does not do anything if --topic is specified") + .help( + "register the schemas of all topics matching the given regex" + + "; does not do anything if --topic is specified", + ) .type(String::class.java) addArgument("-s", "--bootstrap-servers") .help("Kafka hosts, ports and protocols, comma-separated") diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ListCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ListCommand.kt index dac6906d..defb3ed9 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ListCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ListCommand.kt @@ -21,7 +21,7 @@ class ListCommand : SubCommand { out .sorted() .distinct() - .collect(Collectors.joining("\n")) + .collect(Collectors.joining("\n")), ) return 0 } diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SchemaRegistryCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SchemaRegistryCommand.kt index 8a61b6f5..3a9330b1 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SchemaRegistryCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SchemaRegistryCommand.kt @@ -1,11 +1,13 @@ package org.radarbase.schema.tools +import kotlinx.coroutines.runBlocking import net.sourceforge.argparse4j.impl.Arguments import net.sourceforge.argparse4j.inf.ArgumentParser import net.sourceforge.argparse4j.inf.Namespace +import org.radarbase.kotlin.coroutines.forkJoin import org.radarbase.schema.registration.SchemaRegistry -import org.radarbase.schema.specification.config.ToolConfig import org.radarbase.schema.registration.TopicRegistrar +import org.radarbase.schema.specification.config.ToolConfig import org.radarbase.schema.tools.SubCommand.Companion.addRootArgument import org.slf4j.LoggerFactory import java.io.IOException @@ -23,21 +25,28 @@ class SchemaRegistryCommand : SubCommand { ?: System.getenv("SCHEMA_REGISTRY_API_SECRET") val toolConfigFile = options.getString("config") return try { - val registration = createSchemaRegistry(url, apiKey, apiSecret, app.config) - val forced = options.getBoolean("force") - if (forced && !registration.putCompatibility(SchemaRegistry.Compatibility.NONE)) { - return 1 - } - val pattern: Pattern? = TopicRegistrar.matchTopic( - options.getString("topic"), options.getString("match")) - val result = registerSchemas(app, registration, pattern) - if (forced) { - registration.putCompatibility(SchemaRegistry.Compatibility.FULL) + runBlocking { + val registration = createSchemaRegistry(url, apiKey, apiSecret, app.config) + val forced = options.getBoolean("force") + if (forced && !registration.putCompatibility(SchemaRegistry.Compatibility.NONE)) { + return@runBlocking 1 + } + val pattern: Pattern? = TopicRegistrar.matchTopic( + options.getString("topic"), + options.getString("match"), + ) + val result = registerSchemas(app, registration, pattern) + if (forced) { + registration.putCompatibility(SchemaRegistry.Compatibility.FULL) + } + if (result) 0 else 1 } - if (result) 0 else 1 } catch (ex: MalformedURLException) { - logger.error("Schema registry URL {} is invalid: {}", toolConfigFile, - ex.toString()) + logger.error( + "Schema registry URL {} is invalid: {}", + toolConfigFile, + ex.toString(), + ) 1 } catch (ex: IOException) { logger.error("Topic configuration file {} is invalid: {}", url, ex.toString()) @@ -45,9 +54,6 @@ class SchemaRegistryCommand : SubCommand { } catch (ex: IllegalStateException) { logger.error("Cannot reach schema registry. Aborting") 1 - } catch (ex: InterruptedException) { - logger.error("Cannot reach schema registry. Aborting") - 1 } } @@ -61,8 +67,10 @@ class SchemaRegistryCommand : SubCommand { .help("register the schemas of one topic") .type(String::class.java) addArgument("-m", "--match") - .help("register the schemas of all topics matching the given regex" - + "; does not do anything if --topic is specified") + .help( + "register the schemas of all topics matching the given regex" + + "; does not do anything if --topic is specified", + ) .type(String::class.java) addArgument("schemaRegistry") .help("schema registry URL") @@ -76,45 +84,54 @@ class SchemaRegistryCommand : SubCommand { companion object { private val logger = LoggerFactory.getLogger( - SchemaRegistryCommand::class.java) + SchemaRegistryCommand::class.java, + ) @Throws(MalformedURLException::class, InterruptedException::class) - private fun createSchemaRegistry( - url: String, apiKey: String?, apiSecret: String?, - toolConfig: ToolConfig + private suspend fun createSchemaRegistry( + url: String, + apiKey: String?, + apiSecret: String?, + toolConfig: ToolConfig, ): SchemaRegistry { val registry: SchemaRegistry = if (apiKey.isNullOrBlank() || apiSecret.isNullOrBlank()) { logger.info("Initializing standard SchemaRegistration ...") SchemaRegistry(url) } else { logger.info("Initializing SchemaRegistration with authentication...") - SchemaRegistry(url, apiKey, apiSecret, - toolConfig.topics) + SchemaRegistry( + url, + apiKey, + apiSecret, + toolConfig.topics, + ) } registry.initialize() return registry } - private fun registerSchemas( - app: CommandLineApp, registration: SchemaRegistry, - pattern: Pattern? + private suspend fun registerSchemas( + app: CommandLineApp, + registration: SchemaRegistry, + pattern: Pattern?, ): Boolean { return if (pattern == null) { registration.registerSchemas(app.catalogue) } else { - val didUpload = app.catalogue.topics + app.catalogue.topics .filter { pattern.matcher(it.name).find() } - .map(registration::registerSchema) - .reduce { a, b -> a && b } - if (didUpload.isPresent) { - didUpload.get() - } else { - logger.error("Topic {} does not match a known topic." - + " Find the list of acceptable topics" - + " with the `radar-schemas-tools list` command. Aborting.", - pattern) - false - } + .toList() + .forkJoin { registration.registerSchema(it) } + .reduceOrNull { a, b -> a && b } + ?: run { + logger.error( + "Topic {} does not match a known topic." + + " Find the list of acceptable topics" + + " with the `radar-schemas-tools list` command. Aborting.", + pattern, + ) + false + } } } } diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt index b64e697d..60aa81d1 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt @@ -23,7 +23,7 @@ class ValidatorCommand : SubCommand { .flatMap { it.data.asSequence() } .flatMap { d -> try { - d.getTopics(app.catalogue.schemaCatalogue).asSequence() + d.topics(app.catalogue.schemaCatalogue).asSequence() } catch (ex: Exception) { throw IllegalArgumentException(ex) } @@ -49,17 +49,22 @@ class ValidatorCommand : SubCommand { if (options.getBoolean("full")) { exceptionStream = validator.analyseFiles( scope, - app.catalogue.schemaCatalogue) + app.catalogue.schemaCatalogue, + ) } if (options.getBoolean("from_specification")) { exceptionStream = Stream.concat( exceptionStream, - validator.analyseSourceCatalogue(scope, app.catalogue)).distinct() + validator.analyseSourceCatalogue(scope, app.catalogue), + ).distinct() } - resolveValidation(exceptionStream, validator, + resolveValidation( + exceptionStream, + validator, options.getBoolean("verbose"), - options.getBoolean("quiet")) + options.getBoolean("quiet"), + ) } catch (e: IOException) { System.err.println("Failed to load schemas: $e") 1 @@ -92,7 +97,7 @@ class ValidatorCommand : SubCommand { stream: Stream, validator: SchemaValidator, verbose: Boolean, - quiet: Boolean + quiet: Boolean, ): Int = when { !quiet -> { val result = SchemaValidator.format(stream) diff --git a/java-sdk/settings.gradle.kts b/java-sdk/settings.gradle.kts index 699bcc97..123dafbb 100644 --- a/java-sdk/settings.gradle.kts +++ b/java-sdk/settings.gradle.kts @@ -7,16 +7,13 @@ include(":radar-catalog-server") include(":radar-schemas-core") pluginManagement { - val kotlinVersion: String by settings - val dokkaVersion: String by settings - val nexusPluginVersion: String by settings - val dependencyUpdateVersion: String by settings - val avroGeneratorVersion: String by settings - plugins { - kotlin("jvm") version kotlinVersion - id("org.jetbrains.dokka") version dokkaVersion - id("io.github.gradle-nexus.publish-plugin") version nexusPluginVersion - id("com.github.ben-manes.versions") version dependencyUpdateVersion - id("com.github.davidmc24.gradle.plugin.avro-base") version avroGeneratorVersion + repositories { + gradlePluginPortal() + mavenCentral() + maven(url = "https://oss.sonatype.org/content/repositories/snapshots") { + mavenContent { + snapshotsOnly() + } + } } } From 231a8f8016dc6c9471f993e7eb7c4b45d97282b1 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 25 Sep 2023 14:39:41 +0200 Subject: [PATCH 02/27] Use coroutines to do validation --- .../org/radarbase/schema/SchemaCatalogue.kt | 14 +- .../schema/validation/SchemaValidator.kt | 130 ++++++----- .../validation/SpecificationsValidator.kt | 99 ++++---- .../schema/validation/ValidationContext.kt | 52 +++++ .../schema/validation/ValidationHelper.kt | 73 +----- .../validation/rules/RadarSchemaFieldRules.kt | 104 ++++----- .../rules/RadarSchemaMetadataRules.kt | 78 +++---- .../validation/rules/RadarSchemaRules.kt | 220 ++++++++++-------- .../schema/validation/rules/SchemaField.kt | 2 +- .../validation/rules/SchemaFieldRules.kt | 52 ++--- .../validation/rules/SchemaMetadataRules.kt | 57 ++--- .../schema/validation/rules/SchemaRules.kt | 86 +++---- .../schema/validation/rules/Validator.kt | 54 ++--- .../schema/validation/SchemaValidatorTest.kt | 22 +- .../SourceCatalogueValidationTest.kt | 5 +- .../validation/SpecificationsValidatorTest.kt | 26 +-- .../rules/RadarSchemaFieldRulesTest.kt | 39 ++-- .../rules/RadarSchemaMetadataRulesTest.kt | 18 +- .../validation/rules/RadarSchemaRulesTest.kt | 96 ++++---- .../schema/tools/ValidatorCommand.kt | 53 +++-- 20 files changed, 642 insertions(+), 638 deletions(-) create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt index a06858d1..09d07182 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt @@ -95,9 +95,7 @@ class SchemaCatalogue @JvmOverloads constructor( // at all. while (prevSize != schemas.size) { prevSize = schemas.size - val useTypes = schemas - .mapNotNull { (k, v) -> v.schema?.let { k to it } } - .toMap() + val useTypes = schemas.toSchemaMap() val ignoreFiles = schemas.values.asSequence() .map { it.path } .filterNotNullTo(HashSet()) @@ -117,7 +115,7 @@ class SchemaCatalogue @JvmOverloads constructor( ignoreFiles: Set, useTypes: Map, scope: Scope, - ): Unit = customSchemas.asSequence() + ) = customSchemas.asSequence() .filter { (p, _) -> p !in ignoreFiles } .forEach { (p, schema) -> val parser = Schema.Parser() @@ -154,5 +152,13 @@ class SchemaCatalogue @JvmOverloads constructor( companion object { private val logger = LoggerFactory.getLogger(SchemaCatalogue::class.java) + + fun Map.toSchemaMap(): Map = buildMap(size) { + this@toSchemaMap.forEach { (k, v) -> + if (v.schema != null) { + put(k, v.schema) + } + } + } } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt index d799406d..e9a12481 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt @@ -21,7 +21,6 @@ import org.radarbase.schema.Scope import org.radarbase.schema.specification.DataProducer import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.config.SchemaConfig -import org.radarbase.schema.validation.ValidationHelper.matchesExtension import org.radarbase.schema.validation.rules.RadarSchemaMetadataRules import org.radarbase.schema.validation.rules.RadarSchemaRules import org.radarbase.schema.validation.rules.SchemaMetadata @@ -29,10 +28,9 @@ import org.radarbase.schema.validation.rules.SchemaMetadataRules import org.radarbase.schema.validation.rules.Validator import java.nio.file.Path import java.nio.file.PathMatcher -import java.util.Arrays -import java.util.Objects import java.util.stream.Collectors import java.util.stream.Stream +import kotlin.io.path.extension /** * Validator for a set of RADAR-Schemas. @@ -43,13 +41,13 @@ import java.util.stream.Stream class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { val rules: SchemaMetadataRules = RadarSchemaMetadataRules(schemaRoot, config) private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) - private var validator: Validator = rules.getValidator(false) + private var validator: Validator = rules.isSchemaMetadataValid(false) - fun analyseSourceCatalogue( + suspend fun analyseSourceCatalogue( scope: Scope?, catalogue: SourceCatalogue, - ): Stream { - validator = rules.getValidator(true) + ): List { + validator = rules.isSchemaMetadataValid(true) val producers: Stream> = if (scope != null) { catalogue.sources.stream() .filter { it.scope == scope } @@ -57,29 +55,61 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { catalogue.sources.stream() } return try { - producers - .flatMap { it.data.stream() } - .flatMap { topic -> - val (keySchema, valueSchema) = catalogue.schemaCatalogue.getSchemaMetadata(topic) - Stream.of(keySchema, valueSchema).filter { it.schema != null } + validationContext { + val schemas = producers + .flatMap { it.data.stream() } + .flatMap { topic -> + val (keySchema, valueSchema) = catalogue.schemaCatalogue.getSchemaMetadata(topic) + Stream.of(keySchema, valueSchema) + } + .filter { it.schema != null } + .sorted(Comparator.comparing { it.schema!!.fullName }) + .collect(Collectors.toSet()) + + schemas.forEach { metadata -> + if (pathMatcher.matches(metadata.path)) { + validator.launchValidation(metadata) + } } - .sorted(Comparator.comparing { it.schema!!.fullName }) - .distinct() - .flatMap(this::validate) - .distinct() + } } finally { - validator = rules.getValidator(false) + validator = rules.isSchemaMetadataValid(false) } } - fun analyseFiles( - scope: Scope?, + suspend fun analyseFiles( schemaCatalogue: SchemaCatalogue, - ): Stream { + scope: Scope? = null, + ): List = validationContext { if (scope == null) { - return analyseFiles(schemaCatalogue) + Scope.entries.forEach { scope -> analyseFilesInternal(schemaCatalogue, scope) } + } else { + analyseFilesInternal(schemaCatalogue, scope) + } + } + + private fun ValidationContext.analyseFilesInternal( + schemaCatalogue: SchemaCatalogue, + scope: Scope, + ) { + validator = rules.isSchemaMetadataValid(false) + val parsingValidator = parsingValidator(scope, schemaCatalogue) + + schemaCatalogue.unmappedAvroFiles.forEach { metadata -> + parsingValidator.launchValidation(metadata) + } + + schemaCatalogue.schemas.values.forEach { metadata -> + if (pathMatcher.matches(metadata.path)) { + validator.launchValidation(metadata) + } } - validator = rules.getValidator(false) + } + + private fun parsingValidator( + scope: Scope?, + schemaCatalogue: SchemaCatalogue, + ): Validator { val useTypes = buildMap { schemaCatalogue.schemas.forEach { (key, value) -> if (value.scope == scope) { @@ -87,41 +117,27 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { } } } - - return Stream.concat( - schemaCatalogue.unmappedAvroFiles.stream() - .filter { s -> s.scope == scope && s.path != null } - .map { p -> - val parser = Schema.Parser() - parser.addTypes(useTypes) - try { - parser.parse(p.path?.toFile()) - return@map null - } catch (ex: Exception) { - return@map ValidationException("Cannot parse schema", ex) - } - } - .filter(Objects::nonNull) - .map { obj -> requireNotNull(obj) }, - schemaCatalogue.schemas.values.stream() - .flatMap { this.validate(it) }, - ).distinct() + return Validator { metadata -> + val parser = Schema.Parser() + parser.addTypes(useTypes) + try { + parser.parse(metadata.path?.toFile()) + } catch (ex: Exception) { + raise("Cannot parse schema", ex) + } + } } - private fun analyseFiles(schemaCatalogue: SchemaCatalogue): Stream = - Arrays.stream(Scope.entries.toTypedArray()) - .flatMap { scope -> analyseFiles(scope, schemaCatalogue) } /** Validate a single schema in given path. */ - fun validate(schema: Schema, path: Path, scope: Scope): Stream = + fun ValidationContext.validate(schema: Schema, path: Path, scope: Scope) = validate(SchemaMetadata(schema, scope, path)) /** Validate a single schema in given path. */ - private fun validate(schemaMetadata: SchemaMetadata): Stream = + private fun ValidationContext.validate(schemaMetadata: SchemaMetadata) { if (pathMatcher.matches(schemaMetadata.path)) { - validator.validate(schemaMetadata) - } else { - Stream.empty() + validator.launchValidation(schemaMetadata) } + } val validatedSchemas: Map get() = (rules.schemaRules as RadarSchemaRules).schemaStore @@ -130,22 +146,18 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { private const val AVRO_EXTENSION = "avsc" /** Formats a stream of validation exceptions. */ - @JvmStatic - fun format(exceptionStream: Stream): String { - return exceptionStream - .map { ex: ValidationException -> - """ + fun format(exceptions: List): String { + return exceptions.joinToString(separator = "") { ex: ValidationException -> + """ |Validation FAILED: |${ex.message} | | | - """.trimMargin() - } - .collect(Collectors.joining()) + """.trimMargin() + } } - fun isAvscFile(file: Path): Boolean = - matchesExtension(file, AVRO_EXTENSION) + fun isAvscFile(file: Path): Boolean = file.extension.equals(AVRO_EXTENSION, ignoreCase = true) } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt index 25bd2f5c..ebd6ead9 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt @@ -17,35 +17,32 @@ package org.radarbase.schema.validation import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext import org.radarbase.schema.Scope import org.radarbase.schema.specification.config.SchemaConfig import org.radarbase.schema.validation.ValidationHelper.SPECIFICATIONS_PATH -import org.radarbase.schema.validation.ValidationHelper.matchesExtension +import org.radarbase.schema.validation.rules.Validator +import org.radarbase.schema.validation.rules.pathExtensionValidator import org.slf4j.LoggerFactory import java.io.IOException import java.nio.file.Files import java.nio.file.Path import java.nio.file.PathMatcher -import kotlin.io.path.walk +import java.util.stream.Collectors /** * Validates RADAR-Schemas specifications. + * + * @param root RADAR-Schemas directory. + * @param config configuration to exclude certain schemas or fields from validation. + * */ class SpecificationsValidator(root: Path, config: SchemaConfig) { - private val specificationsRoot: Path - private val mapper: ObjectMapper - private val pathMatcher: PathMatcher - - /** - * Specifications validator for given RADAR-Schemas directory. - * @param root RADAR-Schemas directory. - * @param config configuration to exclude certain schemas or fields from validation. - */ - init { - specificationsRoot = root.resolve(SPECIFICATIONS_PATH) - pathMatcher = config.pathMatcher(specificationsRoot) - mapper = ObjectMapper(YAMLFactory()) - } + private val specificationsRoot: Path = root.resolve(SPECIFICATIONS_PATH) + private val mapper: ObjectMapper = ObjectMapper(YAMLFactory()) + private val pathMatcher: PathMatcher = config.pathMatcher(specificationsRoot) /** Check that all files in the specifications directory are YAML files. */ @Throws(IOException::class) @@ -59,10 +56,17 @@ class SpecificationsValidator(root: Path, config: SchemaConfig) { ) return false } - Files.walk(baseFolder).use { walker -> - return walker - .filter { path: Path? -> pathMatcher.matches(path) } - .allMatch { path: Path -> isYmlFile(path) } + return runBlocking { + val paths = baseFolder.fetchChildren() + val exceptions = validationContext { + paths.forEach { isYmlFile.launchValidation(it) } + } + if (exceptions.isEmpty()) { + true + } else { + logger.error("Not all specification files have the right extension: {}", exceptions.joinToString()) + false + } } } @@ -77,32 +81,41 @@ class SpecificationsValidator(root: Path, config: SchemaConfig) { ) return false } - Files.walk(baseFolder).use { walker -> - return walker - .filter { path: Path? -> pathMatcher.matches(path) } - .allMatch { f: Path -> - try { - mapper.readerFor(clazz).readValue(f.toFile()) - return@allMatch true - } catch (ex: IOException) { - logger.error( - "Failed to load configuration {}: {}", - f, - ex.toString(), - ) - return@allMatch false - } - } + val validator = isValidYmlFile(clazz) + + return runBlocking { + val paths = baseFolder.fetchChildren() + val exceptions = validationContext { + paths.forEach { validator.launchValidation(it) } + } + if (exceptions.isEmpty()) { + true + } else { + logger.error("Not all specification files have the right format: {}", exceptions.joinToString()) + false + } } } - companion object { - private val logger = LoggerFactory.getLogger( - SpecificationsValidator::class.java, - ) - const val YML_EXTENSION = "yml" - private fun isYmlFile(path: Path): Boolean { - return matchesExtension(path, YML_EXTENSION) + private suspend fun Path.fetchChildren(): List = withContext(Dispatchers.IO) { + Files.walk(this@fetchChildren).use { walker -> + walker + .filter { pathMatcher.matches(it) } + .collect(Collectors.toList()) } } + + private fun isValidYmlFile(clazz: Class?) = Validator { path -> + try { + mapper.readerFor(clazz).readValue(path.toFile()) + } catch (ex: IOException) { + raise("Failed to load configuration $path", ex) + } + } + + companion object { + private val logger = LoggerFactory.getLogger(SpecificationsValidator::class.java) + + private val isYmlFile: Validator = pathExtensionValidator("yml") + } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt new file mode 100644 index 00000000..2964b62b --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt @@ -0,0 +1,52 @@ +package org.radarbase.schema.validation + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.channels.toList +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch +import org.radarbase.schema.validation.rules.Validator + +interface ValidationContext { + fun raise(message: String, ex: Exception? = null) + + fun Validator.launchValidation(value: T) +} + +private class ValidationContextImpl( + private val coroutineScope: CoroutineScope, +) : ValidationContext { + private val channel = Channel(Channel.UNLIMITED) + private lateinit var producerCoroutineScope: CoroutineScope + + suspend fun runValidation(block: ValidationContext.() -> Unit): List { + coroutineScope.launch { + coroutineScope { + producerCoroutineScope = this + block() + } + channel.close() + } + return channel.toList().distinct() + } + + override fun raise(message: String, ex: Exception?) { + channel.trySend(ValidationException(message, ex)) + } + + override fun Validator.launchValidation(value: T) { + producerCoroutineScope.launch { + runValidation(value) + } + } +} + +suspend fun validationContext(block: ValidationContext.() -> Unit) = + coroutineScope { + val context = ValidationContextImpl(this) + context.runValidation(block) + } + +suspend fun Validator.validate(value: T) = validationContext { + launchValidation(value) +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt index 4addb5f8..a8dc9614 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt @@ -19,9 +19,6 @@ import org.radarbase.schema.Scope import org.radarbase.schema.util.SchemaUtils.projectGroup import org.radarbase.schema.util.SchemaUtils.snakeToCamelCase import java.nio.file.Path -import java.util.Objects -import java.util.function.Predicate -import java.util.regex.Pattern /** * TODO. @@ -31,73 +28,27 @@ object ValidationHelper { const val SPECIFICATIONS_PATH = "specifications" // snake case - private val TOPIC_PATTERN = Pattern.compile( - "[A-Za-z][a-z0-9-]*(_[A-Za-z0-9-]+)*", - ) + private val TOPIC_PATTERN = "[A-Za-z][a-z0-9-]*(_[A-Za-z0-9-]+)*".toRegex() - /** - * TODO. - * @param scope TODO - * @return TODO - */ fun getNamespace(schemaRoot: Path?, schemaPath: Path?, scope: Scope): String { // add subfolder of root to namespace - val rootPath = scope.getPath(schemaRoot) - ?: throw IllegalArgumentException("Scope $scope does not have a commons path") + val rootPath = requireNotNull(scope.getPath(schemaRoot)) { "Scope $scope does not have a commons path" } + requireNotNull(schemaPath) { "Missing schema path" } val relativePath = rootPath.relativize(schemaPath) - val builder = StringBuilder(50) - builder.append(projectGroup).append('.').append(scope.lower) - for (i in 0 until relativePath.nameCount - 1) { - builder.append('.').append(relativePath.getName(i)) + return buildString(50) { + append(projectGroup) + append('.') + append(scope.lower) + for (i in 0 until relativePath.nameCount - 1) { + append('.') + append(relativePath.getName(i)) + } } - return builder.toString() } - /** - * TODO. - * @param path TODO - * @return TODO - */ - @JvmStatic fun getRecordName(path: Path): String { - Objects.requireNonNull(path) return snakeToCamelCase(path.fileName.toString()) } - /** - * TODO. - * @param topicName TODO - * @return TODO - */ - @JvmStatic - fun isValidTopic(topicName: String?): Boolean { - return topicName != null && TOPIC_PATTERN.matcher(topicName).matches() - } - - /** - * TODO. - * @param file TODO. - * @return TODO. - */ - @JvmStatic - fun matchesExtension(file: Path, extension: String): Boolean { - return file.toString().lowercase() - .endsWith("." + extension.lowercase()) - } - - /** - * TODO. - * @param file TODO - * @param extension TODO - * @return TODO - */ - fun equalsFileName(file: Path, extension: String): Predicate { - return Predicate { str: String -> - var fileName = file.fileName.toString() - if (fileName.endsWith(extension)) { - fileName = fileName.substring(0, fileName.length - extension.length) - } - str.equals(fileName, ignoreCase = true) - } - } + fun isValidTopic(topicName: String?): Boolean = topicName?.matches(TOPIC_PATTERN) == true } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt index a69b4b4a..25e8c0b1 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt @@ -7,13 +7,9 @@ import org.apache.avro.Schema.Type.ENUM import org.apache.avro.Schema.Type.NULL import org.apache.avro.Schema.Type.RECORD import org.apache.avro.Schema.Type.UNION -import org.radarbase.schema.validation.ValidationException +import org.radarbase.schema.validation.ValidationContext import org.radarbase.schema.validation.rules.RadarSchemaRules.Companion.validateDocumentation -import org.radarbase.schema.validation.rules.SchemaFieldRules.Companion.message -import org.radarbase.schema.validation.rules.Validator.Companion.check -import org.radarbase.schema.validation.rules.Validator.Companion.validate import java.util.EnumMap -import java.util.stream.Stream /** * Rules for RADAR-Schemas schema fields. @@ -26,88 +22,80 @@ class RadarSchemaFieldRules : SchemaFieldRules { */ init { defaultsValidator = EnumMap(Type::class.java) - defaultsValidator[ENUM] = Validator { validateDefaultEnum(it) } - defaultsValidator[UNION] = Validator { validateDefaultUnion(it) } + defaultsValidator[ENUM] = Validator { isEnumDefaultUnknown(it) } + defaultsValidator[UNION] = Validator { isDefaultUnionCompatible(it) } } override fun validateFieldTypes(schemaRules: SchemaRules): Validator { return Validator { field -> val schema = field.field.schema() val subType = schema.type - return@Validator when (subType) { - UNION -> validateInternalUnion(schemaRules).validate(field) - RECORD -> schemaRules.validateRecord().validate(schema) - ENUM -> schemaRules.validateEnum().validate(schema) - else -> Validator.valid() + when (subType) { + UNION -> validateInternalUnion(schemaRules).launchValidation(field) + RECORD -> schemaRules.isRecordValid.launchValidation(schema) + ENUM -> schemaRules.isEnumValid.launchValidation(schema) + else -> Unit } } } - override fun validateDefault(): Validator { - return Validator { input: SchemaField -> - defaultsValidator - .getOrDefault( - input.field.schema().type, - Validator { validateDefaultOther(it) }, - ) - .validate(input) - } + override val isDefaultValueValid = Validator { input: SchemaField -> + defaultsValidator + .getOrDefault( + input.field.schema().type, + Validator { isDefaultValueNullable(it) }, + ) + .launchValidation(input) } - override fun validateFieldName(): Validator { - return validate( - { f -> f.field.name()?.matches(FIELD_NAME_PATTERN) == true }, - "Field name does not respect lowerCamelCase name convention." + - " Please avoid abbreviations and write out the field name instead.", + override val isNameValid = validator( + predicate = { f -> f.field.name()?.matches(FIELD_NAME_PATTERN) == true }, + message = "Field name does not respect lowerCamelCase name convention." + + " Please avoid abbreviations and write out the field name instead.", + ) + + override val isDocumentationValid = Validator { field: SchemaField -> + validateDocumentation( + doc = field.field.doc(), + raise = ValidationContext::raise, + schema = field, ) } - override fun validateFieldDocumentation(): Validator { - return Validator { field: SchemaField -> - validateDocumentation( - field.field.doc(), - { m, f -> message(f, m) }, + private fun ValidationContext.isEnumDefaultUnknown(field: SchemaField) { + if ( + field.field.schema().enumSymbols.contains(UNKNOWN) && + !(field.field.defaultVal() != null && field.field.defaultVal().toString() == UNKNOWN) + ) { + raise( field, + "Default is \"${field.field.defaultVal()}\". Any Avro enum type that has an \"UNKNOWN\" symbol must set its default value to \"UNKNOWN\".", ) } } - private fun validateDefaultEnum(field: SchemaField): Stream { - return check( - !field.field.schema().enumSymbols.contains(UNKNOWN) || - field.field.defaultVal() != null && field.field.defaultVal() - .toString() == UNKNOWN, - message( - field, - "Default is \"" + field.field.defaultVal() + - "\". Any Avro enum type that has an \"UNKNOWN\" symbol must set its" + - " default value to \"UNKNOWN\".", - ), - ) - } - - private fun validateDefaultUnion(field: SchemaField): Stream { - return check( - !field.field.schema().types.contains(Schema.create(NULL)) || - field.field.defaultVal() != null && field.field.defaultVal() == JsonProperties.NULL_VALUE, - message( + private fun ValidationContext.isDefaultUnionCompatible(field: SchemaField) { + if ( + field.field.schema().types.contains(Schema.create(NULL)) && + !(field.field.defaultVal() != null && field.field.defaultVal() == JsonProperties.NULL_VALUE) + ) { + raise( field, "Default is not null. Any nullable Avro field must" + " specify have its default value set to null.", - ), - ) + ) + } } - private fun validateDefaultOther(field: SchemaField): Stream { - return check( - field.field.defaultVal() == null, - message( + private fun ValidationContext.isDefaultValueNullable(field: SchemaField) { + if (field.field.defaultVal() != null) { + raise( field, "Default of type " + field.field.schema().type + " is set to " + field.field.defaultVal() + ". The only acceptable default values are the" + " \"UNKNOWN\" enum symbol and null.", - ), - ) + ) + } } companion object { diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt index e193d91b..a7979f93 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt @@ -2,10 +2,8 @@ package org.radarbase.schema.validation.rules import org.apache.avro.Schema import org.radarbase.schema.specification.config.SchemaConfig -import org.radarbase.schema.validation.ValidationHelper -import org.radarbase.schema.validation.rules.Validator.Companion.check -import org.radarbase.schema.validation.rules.Validator.Companion.raise -import org.radarbase.schema.validation.rules.Validator.Companion.valid +import org.radarbase.schema.validation.ValidationHelper.getNamespace +import org.radarbase.schema.validation.ValidationHelper.getRecordName import java.nio.file.Path import java.nio.file.PathMatcher @@ -22,54 +20,42 @@ class RadarSchemaMetadataRules( ) : SchemaMetadataRules { private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) - override fun validateSchemaLocation(): Validator = - validateNamespaceSchemaLocation() - .and(validateNameSchemaLocation()) + override val isShemaLocationCorrect = all( + isNamespaceSchemaLocationCorrect(), + isNameSchemaLocationCorrect(), + ) - private fun validateNamespaceSchemaLocation(): Validator = - Validator { metadata -> - try { - val expected = ValidationHelper.getNamespace( - schemaRoot, - metadata.path, - metadata.scope, - ) - val namespace = metadata.schema?.namespace - return@Validator check( - expected.equals(namespace, ignoreCase = true), - message( - metadata, - "Namespace cannot be null and must fully lowercase dot separated without numeric. In this case the expected value is \"$expected\".", - ), - ) - } catch (ex: IllegalArgumentException) { - return@Validator raise( - "Path " + metadata.path + - " is not part of root " + schemaRoot, - ex, + private fun isNamespaceSchemaLocationCorrect() = Validator { metadata -> + try { + val expected = getNamespace(schemaRoot, metadata.path, metadata.scope) + val namespace = metadata.schema?.namespace + if (!expected.equals(namespace, ignoreCase = true)) { + raise( + metadata, + "Namespace cannot be null and must fully lowercase dot separated without numeric. In this case the expected value is \"$expected\".", ) } + } catch (ex: IllegalArgumentException) { + raise("Path ${metadata.path} is not part of root $schemaRoot", ex) } + } - private fun validateNameSchemaLocation(): Validator = - Validator { metadata -> - if (metadata.path == null) { - return@Validator raise(message(metadata, "Missing metadata path")) - } - val expected = ValidationHelper.getRecordName(metadata.path) - if (expected.equals(metadata.schema?.name, ignoreCase = true)) { - valid() - } else { - raise(message(metadata, "Record name should match file name. Expected record name is \"$expected\".")) - } + private fun isNameSchemaLocationCorrect() = Validator { metadata -> + if (metadata.path == null) { + raise(metadata, "Missing metadata path") + return@Validator + } + val expected = getRecordName(metadata.path) + if (!expected.equals(metadata.schema?.name, ignoreCase = true)) { + raise(metadata, "Record name should match file name. Expected record name is \"$expected\".") } + } - override fun schema(validator: Validator): Validator = - Validator { metadata -> - when { - metadata.schema == null -> raise("Missing schema") - pathMatcher.matches(metadata.path) -> validator.validate(metadata.schema) - else -> valid() - } + override fun isSchemaCorrect(validator: Validator) = Validator { metadata -> + when { + metadata.schema == null -> raise("Missing schema") + pathMatcher.matches(metadata.path) -> validator.launchValidation(metadata.schema) + else -> Unit } + } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt index cdfbdab8..63fe6155 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt @@ -22,12 +22,7 @@ import io.confluent.connect.schema.AbstractDataConfig import org.apache.avro.Schema import org.apache.avro.Schema.Type.DOUBLE import org.apache.avro.Schema.Type.RECORD -import org.radarbase.schema.validation.ValidationException -import org.radarbase.schema.validation.rules.Validator.Companion.check -import org.radarbase.schema.validation.rules.Validator.Companion.raise -import org.radarbase.schema.validation.rules.Validator.Companion.valid -import org.radarbase.schema.validation.rules.Validator.Companion.validate -import java.util.stream.Stream +import org.radarbase.schema.validation.ValidationContext /** * Schema validation rules enforced for the RADAR-Schemas repository. @@ -37,97 +32,117 @@ class RadarSchemaRules( ) : SchemaRules { val schemaStore: MutableMap = HashMap() - override fun validateUniqueness() = Validator { schema: Schema -> + override val isUnique = Validator { schema: Schema -> val key = schema.fullName val oldSchema = schemaStore.putIfAbsent(key, schema) - check( - oldSchema == null || oldSchema == schema, - messageSchema(schema, "Schema is already defined elsewhere with a different definition."), - ) + if (oldSchema != null && oldSchema != schema) { + raise( + schema, + "Schema is already defined elsewhere with a different definition.", + ) + } } - override fun validateNameSpace() = validate( - { it.namespace?.matches(NAMESPACE_PATTERN) == true }, - messageSchema("Namespace cannot be null and must fully lowercase, period-separated, without numeric characters."), + override val isNamespaceValid = validator( + predicate = { it.namespace?.matches(NAMESPACE_PATTERN) == true }, + message = schemaErrorMessage("Namespace cannot be null and must fully lowercase, period-separated, without numeric characters."), ) - override fun validateName() = validate( - { it.name?.matches(RECORD_NAME_PATTERN) == true }, - messageSchema("Record names must be camel case."), + override val isNameValid = validator( + predicate = { it.name?.matches(RECORD_NAME_PATTERN) == true }, + message = schemaErrorMessage("Record names must be camel case."), ) - override fun validateSchemaDocumentation() = Validator { schema -> + override val isDocumentationValid = Validator { schema -> validateDocumentation( schema.doc, - { m, t -> messageSchema(t, m) }, + ValidationContext::raise, schema, ) } - override fun validateSymbols() = validate( - { !it.enumSymbols.isNullOrEmpty() }, - messageSchema("Avro Enumerator must have symbol list."), - ).and(validateSymbolNames()) - - private fun validateSymbolNames() = Validator { schema -> - schema.enumSymbols.stream() - .filter { !it.matches(ENUM_SYMBOL_PATTERN) } - .map { s -> - ValidationException( - messageSchema( - schema, - "Symbol $s does not use valid syntax. " + - "Enumerator items should be written in uppercase characters separated by underscores.", - ), + override val isEnumSymbolsValid = Validator { schema -> + if (schema.enumSymbols.isNullOrEmpty()) { + raise(schema, "Avro Enumerator must have symbol list.") + return@Validator + } + schema.enumSymbols.forEach { s -> + if (!s.matches(ENUM_SYMBOL_PATTERN)) { + raise( + schema, + "Symbol $s does not use valid syntax. " + + "Enumerator items should be written in uppercase characters separated by underscores.", ) } + } } + override val hasTime: Validator = validator( + predicate = { it.getField(TIME)?.schema()?.type == DOUBLE }, + message = schemaErrorMessage("Any schema representing collected data must have a \"$TIME$WITH_TYPE_DOUBLE"), + ) + + override val hasTimeCompleted: Validator = validator( + predicate = { it.getField(TIME_COMPLETED)?.schema()?.type == DOUBLE }, + message = schemaErrorMessage("Any ACTIVE schema must have a \"$TIME_COMPLETED$WITH_TYPE_DOUBLE"), + ) + + override val hasNoTimeCompleted: Validator = validator( + predicate = { it.getField(TIME_COMPLETED) == null }, + message = schemaErrorMessage("\"$TIME_COMPLETED\" is allow only in ACTIVE schemas."), + ) + + override val hasTimeReceived: Validator = validator( + predicate = { it.getField(TIME_RECEIVED)?.schema()?.type == DOUBLE }, + message = schemaErrorMessage("Any PASSIVE schema must have a \"$TIME_RECEIVED$WITH_TYPE_DOUBLE"), + ) + + override val hasNoTimeReceived: Validator = validator( + predicate = { it.getField(TIME_RECEIVED) == null }, + message = schemaErrorMessage("\"$TIME_RECEIVED\" is allow only in PASSIVE schemas."), + ) + + override val isAvroConnectCompatible: Validator + /** - * TODO. - * @return TODO + * Validate an enum. */ - override fun validateTime(): Validator = validate( - { it.getField(TIME)?.schema()?.type == DOUBLE }, - messageSchema("Any schema representing collected data must have a \"$TIME$WITH_TYPE_DOUBLE"), + override val isEnumValid: Validator = all( + isUnique, + isNamespaceValid, + isEnumSymbolsValid, + isDocumentationValid, + isNameValid, ) /** - * TODO. - * @return TODO + * Validate a record that is defined inline. */ - override fun validateTimeCompleted(): Validator = validate( - { it.getField(TIME_COMPLETED)?.schema()?.type == DOUBLE }, - messageSchema("Any ACTIVE schema must have a \"$TIME_COMPLETED$WITH_TYPE_DOUBLE"), - ) + override val isRecordValid: Validator /** - * TODO. - * @return TODO + * Validates record schemas of an active source. */ - override fun validateNotTimeCompleted(): Validator = validate( - { it.getField(TIME_COMPLETED) == null }, - messageSchema("\"$TIME_COMPLETED\" is allow only in ACTIVE schemas."), - ) + override val isActiveSourceValid: Validator - override fun validateTimeReceived(): Validator = validate( - { it.getField(TIME_RECEIVED)?.schema()?.type == DOUBLE }, - messageSchema("Any PASSIVE schema must have a \"$TIME_RECEIVED$WITH_TYPE_DOUBLE"), - ) + /** + * Validates schemas of monitor sources. + */ + override val isMonitorSourceValid: Validator - override fun validateNotTimeReceived(): Validator = validate( - { it.getField(TIME_RECEIVED) == null }, - messageSchema("\"$TIME_RECEIVED\" is allow only in PASSIVE schemas."), - ) + /** + * Validates schemas of passive sources. + */ + override val isPassiveSourceValid: Validator - override fun validateAvroData(): Validator { + init { val avroConfig = Builder() .with(AvroDataConfig.CONNECT_META_DATA_CONFIG, false) .with(AbstractDataConfig.SCHEMAS_CACHE_SIZE_CONFIG, 10) .with(AvroDataConfig.ENHANCED_AVRO_SCHEMA_SUPPORT_CONFIG, true) .build() - return Validator { schema: Schema -> + isAvroConnectCompatible = Validator { schema: Schema -> val encoder = AvroData(10) val decoder = AvroData(avroConfig) try { @@ -142,17 +157,33 @@ class RadarSchemaRules( raise("Failed to convert schema back to itself") } } + + isRecordValid = all( + isUnique, + isAvroConnectCompatible, + isNamespaceValid, + isNameValid, + isDocumentationValid, + isFieldsValid(fieldRules.isFieldValid(this)), + ) + + isActiveSourceValid = all(isRecordValid, hasTime) + + isMonitorSourceValid = all(isRecordValid, hasTime) + + isPassiveSourceValid = all(isRecordValid, hasTime, hasTimeReceived, hasNoTimeCompleted) } - override fun fields(validator: Validator): Validator = + override fun isFieldsValid(validator: Validator): Validator = Validator { schema: Schema -> when { schema.type != RECORD -> raise( "Default validation can be applied only to an Avro RECORD, not to ${schema.type} of schema ${schema.fullName}.", ) schema.fields.isEmpty() -> raise("Schema ${schema.fullName} does not contain any fields.") - else -> schema.fields.stream() - .flatMap { field -> validator.validate(SchemaField(schema, field)) } + else -> schema.fields.forEach { field -> + validator.launchValidation(SchemaField(schema, field)) + } } } @@ -172,52 +203,43 @@ class RadarSchemaRules( val ENUM_SYMBOL_PATTERN = "[A-Z][A-Z0-9_]*".toRegex() private const val WITH_TYPE_DOUBLE = "\" field with type \"double\"." - fun validateDocumentation( + fun ValidationContext.validateDocumentation( doc: String?, - message: (String, T) -> String, + raise: ValidationContext.(T, String) -> Unit, schema: T, - ): Stream { + ) { if (doc.isNullOrEmpty()) { - return raise( - message( - "Property \"doc\" is missing. Documentation is" + - " mandatory for all fields. The documentation should report what is being" + - " measured, how, and what units or ranges are applicable. Abbreviations" + - " and acronyms in the documentation should be written out. The sentence" + - " must end with a period '.'. Please add \"doc\" property.", - schema, - ), + raise( + schema, + """Property "doc" is missing. Documentation is mandatory for all fields. + | The documentation should report what is being measured, how, and what + | units or ranges are applicable. Abbreviations and acronyms in the + | documentation should be written out. The sentence must end with a + | period '.'. Please add "doc" property. + """.trimMargin(), ) + return } - var result: Stream = valid() if (doc[doc.length - 1] != '.') { - result = raise( - message( - "Documentation is not terminated with a period. The" + - " documentation should report what is being measured, how, and what units" + - " or ranges are applicable. Abbreviations and acronyms in the" + - " documentation should be written out. Please end the sentence with a" + - " period '.'.", - schema, - ), + raise( + schema, + "Documentation is not terminated with a period. The" + + " documentation should report what is being measured, how, and what units" + + " or ranges are applicable. Abbreviations and acronyms in the" + + " documentation should be written out. Please end the sentence with a" + + " period '.'.", ) } if (!Character.isUpperCase(doc[0])) { - result = Stream.concat( - result, - raise( - message( - "Documentation does not start with a capital letter. The" + - " documentation should report what is being measured, how, and what" + - " units or ranges are applicable. Abbreviations and acronyms in the" + - " documentation should be written out. Please end the sentence with a" + - " period '.'.", - schema, - ), - ), + raise( + schema, + "Documentation does not start with a capital letter. The" + + " documentation should report what is being measured, how, and what" + + " units or ranges are applicable. Abbreviations and acronyms in the" + + " documentation should be written out. Please end the sentence with a" + + " period '.'.", ) } - return result } } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt index af1027f1..503ca8a6 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt @@ -3,4 +3,4 @@ package org.radarbase.schema.validation.rules import org.apache.avro.Schema import org.apache.avro.Schema.Field -data class SchemaField(@JvmField val schema: Schema, @JvmField val field: Field) +data class SchemaField(val schema: Schema, val field: Field) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt index fee9c975..b07ca20f 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt @@ -4,50 +4,44 @@ import org.apache.avro.Schema import org.apache.avro.Schema.Type.ENUM import org.apache.avro.Schema.Type.RECORD import org.apache.avro.Schema.Type.UNION +import org.radarbase.schema.validation.ValidationContext interface SchemaFieldRules { /** Recursively checks field types. */ fun validateFieldTypes(schemaRules: SchemaRules): Validator /** Checks field name format. */ - fun validateFieldName(): Validator + val isNameValid: Validator /** Checks field documentation presence and format. */ - fun validateFieldDocumentation(): Validator + val isDocumentationValid: Validator /** Checks field default values. */ - fun validateDefault(): Validator + val isDefaultValueValid: Validator /** Get a validator for a field. */ - fun getValidator(schemaRules: SchemaRules): Validator { - return validateFieldTypes(schemaRules) - .and(validateFieldName()) - .and(validateDefault()) - .and(validateFieldDocumentation()) - } + fun isFieldValid(schemaRules: SchemaRules): Validator = all( + validateFieldTypes(schemaRules), + isNameValid, + isDefaultValueValid, + isDocumentationValid, + ) /** Get a validator for a union inside a record. */ - fun validateInternalUnion(schemaRules: SchemaRules): Validator { - return Validator { field: SchemaField -> - field.field.schema().types.stream() - .flatMap { schema: Schema -> - val type = schema.type - return@flatMap when (type) { - RECORD -> schemaRules.validateRecord().validate(schema) - ENUM -> schemaRules.validateEnum().validate(schema) - UNION -> Validator.raise( - message(field, "Cannot have a nested union."), - ) - else -> Validator.valid() - } + fun validateInternalUnion(schemaRules: SchemaRules) = Validator { field: SchemaField -> + field.field.schema().types + .forEach { schema: Schema -> + val type = schema.type + when (type) { + RECORD -> schemaRules.isRecordValid.launchValidation(schema) + ENUM -> schemaRules.isEnumValid.launchValidation(schema) + UNION -> raise(field, "Cannot have a nested union.") + else -> Unit } - } + } } +} - companion object { - /** A message function for a field, ending with given text. */ - fun message(field: SchemaField, text: String): String { - return "Field ${field.field.name()} in schema ${field.schema.fullName} is invalid. $text" - } - } +fun ValidationContext.raise(field: SchemaField, text: String) { + raise("Field ${field.field.name()} in schema ${field.schema.fullName} is invalid. $text") } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt index 6bd94bf4..ed0c4acb 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt @@ -2,45 +2,48 @@ package org.radarbase.schema.validation.rules import org.apache.avro.Schema import org.radarbase.schema.Scope +import org.radarbase.schema.validation.ValidationContext interface SchemaMetadataRules { val schemaRules: SchemaRules /** Checks the location of a schema with its internal data. */ - fun validateSchemaLocation(): Validator + val isShemaLocationCorrect: Validator /** * Validates any schema file. It will choose the correct validation method based on the scope * and type of the schema. */ - fun getValidator(validateScopeSpecific: Boolean): Validator = - Validator { metadata -> - if (metadata.schema == null) { - return@Validator Validator.raise("Missing schema") - } - val schemaRules = schemaRules - - var validator = validateSchemaLocation() - validator = if (metadata.schema.type == Schema.Type.ENUM) { - validator.and(schema(schemaRules.validateEnum())) - } else if (validateScopeSpecific) { - when (metadata.scope) { - Scope.ACTIVE -> validator.and(schema(schemaRules.validateActiveSource())) - Scope.MONITOR -> validator.and(schema(schemaRules.validateMonitor())) - Scope.PASSIVE -> validator.and(schema(schemaRules.validatePassive())) - else -> validator.and(schema(schemaRules.validateRecord())) - } - } else { - validator.and(schema(schemaRules.validateRecord())) - } - validator.validate(metadata) + fun isSchemaMetadataValid(scopeSpecificValidation: Boolean) = Validator { metadata -> + if (metadata.schema == null) { + raise("Missing schema") + return@Validator } + val schemaRules = schemaRules - /** Validates schemas without their metadata. */ - fun schema(validator: Validator): Validator = - Validator { metadata -> validator.validate(metadata.schema!!) } + isShemaLocationCorrect.launchValidation(metadata) + + val ruleset = when { + metadata.schema.type == Schema.Type.ENUM -> schemaRules.isEnumValid + !scopeSpecificValidation -> schemaRules.isRecordValid + metadata.scope == Scope.ACTIVE -> schemaRules.isActiveSourceValid + metadata.scope == Scope.MONITOR -> schemaRules.isMonitorSourceValid + metadata.scope == Scope.PASSIVE -> schemaRules.isPassiveSourceValid + else -> schemaRules.isRecordValid + } + isSchemaCorrect(ruleset).launchValidation(metadata) + } - fun message(metadata: SchemaMetadata, text: String): String { - return "Schema ${metadata.schema!!.fullName} at ${metadata.path} is invalid. $text" + /** Validates schemas without their metadata. */ + fun isSchemaCorrect(validator: Validator) = Validator { metadata -> + if (metadata.schema == null) { + raise(metadata, "Schema is empty") + } else { + validator.launchValidation(metadata.schema) + } } } + +fun ValidationContext.raise(metadata: SchemaMetadata, text: String) { + raise("Schema ${metadata.schema?.fullName} at ${metadata.path} is invalid. $text") +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt index 3948db0f..51eb42a0 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt @@ -2,7 +2,7 @@ package org.radarbase.schema.validation.rules import org.apache.avro.Schema import org.apache.avro.Schema.Type.RECORD -import org.radarbase.schema.validation.rules.Validator.Companion.raise +import org.radarbase.schema.validation.ValidationContext interface SchemaRules { val fieldRules: SchemaFieldRules @@ -10,128 +10,102 @@ interface SchemaRules { /** * Checks that schemas are unique compared to already validated schemas. */ - fun validateUniqueness(): Validator + val isUnique: Validator /** * Checks schema namespace format. */ - fun validateNameSpace(): Validator + val isNamespaceValid: Validator /** * Checks schema name format. */ - fun validateName(): Validator + val isNameValid: Validator /** * Checks schema documentation presence and format. */ - fun validateSchemaDocumentation(): Validator + val isDocumentationValid: Validator /** * Checks that the symbols of enums have the required format. */ - fun validateSymbols(): Validator + val isEnumSymbolsValid: Validator /** * Checks that schemas should have a `time` field. */ - fun validateTime(): Validator + val hasTime: Validator /** * Checks that schemas should have a `timeCompleted` field. */ - fun validateTimeCompleted(): Validator + val hasTimeCompleted: Validator /** * Checks that schemas should not have a `timeCompleted` field. */ - fun validateNotTimeCompleted(): Validator + val hasNoTimeCompleted: Validator /** * Checks that schemas should have a `timeReceived` field. */ - fun validateTimeReceived(): Validator + val hasTimeReceived: Validator /** * Checks that schemas should not have a `timeReceived` field. */ - fun validateNotTimeReceived(): Validator + val hasNoTimeReceived: Validator /** * Validate an enum. */ - fun validateEnum(): Validator = validateUniqueness() - .and(validateNameSpace()) - .and(validateSymbols()) - .and(validateSchemaDocumentation()) - .and(validateName()) + val isEnumValid: Validator /** * Validate a record that is defined inline. */ - fun validateRecord(): Validator = validateUniqueness() - .and(validateAvroData()) - .and(validateNameSpace()) - .and(validateName()) - .and(validateSchemaDocumentation()) - .and(fields(fieldRules.getValidator(this))) - - fun validateAvroData(): Validator + val isRecordValid: Validator + val isAvroConnectCompatible: Validator /** * Validates record schemas of an active source. */ - fun validateActiveSource(): Validator = validateRecord() - .and( - validateTime() - .and(validateTimeCompleted()) - .and(validateNotTimeReceived()), - ) + val isActiveSourceValid: Validator /** * Validates schemas of monitor sources. */ - fun validateMonitor(): Validator = validateRecord() - .and(validateTime()) + val isMonitorSourceValid: Validator /** * Validates schemas of passive sources. */ - fun validatePassive(): Validator = validateRecord() - .and(validateTime()) - .and(validateTimeReceived()) - .and(validateNotTimeCompleted()) + val isPassiveSourceValid: Validator - fun messageSchema(text: String): (Schema) -> String { + fun schemaErrorMessage(text: String): (Schema) -> String { return { schema -> "Schema ${schema.fullName} is invalid. $text" } } - fun messageSchema(schema: Schema, text: String): String { - return "Schema ${schema.fullName} is invalid. $text" - } - /** * Validates all fields of records. * Validation will fail on non-record types or records with no fields. */ - fun fields(validator: Validator) = Validator { schema: Schema -> + fun isFieldsValid(validator: Validator) = Validator { schema: Schema -> if (schema.type != RECORD) { - return@Validator raise( - "Default validation can be applied only to an Avro RECORD, not to " + - schema.type + " of schema " + schema.fullName + '.', - ) + raise("Default validation can be applied only to an Avro RECORD, not to ${schema.type} of schema ${schema.fullName}.") + return@Validator } if (schema.fields.isEmpty()) { - return@Validator raise("Schema " + schema.fullName + " does not contain any fields.") + raise("Schema ${schema.fullName} does not contain any fields.") + return@Validator + } + schema.fields.forEach { field -> + validator.launchValidation(SchemaField(schema, field)) } - schema.fields.stream() - .flatMap { field -> - validator.validate( - SchemaField( - schema, - field, - ), - ) - } } } + +fun ValidationContext.raise(schema: Schema, text: String) { + raise("Schema ${schema.fullName} is invalid. $text") +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt index e8817d89..3f34fcea 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt @@ -15,42 +15,36 @@ */ package org.radarbase.schema.validation.rules -import org.radarbase.schema.validation.ValidationException -import java.util.stream.Stream +import org.radarbase.schema.validation.ValidationContext +import java.nio.file.Path +import kotlin.io.path.extension -class Validator( - private val validation: (T) -> Stream, +open class Validator( + private val validation: ValidationContext.(T) -> Unit, ) { - fun and(other: Validator): Validator = Validator { obj -> - Stream.concat( - this.validate(obj), - other.validate(obj), - ) + open fun ValidationContext.runValidation(value: T) { + this.validation(value) } +} - fun validate(value: T): Stream = this.validation.invoke(value) - - companion object { - fun check(test: Boolean, message: String): Stream = - if (test) valid() else raise(message) - - inline fun check(test: Boolean, message: () -> String): Stream { - return if (test) valid() else raise(message()) - } - - fun validate(predicate: (T) -> Boolean, message: String): Validator = - Validator { obj -> - check(predicate(obj), message) - } +fun validator(predicate: (T) -> Boolean, message: String): Validator = + Validator { obj -> + if (!predicate(obj)) raise(message) + } - fun validate(predicate: (T) -> Boolean, message: (T) -> String): Validator = - Validator { obj: T -> - check(predicate(obj), message(obj)) - } +fun validator(predicate: (T) -> Boolean, message: (T) -> String): Validator = + Validator { obj -> + if (!predicate(obj)) raise(message(obj)) + } - fun raise(message: String, ex: Exception? = null): Stream = - Stream.of(ValidationException(message, ex)) +fun all(vararg validators: Validator) = Validator { obj -> + validators.forEach { + it.launchValidation(obj) + } +} - fun valid(): Stream = Stream.empty() +fun pathExtensionValidator(extension: String) = Validator { path -> + if (!path.extension.equals(extension, ignoreCase = true)) { + raise("Path $path does not have extension $extension") } } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt index e1232687..85d7c294 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt @@ -15,6 +15,7 @@ */ package org.radarbase.schema.validation +import kotlinx.coroutines.runBlocking import org.apache.avro.SchemaBuilder import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.fail @@ -36,7 +37,6 @@ import org.radarbase.schema.validation.ValidationHelper.COMMONS_PATH import java.io.IOException import java.nio.file.Path import java.nio.file.Paths -import java.util.stream.Stream class SchemaValidatorTest { private lateinit var validator: SchemaValidator @@ -120,7 +120,7 @@ class SchemaValidatorTest { } @Throws(IOException::class) - private fun testFromSpecification(scope: Scope) { + private fun testFromSpecification(scope: Scope) = runBlocking { val sourceCatalogue = load(ROOT, SchemaConfig(), SourceConfig()) val result = format( validator.analyseSourceCatalogue(scope, sourceCatalogue), @@ -131,14 +131,14 @@ class SchemaValidatorTest { } @Throws(IOException::class) - private fun testScope(scope: Scope) { + private fun testScope(scope: Scope) = runBlocking { val schemaCatalogue = SchemaCatalogue( COMMONS_ROOT, SchemaConfig(), scope, ) val result = format( - validator.analyseFiles(scope, schemaCatalogue), + validator.analyseFiles(schemaCatalogue, scope), ) if (result.isNotEmpty()) { fail(result) @@ -146,7 +146,7 @@ class SchemaValidatorTest { } @Test - fun testEnumerator() { + fun testEnumerator() = runBlocking { val schemaPath = COMMONS_ROOT.resolve( "monitor/application/application_server_status.avsc", ) @@ -156,13 +156,21 @@ class SchemaValidatorTest { .enumeration(name) .doc(documentation) .symbols("CONNECTED", "DISCONNECTED", "UNKNOWN") - var result: Stream = validator.validate(schema, schemaPath, MONITOR) + var result = validationContext { + with(validator) { + validate(schema, schemaPath, MONITOR) + } + } assertEquals(0, result.count()) schema = SchemaBuilder .enumeration(name) .doc(documentation) .symbols("CONNECTED", "DISCONNECTED", "un_known") - result = validator.validate(schema, schemaPath, MONITOR) + result = validationContext { + with(validator) { + validate(schema, schemaPath, MONITOR) + } + } assertEquals(2, result.count()) } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt index de06815d..1c8f216e 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt @@ -43,10 +43,7 @@ class SourceCatalogueValidationTest { @Test fun validateTopicNames() { catalogue.topicNames.forEach { topic: String -> - assertTrue( - isValidTopic(topic), - "$topic is invalid", - ) + assertTrue(isValidTopic(topic), "$topic is invalid") } } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt index 414c4b7c..6606d93f 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt @@ -1,6 +1,6 @@ package org.radarbase.schema.validation -import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.radarbase.schema.Scope.ACTIVE @@ -29,8 +29,8 @@ class SpecificationsValidatorTest { @Test @Throws(IOException::class) fun activeIsYml() { - Assertions.assertTrue(validator.specificationsAreYmlFiles(ACTIVE)) - Assertions.assertTrue( + assertTrue(validator.specificationsAreYmlFiles(ACTIVE)) + assertTrue( validator.checkSpecificationParsing( ACTIVE, ActiveSource::class.java, @@ -41,8 +41,8 @@ class SpecificationsValidatorTest { @Test @Throws(IOException::class) fun monitorIsYml() { - Assertions.assertTrue(validator.specificationsAreYmlFiles(MONITOR)) - Assertions.assertTrue( + assertTrue(validator.specificationsAreYmlFiles(MONITOR)) + assertTrue( validator.checkSpecificationParsing( MONITOR, MonitorSource::class.java, @@ -53,8 +53,8 @@ class SpecificationsValidatorTest { @Test @Throws(IOException::class) fun passiveIsYml() { - Assertions.assertTrue(validator.specificationsAreYmlFiles(PASSIVE)) - Assertions.assertTrue( + assertTrue(validator.specificationsAreYmlFiles(PASSIVE)) + assertTrue( validator.checkSpecificationParsing( PASSIVE, PassiveSource::class.java, @@ -65,8 +65,8 @@ class SpecificationsValidatorTest { @Test @Throws(IOException::class) fun connectorIsYml() { - Assertions.assertTrue(validator.specificationsAreYmlFiles(CONNECTOR)) - Assertions.assertTrue( + assertTrue(validator.specificationsAreYmlFiles(CONNECTOR)) + assertTrue( validator.checkSpecificationParsing( CONNECTOR, ConnectorSource::class.java, @@ -77,15 +77,15 @@ class SpecificationsValidatorTest { @Test @Throws(IOException::class) fun pushIsYml() { - Assertions.assertTrue(validator.specificationsAreYmlFiles(PUSH)) - Assertions.assertTrue(validator.checkSpecificationParsing(PUSH, PushSource::class.java)) + assertTrue(validator.specificationsAreYmlFiles(PUSH)) + assertTrue(validator.checkSpecificationParsing(PUSH, PushSource::class.java)) } @Test @Throws(IOException::class) fun streamIsYml() { - Assertions.assertTrue(validator.specificationsAreYmlFiles(STREAM)) - Assertions.assertTrue( + assertTrue(validator.specificationsAreYmlFiles(STREAM)) + assertTrue( validator.checkSpecificationParsing( STREAM, StreamGroup::class.java, diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt index f983202d..5c06ea64 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt @@ -15,6 +15,7 @@ */ package org.radarbase.schema.validation.rules +import kotlinx.coroutines.runBlocking import org.apache.avro.Schema import org.apache.avro.Schema.Parser import org.apache.avro.SchemaBuilder @@ -23,10 +24,9 @@ import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test -import org.radarbase.schema.validation.ValidationException import org.radarbase.schema.validation.ValidationHelper.getRecordName +import org.radarbase.schema.validation.validate import java.nio.file.Paths -import java.util.stream.Stream /** * TODO. @@ -67,14 +67,13 @@ class RadarSchemaFieldRulesTest { } @Test - fun fieldsTest() { - var result: Stream + fun fieldsTest() = runBlocking { var schema: Schema = SchemaBuilder .builder(MONITOR_NAME_SPACE_MOCK) .record(RECORD_NAME_MOCK) .fields() .endRecord() - result = schemaValidator.fields(validator.validateFieldTypes(schemaValidator)) + var result = schemaValidator.isFieldsValid(validator.validateFieldTypes(schemaValidator)) .validate(schema) Assertions.assertEquals(1, result.count()) schema = SchemaBuilder @@ -83,21 +82,20 @@ class RadarSchemaFieldRulesTest { .fields() .optionalBoolean("optional") .endRecord() - result = schemaValidator.fields(validator.validateFieldTypes(schemaValidator)) + result = schemaValidator.isFieldsValid(validator.validateFieldTypes(schemaValidator)) .validate(schema) Assertions.assertEquals(0, result.count()) } @Test - fun fieldNameTest() { - var result: Stream + fun fieldNameTest() = runBlocking { var schema: Schema = SchemaBuilder .builder(MONITOR_NAME_SPACE_MOCK) .record(RECORD_NAME_MOCK) .fields() .requiredString(FIELD_NUMBER_MOCK) .endRecord() - result = schemaValidator.fields(validator.validateFieldName()).validate(schema) + var result = schemaValidator.isFieldsValid(validator.isNameValid).validate(schema) Assertions.assertEquals(1, result.count()) schema = SchemaBuilder .builder(MONITOR_NAME_SPACE_MOCK) @@ -105,13 +103,12 @@ class RadarSchemaFieldRulesTest { .fields() .requiredDouble("timeReceived") .endRecord() - result = schemaValidator.fields(validator.validateFieldName()).validate(schema) + result = schemaValidator.isFieldsValid(validator.isNameValid).validate(schema) Assertions.assertEquals(0, result.count()) } @Test - fun fieldDocumentationTest() { - var result: Stream + fun fieldDocumentationTest() = runBlocking { var schema: Schema = Parser().parse( """{ |"namespace": "org.radarcns.kafka.key", @@ -124,7 +121,7 @@ class RadarSchemaFieldRulesTest { |} """.trimMargin(), ) - result = schemaValidator.fields(validator.validateFieldDocumentation()).validate(schema) + var result = schemaValidator.isFieldsValid(validator.isDocumentationValid).validate(schema) Assertions.assertEquals(2, result.count()) schema = Parser().parse( """{ @@ -137,14 +134,14 @@ class RadarSchemaFieldRulesTest { |} """.trimMargin(), ) - result = schemaValidator.fields(validator.validateFieldDocumentation()).validate(schema) + result = schemaValidator.isFieldsValid(validator.isDocumentationValid).validate(schema) Assertions.assertEquals(0, result.count()) } @Test - fun defaultValueExceptionTest() { - val result: Stream = schemaValidator.fields( - validator.validateDefault(), + fun defaultValueExceptionTest() = runBlocking { + val result = schemaValidator.isFieldsValid( + validator.isDefaultValueValid, ) .validate( SchemaBuilder.record(RECORD_NAME_MOCK) @@ -161,7 +158,7 @@ class RadarSchemaFieldRulesTest { } @Test // TODO improve test after having define the default guideline - fun defaultValueTest() { + fun defaultValueTest() = runBlocking { val schemaTxtInit = ( "{\"namespace\": \"org.radarcns.test\", " + "\"type\": \"record\", \"name\": \"TestRecord\", \"fields\": " @@ -172,8 +169,8 @@ class RadarSchemaFieldRulesTest { "\"enum\", \"symbols\": [\"Connected\", \"NotConnected\", \"UNKNOWN\"] }, " + "\"default\": \"UNKNOWN\" } ] }", ) - var result: Stream = - schemaValidator.fields(validator.validateDefault()).validate(schema) + var result = + schemaValidator.isFieldsValid(validator.isDefaultValueValid).validate(schema) Assertions.assertEquals(0, result.count()) schema = Parser().parse( schemaTxtInit + @@ -181,7 +178,7 @@ class RadarSchemaFieldRulesTest { "\"enum\", \"symbols\": [\"Connected\", \"NotConnected\", \"UNKNOWN\"] }, " + "\"default\": \"null\" } ] }", ) - result = schemaValidator.fields(validator.validateDefault()).validate(schema) + result = schemaValidator.isFieldsValid(validator.isDefaultValueValid).validate(schema) Assertions.assertEquals(1, result.count()) } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt index 595020f4..54bb823f 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt @@ -15,6 +15,7 @@ */ package org.radarbase.schema.validation.rules +import kotlinx.coroutines.runBlocking import org.apache.avro.SchemaBuilder import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotNull @@ -25,10 +26,9 @@ import org.radarbase.schema.Scope.PASSIVE import org.radarbase.schema.specification.config.SchemaConfig import org.radarbase.schema.validation.SchemaValidator.Companion.format import org.radarbase.schema.validation.SourceCatalogueValidationTest -import org.radarbase.schema.validation.ValidationException import org.radarbase.schema.validation.ValidationHelper +import org.radarbase.schema.validation.validate import java.nio.file.Paths -import java.util.stream.Stream /** * TODO. @@ -59,7 +59,7 @@ class RadarSchemaMetadataRulesTest { } @Test - fun nameSpaceInvalidPlural() { + fun nameSpaceInvalidPlural() = runBlocking { val schema = SchemaBuilder .builder("org.radarcns.monitors.test") .record(RECORD_NAME_MOCK) @@ -69,13 +69,13 @@ class RadarSchemaMetadataRulesTest { MONITOR.getPath(SourceCatalogueValidationTest.BASE_PATH.resolve(ValidationHelper.COMMONS_PATH)) assertNotNull(root) val path = root.resolve("test/record_name.avsc") - val result = validator.validateSchemaLocation() + val result = validator.isShemaLocationCorrect .validate(SchemaMetadata(schema, MONITOR, path)) assertEquals(1, result.count()) } @Test - fun nameSpaceInvalidLastPartPlural() { + fun nameSpaceInvalidLastPartPlural() = runBlocking { val schema = SchemaBuilder .builder("org.radarcns.monitor.tests") .record(RECORD_NAME_MOCK) @@ -85,13 +85,13 @@ class RadarSchemaMetadataRulesTest { MONITOR.getPath(SourceCatalogueValidationTest.BASE_PATH.resolve(ValidationHelper.COMMONS_PATH)) assertNotNull(root) val path = root.resolve("test/record_name.avsc") - val result = validator.validateSchemaLocation() + val result = validator.isShemaLocationCorrect .validate(SchemaMetadata(schema, MONITOR, path)) assertEquals(1, result.count()) } @Test - fun recordNameTest() { + fun recordNameTest() = runBlocking { // misspell aceleration var fieldName = "EmpaticaE4Aceleration" var filePath = Paths.get("/path/to/empatica_e4_acceleration.avsc") @@ -100,7 +100,7 @@ class RadarSchemaMetadataRulesTest { .record(fieldName) .fields() .endRecord() - var result: Stream = validator.validateSchemaLocation() + var result = validator.isShemaLocationCorrect .validate(SchemaMetadata(schema, PASSIVE, filePath)) assertEquals(2, result.count()) fieldName = "EmpaticaE4Acceleration" @@ -111,7 +111,7 @@ class RadarSchemaMetadataRulesTest { .record(fieldName) .fields() .endRecord() - result = validator.validateSchemaLocation() + result = validator.isShemaLocationCorrect .validate(SchemaMetadata(schema, PASSIVE, filePath)) assertEquals("", format(result)) } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt index d99bfb40..4e00c77c 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt @@ -15,6 +15,7 @@ */ package org.radarbase.schema.validation.rules +import kotlinx.coroutines.runBlocking import org.apache.avro.Schema import org.apache.avro.Schema.Parser import org.apache.avro.SchemaBuilder @@ -23,8 +24,7 @@ import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test -import org.radarbase.schema.validation.ValidationException -import java.util.stream.Stream +import org.radarbase.schema.validation.validate /** * TODO. @@ -80,49 +80,47 @@ class RadarSchemaRulesTest { } @Test - fun nameSpaceTest() { + fun nameSpaceTest() = runBlocking { val schema = SchemaBuilder .builder("org.radarcns.active.questionnaire") .record("Questionnaire") .fields() .endRecord() - val result: Stream = validator.validateNameSpace() - .validate(schema) + val result = validator.isNamespaceValid.validate(schema) Assertions.assertEquals(0, result.count()) } @Test - fun nameSpaceInvalidDashTest() { + fun nameSpaceInvalidDashTest() = runBlocking { val schema = SchemaBuilder .builder("org.radar-cns.monitors.test") .record(RECORD_NAME_MOCK) .fields() .endRecord() - val result: Stream = validator.validateNameSpace() + val result = validator.isNamespaceValid .validate(schema) Assertions.assertEquals(1, result.count()) } @Test - fun recordNameTest() { + fun recordNameTest() = runBlocking { val schema = SchemaBuilder .builder("org.radarcns.active.testactive") .record("Schema") .fields() .endRecord() - val result: Stream = validator.validateName() - .validate(schema) + val result = validator.isNameValid.validate(schema) Assertions.assertEquals(0, result.count()) } @Test - fun fieldsTest() { + fun fieldsTest() = runBlocking { var schema: Schema = SchemaBuilder .builder(MONITOR_NAME_SPACE_MOCK) .record(RECORD_NAME_MOCK) .fields() .endRecord() - var result: Stream = validator.fields( + var result = validator.isFieldsValid( validator.fieldRules.validateFieldTypes(validator), ).validate(schema) Assertions.assertEquals(1, result.count()) @@ -132,20 +130,20 @@ class RadarSchemaRulesTest { .fields() .optionalBoolean("optional") .endRecord() - result = validator.fields(validator.fieldRules.validateFieldTypes(validator)) + result = validator.isFieldsValid(validator.fieldRules.validateFieldTypes(validator)) .validate(schema) Assertions.assertEquals(0, result.count()) } @Test - fun timeTest() { + fun timeTest() = runBlocking { var schema: Schema = SchemaBuilder .builder("org.radarcns.time.test") .record(RECORD_NAME_MOCK) .fields() .requiredString("string") .endRecord() - var result: Stream = validator.validateTime().validate(schema) + var result = validator.hasTime.validate(schema) Assertions.assertEquals(1, result.count()) schema = SchemaBuilder .builder("org.radarcns.time.test") @@ -153,21 +151,21 @@ class RadarSchemaRulesTest { .fields() .requiredDouble(RadarSchemaRules.TIME) .endRecord() - result = validator.validateTime().validate(schema) + result = validator.hasTime.validate(schema) Assertions.assertEquals(0, result.count()) } @Test - fun timeCompletedTest() { + fun timeCompletedTest() = runBlocking { var schema: Schema = SchemaBuilder .builder(ACTIVE_NAME_SPACE_MOCK) .record(RECORD_NAME_MOCK) .fields() .requiredString("field") .endRecord() - var result: Stream = validator.validateTimeCompleted().validate(schema) + var result = validator.hasTimeCompleted.validate(schema) Assertions.assertEquals(1, result.count()) - result = validator.validateNotTimeCompleted().validate(schema) + result = validator.hasNoTimeCompleted.validate(schema) Assertions.assertEquals(0, result.count()) schema = SchemaBuilder .builder(ACTIVE_NAME_SPACE_MOCK) @@ -175,23 +173,23 @@ class RadarSchemaRulesTest { .fields() .requiredDouble("timeCompleted") .endRecord() - result = validator.validateTimeCompleted().validate(schema) + result = validator.hasTimeCompleted.validate(schema) Assertions.assertEquals(0, result.count()) - result = validator.validateNotTimeCompleted().validate(schema) + result = validator.hasNoTimeCompleted.validate(schema) Assertions.assertEquals(1, result.count()) } @Test - fun timeReceivedTest() { + fun timeReceivedTest() = runBlocking { var schema: Schema = SchemaBuilder .builder(MONITOR_NAME_SPACE_MOCK) .record(RECORD_NAME_MOCK) .fields() .requiredString("field") .endRecord() - var result: Stream = validator.validateTimeReceived().validate(schema) + var result = validator.hasTimeReceived.validate(schema) Assertions.assertEquals(1, result.count()) - result = validator.validateNotTimeReceived().validate(schema) + result = validator.hasNoTimeReceived.validate(schema) Assertions.assertEquals(0, result.count()) schema = SchemaBuilder .builder(MONITOR_NAME_SPACE_MOCK) @@ -199,20 +197,20 @@ class RadarSchemaRulesTest { .fields() .requiredDouble("timeReceived") .endRecord() - result = validator.validateTimeReceived().validate(schema) + result = validator.hasTimeReceived.validate(schema) Assertions.assertEquals(0, result.count()) - result = validator.validateNotTimeReceived().validate(schema) + result = validator.hasNoTimeReceived.validate(schema) Assertions.assertEquals(1, result.count()) } @Test - fun schemaDocumentationTest() { + fun schemaDocumentationTest() = runBlocking { var schema: Schema = SchemaBuilder .builder(MONITOR_NAME_SPACE_MOCK) .record(RECORD_NAME_MOCK) .fields() .endRecord() - var result: Stream = validator.validateSchemaDocumentation().validate(schema) + var result = validator.isDocumentationValid.validate(schema) Assertions.assertEquals(1, result.count()) schema = SchemaBuilder .builder(MONITOR_NAME_SPACE_MOCK) @@ -220,29 +218,29 @@ class RadarSchemaRulesTest { .doc("Documentation.") .fields() .endRecord() - result = validator.validateSchemaDocumentation().validate(schema) + result = validator.isDocumentationValid.validate(schema) Assertions.assertEquals(0, result.count()) } @Test - fun enumerationSymbolsTest() { + fun enumerationSymbolsTest() = runBlocking { var schema: Schema = SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK) .symbols("TEST", UNKNOWN_MOCK) - var result: Stream = validator.validateSymbols().validate(schema) + var result = validator.isEnumSymbolsValid.validate(schema) Assertions.assertEquals(0, result.count()) schema = SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK).symbols() - result = validator.validateSymbols().validate(schema) + result = validator.isEnumSymbolsValid.validate(schema) Assertions.assertEquals(1, result.count()) } @Test - fun enumerationSymbolTest() { + fun enumerationSymbolTest() = runBlocking { val enumName = "org.radarcns.monitor.application.ApplicationServerStatus" val connected = "CONNECTED" var schema: Schema = SchemaBuilder .enumeration(enumName) .symbols(connected, "DISCONNECTED", UNKNOWN_MOCK) - var result: Stream = validator.validateSymbols().validate(schema) + var result = validator.isEnumSymbolsValid.validate(schema) Assertions.assertEquals(0, result.count()) val schemaTxtInit = ( "{\"namespace\": \"org.radarcns.monitor.application\", " + @@ -254,39 +252,39 @@ class RadarSchemaRulesTest { schemaTxtInit + "\"CONNECTED\", \"NOT_CONNECTED\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd, ) - result = validator.validateSymbols().validate(schema) + result = validator.isEnumSymbolsValid.validate(schema) Assertions.assertEquals(0, result.count()) schema = SchemaBuilder .enumeration(enumName) .symbols(connected, "disconnected", UNKNOWN_MOCK) - result = validator.validateSymbols().validate(schema) + result = validator.isEnumSymbolsValid.validate(schema) Assertions.assertEquals(1, result.count()) schema = SchemaBuilder .enumeration(enumName) .symbols(connected, "Not_Connected", UNKNOWN_MOCK) - result = validator.validateSymbols().validate(schema) + result = validator.isEnumSymbolsValid.validate(schema) Assertions.assertEquals(1, result.count()) schema = SchemaBuilder .enumeration(enumName) .symbols(connected, "NotConnected", UNKNOWN_MOCK) - result = validator.validateSymbols().validate(schema) + result = validator.isEnumSymbolsValid.validate(schema) Assertions.assertEquals(1, result.count()) schema = Parser().parse( schemaTxtInit + "\"CONNECTED\", \"Not_Connected\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd, ) - result = validator.validateSymbols().validate(schema) + result = validator.isEnumSymbolsValid.validate(schema) Assertions.assertEquals(1, result.count()) schema = Parser().parse( schemaTxtInit + "\"Connected\", \"NotConnected\", \"" + UNKNOWN_MOCK + "\"" + schemaTxtEnd, ) - result = validator.validateSymbols().validate(schema) + result = validator.isEnumSymbolsValid.validate(schema) Assertions.assertEquals(2, result.count()) } @Test - fun testUniqueness() { + fun testUniqueness() = runBlocking { val prefix = ( "{\"namespace\": \"org.radarcns.monitor.application\", " + "\"name\": \"" @@ -297,33 +295,33 @@ class RadarSchemaRulesTest { prefix + "ServerStatus" + infix + "[\"A\", \"B\"]" + suffix, ) - var result: Stream = validator.validateUniqueness().validate(schema) + var result = validator.isUnique.validate(schema) Assertions.assertEquals(0, result.count()) - result = validator.validateUniqueness().validate(schema) + result = validator.isUnique.validate(schema) Assertions.assertEquals(0, result.count()) val schemaAlt = Parser().parse( prefix + "ServerStatus" + infix + "[\"A\", \"B\", \"C\"]" + suffix, ) - result = validator.validateUniqueness().validate(schemaAlt) + result = validator.isUnique.validate(schemaAlt) Assertions.assertEquals(1, result.count()) - result = validator.validateUniqueness().validate(schemaAlt) + result = validator.isUnique.validate(schemaAlt) Assertions.assertEquals(1, result.count()) val schema2 = Parser().parse( prefix + "ServerStatus2" + infix + "[\"A\", \"B\"]" + suffix, ) - result = validator.validateUniqueness().validate(schema2) + result = validator.isUnique.validate(schema2) Assertions.assertEquals(0, result.count()) val schema3 = Parser().parse( prefix + "ServerStatus" + infix + "[\"A\", \"B\"]" + suffix, ) - result = validator.validateUniqueness().validate(schema3) + result = validator.isUnique.validate(schema3) Assertions.assertEquals(0, result.count()) - result = validator.validateUniqueness().validate(schema3) + result = validator.isUnique.validate(schema3) Assertions.assertEquals(0, result.count()) - result = validator.validateUniqueness().validate(schemaAlt) + result = validator.isUnique.validate(schemaAlt) Assertions.assertEquals(1, result.count()) } diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt index 60aa81d1..6b7f454d 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt @@ -1,5 +1,7 @@ package org.radarbase.schema.tools +import kotlinx.coroutines.async +import kotlinx.coroutines.runBlocking import net.sourceforge.argparse4j.impl.Arguments import net.sourceforge.argparse4j.inf.ArgumentParser import net.sourceforge.argparse4j.inf.Namespace @@ -9,7 +11,6 @@ import org.radarbase.schema.validation.SchemaValidator import org.radarbase.schema.validation.ValidationException import org.radarbase.schema.validation.ValidationHelper.COMMONS_PATH import java.io.IOException -import java.util.stream.Stream import kotlin.streams.asSequence class ValidatorCommand : SubCommand { @@ -45,26 +46,34 @@ class ValidatorCommand : SubCommand { return try { val validator = SchemaValidator(app.root.resolve(COMMONS_PATH), app.config.schemas) - var exceptionStream = Stream.empty() - if (options.getBoolean("full")) { - exceptionStream = validator.analyseFiles( - scope, - app.catalogue.schemaCatalogue, + runBlocking { + val fullValidationJob = async { + if (options.getBoolean("full")) { + if (scope == null) { + validator.analyseFiles(app.catalogue.schemaCatalogue) + } else { + validator.analyseFiles(app.catalogue.schemaCatalogue, scope) + } + } else { + emptyList() + } + } + val fromSpecJob = async { + if (options.getBoolean("from_specification")) { + validator.analyseSourceCatalogue(scope, app.catalogue) + } else { + emptyList() + } + } + val exceptions = fullValidationJob.await() + fromSpecJob.await() + + resolveValidation( + exceptions, + validator, + options.getBoolean("verbose"), + options.getBoolean("quiet"), ) } - if (options.getBoolean("from_specification")) { - exceptionStream = Stream.concat( - exceptionStream, - validator.analyseSourceCatalogue(scope, app.catalogue), - ).distinct() - } - - resolveValidation( - exceptionStream, - validator, - options.getBoolean("verbose"), - options.getBoolean("quiet"), - ) } catch (e: IOException) { System.err.println("Failed to load schemas: $e") 1 @@ -76,7 +85,7 @@ class ValidatorCommand : SubCommand { description("Validate a set of specifications.") addArgument("-s", "--scope") .help("type of specifications to validate") - .choices(*Scope.values()) + .choices(Scope.entries) addArgument("-v", "--verbose") .help("verbose validation message") .action(Arguments.storeTrue()) @@ -94,7 +103,7 @@ class ValidatorCommand : SubCommand { } private fun resolveValidation( - stream: Stream, + stream: List, validator: SchemaValidator, verbose: Boolean, quiet: Boolean, @@ -111,7 +120,7 @@ class ValidatorCommand : SubCommand { } if (result.isNotEmpty()) 1 else 0 } - stream.count() > 0 -> 1 + stream.isNotEmpty() -> 1 else -> 0 } } From 3c38648de6165ee779bb4a1652eb82d48f1b1f86 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 25 Sep 2023 15:00:49 +0200 Subject: [PATCH 03/27] Simplified SpecificationsValidator --- .../validation/SpecificationsValidator.kt | 70 ++++++------------ .../validation/SpecificationsValidatorTest.kt | 73 +++++++------------ 2 files changed, 49 insertions(+), 94 deletions(-) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt index ebd6ead9..8babbb66 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt @@ -18,11 +18,9 @@ package org.radarbase.schema.validation import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.dataformat.yaml.YAMLFactory import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withContext import org.radarbase.schema.Scope import org.radarbase.schema.specification.config.SchemaConfig -import org.radarbase.schema.validation.ValidationHelper.SPECIFICATIONS_PATH import org.radarbase.schema.validation.rules.Validator import org.radarbase.schema.validation.rules.pathExtensionValidator import org.slf4j.LoggerFactory @@ -35,64 +33,38 @@ import java.util.stream.Collectors /** * Validates RADAR-Schemas specifications. * - * @param root RADAR-Schemas directory. + * @param root RADAR-Schemas specifications directory. * @param config configuration to exclude certain schemas or fields from validation. * */ -class SpecificationsValidator(root: Path, config: SchemaConfig) { - private val specificationsRoot: Path = root.resolve(SPECIFICATIONS_PATH) +class SpecificationsValidator( + private val root: Path, + private val config: SchemaConfig, +) { private val mapper: ObjectMapper = ObjectMapper(YAMLFactory()) - private val pathMatcher: PathMatcher = config.pathMatcher(specificationsRoot) + private val pathMatcher: PathMatcher = config.pathMatcher(root) - /** Check that all files in the specifications directory are YAML files. */ - @Throws(IOException::class) - fun specificationsAreYmlFiles(scope: Scope): Boolean { - val baseFolder = scope.getPath(specificationsRoot) - if (baseFolder == null) { + fun ofScope(scope: Scope): SpecificationsValidator? { + val baseFolder = scope.getPath(root) + return if (baseFolder == null) { logger.info( "{} sources folder not present at {}", scope, - specificationsRoot.resolve(scope.lower), + root.resolve(scope.lower), ) - return false - } - return runBlocking { - val paths = baseFolder.fetchChildren() - val exceptions = validationContext { - paths.forEach { isYmlFile.launchValidation(it) } - } - if (exceptions.isEmpty()) { - true - } else { - logger.error("Not all specification files have the right extension: {}", exceptions.joinToString()) - false - } + null + } else { + SpecificationsValidator(baseFolder, config) } } - @Throws(IOException::class) - fun checkSpecificationParsing(scope: Scope, clazz: Class?): Boolean { - val baseFolder = scope.getPath(specificationsRoot) - if (baseFolder == null) { - logger.info( - "{} sources folder not present at {}", - scope, - specificationsRoot.resolve(scope.lower), - ) - return false - } - val validator = isValidYmlFile(clazz) - - return runBlocking { - val paths = baseFolder.fetchChildren() - val exceptions = validationContext { - paths.forEach { validator.launchValidation(it) } - } - if (exceptions.isEmpty()) { - true - } else { - logger.error("Not all specification files have the right format: {}", exceptions.joinToString()) - false + suspend fun isValidSpecification(clazz: Class?): List { + val paths = root.fetchChildren() + return validationContext { + val isParseableAsClass = isYmlFileParseable(clazz) + paths.forEach { p -> + isYmlFile.launchValidation(p) + isParseableAsClass.launchValidation(p) } } } @@ -105,7 +77,7 @@ class SpecificationsValidator(root: Path, config: SchemaConfig) { } } - private fun isValidYmlFile(clazz: Class?) = Validator { path -> + private fun isYmlFileParseable(clazz: Class?) = Validator { path -> try { mapper.readerFor(clazz).readValue(path.toFile()) } catch (ex: IOException) { diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt index 6606d93f..23dbbe15 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt @@ -1,6 +1,7 @@ package org.radarbase.schema.validation -import org.junit.jupiter.api.Assertions.assertTrue +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.radarbase.schema.Scope.ACTIVE @@ -16,6 +17,7 @@ import org.radarbase.schema.specification.monitor.MonitorSource import org.radarbase.schema.specification.passive.PassiveSource import org.radarbase.schema.specification.push.PushSource import org.radarbase.schema.specification.stream.StreamGroup +import org.radarbase.schema.validation.ValidationHelper.SPECIFICATIONS_PATH import java.io.IOException class SpecificationsValidatorTest { @@ -23,73 +25,54 @@ class SpecificationsValidatorTest { @BeforeEach fun setUp() { - validator = SpecificationsValidator(SourceCatalogueValidationTest.BASE_PATH, SchemaConfig()) + validator = SpecificationsValidator(SourceCatalogueValidationTest.BASE_PATH.resolve(SPECIFICATIONS_PATH), SchemaConfig()) } @Test @Throws(IOException::class) - fun activeIsYml() { - assertTrue(validator.specificationsAreYmlFiles(ACTIVE)) - assertTrue( - validator.checkSpecificationParsing( - ACTIVE, - ActiveSource::class.java, - ), - ) + fun activeIsYml() = runBlocking { + val validator = validator.ofScope(ACTIVE) ?: return@runBlocking + val result = validator.isValidSpecification(ActiveSource::class.java) + assertEquals("", SchemaValidator.format(result)) } @Test @Throws(IOException::class) - fun monitorIsYml() { - assertTrue(validator.specificationsAreYmlFiles(MONITOR)) - assertTrue( - validator.checkSpecificationParsing( - MONITOR, - MonitorSource::class.java, - ), - ) + fun monitorIsYml() = runBlocking { + val validator = validator.ofScope(MONITOR) ?: return@runBlocking + val result = validator.isValidSpecification(MonitorSource::class.java) + assertEquals("", SchemaValidator.format(result)) } @Test @Throws(IOException::class) - fun passiveIsYml() { - assertTrue(validator.specificationsAreYmlFiles(PASSIVE)) - assertTrue( - validator.checkSpecificationParsing( - PASSIVE, - PassiveSource::class.java, - ), - ) + fun passiveIsYml() = runBlocking { + val validator = validator.ofScope(PASSIVE) ?: return@runBlocking + val result = validator.isValidSpecification(PassiveSource::class.java) + assertEquals("", SchemaValidator.format(result)) } @Test @Throws(IOException::class) - fun connectorIsYml() { - assertTrue(validator.specificationsAreYmlFiles(CONNECTOR)) - assertTrue( - validator.checkSpecificationParsing( - CONNECTOR, - ConnectorSource::class.java, - ), - ) + fun connectorIsYml() = runBlocking { + val validator = validator.ofScope(CONNECTOR) ?: return@runBlocking + val result = validator.isValidSpecification(ConnectorSource::class.java) + assertEquals("", SchemaValidator.format(result)) } @Test @Throws(IOException::class) - fun pushIsYml() { - assertTrue(validator.specificationsAreYmlFiles(PUSH)) - assertTrue(validator.checkSpecificationParsing(PUSH, PushSource::class.java)) + fun pushIsYml() = runBlocking { + val validator = validator.ofScope(PUSH) ?: return@runBlocking + val result = validator.isValidSpecification(PushSource::class.java) + assertEquals("", SchemaValidator.format(result)) } @Test @Throws(IOException::class) - fun streamIsYml() { - assertTrue(validator.specificationsAreYmlFiles(STREAM)) - assertTrue( - validator.checkSpecificationParsing( - STREAM, - StreamGroup::class.java, - ), - ) + fun streamIsYml() = runBlocking { + val validator = validator.ofScope(STREAM) ?: return@runBlocking + val result = validator.isValidSpecification(StreamGroup::class.java) + assertEquals("", SchemaValidator.format(result)) } } From dee25e4709c31bbea9f770b55b795efbf5900872 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 25 Sep 2023 15:05:33 +0200 Subject: [PATCH 04/27] Simplify schema validator setup --- .../schema/validation/SchemaValidator.kt | 36 +++++++++---------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt index e9a12481..681c85c7 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt @@ -41,39 +41,34 @@ import kotlin.io.path.extension class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { val rules: SchemaMetadataRules = RadarSchemaMetadataRules(schemaRoot, config) private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) - private var validator: Validator = rules.isSchemaMetadataValid(false) suspend fun analyseSourceCatalogue( scope: Scope?, catalogue: SourceCatalogue, ): List { - validator = rules.isSchemaMetadataValid(true) + val validator = rules.isSchemaMetadataValid(true) val producers: Stream> = if (scope != null) { catalogue.sources.stream() .filter { it.scope == scope } } else { catalogue.sources.stream() } - return try { - validationContext { - val schemas = producers - .flatMap { it.data.stream() } - .flatMap { topic -> - val (keySchema, valueSchema) = catalogue.schemaCatalogue.getSchemaMetadata(topic) - Stream.of(keySchema, valueSchema) - } - .filter { it.schema != null } - .sorted(Comparator.comparing { it.schema!!.fullName }) - .collect(Collectors.toSet()) + return validationContext { + val schemas = producers + .flatMap { it.data.stream() } + .flatMap { topic -> + val (keySchema, valueSchema) = catalogue.schemaCatalogue.getSchemaMetadata(topic) + Stream.of(keySchema, valueSchema) + } + .filter { it.schema != null } + .sorted(Comparator.comparing { it.schema!!.fullName }) + .collect(Collectors.toSet()) - schemas.forEach { metadata -> - if (pathMatcher.matches(metadata.path)) { - validator.launchValidation(metadata) - } + schemas.forEach { metadata -> + if (pathMatcher.matches(metadata.path)) { + validator.launchValidation(metadata) } } - } finally { - validator = rules.isSchemaMetadataValid(false) } } @@ -92,7 +87,7 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { schemaCatalogue: SchemaCatalogue, scope: Scope, ) { - validator = rules.isSchemaMetadataValid(false) + val validator = rules.isSchemaMetadataValid(false) val parsingValidator = parsingValidator(scope, schemaCatalogue) schemaCatalogue.unmappedAvroFiles.forEach { metadata -> @@ -134,6 +129,7 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { /** Validate a single schema in given path. */ private fun ValidationContext.validate(schemaMetadata: SchemaMetadata) { + val validator = rules.isSchemaMetadataValid(false) if (pathMatcher.matches(schemaMetadata.path)) { validator.launchValidation(schemaMetadata) } From 6d545759de91e50cd0d01b0d9c1606df76808cdc Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 26 Sep 2023 17:16:13 +0200 Subject: [PATCH 05/27] Fix docker build --- Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 998281e2..09e4299f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,18 @@ -FROM --platform=$BUILDPLATFORM gradle:7.5-jdk17 as builder +FROM --platform=$BUILDPLATFORM gradle:8.3-jdk17 as builder RUN mkdir -p /code/java-sdk WORKDIR /code/java-sdk ENV GRADLE_USER_HOME=/code/.gradlecache \ - GRADLE_OPTS=-Djdk.lang.Process.launchMechanism=vfork + GRADLE_OPTS="-Djdk.lang.Process.launchMechanism=vfork -Dorg.gradle.vfs.watch=false" +COPY java-sdk/buildSrc /code/java-sdk/buildSrc COPY java-sdk/*.gradle.kts java-sdk/gradle.properties /code/java-sdk/ COPY java-sdk/radar-schemas-commons/build.gradle.kts /code/java-sdk/radar-schemas-commons/ COPY java-sdk/radar-schemas-core/build.gradle.kts /code/java-sdk/radar-schemas-core/ COPY java-sdk/radar-schemas-registration/build.gradle.kts /code/java-sdk/radar-schemas-registration/ COPY java-sdk/radar-schemas-tools/build.gradle.kts /code/java-sdk/radar-schemas-tools/ COPY java-sdk/radar-catalog-server/build.gradle.kts /code/java-sdk/radar-catalog-server/ -RUN gradle downloadDependencies copyDependencies startScripts --no-watch-fs +RUN gradle downloadDependencies copyDependencies startScripts COPY commons /code/commons COPY specifications /code/specifications @@ -22,7 +23,7 @@ COPY java-sdk/radar-schemas-registration/src /code/java-sdk/radar-schemas-regist COPY java-sdk/radar-schemas-tools/src /code/java-sdk/radar-schemas-tools/src COPY java-sdk/radar-catalog-server/src /code/java-sdk/radar-catalog-server/src -RUN gradle jar --no-watch-fs +RUN gradle jar FROM eclipse-temurin:17-jre From 8587e9554dd7cca2182b793d71fc34c37d348ece Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 26 Sep 2023 17:16:42 +0200 Subject: [PATCH 06/27] Removed old comments and simplified test --- .../specification/active/ActiveSource.kt | 6 +- .../questionnaire/QuestionnaireDataTopic.kt | 3 - .../specification/passive/PassiveDataTopic.kt | 3 - .../specification/passive/PassiveSource.kt | 3 - .../specification/stream/StreamDataTopic.kt | 4 +- .../org/radarbase/schema/util/SchemaUtils.kt | 104 ++++------- .../schema/validation/SchemaValidator.kt | 12 +- .../schema/validation/ValidationContext.kt | 4 +- .../schema/validation/ValidationException.kt | 3 - .../schema/validation/ValidationHelper.kt | 3 - .../SourceCatalogueValidationTest.kt | 3 - .../rules/RadarSchemaFieldRulesTest.kt | 166 ++++++------------ .../rules/RadarSchemaMetadataRulesTest.kt | 3 - .../validation/rules/RadarSchemaRulesTest.kt | 3 - ...chemaUtilsTest.java => SchemaUtilsTest.kt} | 20 +-- .../schema/registration/SchemaRegistry.kt | 9 +- 16 files changed, 122 insertions(+), 227 deletions(-) rename java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/util/{SchemaUtilsTest.java => SchemaUtilsTest.kt} (64%) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.kt index 8ead1648..1f2d370e 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/ActiveSource.kt @@ -29,9 +29,6 @@ import org.radarbase.schema.specification.DataProducer import org.radarbase.schema.specification.DataTopic import org.radarbase.schema.specification.active.questionnaire.QuestionnaireSource -/** - * TODO. - */ @JsonTypeInfo(use = NAME, property = "assessment_type") @JsonSubTypes( value = [ @@ -63,6 +60,5 @@ open class ActiveSource : DataProducer() { @JsonProperty val version: String? = null - override val scope: Scope - get() = ACTIVE + override val scope: Scope = ACTIVE } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.kt index 01e83135..9a562547 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/active/questionnaire/QuestionnaireDataTopic.kt @@ -22,9 +22,6 @@ import org.radarbase.config.OpenConfig import org.radarbase.schema.specification.DataTopic import java.net.URL -/** - * TODO. - */ @JsonInclude(NON_NULL) @OpenConfig class QuestionnaireDataTopic : DataTopic() { diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.kt index 46743be8..6ec8a9cf 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveDataTopic.kt @@ -22,9 +22,6 @@ import org.radarbase.schema.specification.AppDataTopic import org.radarcns.catalogue.ProcessingState import java.util.Objects -/** - * TODO. - */ @JsonInclude(NON_NULL) class PassiveDataTopic : AppDataTopic() { @JsonProperty("processing_state") diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.kt index 9e57225d..fca83c92 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/passive/PassiveSource.kt @@ -24,9 +24,6 @@ import org.radarbase.schema.Scope import org.radarbase.schema.Scope.PASSIVE import org.radarbase.schema.specification.AppSource -/** - * TODO. - */ @JsonInclude(NON_NULL) @OpenConfig class PassiveSource : AppSource() { diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt index 9a07c9c7..bc2a51db 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt @@ -9,7 +9,7 @@ import org.radarbase.config.AvroTopicConfig import org.radarbase.config.OpenConfig import org.radarbase.schema.SchemaCatalogue import org.radarbase.schema.specification.DataTopic -import org.radarbase.schema.util.SchemaUtils +import org.radarbase.schema.util.SchemaUtils.applyOrEmpty import org.radarbase.stream.TimeWindowMetadata import org.radarbase.topic.AvroTopic import org.radarcns.kafka.AggregateKey @@ -81,7 +81,7 @@ class StreamDataTopic : DataTopic() { override fun topics(schemaCatalogue: SchemaCatalogue): Stream> { return topicNames .flatMap( - SchemaUtils.applyOrEmpty { topic -> + applyOrEmpty { topic -> val config = AvroTopicConfig() config.topic = topic config.keySchema = keySchema diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt index d014bb7e..bb1e9222 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt @@ -21,50 +21,38 @@ import java.util.Properties import java.util.function.Function import java.util.stream.Stream -/** - * TODO. - */ object SchemaUtils { private val logger = LoggerFactory.getLogger(SchemaUtils::class.java) private const val GRADLE_PROPERTIES = "exchange.properties" private const val GROUP_PROPERTY = "project.group" - @JvmStatic - @get:Synchronized - var projectGroup: String? = null - /** - * TODO. - * @return TODO - */ - get() { - if (field == null) { - val prop = Properties() - val loader = ClassLoader.getSystemClassLoader() - try { - loader.getResourceAsStream(GRADLE_PROPERTIES).use { `in` -> - if (`in` == null) { - field = "org.radarcns" - logger.debug("Project group not specified. Using \"{}\".", field) - } else { - prop.load(`in`) - field = prop.getProperty(GROUP_PROPERTY) - if (field == null) { - field = "org.radarcns" - logger.debug("Project group not specified. Using \"{}\".", field) - } - } + val projectGroup: String by lazy { + val prop = Properties() + val loader = ClassLoader.getSystemClassLoader() + try { + loader.getResourceAsStream(GRADLE_PROPERTIES).use { inputStream -> + var result = "org.radarcns" + if (inputStream == null) { + logger.debug("Project group not specified. Using \"{}\".", result) + } else { + prop.load(inputStream) + val groupProp = prop.getProperty(GROUP_PROPERTY) + if (groupProp == null) { + logger.debug("Project group not specified. Using \"{}\".", result) + } else { + result = groupProp } - } catch (exc: IOException) { - throw IllegalStateException( - GROUP_PROPERTY + - " cannot be extracted from " + GRADLE_PROPERTIES, - exc, - ) } + result } - return field + } catch (exc: IOException) { + throw IllegalStateException( + GROUP_PROPERTY + + " cannot be extracted from " + GRADLE_PROPERTIES, + exc, + ) } - private set + } /** * Expand a class name with the group name if it starts with a dot. @@ -86,28 +74,27 @@ object SchemaUtils { * @param value file name in snake_case * @return main part of file name in CamelCase. */ - @JvmStatic fun snakeToCamelCase(value: String): String { val fileName = value.toCharArray() - val builder = StringBuilder(fileName.size) - var nextIsUpperCase = true - for (c in fileName) { - when (c) { - '_' -> nextIsUpperCase = true - '.' -> return builder.toString() - else -> if (nextIsUpperCase) { - builder.append(c.toString().uppercase()) - nextIsUpperCase = false - } else { - builder.append(c) + return buildString(fileName.size) { + var nextIsUpperCase = true + for (c in fileName) { + when (c) { + '_' -> nextIsUpperCase = true + '.' -> return@buildString + else -> if (nextIsUpperCase) { + append(c.toString().uppercase()) + nextIsUpperCase = false + } else { + append(c) + } } } } - return builder.toString() } /** Apply a throwing function, and if it throws, log it and let it return an empty Stream. */ - fun applyOrEmpty(func: ThrowingFunction?>): Function?> { + fun applyOrEmpty(func: ThrowingFunction>): Function> { return Function { t: T -> try { return@Function func.apply(t) @@ -118,24 +105,11 @@ object SchemaUtils { } } - /** Apply a throwing function, and if it throws, log it and let it return an empty Stream. */ - fun applyOrIllegalException( - func: ThrowingFunction?>, - ): Function?> { - return Function { t: T -> - try { - return@Function func.apply(t) - } catch (ex: Exception) { - throw IllegalStateException(ex.message, ex) - } - } - } - /** * Function that may throw an exception. - * @param type of value taken. - * @param type of value returned. - */ + * @param T type of value taken. + * @param R type of value returned. + */ fun interface ThrowingFunction { /** * Apply containing function. diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt index 681c85c7..7d55910c 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt @@ -15,6 +15,8 @@ */ package org.radarbase.schema.validation +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch import org.apache.avro.Schema import org.radarbase.schema.SchemaCatalogue import org.radarbase.schema.Scope @@ -115,10 +117,12 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { return Validator { metadata -> val parser = Schema.Parser() parser.addTypes(useTypes) - try { - parser.parse(metadata.path?.toFile()) - } catch (ex: Exception) { - raise("Cannot parse schema", ex) + coroutineScope.launch(Dispatchers.IO) { + try { + parser.parse(metadata.path?.toFile()) + } catch (ex: Exception) { + raise("Cannot parse schema", ex) + } } } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt index 2964b62b..3e6f61ac 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt @@ -8,13 +8,15 @@ import kotlinx.coroutines.launch import org.radarbase.schema.validation.rules.Validator interface ValidationContext { + val coroutineScope: CoroutineScope + fun raise(message: String, ex: Exception? = null) fun Validator.launchValidation(value: T) } private class ValidationContextImpl( - private val coroutineScope: CoroutineScope, + override val coroutineScope: CoroutineScope, ) : ValidationContext { private val channel = Channel(Channel.UNLIMITED) private lateinit var producerCoroutineScope: CoroutineScope diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt index 579a2dc2..cb4b20f4 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt @@ -15,9 +15,6 @@ */ package org.radarbase.schema.validation -/** - * TODO. - */ class ValidationException : RuntimeException { constructor(message: String?) : super(message) constructor(message: String?, exception: Throwable?) : super(message, exception) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt index a8dc9614..32f00673 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt @@ -20,9 +20,6 @@ import org.radarbase.schema.util.SchemaUtils.projectGroup import org.radarbase.schema.util.SchemaUtils.snakeToCamelCase import java.nio.file.Path -/** - * TODO. - */ object ValidationHelper { const val COMMONS_PATH = "commons" const val SPECIFICATIONS_PATH = "specifications" diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt index 1c8f216e..a1d58f6b 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt @@ -36,9 +36,6 @@ import java.util.Objects import java.util.stream.Collectors import java.util.stream.Stream -/** - * TODO. - */ class SourceCatalogueValidationTest { @Test fun validateTopicNames() { diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt index 5c06ea64..c6d1f46c 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt @@ -17,20 +17,18 @@ package org.radarbase.schema.validation.rules import kotlinx.coroutines.runBlocking import org.apache.avro.Schema -import org.apache.avro.Schema.Parser import org.apache.avro.SchemaBuilder -import org.junit.jupiter.api.Assertions +import org.apache.avro.SchemaBuilder.FieldAssembler +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test +import org.radarbase.schema.validation.SchemaValidator import org.radarbase.schema.validation.ValidationHelper.getRecordName import org.radarbase.schema.validation.validate import java.nio.file.Paths -/** - * TODO. - */ class RadarSchemaFieldRulesTest { private lateinit var validator: RadarSchemaFieldRules private lateinit var schemaValidator: RadarSchemaRules @@ -43,11 +41,11 @@ class RadarSchemaFieldRulesTest { @Test fun fileNameTest() { - Assertions.assertEquals( + assertEquals( "Questionnaire", getRecordName(Paths.get("/path/to/questionnaire.avsc")), ) - Assertions.assertEquals( + assertEquals( "ApplicationExternalTime", getRecordName( Paths.get("/path/to/application_external_time.avsc"), @@ -68,125 +66,73 @@ class RadarSchemaFieldRulesTest { @Test fun fieldsTest() = runBlocking { - var schema: Schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .endRecord() - var result = schemaValidator.isFieldsValid(validator.validateFieldTypes(schemaValidator)) - .validate(schema) - Assertions.assertEquals(1, result.count()) - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .optionalBoolean("optional") - .endRecord() - result = schemaValidator.isFieldsValid(validator.validateFieldTypes(schemaValidator)) - .validate(schema) - Assertions.assertEquals(0, result.count()) + assertFieldsErrorCount(1, validator.validateFieldTypes(schemaValidator), "Should have at least one field") + + assertFieldsErrorCount(0, validator.validateFieldTypes(schemaValidator), "Single optional field should be fine") { + optionalBoolean("optional") + } + } + + private suspend fun assertFieldsErrorCount( + count: Int, + fieldValidator: Validator, + message: String, + schemaBuilder: FieldAssembler.() -> Unit = {}, + ) { + val result = schemaValidator.isFieldsValid(fieldValidator) + .validate( + SchemaBuilder.builder("org.radarcns.monitor.test") + .record("RecordName") + .fields() + .apply(schemaBuilder) + .endRecord(), + ) + assertEquals(count, result.size) { message + SchemaValidator.format(result) } } @Test fun fieldNameTest() = runBlocking { - var schema: Schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredString(FIELD_NUMBER_MOCK) - .endRecord() - var result = schemaValidator.isFieldsValid(validator.isNameValid).validate(schema) - Assertions.assertEquals(1, result.count()) - schema = SchemaBuilder - .builder(MONITOR_NAME_SPACE_MOCK) - .record(RECORD_NAME_MOCK) - .fields() - .requiredDouble("timeReceived") - .endRecord() - result = schemaValidator.isFieldsValid(validator.isNameValid).validate(schema) - Assertions.assertEquals(0, result.count()) + assertFieldsErrorCount(1, validator.isNameValid, "Field names should not start with uppercase") { + requiredString("Field1") + } + assertFieldsErrorCount(0, validator.isNameValid, "Field name timeReceived is correct") { + requiredDouble("timeReceived") + } } @Test fun fieldDocumentationTest() = runBlocking { - var schema: Schema = Parser().parse( - """{ - |"namespace": "org.radarcns.kafka.key", - |"type": "record", - |"name": "key", "type": - |"record", - |"fields": [ - |{"name": "userId", "type": "string" , "doc": "Documentation"}, - |{"name": "sourceId", "type": "string"} ] - |} - """.trimMargin(), - ) - var result = schemaValidator.isFieldsValid(validator.isDocumentationValid).validate(schema) - Assertions.assertEquals(2, result.count()) - schema = Parser().parse( - """{ - |"namespace": "org.radarcns.kafka.key", - |"type": "record", - |"name": "key", - |"type": "record", - |"fields": [ - |{"name": "userId", "type": "string" , "doc": "Documentation."}] - |} - """.trimMargin(), - ) - result = schemaValidator.isFieldsValid(validator.isDocumentationValid).validate(schema) - Assertions.assertEquals(0, result.count()) + assertFieldsErrorCount(2, validator.isDocumentationValid, "Documentation should be reported missing or incorrectly formatted.") { + name("userId").doc("Documentation").type("string").noDefault() + name("sourceId").type("string").noDefault() + } + assertFieldsErrorCount(0, validator.isDocumentationValid, "Documentation should be valid") { + name("userId").doc("Documentation.").type("string").noDefault() + } } @Test fun defaultValueExceptionTest() = runBlocking { - val result = schemaValidator.isFieldsValid( - validator.isDefaultValueValid, - ) - .validate( - SchemaBuilder.record(RECORD_NAME_MOCK) - .fields() - .name(FIELD_NUMBER_MOCK) - .type( - SchemaBuilder.enumeration(ENUMERATOR_NAME_SPACE_MOCK) - .symbols("VAL", UNKNOWN_MOCK), - ) - .noDefault() - .endRecord(), - ) - Assertions.assertEquals(1, result.count()) + assertFieldsErrorCount(1, validator.isDefaultValueValid, "Enum fields should have a default.") { + name("Field1") + .type( + SchemaBuilder.enumeration("org.radarcns.test.EnumeratorTest") + .symbols("VAL", "UNKNOWN"), + ) + .noDefault() + } } @Test // TODO improve test after having define the default guideline fun defaultValueTest() = runBlocking { - val schemaTxtInit = ( - "{\"namespace\": \"org.radarcns.test\", " + - "\"type\": \"record\", \"name\": \"TestRecord\", \"fields\": " - ) - var schema: Schema = Parser().parse( - schemaTxtInit + - "[ {\"name\": \"serverStatus\", \"type\": {\"name\": \"ServerStatus\", \"type\": " + - "\"enum\", \"symbols\": [\"Connected\", \"NotConnected\", \"UNKNOWN\"] }, " + - "\"default\": \"UNKNOWN\" } ] }", - ) - var result = - schemaValidator.isFieldsValid(validator.isDefaultValueValid).validate(schema) - Assertions.assertEquals(0, result.count()) - schema = Parser().parse( - schemaTxtInit + - "[ {\"name\": \"serverStatus\", \"type\": {\"name\": \"ServerStatus\", \"type\": " + - "\"enum\", \"symbols\": [\"Connected\", \"NotConnected\", \"UNKNOWN\"] }, " + - "\"default\": \"null\" } ] }", - ) - result = schemaValidator.isFieldsValid(validator.isDefaultValueValid).validate(schema) - Assertions.assertEquals(1, result.count()) - } + val serverStatusEnum = SchemaBuilder.enumeration("org.radarcns.monitor.test.ServerStatus") + .symbols("Connected", "NotConnected", "UNKNOWN") - companion object { - private const val MONITOR_NAME_SPACE_MOCK = "org.radarcns.monitor.test" - private const val ENUMERATOR_NAME_SPACE_MOCK = "org.radarcns.test.EnumeratorTest" - private const val UNKNOWN_MOCK = "UNKNOWN" - private const val RECORD_NAME_MOCK = "RecordName" - private const val FIELD_NUMBER_MOCK = "Field1" + assertFieldsErrorCount(0, validator.isDefaultValueValid, "Enum fields should have an UNKNOWN default.") { + name("serverStatus").type(serverStatusEnum).withDefault("UNKNOWN") + } + assertFieldsErrorCount(1, validator.isDefaultValueValid, "Enum fields with no UNKNOWN default should be reported.") { + name("serverStatus").type(serverStatusEnum).noDefault() + } } } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt index 54bb823f..cb3075ed 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt @@ -30,9 +30,6 @@ import org.radarbase.schema.validation.ValidationHelper import org.radarbase.schema.validation.validate import java.nio.file.Paths -/** - * TODO. - */ class RadarSchemaMetadataRulesTest { private lateinit var validator: RadarSchemaMetadataRules diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt index 4e00c77c..89a6c573 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt @@ -26,9 +26,6 @@ import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.radarbase.schema.validation.validate -/** - * TODO. - */ class RadarSchemaRulesTest { private lateinit var validator: RadarSchemaRules diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/util/SchemaUtilsTest.java b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/util/SchemaUtilsTest.kt similarity index 64% rename from java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/util/SchemaUtilsTest.java rename to java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/util/SchemaUtilsTest.kt index f5061741..a60028ed 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/util/SchemaUtilsTest.java +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/util/SchemaUtilsTest.kt @@ -13,21 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.radarbase.schema.validation.util -package org.radarbase.schema.validation.util; - -import org.junit.jupiter.api.Test; -import org.radarbase.schema.util.SchemaUtils; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * TODO. - */ -public class SchemaUtilsTest { +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.radarbase.schema.util.SchemaUtils.projectGroup +class SchemaUtilsTest { @Test - public void projectGroupTest() { - assertEquals("org.radarcns", SchemaUtils.getProjectGroup()); + fun projectGroupTest() { + assertEquals("org.radarcns", projectGroup) } } diff --git a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/SchemaRegistry.kt b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/SchemaRegistry.kt index 4f8574c1..4d235091 100644 --- a/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/SchemaRegistry.kt +++ b/java-sdk/radar-schemas-registration/src/main/java/org/radarbase/schema/registration/SchemaRegistry.kt @@ -128,9 +128,12 @@ class SchemaRegistry( } .toList() - val remainingTopics = topicConfiguration.toMutableMap() - sourceTopics.forEach { remainingTopics -= it.name } - + val remainingTopics = buildMap(topicConfiguration.size) { + putAll(topicConfiguration) + sourceTopics.forEach { + remove(it.name) + } + } val configuredTopics = remainingTopics .mapNotNull { (name, topicConfig) -> loadAvroTopic(name, topicConfig) } From 37e7a2adbc3f8d777a89b98a19eaee4e53017f22 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 27 Sep 2023 12:17:00 +0200 Subject: [PATCH 07/27] Sanity check --- .../radar-catalog-server/build.gradle.kts | 2 +- .../radar-schemas-commons/build.gradle.kts | 35 +++----- java-sdk/radar-schemas-core/build.gradle.kts | 1 + .../org/radarbase/schema/SchemaCatalogue.kt | 86 +++++++++---------- .../schema/specification/SourceCatalogue.kt | 72 ++++++++++------ .../org/radarbase/schema/util/SchemaUtils.kt | 14 +++ .../schema/validation/SchemaValidator.kt | 28 +++--- .../validation/SpecificationsValidator.kt | 15 +--- .../schema/validation/ValidationContext.kt | 51 ++++++----- .../rules/RadarSchemaMetadataRules.kt | 16 ++-- .../schema/validation/rules/SchemaMetadata.kt | 9 +- .../validation/rules/SchemaMetadataRules.kt | 18 +--- .../specification/config/SchemaConfigTest.kt | 2 +- .../rules/RadarSchemaMetadataRulesTest.kt | 8 +- 14 files changed, 185 insertions(+), 172 deletions(-) diff --git a/java-sdk/radar-catalog-server/build.gradle.kts b/java-sdk/radar-catalog-server/build.gradle.kts index 77bcf30c..58d827a1 100644 --- a/java-sdk/radar-catalog-server/build.gradle.kts +++ b/java-sdk/radar-catalog-server/build.gradle.kts @@ -3,10 +3,10 @@ description = "RADAR Schemas specification and validation tools." dependencies { implementation("org.radarbase:radar-jersey:${Versions.radarJersey}") implementation(project(":radar-schemas-core")) + implementation("org.radarbase:radar-commons-kotlin:${Versions.radarCommons}") implementation("net.sourceforge.argparse4j:argparse4j:${Versions.argparse}") - testImplementation(platform("io.ktor:ktor-bom:${Versions.ktor}")) testImplementation("io.ktor:ktor-client-content-negotiation") testImplementation("io.ktor:ktor-serialization-kotlinx-json") } diff --git a/java-sdk/radar-schemas-commons/build.gradle.kts b/java-sdk/radar-schemas-commons/build.gradle.kts index 16aa04d5..e6525342 100644 --- a/java-sdk/radar-schemas-commons/build.gradle.kts +++ b/java-sdk/radar-schemas-commons/build.gradle.kts @@ -4,14 +4,23 @@ plugins { id("com.github.davidmc24.gradle.plugin.avro-base") } -// Generated avro files -val avroOutputDir = file("$projectDir/src/generated/java") - description = "RADAR Schemas Commons SDK" +// ---------------------------------------------------------------------------// +// AVRO file manipulation // +// ---------------------------------------------------------------------------// +val generateAvro by tasks.registering(GenerateAvroJavaTask::class) { + source( + rootProject.fileTree("../commons") { + include("**/*.avsc") + }, + ) + setOutputDir(layout.projectDirectory.dir("src/generated/java").asFile) +} + sourceSets { main { - java.srcDir(avroOutputDir) + java.srcDir(generateAvro.map { it.outputs }) } } @@ -30,21 +39,5 @@ dependencies { // Clean settings // // ---------------------------------------------------------------------------// tasks.clean { - delete(avroOutputDir) + delete(generateAvro.map { it.outputs }) } - -// ---------------------------------------------------------------------------// -// AVRO file manipulation // -// ---------------------------------------------------------------------------// -val generateAvro by tasks.registering(GenerateAvroJavaTask::class) { - source( - rootProject.fileTree("../commons") { - include("**/*.avsc") - }, - ) - setOutputDir(avroOutputDir) -} - -tasks["compileJava"].dependsOn(generateAvro) -tasks["compileKotlin"].dependsOn(generateAvro) -tasks["dokkaJavadoc"].dependsOn(generateAvro) diff --git a/java-sdk/radar-schemas-core/build.gradle.kts b/java-sdk/radar-schemas-core/build.gradle.kts index c934184b..36d6c450 100644 --- a/java-sdk/radar-schemas-core/build.gradle.kts +++ b/java-sdk/radar-schemas-core/build.gradle.kts @@ -13,6 +13,7 @@ dependencies { } api("jakarta.validation:jakarta.validation-api:${Versions.jakartaValidation}") api(project(":radar-schemas-commons")) + implementation("org.radarbase:radar-commons-kotlin:${Versions.radarCommons}") api(platform("com.fasterxml.jackson:jackson-bom:${Versions.jackson}")) api("com.fasterxml.jackson.core:jackson-databind") diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt index 09d07182..049986d8 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt @@ -1,20 +1,23 @@ package org.radarbase.schema +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext import org.apache.avro.Schema import org.apache.avro.generic.GenericRecord import org.radarbase.config.AvroTopicConfig +import org.radarbase.kotlin.coroutines.forkJoin import org.radarbase.schema.specification.config.SchemaConfig -import org.radarbase.schema.validation.SchemaValidator +import org.radarbase.schema.util.SchemaUtils.listRecursive +import org.radarbase.schema.validation.SchemaValidator.Companion.isAvscFile +import org.radarbase.schema.validation.rules.FailedSchemaMetadata import org.radarbase.schema.validation.rules.SchemaMetadata import org.radarbase.topic.AvroTopic import org.slf4j.LoggerFactory import java.io.IOException -import java.nio.file.Files import java.nio.file.Path import java.nio.file.PathMatcher import java.util.* -import kotlin.collections.HashMap -import kotlin.collections.HashSet import kotlin.io.path.exists import kotlin.io.path.inputStream @@ -24,17 +27,19 @@ class SchemaCatalogue @JvmOverloads constructor( scope: Scope? = null, ) { val schemas: Map - val unmappedAvroFiles: List + val unmappedAvroFiles: List init { val schemaTemp = HashMap() - val unmappedTemp = mutableListOf() + val unmappedTemp = mutableListOf() val matcher = config.pathMatcher(schemaRoot) - if (scope != null) { - loadSchemas(schemaTemp, unmappedTemp, scope, matcher, config) - } else { - for (useScope in Scope.entries) { - loadSchemas(schemaTemp, unmappedTemp, useScope, matcher, config) + runBlocking { + if (scope != null) { + loadSchemas(schemaTemp, unmappedTemp, scope, matcher, config) + } else { + for (useScope in Scope.entries) { + loadSchemas(schemaTemp, unmappedTemp, useScope, matcher, config) + } } } schemas = schemaTemp.toMap() @@ -61,9 +66,9 @@ class SchemaCatalogue @JvmOverloads constructor( } @Throws(IOException::class) - private fun loadSchemas( + private suspend fun loadSchemas( schemas: MutableMap, - unmappedFiles: MutableList, + unmappedFiles: MutableList, scope: Scope, matcher: PathMatcher, config: SchemaConfig, @@ -71,17 +76,14 @@ class SchemaCatalogue @JvmOverloads constructor( val walkRoot = schemaRoot.resolve(scope.lower) val avroFiles = buildMap { if (walkRoot.exists()) { - Files.walk(walkRoot).use { walker -> - walker - .filter { p -> - matcher.matches(p) && SchemaValidator.isAvscFile(p) - } - .forEach { p -> - p.inputStream().reader().use { - put(p, it.readText()) - } + walkRoot + .listRecursive { matcher.matches(it) && it.isAvscFile() } + .forkJoin(Dispatchers.IO) { p -> + p.inputStream().reader().use { + p to it.readText() } - } + } + .toMap(this@buildMap) } config.schemas(scope).forEach { (key, value) -> put(walkRoot.resolve(key), value) @@ -95,10 +97,8 @@ class SchemaCatalogue @JvmOverloads constructor( // at all. while (prevSize != schemas.size) { prevSize = schemas.size - val useTypes = schemas.toSchemaMap() - val ignoreFiles = schemas.values.asSequence() - .map { it.path } - .filterNotNullTo(HashSet()) + val useTypes = schemas.mapValues { (_, v) -> v.schema } + val ignoreFiles = schemas.values.mapTo(HashSet()) { it.path } schemas.putParsedSchemas(avroFiles, ignoreFiles, useTypes, scope) } @@ -107,26 +107,32 @@ class SchemaCatalogue @JvmOverloads constructor( avroFiles.keys.asSequence() .filter { it !in mappedPaths } .distinct() - .mapTo(unmappedFiles) { p -> SchemaMetadata(null, scope, p) } + .mapTo(unmappedFiles) { p -> FailedSchemaMetadata(scope, p) } } - private fun MutableMap.putParsedSchemas( + private suspend fun MutableMap.putParsedSchemas( customSchemas: Map, ignoreFiles: Set, useTypes: Map, scope: Scope, - ) = customSchemas.asSequence() + ) = customSchemas .filter { (p, _) -> p !in ignoreFiles } - .forEach { (p, schema) -> + .entries + .forkJoin { (p, schema) -> val parser = Schema.Parser() parser.addTypes(useTypes) - try { - val parsedSchema = parser.parse(schema) - put(parsedSchema.fullName, SchemaMetadata(parsedSchema, scope, p)) - } catch (ex: Exception) { - logger.debug("Cannot parse schema {}: {}", p, ex.toString()) + withContext(Dispatchers.IO) { + try { + val parsedSchema = parser.parse(schema) + parsedSchema.fullName to SchemaMetadata(parsedSchema, scope, p) + } catch (ex: Exception) { + logger.debug("Cannot parse schema {}: {}", p, ex.toString()) + null + } } } + .filterNotNull() + .toMap(this@putParsedSchemas) /** * Returns an avro topic with the schemas from this catalogue. @@ -152,13 +158,5 @@ class SchemaCatalogue @JvmOverloads constructor( companion object { private val logger = LoggerFactory.getLogger(SchemaCatalogue::class.java) - - fun Map.toSchemaMap(): Map = buildMap(size) { - this@toSchemaMap.forEach { (k, v) -> - if (v.schema != null) { - put(k, v.schema) - } - } - } } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt index 360e2d1e..ec4b25f8 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt @@ -19,6 +19,10 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.PropertyNamingStrategies import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.runBlocking +import org.radarbase.kotlin.coroutines.forkJoin import org.radarbase.schema.SchemaCatalogue import org.radarbase.schema.Scope import org.radarbase.schema.specification.active.ActiveSource @@ -29,18 +33,17 @@ import org.radarbase.schema.specification.monitor.MonitorSource import org.radarbase.schema.specification.passive.PassiveSource import org.radarbase.schema.specification.push.PushSource import org.radarbase.schema.specification.stream.StreamGroup +import org.radarbase.schema.util.SchemaUtils.listRecursive import org.radarbase.schema.validation.ValidationHelper import org.radarbase.schema.validation.ValidationHelper.SPECIFICATIONS_PATH import org.radarbase.topic.AvroTopic import org.slf4j.LoggerFactory import java.io.IOException -import java.nio.file.Files import java.nio.file.InvalidPathException import java.nio.file.Path import java.nio.file.PathMatcher import java.util.stream.Stream import kotlin.io.path.exists -import kotlin.streams.asSequence class SourceCatalogue internal constructor( val schemaCatalogue: SchemaCatalogue, @@ -104,19 +107,41 @@ class SourceCatalogue internal constructor( schemaConfig, ) val pathMatcher = sourceConfig.pathMatcher(specRoot) - return SourceCatalogue( - schemaCatalogue, - initSources(mapper, specRoot, Scope.ACTIVE, pathMatcher, sourceConfig.active), - initSources(mapper, specRoot, Scope.MONITOR, pathMatcher, sourceConfig.monitor), - initSources(mapper, specRoot, Scope.PASSIVE, pathMatcher, sourceConfig.passive), - initSources(mapper, specRoot, Scope.STREAM, pathMatcher, sourceConfig.stream), - initSources(mapper, specRoot, Scope.CONNECTOR, pathMatcher, sourceConfig.connector), - initSources(mapper, specRoot, Scope.PUSH, pathMatcher, sourceConfig.push), - ) + + return runBlocking { + val activeJob = async { + initSources(mapper, specRoot, Scope.ACTIVE, pathMatcher, sourceConfig.active) + } + val monitorJob = async { + initSources(mapper, specRoot, Scope.MONITOR, pathMatcher, sourceConfig.monitor) + } + val passiveJob = async { + initSources(mapper, specRoot, Scope.PASSIVE, pathMatcher, sourceConfig.passive) + } + val streamJob = async { + initSources(mapper, specRoot, Scope.STREAM, pathMatcher, sourceConfig.stream) + } + val connectorJob = async { + initSources(mapper, specRoot, Scope.CONNECTOR, pathMatcher, sourceConfig.connector) + } + val pushJob = async { + initSources(mapper, specRoot, Scope.PUSH, pathMatcher, sourceConfig.push) + } + + SourceCatalogue( + schemaCatalogue, + activeSources = activeJob.await(), + monitorSources = monitorJob.await(), + passiveSources = passiveJob.await(), + streamGroups = streamJob.await(), + connectorSources = connectorJob.await(), + pushSources = pushJob.await(), + ) + } } @Throws(IOException::class) - private inline fun initSources( + private suspend inline fun initSources( mapper: ObjectMapper, root: Path, scope: Scope, @@ -129,19 +154,18 @@ class SourceCatalogue internal constructor( return otherSources } val reader = mapper.readerFor(T::class.java) - return buildList { - Files.walk(baseFolder).use { walker -> - walker - .asSequence() - .filter(sourceRootPathMatcher::matches) - .forEach { p -> - try { - add(reader.readValue(p.toFile())) - } catch (ex: IOException) { - logger.error("Failed to load configuration {}: {}", p, ex.toString()) - } + val fileList = baseFolder.listRecursive(sourceRootPathMatcher::matches) + return buildList(fileList.size + otherSources.size) { + fileList + .forkJoin(Dispatchers.IO) { p -> + try { + reader.readValue(p.toFile()) + } catch (ex: IOException) { + logger.error("Failed to load configuration {}: {}", p, ex.toString()) + null } - } + } + .filterIsInstanceTo(this@buildList) addAll(otherSources) } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt index bb1e9222..1718599d 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/util/SchemaUtils.kt @@ -15,11 +15,17 @@ */ package org.radarbase.schema.util +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext import org.slf4j.LoggerFactory import java.io.IOException +import java.nio.file.Files +import java.nio.file.Path import java.util.Properties import java.util.function.Function +import java.util.stream.Collectors import java.util.stream.Stream +import kotlin.io.path.walk object SchemaUtils { private val logger = LoggerFactory.getLogger(SchemaUtils::class.java) @@ -105,6 +111,14 @@ object SchemaUtils { } } + suspend fun Path.listRecursive(pathMatcher: (Path) -> Boolean): List = withContext(Dispatchers.IO) { + Files.walk(this@listRecursive).use { walker -> + walker + .filter(pathMatcher) + .collect(Collectors.toList()) + } + } + /** * Function that may throw an exception. * @param T type of value taken. diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt index 7d55910c..1ba073c1 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt @@ -16,13 +16,13 @@ package org.radarbase.schema.validation import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch import org.apache.avro.Schema import org.radarbase.schema.SchemaCatalogue import org.radarbase.schema.Scope import org.radarbase.schema.specification.DataProducer import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.validation.rules.FailedSchemaMetadata import org.radarbase.schema.validation.rules.RadarSchemaMetadataRules import org.radarbase.schema.validation.rules.RadarSchemaRules import org.radarbase.schema.validation.rules.SchemaMetadata @@ -55,17 +55,15 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { } else { catalogue.sources.stream() } - return validationContext { - val schemas = producers - .flatMap { it.data.stream() } - .flatMap { topic -> - val (keySchema, valueSchema) = catalogue.schemaCatalogue.getSchemaMetadata(topic) - Stream.of(keySchema, valueSchema) - } - .filter { it.schema != null } - .sorted(Comparator.comparing { it.schema!!.fullName }) - .collect(Collectors.toSet()) + val schemas = producers + .flatMap { it.data.stream() } + .flatMap { topic -> + val (keySchema, valueSchema) = catalogue.schemaCatalogue.getSchemaMetadata(topic) + Stream.of(keySchema, valueSchema) + } + .collect(Collectors.toSet()) + return validationContext { schemas.forEach { metadata -> if (pathMatcher.matches(metadata.path)) { validator.launchValidation(metadata) @@ -106,7 +104,7 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { private fun parsingValidator( scope: Scope?, schemaCatalogue: SchemaCatalogue, - ): Validator { + ): Validator { val useTypes = buildMap { schemaCatalogue.schemas.forEach { (key, value) -> if (value.scope == scope) { @@ -117,9 +115,9 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { return Validator { metadata -> val parser = Schema.Parser() parser.addTypes(useTypes) - coroutineScope.launch(Dispatchers.IO) { + launchValidation(Dispatchers.IO) { try { - parser.parse(metadata.path?.toFile()) + parser.parse(metadata.path.toFile()) } catch (ex: Exception) { raise("Cannot parse schema", ex) } @@ -158,6 +156,6 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { } } - fun isAvscFile(file: Path): Boolean = file.extension.equals(AVRO_EXTENSION, ignoreCase = true) + fun Path.isAvscFile(): Boolean = extension.equals(AVRO_EXTENSION, ignoreCase = true) } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt index 8babbb66..9ebb9fd8 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt @@ -17,18 +17,15 @@ package org.radarbase.schema.validation import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.dataformat.yaml.YAMLFactory -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext import org.radarbase.schema.Scope import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.util.SchemaUtils.listRecursive import org.radarbase.schema.validation.rules.Validator import org.radarbase.schema.validation.rules.pathExtensionValidator import org.slf4j.LoggerFactory import java.io.IOException -import java.nio.file.Files import java.nio.file.Path import java.nio.file.PathMatcher -import java.util.stream.Collectors /** * Validates RADAR-Schemas specifications. @@ -59,7 +56,7 @@ class SpecificationsValidator( } suspend fun isValidSpecification(clazz: Class?): List { - val paths = root.fetchChildren() + val paths = root.listRecursive { pathMatcher.matches(it) } return validationContext { val isParseableAsClass = isYmlFileParseable(clazz) paths.forEach { p -> @@ -69,14 +66,6 @@ class SpecificationsValidator( } } - private suspend fun Path.fetchChildren(): List = withContext(Dispatchers.IO) { - Files.walk(this@fetchChildren).use { walker -> - walker - .filter { pathMatcher.matches(it) } - .collect(Collectors.toList()) - } - } - private fun isYmlFileParseable(clazz: Class?) = Validator { path -> try { mapper.readerFor(clazz).readValue(path.toFile()) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt index 3e6f61ac..3709ba4b 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt @@ -2,53 +2,60 @@ package org.radarbase.schema.validation import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.channels.Channel -import kotlinx.coroutines.channels.toList +import kotlinx.coroutines.channels.Channel.Factory.UNLIMITED +import kotlinx.coroutines.channels.SendChannel +import kotlinx.coroutines.channels.consumeEach import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.launch import org.radarbase.schema.validation.rules.Validator +import kotlin.coroutines.CoroutineContext +import kotlin.coroutines.EmptyCoroutineContext interface ValidationContext { - val coroutineScope: CoroutineScope - fun raise(message: String, ex: Exception? = null) fun Validator.launchValidation(value: T) + + fun launchValidation(context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.() -> Unit) } private class ValidationContextImpl( - override val coroutineScope: CoroutineScope, + private val channel: SendChannel, + private val coroutineScope: CoroutineScope, ) : ValidationContext { - private val channel = Channel(Channel.UNLIMITED) - private lateinit var producerCoroutineScope: CoroutineScope - - suspend fun runValidation(block: ValidationContext.() -> Unit): List { - coroutineScope.launch { - coroutineScope { - producerCoroutineScope = this - block() - } - channel.close() - } - return channel.toList().distinct() - } override fun raise(message: String, ex: Exception?) { channel.trySend(ValidationException(message, ex)) } override fun Validator.launchValidation(value: T) { - producerCoroutineScope.launch { + coroutineScope.launch { runValidation(value) } } + + override fun launchValidation(context: CoroutineContext, block: suspend CoroutineScope.() -> Unit) { + coroutineScope.launch(context, block = block) + } } -suspend fun validationContext(block: ValidationContext.() -> Unit) = +suspend fun validationContext(block: ValidationContext.() -> Unit): List { + val channel = Channel(UNLIMITED) coroutineScope { - val context = ValidationContextImpl(this) - context.runValidation(block) + val producerJob = launch { + with(ValidationContextImpl(channel, this@launch)) { + block() + } + } + producerJob.join() + channel.close() } + return buildSet { + channel.consumeEach { add(it) } + }.toList() +} + suspend fun Validator.validate(value: T) = validationContext { - launchValidation(value) + launchValidation(value = value) } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt index a7979f93..111a1aa1 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt @@ -20,7 +20,7 @@ class RadarSchemaMetadataRules( ) : SchemaMetadataRules { private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) - override val isShemaLocationCorrect = all( + override val isSchemaLocationCorrect = all( isNamespaceSchemaLocationCorrect(), isNameSchemaLocationCorrect(), ) @@ -28,7 +28,7 @@ class RadarSchemaMetadataRules( private fun isNamespaceSchemaLocationCorrect() = Validator { metadata -> try { val expected = getNamespace(schemaRoot, metadata.path, metadata.scope) - val namespace = metadata.schema?.namespace + val namespace = metadata.schema.namespace if (!expected.equals(namespace, ignoreCase = true)) { raise( metadata, @@ -41,21 +41,15 @@ class RadarSchemaMetadataRules( } private fun isNameSchemaLocationCorrect() = Validator { metadata -> - if (metadata.path == null) { - raise(metadata, "Missing metadata path") - return@Validator - } val expected = getRecordName(metadata.path) - if (!expected.equals(metadata.schema?.name, ignoreCase = true)) { + if (!expected.equals(metadata.schema.name, ignoreCase = true)) { raise(metadata, "Record name should match file name. Expected record name is \"$expected\".") } } override fun isSchemaCorrect(validator: Validator) = Validator { metadata -> - when { - metadata.schema == null -> raise("Missing schema") - pathMatcher.matches(metadata.path) -> validator.launchValidation(metadata.schema) - else -> Unit + if (pathMatcher.matches(metadata.path)) { + validator.launchValidation(metadata.schema) } } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.kt index a4dc7d24..0a5eb6be 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadata.kt @@ -8,7 +8,12 @@ import java.nio.file.Path * Schema with metadata. */ data class SchemaMetadata( - val schema: Schema?, + val schema: Schema, val scope: Scope, - val path: Path?, + val path: Path, +) + +data class FailedSchemaMetadata( + val scope: Scope, + val path: Path, ) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt index ed0c4acb..4fdd67b2 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt @@ -8,20 +8,14 @@ interface SchemaMetadataRules { val schemaRules: SchemaRules /** Checks the location of a schema with its internal data. */ - val isShemaLocationCorrect: Validator + val isSchemaLocationCorrect: Validator /** * Validates any schema file. It will choose the correct validation method based on the scope * and type of the schema. */ fun isSchemaMetadataValid(scopeSpecificValidation: Boolean) = Validator { metadata -> - if (metadata.schema == null) { - raise("Missing schema") - return@Validator - } - val schemaRules = schemaRules - - isShemaLocationCorrect.launchValidation(metadata) + isSchemaLocationCorrect.launchValidation(metadata) val ruleset = when { metadata.schema.type == Schema.Type.ENUM -> schemaRules.isEnumValid @@ -36,14 +30,10 @@ interface SchemaMetadataRules { /** Validates schemas without their metadata. */ fun isSchemaCorrect(validator: Validator) = Validator { metadata -> - if (metadata.schema == null) { - raise(metadata, "Schema is empty") - } else { - validator.launchValidation(metadata.schema) - } + validator.launchValidation(metadata.schema) } } fun ValidationContext.raise(metadata: SchemaMetadata, text: String) { - raise("Schema ${metadata.schema?.fullName} at ${metadata.path} is invalid. $text") + raise("Schema ${metadata.schema.fullName} at ${metadata.path} is invalid. $text") } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt index c8fd1dcf..565f30b3 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt @@ -36,7 +36,7 @@ internal class SchemaConfigTest { assertEquals(1, schemaCatalogue.schemas.size) val (fullName, schemaMetadata) = schemaCatalogue.schemas.entries.first() assertEquals("org.radarcns.monitor.application.ApplicationUptime2", fullName) - assertEquals("org.radarcns.monitor.application.ApplicationUptime2", schemaMetadata.schema!!.fullName) + assertEquals("org.radarcns.monitor.application.ApplicationUptime2", schemaMetadata.schema.fullName) assertEquals(commonsRoot.resolve("monitor/application/test.avsc"), schemaMetadata.path) assertEquals(Scope.MONITOR, schemaMetadata.scope) } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt index cb3075ed..c5e08a03 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt @@ -66,7 +66,7 @@ class RadarSchemaMetadataRulesTest { MONITOR.getPath(SourceCatalogueValidationTest.BASE_PATH.resolve(ValidationHelper.COMMONS_PATH)) assertNotNull(root) val path = root.resolve("test/record_name.avsc") - val result = validator.isShemaLocationCorrect + val result = validator.isSchemaLocationCorrect .validate(SchemaMetadata(schema, MONITOR, path)) assertEquals(1, result.count()) } @@ -82,7 +82,7 @@ class RadarSchemaMetadataRulesTest { MONITOR.getPath(SourceCatalogueValidationTest.BASE_PATH.resolve(ValidationHelper.COMMONS_PATH)) assertNotNull(root) val path = root.resolve("test/record_name.avsc") - val result = validator.isShemaLocationCorrect + val result = validator.isSchemaLocationCorrect .validate(SchemaMetadata(schema, MONITOR, path)) assertEquals(1, result.count()) } @@ -97,7 +97,7 @@ class RadarSchemaMetadataRulesTest { .record(fieldName) .fields() .endRecord() - var result = validator.isShemaLocationCorrect + var result = validator.isSchemaLocationCorrect .validate(SchemaMetadata(schema, PASSIVE, filePath)) assertEquals(2, result.count()) fieldName = "EmpaticaE4Acceleration" @@ -108,7 +108,7 @@ class RadarSchemaMetadataRulesTest { .record(fieldName) .fields() .endRecord() - result = validator.isShemaLocationCorrect + result = validator.isSchemaLocationCorrect .validate(SchemaMetadata(schema, PASSIVE, filePath)) assertEquals("", format(result)) } From 8d2ba3093c0701d30e0313af440edead242f90b5 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 27 Sep 2023 13:42:09 +0200 Subject: [PATCH 08/27] Misc migration to coroutines --- .../schema/service/SourceCatalogueServer.kt | 16 +- .../service/SourceCatalogueServerTest.kt | 6 +- .../org/radarbase/schema/SchemaCatalogue.kt | 198 +++++++++--------- .../schema/specification/DataTopic.kt | 2 +- .../schema/specification/SourceCatalogue.kt | 186 ++++++++-------- .../specification/stream/StreamDataTopic.kt | 2 +- .../schema/validation/SchemaValidator.kt | 17 +- .../schema/validation/ValidationContext.kt | 24 +++ .../schema/validation/ValidationException.kt | 14 ++ .../schema/validation/ValidationHelper.kt | 4 +- .../rules/RadarSchemaMetadataRules.kt | 4 +- .../specification/config/SchemaConfigTest.kt | 5 +- .../schema/validation/SchemaValidatorTest.kt | 13 +- .../SourceCatalogueValidationTest.kt | 9 +- .../validation/SpecificationsValidatorTest.kt | 12 +- .../rules/RadarSchemaFieldRulesTest.kt | 12 +- .../rules/RadarSchemaMetadataRulesTest.kt | 11 +- .../radarbase/schema/tools/CommandLineApp.kt | 37 ++-- .../schema/tools/KafkaTopicsCommand.kt | 33 ++- .../org/radarbase/schema/tools/ListCommand.kt | 2 +- .../schema/tools/SchemaRegistryCommand.kt | 33 ++- .../org/radarbase/schema/tools/SubCommand.kt | 2 +- .../schema/tools/ValidatorCommand.kt | 9 +- 23 files changed, 341 insertions(+), 310 deletions(-) diff --git a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueServer.kt b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueServer.kt index 4f56eebc..ba98c26f 100644 --- a/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueServer.kt +++ b/java-sdk/radar-catalog-server/src/main/java/org/radarbase/schema/service/SourceCatalogueServer.kt @@ -1,5 +1,6 @@ package org.radarbase.schema.service +import kotlinx.coroutines.runBlocking import net.sourceforge.argparse4j.ArgumentParsers import net.sourceforge.argparse4j.helper.HelpScreenException import net.sourceforge.argparse4j.inf.ArgumentParserException @@ -10,7 +11,6 @@ import org.radarbase.jersey.enhancer.Enhancers.exception import org.radarbase.jersey.enhancer.Enhancers.health import org.radarbase.jersey.enhancer.Enhancers.mapper import org.radarbase.schema.specification.SourceCatalogue -import org.radarbase.schema.specification.SourceCatalogue.Companion.load import org.radarbase.schema.specification.config.ToolConfig import org.radarbase.schema.specification.config.loadToolConfig import org.slf4j.LoggerFactory @@ -50,7 +50,7 @@ class SourceCatalogueServer( private val logger = LoggerFactory.getLogger(SourceCatalogueServer::class.java) @JvmStatic - fun main(args: Array) { + fun main(vararg args: String) { val logger = LoggerFactory.getLogger(SourceCatalogueServer::class.java) val parser = ArgumentParsers.newFor("radar-catalog-server") .addHelp(true) @@ -77,11 +77,13 @@ class SourceCatalogueServer( } val config = loadConfig(parsedArgs.getString("config")) val sourceCatalogue: SourceCatalogue = try { - load( - Paths.get(parsedArgs.getString("root")), - schemaConfig = config.schemas, - sourceConfig = config.sources, - ) + runBlocking { + SourceCatalogue( + Paths.get(parsedArgs.getString("root")), + schemaConfig = config.schemas, + sourceConfig = config.sources, + ) + } } catch (e: IOException) { logger.error("Failed to load source catalogue", e) logger.error(parser.formatUsage()) diff --git a/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.kt b/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.kt index 520924b6..ac855ef6 100644 --- a/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.kt +++ b/java-sdk/radar-catalog-server/src/test/java/org/radarbase/schema/service/SourceCatalogueServerTest.kt @@ -20,7 +20,7 @@ import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test -import org.radarbase.schema.specification.SourceCatalogue.Companion.load +import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.config.SchemaConfig import org.radarbase.schema.specification.config.SourceConfig import java.nio.file.Paths @@ -37,7 +37,9 @@ internal class SourceCatalogueServerTest { server = SourceCatalogueServer(9876) serverThread = Thread { try { - val sourceCatalog = load(Paths.get("../.."), SchemaConfig(), SourceConfig()) + val sourceCatalog = runBlocking { + SourceCatalogue(Paths.get("../.."), SchemaConfig(), SourceConfig()) + } server.start(sourceCatalog) } catch (e: IllegalStateException) { // this is acceptable diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt index 049986d8..c58a4174 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/SchemaCatalogue.kt @@ -18,34 +18,14 @@ import java.io.IOException import java.nio.file.Path import java.nio.file.PathMatcher import java.util.* +import kotlin.collections.HashSet import kotlin.io.path.exists -import kotlin.io.path.inputStream +import kotlin.io.path.readText -class SchemaCatalogue @JvmOverloads constructor( - private val schemaRoot: Path, - config: SchemaConfig, - scope: Scope? = null, +class SchemaCatalogue( + val schemas: Map, + val unmappedSchemas: List, ) { - val schemas: Map - val unmappedAvroFiles: List - - init { - val schemaTemp = HashMap() - val unmappedTemp = mutableListOf() - val matcher = config.pathMatcher(schemaRoot) - runBlocking { - if (scope != null) { - loadSchemas(schemaTemp, unmappedTemp, scope, matcher, config) - } else { - for (useScope in Scope.entries) { - loadSchemas(schemaTemp, unmappedTemp, useScope, matcher, config) - } - } - } - schemas = schemaTemp.toMap() - unmappedAvroFiles = unmappedTemp.toList() - } - /** * Returns an avro topic with the schemas from this catalogue. * @param config avro topic configuration @@ -54,8 +34,8 @@ class SchemaCatalogue @JvmOverloads constructor( * @throws NullPointerException if the key or value schema configurations are null * @throws IllegalArgumentException if the topic configuration is null */ - fun getGenericAvroTopic(config: AvroTopicConfig): AvroTopic { - val (keySchema, valueSchema) = getSchemaMetadata(config) + fun genericAvroTopic(config: AvroTopicConfig): AvroTopic { + val (keySchema, valueSchema) = topicSchemas(config) return AvroTopic( requireNotNull(config.topic) { "Missing Avro topic in configuration" }, requireNotNull(keySchema.schema) { "Missing Avro key schema" }, @@ -65,75 +45,6 @@ class SchemaCatalogue @JvmOverloads constructor( ) } - @Throws(IOException::class) - private suspend fun loadSchemas( - schemas: MutableMap, - unmappedFiles: MutableList, - scope: Scope, - matcher: PathMatcher, - config: SchemaConfig, - ) { - val walkRoot = schemaRoot.resolve(scope.lower) - val avroFiles = buildMap { - if (walkRoot.exists()) { - walkRoot - .listRecursive { matcher.matches(it) && it.isAvscFile() } - .forkJoin(Dispatchers.IO) { p -> - p.inputStream().reader().use { - p to it.readText() - } - } - .toMap(this@buildMap) - } - config.schemas(scope).forEach { (key, value) -> - put(walkRoot.resolve(key), value) - } - } - - var prevSize = -1 - - // Recursively parse all schemas. - // If the parsed schema size does not change anymore, the final schemas cannot be parsed - // at all. - while (prevSize != schemas.size) { - prevSize = schemas.size - val useTypes = schemas.mapValues { (_, v) -> v.schema } - val ignoreFiles = schemas.values.mapTo(HashSet()) { it.path } - - schemas.putParsedSchemas(avroFiles, ignoreFiles, useTypes, scope) - } - val mappedPaths = schemas.values.mapTo(HashSet()) { it.path } - - avroFiles.keys.asSequence() - .filter { it !in mappedPaths } - .distinct() - .mapTo(unmappedFiles) { p -> FailedSchemaMetadata(scope, p) } - } - - private suspend fun MutableMap.putParsedSchemas( - customSchemas: Map, - ignoreFiles: Set, - useTypes: Map, - scope: Scope, - ) = customSchemas - .filter { (p, _) -> p !in ignoreFiles } - .entries - .forkJoin { (p, schema) -> - val parser = Schema.Parser() - parser.addTypes(useTypes) - withContext(Dispatchers.IO) { - try { - val parsedSchema = parser.parse(schema) - parsedSchema.fullName to SchemaMetadata(parsedSchema, scope, p) - } catch (ex: Exception) { - logger.debug("Cannot parse schema {}: {}", p, ex.toString()) - null - } - } - } - .filterNotNull() - .toMap(this@putParsedSchemas) - /** * Returns an avro topic with the schemas from this catalogue. * @param config avro topic configuration @@ -142,7 +53,7 @@ class SchemaCatalogue @JvmOverloads constructor( * @throws NullPointerException if the key or value schema configurations are null * @throws IllegalArgumentException if the topic configuration is null */ - fun getSchemaMetadata(config: AvroTopicConfig): Pair { + fun topicSchemas(config: AvroTopicConfig): Pair { val parsedKeySchema = schemas[config.keySchema] ?: throw NoSuchElementException( "Key schema " + config.keySchema + @@ -155,8 +66,97 @@ class SchemaCatalogue @JvmOverloads constructor( ) return Pair(parsedKeySchema, parsedValueSchema) } +} - companion object { - private val logger = LoggerFactory.getLogger(SchemaCatalogue::class.java) +private val logger = LoggerFactory.getLogger(SchemaCatalogue::class.java) + +/** + * Load a schema catalogue. + * @param schemaRoot root of schema directory. + * @param config schema configuration + * @param scope scope to read. If null, all scopes are read. + */ +suspend fun SchemaCatalogue( + schemaRoot: Path, + config: SchemaConfig, + scope: Scope? = null, +): SchemaCatalogue { + val matcher = config.pathMatcher(schemaRoot) + val (schemas, unmapped) = runBlocking { + if (scope != null) { + loadSchemas(schemaRoot, scope, matcher, config) + } else { + Scope.entries + .forkJoin { s -> loadSchemas(schemaRoot, s, matcher, config) } + .reduce { (m1, l1), (m2, l2) -> Pair(m1 + m2, l1 + l2) } + } + } + return SchemaCatalogue(schemas, unmapped) +} + +@Throws(IOException::class) +private suspend fun loadSchemas( + schemaRoot: Path, + scope: Scope, + matcher: PathMatcher, + config: SchemaConfig, +): Pair, List> { + val scopeRoot = schemaRoot.resolve(scope.lower) + val avroFiles = buildMap { + if (scopeRoot.exists()) { + scopeRoot + .listRecursive { matcher.matches(it) && it.isAvscFile() } + .forkJoin(Dispatchers.IO) { p -> + p to p.readText() + } + .toMap(this@buildMap) + } + config.schemas(scope).forEach { (key, value) -> + put(scopeRoot.resolve(key), value) + } } + + var prevSize = -1 + + // Recursively parse all schemas. + // If the parsed schema size does not change anymore, the final schemas cannot be parsed + // at all. + val schemas = buildMap { + while (prevSize != size) { + prevSize = size + val useTypes = mapValues { (_, v) -> v.schema } + val ignoreFiles = values.mapTo(HashSet()) { it.path } + + putAll(avroFiles.parseSchemas(ignoreFiles, useTypes, scope)) + } + } + val mappedPaths = schemas.values.mapTo(HashSet()) { it.path } + + val unmapped = avroFiles.keys + .filterTo(HashSet()) { it !in mappedPaths } + .map { p -> FailedSchemaMetadata(scope, p) } + + return Pair(schemas, unmapped) } + +private suspend fun Map.parseSchemas( + ignoreFiles: Set, + useTypes: Map, + scope: Scope, +) = filter { (p, _) -> p !in ignoreFiles } + .entries + .forkJoin { (p, schema) -> + val parser = Schema.Parser() + parser.addTypes(useTypes) + withContext(Dispatchers.IO) { + try { + val parsedSchema = parser.parse(schema) + parsedSchema.fullName to SchemaMetadata(parsedSchema, scope, p) + } catch (ex: Exception) { + logger.debug("Cannot parse schema {}: {}", p, ex.toString()) + null + } + } + } + .filterNotNull() + .toMap() diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.kt index c77f69f5..f1644882 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/DataTopic.kt @@ -55,7 +55,7 @@ class DataTopic : AvroTopicConfig() { @JsonIgnore @Throws(IOException::class) fun topics(schemaCatalogue: SchemaCatalogue): Stream> { - return Stream.of(schemaCatalogue.getGenericAvroTopic(this)) + return Stream.of(schemaCatalogue.genericAvroTopic(this)) } @JsonProperty("key_schema") diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt index ec4b25f8..c670f8c0 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/SourceCatalogue.kt @@ -21,7 +21,7 @@ import com.fasterxml.jackson.databind.PropertyNamingStrategies import com.fasterxml.jackson.dataformat.yaml.YAMLFactory import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async -import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.coroutineScope import org.radarbase.kotlin.coroutines.forkJoin import org.radarbase.schema.SchemaCatalogue import org.radarbase.schema.Scope @@ -72,102 +72,108 @@ class SourceCatalogue internal constructor( val topics: Stream> get() = sources.stream() .flatMap { it.topics(schemaCatalogue) } +} - companion object { - private val logger = LoggerFactory.getLogger(SourceCatalogue::class.java) +private val logger = LoggerFactory.getLogger(SourceCatalogue::class.java) - /** - * Load the source catalogue based at the given root directory. - * @param root Directory containing a specifications subdirectory. - * @return parsed source catalogue. - * @throws InvalidPathException if the `specifications` directory cannot be found in given - * root. - * @throws IOException if the source catalogue could not be read. - */ - @Throws(IOException::class, InvalidPathException::class) - fun load( - root: Path, - schemaConfig: SchemaConfig, - sourceConfig: SourceConfig, - ): SourceCatalogue { - val specRoot = root.resolve(SPECIFICATIONS_PATH) - val mapper = ObjectMapper(YAMLFactory()).apply { - propertyNamingStrategy = PropertyNamingStrategies.SNAKE_CASE - setVisibility( - serializationConfig.defaultVisibilityChecker - .withFieldVisibility(JsonAutoDetect.Visibility.ANY) - .withGetterVisibility(JsonAutoDetect.Visibility.NONE) - .withIsGetterVisibility(JsonAutoDetect.Visibility.NONE) - .withSetterVisibility(JsonAutoDetect.Visibility.NONE) - .withCreatorVisibility(JsonAutoDetect.Visibility.NONE), - ) - } - val schemaCatalogue = SchemaCatalogue( +/** + * Load the source catalogue based at the given root directory. + * @param root Directory containing a specifications subdirectory. + * @return parsed source catalogue. + * @throws InvalidPathException if the `specifications` directory cannot be found in given + * root. + * @throws IOException if the source catalogue could not be read. + */ +@Throws(IOException::class, InvalidPathException::class) +suspend fun SourceCatalogue( + root: Path, + schemaConfig: SchemaConfig, + sourceConfig: SourceConfig, +): SourceCatalogue { + val specRoot = root.resolve(SPECIFICATIONS_PATH) + val mapper = ObjectMapper(YAMLFactory()).apply { + propertyNamingStrategy = PropertyNamingStrategies.SNAKE_CASE + setVisibility( + serializationConfig.defaultVisibilityChecker + .withFieldVisibility(JsonAutoDetect.Visibility.ANY) + .withGetterVisibility(JsonAutoDetect.Visibility.NONE) + .withIsGetterVisibility(JsonAutoDetect.Visibility.NONE) + .withSetterVisibility(JsonAutoDetect.Visibility.NONE) + .withCreatorVisibility(JsonAutoDetect.Visibility.NONE), + ) + } + val pathMatcher = sourceConfig.pathMatcher(specRoot) + + return coroutineScope { + val schemaCatalogueJob = async { + SchemaCatalogue( root.resolve(ValidationHelper.COMMONS_PATH), schemaConfig, ) - val pathMatcher = sourceConfig.pathMatcher(specRoot) - - return runBlocking { - val activeJob = async { - initSources(mapper, specRoot, Scope.ACTIVE, pathMatcher, sourceConfig.active) - } - val monitorJob = async { - initSources(mapper, specRoot, Scope.MONITOR, pathMatcher, sourceConfig.monitor) - } - val passiveJob = async { - initSources(mapper, specRoot, Scope.PASSIVE, pathMatcher, sourceConfig.passive) - } - val streamJob = async { - initSources(mapper, specRoot, Scope.STREAM, pathMatcher, sourceConfig.stream) - } - val connectorJob = async { - initSources(mapper, specRoot, Scope.CONNECTOR, pathMatcher, sourceConfig.connector) - } - val pushJob = async { - initSources(mapper, specRoot, Scope.PUSH, pathMatcher, sourceConfig.push) - } - - SourceCatalogue( - schemaCatalogue, - activeSources = activeJob.await(), - monitorSources = monitorJob.await(), - passiveSources = passiveJob.await(), - streamGroups = streamJob.await(), - connectorSources = connectorJob.await(), - pushSources = pushJob.await(), - ) - } + } + val activeJob = async { + initSources(mapper, specRoot, Scope.ACTIVE, pathMatcher, sourceConfig.active) + } + val monitorJob = async { + initSources(mapper, specRoot, Scope.MONITOR, pathMatcher, sourceConfig.monitor) + } + val passiveJob = async { + initSources(mapper, specRoot, Scope.PASSIVE, pathMatcher, sourceConfig.passive) + } + val streamJob = async { + initSources(mapper, specRoot, Scope.STREAM, pathMatcher, sourceConfig.stream) + } + val connectorJob = async { + initSources( + mapper, + specRoot, + Scope.CONNECTOR, + pathMatcher, + sourceConfig.connector, + ) + } + val pushJob = async { + initSources(mapper, specRoot, Scope.PUSH, pathMatcher, sourceConfig.push) } - @Throws(IOException::class) - private suspend inline fun initSources( - mapper: ObjectMapper, - root: Path, - scope: Scope, - sourceRootPathMatcher: PathMatcher, - otherSources: List, - ): List { - val baseFolder = root.resolve(scope.lower) - if (!baseFolder.exists()) { - logger.info("{} sources folder not present at {}", scope, baseFolder) - return otherSources - } - val reader = mapper.readerFor(T::class.java) - val fileList = baseFolder.listRecursive(sourceRootPathMatcher::matches) - return buildList(fileList.size + otherSources.size) { - fileList - .forkJoin(Dispatchers.IO) { p -> - try { - reader.readValue(p.toFile()) - } catch (ex: IOException) { - logger.error("Failed to load configuration {}: {}", p, ex.toString()) - null - } - } - .filterIsInstanceTo(this@buildList) - addAll(otherSources) + SourceCatalogue( + schemaCatalogueJob.await(), + activeSources = activeJob.await(), + monitorSources = monitorJob.await(), + passiveSources = passiveJob.await(), + streamGroups = streamJob.await(), + connectorSources = connectorJob.await(), + pushSources = pushJob.await(), + ) + } +} + +@Throws(IOException::class) +private suspend inline fun initSources( + mapper: ObjectMapper, + root: Path, + scope: Scope, + sourceRootPathMatcher: PathMatcher, + otherSources: List, +): List { + val baseFolder = root.resolve(scope.lower) + if (!baseFolder.exists()) { + logger.info("{} sources folder not present at {}", scope, baseFolder) + return otherSources + } + val reader = mapper.readerFor(T::class.java) + val fileList = baseFolder.listRecursive(sourceRootPathMatcher::matches) + return buildList(fileList.size + otherSources.size) { + fileList + .forkJoin(Dispatchers.IO) { p -> + try { + reader.readValue(p.toFile()) + } catch (ex: IOException) { + logger.error("Failed to load configuration {}: {}", p, ex.toString()) + null + } } - } + .filterIsInstanceTo(this@buildList) + addAll(otherSources) } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt index bc2a51db..1b43c70d 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/specification/stream/StreamDataTopic.kt @@ -86,7 +86,7 @@ class StreamDataTopic : DataTopic() { config.topic = topic config.keySchema = keySchema config.valueSchema = valueSchema - Stream.of(schemaCatalogue.getGenericAvroTopic(config)) + Stream.of(schemaCatalogue.genericAvroTopic(config)) }, ) } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt index 1ba073c1..1699323e 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt @@ -58,7 +58,7 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { val schemas = producers .flatMap { it.data.stream() } .flatMap { topic -> - val (keySchema, valueSchema) = catalogue.schemaCatalogue.getSchemaMetadata(topic) + val (keySchema, valueSchema) = catalogue.schemaCatalogue.topicSchemas(topic) Stream.of(keySchema, valueSchema) } .collect(Collectors.toSet()) @@ -90,7 +90,7 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { val validator = rules.isSchemaMetadataValid(false) val parsingValidator = parsingValidator(scope, schemaCatalogue) - schemaCatalogue.unmappedAvroFiles.forEach { metadata -> + schemaCatalogue.unmappedSchemas.forEach { metadata -> parsingValidator.launchValidation(metadata) } @@ -143,19 +143,6 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { companion object { private const val AVRO_EXTENSION = "avsc" - /** Formats a stream of validation exceptions. */ - fun format(exceptions: List): String { - return exceptions.joinToString(separator = "") { ex: ValidationException -> - """ - |Validation FAILED: - |${ex.message} - | - | - | - """.trimMargin() - } - } - fun Path.isAvscFile(): Boolean = extension.equals(AVRO_EXTENSION, ignoreCase = true) } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt index 3709ba4b..0f817801 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt @@ -11,16 +11,31 @@ import org.radarbase.schema.validation.rules.Validator import kotlin.coroutines.CoroutineContext import kotlin.coroutines.EmptyCoroutineContext +/** + * Context that validators run in. As part of the context, they can raise errors and launch + * validations in additional coroutines. + */ interface ValidationContext { + /** Raise a validation exception. */ fun raise(message: String, ex: Exception? = null) + /** Launch a validation by a validator in a new coroutine. */ fun Validator.launchValidation(value: T) + /** + * Launch an inline validation in a new coroutine. By passing [context], the validation is run + * in a different sub-context. + */ fun launchValidation(context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.() -> Unit) } +/** + * Implementation of a validation context that raises exceptions in a [Channel]. + */ private class ValidationContextImpl( + /** Channel that will receive validation exceptions. */ private val channel: SendChannel, + /** Scope that the validation will run in. */ private val coroutineScope: CoroutineScope, ) : ValidationContext { @@ -39,6 +54,11 @@ private class ValidationContextImpl( } } +/** + * Create a ValidationContext to launch validations in. + * + * @return validation exceptions that were raised as within the validation context. + */ suspend fun validationContext(block: ValidationContext.() -> Unit): List { val channel = Channel(UNLIMITED) coroutineScope { @@ -56,6 +76,10 @@ suspend fun validationContext(block: ValidationContext.() -> Unit): List Validator.validate(value: T) = validationContext { launchValidation(value = value) } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt index cb4b20f4..ac6ee499 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt @@ -15,7 +15,21 @@ */ package org.radarbase.schema.validation +/** Exception raised by a validtor. */ class ValidationException : RuntimeException { constructor(message: String?) : super(message) constructor(message: String?, exception: Throwable?) : super(message, exception) } + +/** Formats a stream of validation exceptions. */ +fun List.toFormattedString(): String { + return joinToString(separator = "") { ex: ValidationException -> + """ + |Validation FAILED: + |${ex.message} + | + | + | + """.trimMargin() + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt index 32f00673..990629a9 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationHelper.kt @@ -43,9 +43,7 @@ object ValidationHelper { } } - fun getRecordName(path: Path): String { - return snakeToCamelCase(path.fileName.toString()) - } + fun Path.toRecordName(): String = snakeToCamelCase(fileName.toString()) fun isValidTopic(topicName: String?): Boolean = topicName?.matches(TOPIC_PATTERN) == true } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt index 111a1aa1..d850c9bc 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt @@ -3,7 +3,7 @@ package org.radarbase.schema.validation.rules import org.apache.avro.Schema import org.radarbase.schema.specification.config.SchemaConfig import org.radarbase.schema.validation.ValidationHelper.getNamespace -import org.radarbase.schema.validation.ValidationHelper.getRecordName +import org.radarbase.schema.validation.ValidationHelper.toRecordName import java.nio.file.Path import java.nio.file.PathMatcher @@ -41,7 +41,7 @@ class RadarSchemaMetadataRules( } private fun isNameSchemaLocationCorrect() = Validator { metadata -> - val expected = getRecordName(metadata.path) + val expected = metadata.path.toRecordName() if (!expected.equals(metadata.schema.name, ignoreCase = true)) { raise(metadata, "Record name should match file name. Expected record name is \"$expected\".") } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt index 565f30b3..3db14717 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/specification/config/SchemaConfigTest.kt @@ -1,5 +1,6 @@ package org.radarbase.schema.specification.config +import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test import org.radarbase.schema.SchemaCatalogue @@ -32,7 +33,9 @@ internal class SchemaConfigTest { val commonsRoot = Paths.get("../..").resolve(COMMONS_PATH) .absolute() .normalize() - val schemaCatalogue = SchemaCatalogue(commonsRoot, config) + val schemaCatalogue = runBlocking { + SchemaCatalogue(commonsRoot, config) + } assertEquals(1, schemaCatalogue.schemas.size) val (fullName, schemaMetadata) = schemaCatalogue.schemas.entries.first() assertEquals("org.radarcns.monitor.application.ApplicationUptime2", fullName) diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt index 85d7c294..38336642 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SchemaValidatorTest.kt @@ -29,10 +29,9 @@ import org.radarbase.schema.Scope.CONNECTOR import org.radarbase.schema.Scope.KAFKA import org.radarbase.schema.Scope.MONITOR import org.radarbase.schema.Scope.PASSIVE -import org.radarbase.schema.specification.SourceCatalogue.Companion.load +import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.config.SchemaConfig import org.radarbase.schema.specification.config.SourceConfig -import org.radarbase.schema.validation.SchemaValidator.Companion.format import org.radarbase.schema.validation.ValidationHelper.COMMONS_PATH import java.io.IOException import java.nio.file.Path @@ -121,10 +120,8 @@ class SchemaValidatorTest { @Throws(IOException::class) private fun testFromSpecification(scope: Scope) = runBlocking { - val sourceCatalogue = load(ROOT, SchemaConfig(), SourceConfig()) - val result = format( - validator.analyseSourceCatalogue(scope, sourceCatalogue), - ) + val sourceCatalogue = SourceCatalogue(ROOT, SchemaConfig(), SourceConfig()) + val result = validator.analyseSourceCatalogue(scope, sourceCatalogue).toFormattedString() if (result.isNotEmpty()) { fail(result) } @@ -137,9 +134,7 @@ class SchemaValidatorTest { SchemaConfig(), scope, ) - val result = format( - validator.analyseFiles(schemaCatalogue, scope), - ) + val result = validator.analyseFiles(schemaCatalogue, scope).toFormattedString() if (result.isNotEmpty()) { fail(result) } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt index a1d58f6b..faa10a0f 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SourceCatalogueValidationTest.kt @@ -16,6 +16,7 @@ package org.radarbase.schema.validation import com.fasterxml.jackson.databind.ObjectMapper +import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue @@ -26,11 +27,11 @@ import org.opentest4j.MultipleFailuresError import org.radarbase.schema.specification.DataProducer import org.radarbase.schema.specification.DataTopic import org.radarbase.schema.specification.SourceCatalogue -import org.radarbase.schema.specification.SourceCatalogue.Companion.load import org.radarbase.schema.specification.config.SchemaConfig import org.radarbase.schema.specification.config.SourceConfig import org.radarbase.schema.validation.ValidationHelper.isValidTopic import java.io.IOException +import java.nio.file.Path import java.nio.file.Paths import java.util.Objects import java.util.stream.Collectors @@ -106,13 +107,15 @@ class SourceCatalogueValidationTest { companion object { private lateinit var catalogue: SourceCatalogue - val BASE_PATH = Paths.get("../..").toAbsolutePath().normalize() + val BASE_PATH: Path = Paths.get("../..").toAbsolutePath().normalize() @BeforeAll @JvmStatic @Throws(IOException::class) fun setUp() { - catalogue = load(BASE_PATH, SchemaConfig(), SourceConfig()) + catalogue = runBlocking { + SourceCatalogue(BASE_PATH, SchemaConfig(), SourceConfig()) + } } } } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt index 23dbbe15..868a9bfc 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/SpecificationsValidatorTest.kt @@ -33,7 +33,7 @@ class SpecificationsValidatorTest { fun activeIsYml() = runBlocking { val validator = validator.ofScope(ACTIVE) ?: return@runBlocking val result = validator.isValidSpecification(ActiveSource::class.java) - assertEquals("", SchemaValidator.format(result)) + assertEquals("", result.toFormattedString()) } @Test @@ -41,7 +41,7 @@ class SpecificationsValidatorTest { fun monitorIsYml() = runBlocking { val validator = validator.ofScope(MONITOR) ?: return@runBlocking val result = validator.isValidSpecification(MonitorSource::class.java) - assertEquals("", SchemaValidator.format(result)) + assertEquals("", result.toFormattedString()) } @Test @@ -49,7 +49,7 @@ class SpecificationsValidatorTest { fun passiveIsYml() = runBlocking { val validator = validator.ofScope(PASSIVE) ?: return@runBlocking val result = validator.isValidSpecification(PassiveSource::class.java) - assertEquals("", SchemaValidator.format(result)) + assertEquals("", result.toFormattedString()) } @Test @@ -57,7 +57,7 @@ class SpecificationsValidatorTest { fun connectorIsYml() = runBlocking { val validator = validator.ofScope(CONNECTOR) ?: return@runBlocking val result = validator.isValidSpecification(ConnectorSource::class.java) - assertEquals("", SchemaValidator.format(result)) + assertEquals("", result.toFormattedString()) } @Test @@ -65,7 +65,7 @@ class SpecificationsValidatorTest { fun pushIsYml() = runBlocking { val validator = validator.ofScope(PUSH) ?: return@runBlocking val result = validator.isValidSpecification(PushSource::class.java) - assertEquals("", SchemaValidator.format(result)) + assertEquals("", result.toFormattedString()) } @Test @@ -73,6 +73,6 @@ class SpecificationsValidatorTest { fun streamIsYml() = runBlocking { val validator = validator.ofScope(STREAM) ?: return@runBlocking val result = validator.isValidSpecification(StreamGroup::class.java) - assertEquals("", SchemaValidator.format(result)) + assertEquals("", result.toFormattedString()) } } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt index c6d1f46c..a03ab9f3 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt @@ -24,8 +24,8 @@ import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test -import org.radarbase.schema.validation.SchemaValidator -import org.radarbase.schema.validation.ValidationHelper.getRecordName +import org.radarbase.schema.validation.ValidationHelper.toRecordName +import org.radarbase.schema.validation.toFormattedString import org.radarbase.schema.validation.validate import java.nio.file.Paths @@ -43,13 +43,11 @@ class RadarSchemaFieldRulesTest { fun fileNameTest() { assertEquals( "Questionnaire", - getRecordName(Paths.get("/path/to/questionnaire.avsc")), + Paths.get("/path/to/questionnaire.avsc").toRecordName(), ) assertEquals( "ApplicationExternalTime", - getRecordName( - Paths.get("/path/to/application_external_time.avsc"), - ), + Paths.get("/path/to/application_external_time.avsc").toRecordName(), ) } @@ -87,7 +85,7 @@ class RadarSchemaFieldRulesTest { .apply(schemaBuilder) .endRecord(), ) - assertEquals(count, result.size) { message + SchemaValidator.format(result) } + assertEquals(count, result.size) { message + result.toFormattedString() } } @Test diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt index c5e08a03..a93aadbc 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt @@ -24,9 +24,10 @@ import org.junit.jupiter.api.Test import org.radarbase.schema.Scope.MONITOR import org.radarbase.schema.Scope.PASSIVE import org.radarbase.schema.specification.config.SchemaConfig -import org.radarbase.schema.validation.SchemaValidator.Companion.format import org.radarbase.schema.validation.SourceCatalogueValidationTest import org.radarbase.schema.validation.ValidationHelper +import org.radarbase.schema.validation.ValidationHelper.toRecordName +import org.radarbase.schema.validation.toFormattedString import org.radarbase.schema.validation.validate import java.nio.file.Paths @@ -45,13 +46,11 @@ class RadarSchemaMetadataRulesTest { fun fileNameTest() { assertEquals( "Questionnaire", - ValidationHelper.getRecordName(Paths.get("/path/to/questionnaire.avsc")), + Paths.get("/path/to/questionnaire.avsc").toRecordName(), ) assertEquals( "ApplicationExternalTime", - ValidationHelper.getRecordName( - Paths.get("/path/to/application_external_time.avsc"), - ), + Paths.get("/path/to/application_external_time.avsc").toRecordName(), ) } @@ -110,7 +109,7 @@ class RadarSchemaMetadataRulesTest { .endRecord() result = validator.isSchemaLocationCorrect .validate(SchemaMetadata(schema, PASSIVE, filePath)) - assertEquals("", format(result)) + assertEquals("", result.toFormattedString()) } companion object { diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/CommandLineApp.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/CommandLineApp.kt index 024d9179..0c76e2cf 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/CommandLineApp.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/CommandLineApp.kt @@ -15,6 +15,7 @@ */ package org.radarbase.schema.tools +import kotlinx.coroutines.runBlocking import net.sourceforge.argparse4j.ArgumentParsers import net.sourceforge.argparse4j.helper.HelpScreenException import net.sourceforge.argparse4j.inf.ArgumentParser @@ -45,9 +46,8 @@ import kotlin.system.exitProcess class CommandLineApp( val root: Path, val config: ToolConfig, + val catalogue: SourceCatalogue, ) { - val catalogue: SourceCatalogue = SourceCatalogue.load(root, config.schemas, config.sources) - init { logger.info("radar-schema-tools is initialized with root directory {}", this.root) } @@ -131,22 +131,27 @@ class CommandLineApp( val toolConfig = loadConfig(ns.getString("config")) logger.info("Loading radar-schemas-tools with configuration {}", toolConfig) - - val app: CommandLineApp = try { - CommandLineApp(root, toolConfig) - } catch (e: IOException) { - logger.error("Failed to load catalog from root.") - exitProcess(1) - } - val subparser = ns.getString("subparser") - val command = subCommands.find { it.name == subparser } - ?: run { - parser.handleError( - ArgumentParserException("Subcommand $subparser not implemented", parser), - ) + runBlocking { + val app: CommandLineApp = try { + val catalogue = SourceCatalogue(root, toolConfig.schemas, toolConfig.sources) + CommandLineApp(root, toolConfig, catalogue) + } catch (e: IOException) { + logger.error("Failed to load catalog from root.") exitProcess(1) } - exitProcess(command.execute(ns, app)) + val subparser = ns.getString("subparser") + val command = subCommands.find { it.name == subparser } + ?: run { + parser.handleError( + ArgumentParserException( + "Subcommand $subparser not implemented", + parser, + ), + ) + exitProcess(1) + } + exitProcess(command.execute(ns, app)) + } } private fun loadConfig(fileName: String): ToolConfig = try { diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/KafkaTopicsCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/KafkaTopicsCommand.kt index a97be2a0..7703a0b4 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/KafkaTopicsCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/KafkaTopicsCommand.kt @@ -1,6 +1,5 @@ package org.radarbase.schema.tools -import kotlinx.coroutines.runBlocking import net.sourceforge.argparse4j.inf.ArgumentParser import net.sourceforge.argparse4j.inf.Namespace import org.radarbase.schema.registration.KafkaTopics @@ -15,7 +14,7 @@ import org.slf4j.LoggerFactory class KafkaTopicsCommand : SubCommand { override val name = "create" - override fun execute(options: Namespace, app: CommandLineApp): Int { + override suspend fun execute(options: Namespace, app: CommandLineApp): Int { val brokers = options.getInt("brokers") val replication = options.getShort("replication") ?: 3 if (brokers < replication) { @@ -30,23 +29,21 @@ class KafkaTopicsCommand : SubCommand { val toolConfig: ToolConfig = app.config .configureKafka(bootstrapServers = options.getString("bootstrap_servers")) - return runBlocking { - KafkaTopics(toolConfig).use { topics -> - try { - val numTries = options.getInt("num_tries") - topics.initialize(brokers, numTries) - } catch (ex: IllegalStateException) { - logger.error("Kafka brokers not yet available. Aborting.") - return@use 1 - } - topics.createTopics( - app.catalogue, - options.getInt("partitions") ?: 3, - replication, - options.getString("topic"), - options.getString("match"), - ) + return KafkaTopics(toolConfig).use { topics -> + try { + val numTries = options.getInt("num_tries") + topics.initialize(brokers, numTries) + } catch (ex: IllegalStateException) { + logger.error("Kafka brokers not yet available. Aborting.") + return@use 1 } + topics.createTopics( + app.catalogue, + options.getInt("partitions") ?: 3, + replication, + options.getString("topic"), + options.getString("match"), + ) } } diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ListCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ListCommand.kt index defb3ed9..64ec885a 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ListCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ListCommand.kt @@ -10,7 +10,7 @@ import java.util.stream.Stream class ListCommand : SubCommand { override val name: String = "list" - override fun execute(options: Namespace, app: CommandLineApp): Int { + override suspend fun execute(options: Namespace, app: CommandLineApp): Int { val out: Stream = when { options.getBoolean("raw") -> app.rawTopics options.getBoolean("stream") -> app.resultsCacheTopics diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SchemaRegistryCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SchemaRegistryCommand.kt index 3a9330b1..e55f52e1 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SchemaRegistryCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SchemaRegistryCommand.kt @@ -1,6 +1,5 @@ package org.radarbase.schema.tools -import kotlinx.coroutines.runBlocking import net.sourceforge.argparse4j.impl.Arguments import net.sourceforge.argparse4j.inf.ArgumentParser import net.sourceforge.argparse4j.inf.Namespace @@ -17,7 +16,7 @@ import java.util.regex.Pattern class SchemaRegistryCommand : SubCommand { override val name = "register" - override fun execute(options: Namespace, app: CommandLineApp): Int { + override suspend fun execute(options: Namespace, app: CommandLineApp): Int { val url = options.get("schemaRegistry") val apiKey = options.getString("api_key") ?: System.getenv("SCHEMA_REGISTRY_API_KEY") @@ -25,22 +24,20 @@ class SchemaRegistryCommand : SubCommand { ?: System.getenv("SCHEMA_REGISTRY_API_SECRET") val toolConfigFile = options.getString("config") return try { - runBlocking { - val registration = createSchemaRegistry(url, apiKey, apiSecret, app.config) - val forced = options.getBoolean("force") - if (forced && !registration.putCompatibility(SchemaRegistry.Compatibility.NONE)) { - return@runBlocking 1 - } - val pattern: Pattern? = TopicRegistrar.matchTopic( - options.getString("topic"), - options.getString("match"), - ) - val result = registerSchemas(app, registration, pattern) - if (forced) { - registration.putCompatibility(SchemaRegistry.Compatibility.FULL) - } - if (result) 0 else 1 + val registration = SchemaRegistry(url, apiKey, apiSecret, app.config) + val forced = options.getBoolean("force") + if (forced && !registration.putCompatibility(SchemaRegistry.Compatibility.NONE)) { + return 1 + } + val pattern: Pattern? = TopicRegistrar.matchTopic( + options.getString("topic"), + options.getString("match"), + ) + val result = registerSchemas(app, registration, pattern) + if (forced) { + registration.putCompatibility(SchemaRegistry.Compatibility.FULL) } + if (result) 0 else 1 } catch (ex: MalformedURLException) { logger.error( "Schema registry URL {} is invalid: {}", @@ -88,7 +85,7 @@ class SchemaRegistryCommand : SubCommand { ) @Throws(MalformedURLException::class, InterruptedException::class) - private suspend fun createSchemaRegistry( + private suspend fun SchemaRegistry( url: String, apiKey: String?, apiSecret: String?, diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SubCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SubCommand.kt index eec39798..7fa8b0df 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SubCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/SubCommand.kt @@ -20,7 +20,7 @@ interface SubCommand { * @param app application with source catalogue. * @return command exit code. */ - fun execute(options: Namespace, app: CommandLineApp): Int + suspend fun execute(options: Namespace, app: CommandLineApp): Int /** * Add the description and arguments for this sub-command to the argument parser. The values of diff --git a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt index 6b7f454d..e6475698 100644 --- a/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt +++ b/java-sdk/radar-schemas-tools/src/main/java/org/radarbase/schema/tools/ValidatorCommand.kt @@ -1,7 +1,7 @@ package org.radarbase.schema.tools import kotlinx.coroutines.async -import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.coroutineScope import net.sourceforge.argparse4j.impl.Arguments import net.sourceforge.argparse4j.inf.ArgumentParser import net.sourceforge.argparse4j.inf.Namespace @@ -10,13 +10,14 @@ import org.radarbase.schema.tools.SubCommand.Companion.addRootArgument import org.radarbase.schema.validation.SchemaValidator import org.radarbase.schema.validation.ValidationException import org.radarbase.schema.validation.ValidationHelper.COMMONS_PATH +import org.radarbase.schema.validation.toFormattedString import java.io.IOException import kotlin.streams.asSequence class ValidatorCommand : SubCommand { override val name: String = "validate" - override fun execute(options: Namespace, app: CommandLineApp): Int { + override suspend fun execute(options: Namespace, app: CommandLineApp): Int { try { println() println("Validated topics:") @@ -46,7 +47,7 @@ class ValidatorCommand : SubCommand { return try { val validator = SchemaValidator(app.root.resolve(COMMONS_PATH), app.config.schemas) - runBlocking { + coroutineScope { val fullValidationJob = async { if (options.getBoolean("full")) { if (scope == null) { @@ -109,7 +110,7 @@ class ValidatorCommand : SubCommand { quiet: Boolean, ): Int = when { !quiet -> { - val result = SchemaValidator.format(stream) + val result = stream.toFormattedString() println(result) if (verbose) { println("Validated schemas:") From e0fe2bf538951284f6db24b0cc60b54d549f0968 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 27 Sep 2023 14:19:24 +0200 Subject: [PATCH 09/27] Simplified validator initialization --- .../schema/validation/SchemaValidator.kt | 6 +- .../schema/validation/ValidationException.kt | 2 +- .../validation/rules/RadarSchemaFieldRules.kt | 50 ++++++-- .../rules/RadarSchemaMetadataRules.kt | 32 ++++- .../validation/rules/RadarSchemaRules.kt | 96 ++++++++------- .../schema/validation/rules/SchemaField.kt | 5 +- .../validation/rules/SchemaFieldRules.kt | 47 -------- .../validation/rules/SchemaMetadataRules.kt | 39 ------ .../schema/validation/rules/SchemaRules.kt | 111 ------------------ .../rules/RadarSchemaFieldRulesTest.kt | 4 +- .../validation/rules/RadarSchemaRulesTest.kt | 4 +- 11 files changed, 129 insertions(+), 267 deletions(-) delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt delete mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt index 1699323e..c0c8915e 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt @@ -24,9 +24,7 @@ import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.config.SchemaConfig import org.radarbase.schema.validation.rules.FailedSchemaMetadata import org.radarbase.schema.validation.rules.RadarSchemaMetadataRules -import org.radarbase.schema.validation.rules.RadarSchemaRules import org.radarbase.schema.validation.rules.SchemaMetadata -import org.radarbase.schema.validation.rules.SchemaMetadataRules import org.radarbase.schema.validation.rules.Validator import java.nio.file.Path import java.nio.file.PathMatcher @@ -41,7 +39,7 @@ import kotlin.io.path.extension * @param config configuration to exclude certain schemas or fields from validation. */ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { - val rules: SchemaMetadataRules = RadarSchemaMetadataRules(schemaRoot, config) + val rules = RadarSchemaMetadataRules(schemaRoot, config) private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) suspend fun analyseSourceCatalogue( @@ -138,7 +136,7 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { } val validatedSchemas: Map - get() = (rules.schemaRules as RadarSchemaRules).schemaStore + get() = rules.schemaRules.schemaStore companion object { private const val AVRO_EXTENSION = "avsc" diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt index ac6ee499..3e971cab 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationException.kt @@ -30,6 +30,6 @@ fun List.toFormattedString(): String { | | | - """.trimMargin() + """.trimMargin() } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt index 25e8c0b1..aeb63de2 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt @@ -14,8 +14,9 @@ import java.util.EnumMap /** * Rules for RADAR-Schemas schema fields. */ -class RadarSchemaFieldRules : SchemaFieldRules { +class RadarSchemaFieldRules { private val defaultsValidator: MutableMap> + internal lateinit var schemaRules: RadarSchemaRules /** * Rules for RADAR-Schemas schema fields. @@ -26,20 +27,32 @@ class RadarSchemaFieldRules : SchemaFieldRules { defaultsValidator[UNION] = Validator { isDefaultUnionCompatible(it) } } - override fun validateFieldTypes(schemaRules: SchemaRules): Validator { - return Validator { field -> - val schema = field.field.schema() - val subType = schema.type - when (subType) { - UNION -> validateInternalUnion(schemaRules).launchValidation(field) - RECORD -> schemaRules.isRecordValid.launchValidation(schema) - ENUM -> schemaRules.isEnumValid.launchValidation(schema) - else -> Unit + /** Get a validator for a union inside a record. */ + private val validateInternalUnion = Validator { field -> + field.field.schema().types + .forEach { schema: Schema -> + val type = schema.type + when (type) { + RECORD -> schemaRules.isRecordValid.launchValidation(schema) + ENUM -> schemaRules.isEnumValid.launchValidation(schema) + UNION -> raise(field, "Cannot have a nested union.") + else -> Unit + } } + } + + val isFieldTypeValid: Validator = Validator { field -> + val schema = field.field.schema() + val subType = schema.type + when (subType) { + UNION -> validateInternalUnion.launchValidation(field) + RECORD -> schemaRules.isRecordValid.launchValidation(schema) + ENUM -> schemaRules.isEnumValid.launchValidation(schema) + else -> Unit } } - override val isDefaultValueValid = Validator { input: SchemaField -> + val isDefaultValueValid = Validator { input: SchemaField -> defaultsValidator .getOrDefault( input.field.schema().type, @@ -48,13 +61,13 @@ class RadarSchemaFieldRules : SchemaFieldRules { .launchValidation(input) } - override val isNameValid = validator( + val isNameValid = validator( predicate = { f -> f.field.name()?.matches(FIELD_NAME_PATTERN) == true }, message = "Field name does not respect lowerCamelCase name convention." + " Please avoid abbreviations and write out the field name instead.", ) - override val isDocumentationValid = Validator { field: SchemaField -> + val isDocumentationValid = Validator { field: SchemaField -> validateDocumentation( doc = field.field.doc(), raise = ValidationContext::raise, @@ -62,6 +75,13 @@ class RadarSchemaFieldRules : SchemaFieldRules { ) } + val isFieldValid: Validator = all( + isFieldTypeValid, + isNameValid, + isDefaultValueValid, + isDocumentationValid, + ) + private fun ValidationContext.isEnumDefaultUnknown(field: SchemaField) { if ( field.field.schema().enumSymbols.contains(UNKNOWN) && @@ -105,3 +125,7 @@ class RadarSchemaFieldRules : SchemaFieldRules { internal val FIELD_NAME_PATTERN = "[a-z][a-z0-9]*([a-z0-9][A-Z][a-z0-9]+)?([A-Z][a-z0-9]+)*[A-Z]?".toRegex() } } + +fun ValidationContext.raise(field: SchemaField, text: String) { + raise("Field ${field.field.name()} in schema ${field.schema.fullName} is invalid. $text") +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt index d850c9bc..fb36691e 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt @@ -1,7 +1,9 @@ package org.radarbase.schema.validation.rules import org.apache.avro.Schema +import org.radarbase.schema.Scope import org.radarbase.schema.specification.config.SchemaConfig +import org.radarbase.schema.validation.ValidationContext import org.radarbase.schema.validation.ValidationHelper.getNamespace import org.radarbase.schema.validation.ValidationHelper.toRecordName import java.nio.file.Path @@ -16,15 +18,33 @@ import java.nio.file.PathMatcher class RadarSchemaMetadataRules( private val schemaRoot: Path, config: SchemaConfig, - override val schemaRules: SchemaRules = RadarSchemaRules(), -) : SchemaMetadataRules { + val schemaRules: RadarSchemaRules = RadarSchemaRules(), +) { private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) - override val isSchemaLocationCorrect = all( + val isSchemaLocationCorrect = all( isNamespaceSchemaLocationCorrect(), isNameSchemaLocationCorrect(), ) + /** + * Validates any schema file. It will choose the correct validation method based on the scope + * and type of the schema. + */ + fun isSchemaMetadataValid(scopeSpecificValidation: Boolean) = Validator { metadata -> + isSchemaLocationCorrect.launchValidation(metadata) + + val ruleset = when { + metadata.schema.type == Schema.Type.ENUM -> schemaRules.isEnumValid + !scopeSpecificValidation -> schemaRules.isRecordValid + metadata.scope == Scope.ACTIVE -> schemaRules.isActiveSourceValid + metadata.scope == Scope.MONITOR -> schemaRules.isMonitorSourceValid + metadata.scope == Scope.PASSIVE -> schemaRules.isPassiveSourceValid + else -> schemaRules.isRecordValid + } + isSchemaCorrect(ruleset).launchValidation(metadata) + } + private fun isNamespaceSchemaLocationCorrect() = Validator { metadata -> try { val expected = getNamespace(schemaRoot, metadata.path, metadata.scope) @@ -47,9 +67,13 @@ class RadarSchemaMetadataRules( } } - override fun isSchemaCorrect(validator: Validator) = Validator { metadata -> + fun isSchemaCorrect(validator: Validator) = Validator { metadata -> if (pathMatcher.matches(metadata.path)) { validator.launchValidation(metadata.schema) } } } + +fun ValidationContext.raise(metadata: SchemaMetadata, text: String) { + raise("Schema ${metadata.schema.fullName} at ${metadata.path} is invalid. $text") +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt index 63fe6155..2c1ab78c 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt @@ -28,11 +28,11 @@ import org.radarbase.schema.validation.ValidationContext * Schema validation rules enforced for the RADAR-Schemas repository. */ class RadarSchemaRules( - override val fieldRules: RadarSchemaFieldRules = RadarSchemaFieldRules(), -) : SchemaRules { + val fieldRules: RadarSchemaFieldRules = RadarSchemaFieldRules(), +) { val schemaStore: MutableMap = HashMap() - override val isUnique = Validator { schema: Schema -> + val isUnique = Validator { schema: Schema -> val key = schema.fullName val oldSchema = schemaStore.putIfAbsent(key, schema) if (oldSchema != null && oldSchema != schema) { @@ -43,17 +43,17 @@ class RadarSchemaRules( } } - override val isNamespaceValid = validator( + val isNamespaceValid = validator( predicate = { it.namespace?.matches(NAMESPACE_PATTERN) == true }, message = schemaErrorMessage("Namespace cannot be null and must fully lowercase, period-separated, without numeric characters."), ) - override val isNameValid = validator( + val isNameValid = validator( predicate = { it.name?.matches(RECORD_NAME_PATTERN) == true }, message = schemaErrorMessage("Record names must be camel case."), ) - override val isDocumentationValid = Validator { schema -> + val isDocumentationValid = Validator { schema -> validateDocumentation( schema.doc, ValidationContext::raise, @@ -61,7 +61,7 @@ class RadarSchemaRules( ) } - override val isEnumSymbolsValid = Validator { schema -> + val isEnumSymbolsValid = Validator { schema -> if (schema.enumSymbols.isNullOrEmpty()) { raise(schema, "Avro Enumerator must have symbol list.") return@Validator @@ -77,37 +77,35 @@ class RadarSchemaRules( } } - override val hasTime: Validator = validator( + val hasTime: Validator = validator( predicate = { it.getField(TIME)?.schema()?.type == DOUBLE }, message = schemaErrorMessage("Any schema representing collected data must have a \"$TIME$WITH_TYPE_DOUBLE"), ) - override val hasTimeCompleted: Validator = validator( + val hasTimeCompleted: Validator = validator( predicate = { it.getField(TIME_COMPLETED)?.schema()?.type == DOUBLE }, message = schemaErrorMessage("Any ACTIVE schema must have a \"$TIME_COMPLETED$WITH_TYPE_DOUBLE"), ) - override val hasNoTimeCompleted: Validator = validator( + val hasNoTimeCompleted: Validator = validator( predicate = { it.getField(TIME_COMPLETED) == null }, message = schemaErrorMessage("\"$TIME_COMPLETED\" is allow only in ACTIVE schemas."), ) - override val hasTimeReceived: Validator = validator( + val hasTimeReceived: Validator = validator( predicate = { it.getField(TIME_RECEIVED)?.schema()?.type == DOUBLE }, message = schemaErrorMessage("Any PASSIVE schema must have a \"$TIME_RECEIVED$WITH_TYPE_DOUBLE"), ) - override val hasNoTimeReceived: Validator = validator( + val hasNoTimeReceived: Validator = validator( predicate = { it.getField(TIME_RECEIVED) == null }, message = schemaErrorMessage("\"$TIME_RECEIVED\" is allow only in PASSIVE schemas."), ) - override val isAvroConnectCompatible: Validator - /** * Validate an enum. */ - override val isEnumValid: Validator = all( + val isEnumValid: Validator = all( isUnique, isNamespaceValid, isEnumSymbolsValid, @@ -118,53 +116,33 @@ class RadarSchemaRules( /** * Validate a record that is defined inline. */ - override val isRecordValid: Validator + val isRecordValid: Validator /** * Validates record schemas of an active source. */ - override val isActiveSourceValid: Validator + val isActiveSourceValid: Validator /** * Validates schemas of monitor sources. */ - override val isMonitorSourceValid: Validator + val isMonitorSourceValid: Validator /** * Validates schemas of passive sources. */ - override val isPassiveSourceValid: Validator + val isPassiveSourceValid: Validator init { - val avroConfig = Builder() - .with(AvroDataConfig.CONNECT_META_DATA_CONFIG, false) - .with(AbstractDataConfig.SCHEMAS_CACHE_SIZE_CONFIG, 10) - .with(AvroDataConfig.ENHANCED_AVRO_SCHEMA_SUPPORT_CONFIG, true) - .build() - - isAvroConnectCompatible = Validator { schema: Schema -> - val encoder = AvroData(10) - val decoder = AvroData(avroConfig) - try { - val connectSchema = encoder.toConnectSchema(schema) - val originalSchema = decoder.fromConnectSchema(connectSchema) - check(schema == originalSchema) { - "Schema changed by validation: " + - schema.toString(true) + " is not equal to " + - originalSchema.toString(true) - } - } catch (ex: Exception) { - raise("Failed to convert schema back to itself") - } - } + fieldRules.schemaRules = this isRecordValid = all( isUnique, - isAvroConnectCompatible, + isAvroConnectCompatible(), isNamespaceValid, isNameValid, isDocumentationValid, - isFieldsValid(fieldRules.isFieldValid(this)), + isFieldsValid(fieldRules.isFieldValid), ) isActiveSourceValid = all(isRecordValid, hasTime) @@ -174,7 +152,7 @@ class RadarSchemaRules( isPassiveSourceValid = all(isRecordValid, hasTime, hasTimeReceived, hasNoTimeCompleted) } - override fun isFieldsValid(validator: Validator): Validator = + fun isFieldsValid(validator: Validator): Validator = Validator { schema: Schema -> when { schema.type != RECORD -> raise( @@ -187,6 +165,34 @@ class RadarSchemaRules( } } + private fun isAvroConnectCompatible(): Validator { + val avroConfig = Builder() + .with(AvroDataConfig.CONNECT_META_DATA_CONFIG, false) + .with(AbstractDataConfig.SCHEMAS_CACHE_SIZE_CONFIG, 10) + .with(AvroDataConfig.ENHANCED_AVRO_SCHEMA_SUPPORT_CONFIG, true) + .build() + + return Validator { schema: Schema -> + val encoder = AvroData(10) + val decoder = AvroData(avroConfig) + try { + val connectSchema = encoder.toConnectSchema(schema) + val originalSchema = decoder.fromConnectSchema(connectSchema) + check(schema == originalSchema) { + "Schema changed by validation: " + + schema.toString(true) + " is not equal to " + + originalSchema.toString(true) + } + } catch (ex: Exception) { + raise("Failed to convert schema back to itself") + } + } + } + + private fun schemaErrorMessage(text: String): (Schema) -> String { + return { schema -> "Schema ${schema.fullName} is invalid. $text" } + } + companion object { // used in testing const val TIME = "time" @@ -243,3 +249,7 @@ class RadarSchemaRules( } } } + +fun ValidationContext.raise(schema: Schema, text: String) { + raise("Schema ${schema.fullName} is invalid. $text") +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt index 503ca8a6..ebc0e93d 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaField.kt @@ -3,4 +3,7 @@ package org.radarbase.schema.validation.rules import org.apache.avro.Schema import org.apache.avro.Schema.Field -data class SchemaField(val schema: Schema, val field: Field) +data class SchemaField( + val schema: Schema, + val field: Field, +) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt deleted file mode 100644 index b07ca20f..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt +++ /dev/null @@ -1,47 +0,0 @@ -package org.radarbase.schema.validation.rules - -import org.apache.avro.Schema -import org.apache.avro.Schema.Type.ENUM -import org.apache.avro.Schema.Type.RECORD -import org.apache.avro.Schema.Type.UNION -import org.radarbase.schema.validation.ValidationContext - -interface SchemaFieldRules { - /** Recursively checks field types. */ - fun validateFieldTypes(schemaRules: SchemaRules): Validator - - /** Checks field name format. */ - val isNameValid: Validator - - /** Checks field documentation presence and format. */ - val isDocumentationValid: Validator - - /** Checks field default values. */ - val isDefaultValueValid: Validator - - /** Get a validator for a field. */ - fun isFieldValid(schemaRules: SchemaRules): Validator = all( - validateFieldTypes(schemaRules), - isNameValid, - isDefaultValueValid, - isDocumentationValid, - ) - - /** Get a validator for a union inside a record. */ - fun validateInternalUnion(schemaRules: SchemaRules) = Validator { field: SchemaField -> - field.field.schema().types - .forEach { schema: Schema -> - val type = schema.type - when (type) { - RECORD -> schemaRules.isRecordValid.launchValidation(schema) - ENUM -> schemaRules.isEnumValid.launchValidation(schema) - UNION -> raise(field, "Cannot have a nested union.") - else -> Unit - } - } - } -} - -fun ValidationContext.raise(field: SchemaField, text: String) { - raise("Field ${field.field.name()} in schema ${field.schema.fullName} is invalid. $text") -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt deleted file mode 100644 index 4fdd67b2..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt +++ /dev/null @@ -1,39 +0,0 @@ -package org.radarbase.schema.validation.rules - -import org.apache.avro.Schema -import org.radarbase.schema.Scope -import org.radarbase.schema.validation.ValidationContext - -interface SchemaMetadataRules { - val schemaRules: SchemaRules - - /** Checks the location of a schema with its internal data. */ - val isSchemaLocationCorrect: Validator - - /** - * Validates any schema file. It will choose the correct validation method based on the scope - * and type of the schema. - */ - fun isSchemaMetadataValid(scopeSpecificValidation: Boolean) = Validator { metadata -> - isSchemaLocationCorrect.launchValidation(metadata) - - val ruleset = when { - metadata.schema.type == Schema.Type.ENUM -> schemaRules.isEnumValid - !scopeSpecificValidation -> schemaRules.isRecordValid - metadata.scope == Scope.ACTIVE -> schemaRules.isActiveSourceValid - metadata.scope == Scope.MONITOR -> schemaRules.isMonitorSourceValid - metadata.scope == Scope.PASSIVE -> schemaRules.isPassiveSourceValid - else -> schemaRules.isRecordValid - } - isSchemaCorrect(ruleset).launchValidation(metadata) - } - - /** Validates schemas without their metadata. */ - fun isSchemaCorrect(validator: Validator) = Validator { metadata -> - validator.launchValidation(metadata.schema) - } -} - -fun ValidationContext.raise(metadata: SchemaMetadata, text: String) { - raise("Schema ${metadata.schema.fullName} at ${metadata.path} is invalid. $text") -} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt deleted file mode 100644 index 51eb42a0..00000000 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt +++ /dev/null @@ -1,111 +0,0 @@ -package org.radarbase.schema.validation.rules - -import org.apache.avro.Schema -import org.apache.avro.Schema.Type.RECORD -import org.radarbase.schema.validation.ValidationContext - -interface SchemaRules { - val fieldRules: SchemaFieldRules - - /** - * Checks that schemas are unique compared to already validated schemas. - */ - val isUnique: Validator - - /** - * Checks schema namespace format. - */ - val isNamespaceValid: Validator - - /** - * Checks schema name format. - */ - val isNameValid: Validator - - /** - * Checks schema documentation presence and format. - */ - val isDocumentationValid: Validator - - /** - * Checks that the symbols of enums have the required format. - */ - val isEnumSymbolsValid: Validator - - /** - * Checks that schemas should have a `time` field. - */ - val hasTime: Validator - - /** - * Checks that schemas should have a `timeCompleted` field. - */ - val hasTimeCompleted: Validator - - /** - * Checks that schemas should not have a `timeCompleted` field. - */ - val hasNoTimeCompleted: Validator - - /** - * Checks that schemas should have a `timeReceived` field. - */ - val hasTimeReceived: Validator - - /** - * Checks that schemas should not have a `timeReceived` field. - */ - val hasNoTimeReceived: Validator - - /** - * Validate an enum. - */ - val isEnumValid: Validator - - /** - * Validate a record that is defined inline. - */ - val isRecordValid: Validator - val isAvroConnectCompatible: Validator - - /** - * Validates record schemas of an active source. - */ - val isActiveSourceValid: Validator - - /** - * Validates schemas of monitor sources. - */ - val isMonitorSourceValid: Validator - - /** - * Validates schemas of passive sources. - */ - val isPassiveSourceValid: Validator - - fun schemaErrorMessage(text: String): (Schema) -> String { - return { schema -> "Schema ${schema.fullName} is invalid. $text" } - } - - /** - * Validates all fields of records. - * Validation will fail on non-record types or records with no fields. - */ - fun isFieldsValid(validator: Validator) = Validator { schema: Schema -> - if (schema.type != RECORD) { - raise("Default validation can be applied only to an Avro RECORD, not to ${schema.type} of schema ${schema.fullName}.") - return@Validator - } - if (schema.fields.isEmpty()) { - raise("Schema ${schema.fullName} does not contain any fields.") - return@Validator - } - schema.fields.forEach { field -> - validator.launchValidation(SchemaField(schema, field)) - } - } -} - -fun ValidationContext.raise(schema: Schema, text: String) { - raise("Schema ${schema.fullName} is invalid. $text") -} diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt index a03ab9f3..98dea5f5 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt @@ -64,9 +64,9 @@ class RadarSchemaFieldRulesTest { @Test fun fieldsTest() = runBlocking { - assertFieldsErrorCount(1, validator.validateFieldTypes(schemaValidator), "Should have at least one field") + assertFieldsErrorCount(1, validator.isFieldTypeValid, "Should have at least one field") - assertFieldsErrorCount(0, validator.validateFieldTypes(schemaValidator), "Single optional field should be fine") { + assertFieldsErrorCount(0, validator.isFieldTypeValid, "Single optional field should be fine") { optionalBoolean("optional") } } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt index 89a6c573..5c183c10 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt @@ -118,7 +118,7 @@ class RadarSchemaRulesTest { .fields() .endRecord() var result = validator.isFieldsValid( - validator.fieldRules.validateFieldTypes(validator), + validator.fieldRules.isFieldTypeValid, ).validate(schema) Assertions.assertEquals(1, result.count()) schema = SchemaBuilder @@ -127,7 +127,7 @@ class RadarSchemaRulesTest { .fields() .optionalBoolean("optional") .endRecord() - result = validator.isFieldsValid(validator.fieldRules.validateFieldTypes(validator)) + result = validator.isFieldsValid(validator.fieldRules.isFieldTypeValid) .validate(schema) Assertions.assertEquals(0, result.count()) } From 034e990c6f79bdb42e74a133bb1598aaf49b5eda Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 27 Sep 2023 15:32:51 +0200 Subject: [PATCH 10/27] Made specific subclasses to Validator --- .../schema/validation/SchemaValidator.kt | 48 ++++++-------- .../validation/SpecificationsValidator.kt | 11 ++-- .../schema/validation/ValidationContext.kt | 54 ++++++++------- .../schema/validation/rules/AllValidator.kt | 17 +++++ .../validation/rules/DirectValidator.kt | 15 +++++ .../schema/validation/rules/PathRules.kt | 11 ++++ .../validation/rules/PredicateValidator.kt | 20 ++++++ ...chemaFieldRules.kt => SchemaFieldRules.kt} | 51 +++++++------- ...etadataRules.kt => SchemaMetadataRules.kt} | 16 ++--- .../{RadarSchemaRules.kt => SchemaRules.kt} | 8 +-- .../schema/validation/rules/Validator.kt | 33 +--------- .../rules/RadarSchemaMetadataRulesTest.kt | 4 +- .../validation/rules/RadarSchemaRulesTest.kt | 66 +++++++++---------- ...ldRulesTest.kt => SchemaFieldRulesTest.kt} | 24 +++---- 14 files changed, 201 insertions(+), 177 deletions(-) create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/AllValidator.kt create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/DirectValidator.kt create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PathRules.kt create mode 100644 java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PredicateValidator.kt rename java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/{RadarSchemaFieldRules.kt => SchemaFieldRules.kt} (73%) rename java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/{RadarSchemaMetadataRules.kt => SchemaMetadataRules.kt} (89%) rename java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/{RadarSchemaRules.kt => SchemaRules.kt} (97%) rename java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/{RadarSchemaFieldRulesTest.kt => SchemaFieldRulesTest.kt} (84%) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt index c0c8915e..633d9a49 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SchemaValidator.kt @@ -23,11 +23,10 @@ import org.radarbase.schema.specification.DataProducer import org.radarbase.schema.specification.SourceCatalogue import org.radarbase.schema.specification.config.SchemaConfig import org.radarbase.schema.validation.rules.FailedSchemaMetadata -import org.radarbase.schema.validation.rules.RadarSchemaMetadataRules import org.radarbase.schema.validation.rules.SchemaMetadata +import org.radarbase.schema.validation.rules.SchemaMetadataRules import org.radarbase.schema.validation.rules.Validator import java.nio.file.Path -import java.nio.file.PathMatcher import java.util.stream.Collectors import java.util.stream.Stream import kotlin.io.path.extension @@ -38,9 +37,11 @@ import kotlin.io.path.extension * @param schemaRoot RADAR-Schemas commons directory. * @param config configuration to exclude certain schemas or fields from validation. */ -class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { - val rules = RadarSchemaMetadataRules(schemaRoot, config) - private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) +class SchemaValidator( + schemaRoot: Path, + config: SchemaConfig, +) { + val rules = SchemaMetadataRules(schemaRoot, config) suspend fun analyseSourceCatalogue( scope: Scope?, @@ -61,13 +62,7 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { } .collect(Collectors.toSet()) - return validationContext { - schemas.forEach { metadata -> - if (pathMatcher.matches(metadata.path)) { - validator.launchValidation(metadata) - } - } - } + return validator.validateAll(schemas) } suspend fun analyseFiles( @@ -75,7 +70,11 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { scope: Scope? = null, ): List = validationContext { if (scope == null) { - Scope.entries.forEach { scope -> analyseFilesInternal(schemaCatalogue, scope) } + Scope.entries.forEach { scope -> + launch { + analyseFilesInternal(schemaCatalogue, scope) + } + } } else { analyseFilesInternal(schemaCatalogue, scope) } @@ -85,18 +84,11 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { schemaCatalogue: SchemaCatalogue, scope: Scope, ) { - val validator = rules.isSchemaMetadataValid(false) - val parsingValidator = parsingValidator(scope, schemaCatalogue) + parsingValidator(scope, schemaCatalogue) + .validateAll(schemaCatalogue.unmappedSchemas) - schemaCatalogue.unmappedSchemas.forEach { metadata -> - parsingValidator.launchValidation(metadata) - } - - schemaCatalogue.schemas.values.forEach { metadata -> - if (pathMatcher.matches(metadata.path)) { - validator.launchValidation(metadata) - } - } + rules.isSchemaMetadataValid(false) + .validateAll(schemaCatalogue.schemas.values) } private fun parsingValidator( @@ -113,7 +105,7 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { return Validator { metadata -> val parser = Schema.Parser() parser.addTypes(useTypes) - launchValidation(Dispatchers.IO) { + launch(Dispatchers.IO) { try { parser.parse(metadata.path.toFile()) } catch (ex: Exception) { @@ -129,10 +121,8 @@ class SchemaValidator(schemaRoot: Path, config: SchemaConfig) { /** Validate a single schema in given path. */ private fun ValidationContext.validate(schemaMetadata: SchemaMetadata) { - val validator = rules.isSchemaMetadataValid(false) - if (pathMatcher.matches(schemaMetadata.path)) { - validator.launchValidation(schemaMetadata) - } + rules.isSchemaMetadataValid(false) + .validate(schemaMetadata) } val validatedSchemas: Map diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt index 9ebb9fd8..72ef935e 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/SpecificationsValidator.kt @@ -21,7 +21,7 @@ import org.radarbase.schema.Scope import org.radarbase.schema.specification.config.SchemaConfig import org.radarbase.schema.util.SchemaUtils.listRecursive import org.radarbase.schema.validation.rules.Validator -import org.radarbase.schema.validation.rules.pathExtensionValidator +import org.radarbase.schema.validation.rules.hasExtension import org.slf4j.LoggerFactory import java.io.IOException import java.nio.file.Path @@ -58,11 +58,8 @@ class SpecificationsValidator( suspend fun isValidSpecification(clazz: Class?): List { val paths = root.listRecursive { pathMatcher.matches(it) } return validationContext { - val isParseableAsClass = isYmlFileParseable(clazz) - paths.forEach { p -> - isYmlFile.launchValidation(p) - isParseableAsClass.launchValidation(p) - } + isYmlFile.validateAll(paths) + isYmlFileParseable(clazz).validateAll(paths) } } @@ -77,6 +74,6 @@ class SpecificationsValidator( companion object { private val logger = LoggerFactory.getLogger(SpecificationsValidator::class.java) - private val isYmlFile: Validator = pathExtensionValidator("yml") + private val isYmlFile: Validator = hasExtension("yml") } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt index 0f817801..4333d594 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/ValidationContext.kt @@ -15,41 +15,39 @@ import kotlin.coroutines.EmptyCoroutineContext * Context that validators run in. As part of the context, they can raise errors and launch * validations in additional coroutines. */ -interface ValidationContext { - /** Raise a validation exception. */ - fun raise(message: String, ex: Exception? = null) - - /** Launch a validation by a validator in a new coroutine. */ - fun Validator.launchValidation(value: T) - - /** - * Launch an inline validation in a new coroutine. By passing [context], the validation is run - * in a different sub-context. - */ - fun launchValidation(context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.() -> Unit) -} - -/** - * Implementation of a validation context that raises exceptions in a [Channel]. - */ -private class ValidationContextImpl( +class ValidationContext( /** Channel that will receive validation exceptions. */ private val channel: SendChannel, /** Scope that the validation will run in. */ private val coroutineScope: CoroutineScope, -) : ValidationContext { - - override fun raise(message: String, ex: Exception?) { +) { + /** Raise a validation exception. */ + fun raise(message: String, ex: Exception? = null) { channel.trySend(ValidationException(message, ex)) } - override fun Validator.launchValidation(value: T) { + /** Launch a validation by a validator in a new coroutine. */ + fun Validator.launchValidation(value: T) { coroutineScope.launch { runValidation(value) } } - override fun launchValidation(context: CoroutineContext, block: suspend CoroutineScope.() -> Unit) { + /** Launch a validation by a validator in the same coroutine. */ + fun Validator.validate(value: T) { + runValidation(value) + } + + /** Validate all given values. */ + fun Validator.validateAll(values: Iterable) { + values.forEach { launchValidation(it) } + } + + /** + * Launch an inline validation in a new coroutine. By passing [context], the validation is run + * in a different sub-context. + */ + fun launch(context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.() -> Unit) { coroutineScope.launch(context, block = block) } } @@ -63,7 +61,7 @@ suspend fun validationContext(block: ValidationContext.() -> Unit): List(UNLIMITED) coroutineScope { val producerJob = launch { - with(ValidationContextImpl(channel, this@launch)) { + with(ValidationContext(channel, this@launch)) { block() } } @@ -83,3 +81,11 @@ suspend fun validationContext(block: ValidationContext.() -> Unit): List Validator.validate(value: T) = validationContext { launchValidation(value = value) } + +/** + * Run a validation inside its own context. This can be used for one-off validations. Otherwise, a + * separate validationContext should be created. + */ +suspend fun Validator.validateAll(values: Iterable) = validationContext { + validateAll(values = values) +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/AllValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/AllValidator.kt new file mode 100644 index 00000000..cf1ddfbe --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/AllValidator.kt @@ -0,0 +1,17 @@ +package org.radarbase.schema.validation.rules + +import org.radarbase.schema.validation.ValidationContext + +/** Validator that checks all validator in its list. */ +class AllValidator( + private val subValidators: List>, +) : Validator { + override fun ValidationContext.runValidation(value: T) { + subValidators.forEach { + it.launchValidation(value) + } + } +} + +/** Create a new validator that combines the validation of underlying validators. */ +fun all(vararg validators: Validator) = AllValidator(validators.toList()) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/DirectValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/DirectValidator.kt new file mode 100644 index 00000000..0dc2d3ef --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/DirectValidator.kt @@ -0,0 +1,15 @@ +package org.radarbase.schema.validation.rules + +import org.radarbase.schema.validation.ValidationContext + +/** Validator that checks given predicate. */ +class DirectValidator( + private val validation: ValidationContext.(T) -> Unit, +) : Validator { + override fun ValidationContext.runValidation(value: T) { + validation(value) + } +} + +/** Implementation of validator that passes given function as in a new Validator object. */ +fun Validator(validation: ValidationContext.(T) -> Unit): Validator = DirectValidator(validation) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PathRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PathRules.kt new file mode 100644 index 00000000..7401bef7 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PathRules.kt @@ -0,0 +1,11 @@ +package org.radarbase.schema.validation.rules + +import java.nio.file.Path +import kotlin.io.path.extension + +/** Validator that checks if a path has given extension. */ +fun hasExtension(extension: String) = Validator { path -> + if (!path.extension.equals(extension, ignoreCase = true)) { + raise("Path $path does not have extension $extension") + } +} diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PredicateValidator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PredicateValidator.kt new file mode 100644 index 00000000..e7b31169 --- /dev/null +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/PredicateValidator.kt @@ -0,0 +1,20 @@ +package org.radarbase.schema.validation.rules + +import org.radarbase.schema.validation.ValidationContext + +/** Validator that checks given [predicate] and raises with [message] if it fails. */ +class PredicateValidator( + private val predicate: (T) -> Boolean, + private val message: (T) -> String, +) : Validator { + override fun ValidationContext.runValidation(value: T) { + if (!predicate(value)) { + raise(message(value)) + } + } +} + +/** Create a validator that checks given predicate and raises with message if it does not match. */ +fun validator(predicate: (T) -> Boolean, message: String): Validator = PredicateValidator(predicate) { message } + +fun validator(predicate: (T) -> Boolean, message: (T) -> String): Validator = PredicateValidator(predicate, message) diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt similarity index 73% rename from java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt rename to java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt index aeb63de2..5d9c805c 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaFieldRules.kt @@ -8,15 +8,15 @@ import org.apache.avro.Schema.Type.NULL import org.apache.avro.Schema.Type.RECORD import org.apache.avro.Schema.Type.UNION import org.radarbase.schema.validation.ValidationContext -import org.radarbase.schema.validation.rules.RadarSchemaRules.Companion.validateDocumentation +import org.radarbase.schema.validation.rules.SchemaRules.Companion.validateDocumentation import java.util.EnumMap /** * Rules for RADAR-Schemas schema fields. */ -class RadarSchemaFieldRules { +class SchemaFieldRules { private val defaultsValidator: MutableMap> - internal lateinit var schemaRules: RadarSchemaRules + internal lateinit var schemaRules: SchemaRules /** * Rules for RADAR-Schemas schema fields. @@ -45,26 +45,36 @@ class RadarSchemaFieldRules { val schema = field.field.schema() val subType = schema.type when (subType) { - UNION -> validateInternalUnion.launchValidation(field) - RECORD -> schemaRules.isRecordValid.launchValidation(schema) - ENUM -> schemaRules.isEnumValid.launchValidation(schema) + UNION -> validateInternalUnion.validate(field) + RECORD -> schemaRules.isRecordValid.validate(schema) + ENUM -> schemaRules.isEnumValid.validate(schema) else -> Unit } } + private val isDefaultValueNullable = Validator { field -> + if (field.field.defaultVal() != null) { + raise( + field, + "Default of type ${field.field.schema().type} is set to ${field.field.defaultVal()}. " + + "The only acceptable default values are the \"UNKNOWN\" enum symbol and null.", + ) + } + } + val isDefaultValueValid = Validator { input: SchemaField -> defaultsValidator .getOrDefault( input.field.schema().type, - Validator { isDefaultValueNullable(it) }, + isDefaultValueNullable, ) - .launchValidation(input) + .validate(input) } val isNameValid = validator( predicate = { f -> f.field.name()?.matches(FIELD_NAME_PATTERN) == true }, - message = "Field name does not respect lowerCamelCase name convention." + - " Please avoid abbreviations and write out the field name instead.", + message = "Field name does not respect lowerCamelCase name convention. " + + "Please avoid abbreviations and write out the field name instead.", ) val isDocumentationValid = Validator { field: SchemaField -> @@ -85,7 +95,7 @@ class RadarSchemaFieldRules { private fun ValidationContext.isEnumDefaultUnknown(field: SchemaField) { if ( field.field.schema().enumSymbols.contains(UNKNOWN) && - !(field.field.defaultVal() != null && field.field.defaultVal().toString() == UNKNOWN) + field.field.defaultVal()?.toString() != UNKNOWN ) { raise( field, @@ -96,30 +106,19 @@ class RadarSchemaFieldRules { private fun ValidationContext.isDefaultUnionCompatible(field: SchemaField) { if ( - field.field.schema().types.contains(Schema.create(NULL)) && - !(field.field.defaultVal() != null && field.field.defaultVal() == JsonProperties.NULL_VALUE) + field.field.schema().types.contains(NULL_SCHEMA) && + field.field.defaultVal() != JsonProperties.NULL_VALUE ) { raise( field, - "Default is not null. Any nullable Avro field must" + - " specify have its default value set to null.", - ) - } - } - - private fun ValidationContext.isDefaultValueNullable(field: SchemaField) { - if (field.field.defaultVal() != null) { - raise( - field, - "Default of type " + field.field.schema().type + " is set to " + - field.field.defaultVal() + ". The only acceptable default values are the" + - " \"UNKNOWN\" enum symbol and null.", + "Default is not null. Any nullable Avro field must specify have its default value set to null.", ) } } companion object { private const val UNKNOWN = "UNKNOWN" + private val NULL_SCHEMA = Schema.create(NULL) // lowerCamelCase internal val FIELD_NAME_PATTERN = "[a-z][a-z0-9]*([a-z0-9][A-Z][a-z0-9]+)?([A-Z][a-z0-9]+)*[A-Z]?".toRegex() diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt similarity index 89% rename from java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt rename to java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt index fb36691e..ac244590 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaMetadataRules.kt @@ -15,10 +15,10 @@ import java.nio.file.PathMatcher * @param config configuration for excluding schemas from validation. * @param schemaRules schema rules implementation. */ -class RadarSchemaMetadataRules( +class SchemaMetadataRules( private val schemaRoot: Path, config: SchemaConfig, - val schemaRules: RadarSchemaRules = RadarSchemaRules(), + val schemaRules: SchemaRules = SchemaRules(), ) { private val pathMatcher: PathMatcher = config.pathMatcher(schemaRoot) @@ -32,6 +32,10 @@ class RadarSchemaMetadataRules( * and type of the schema. */ fun isSchemaMetadataValid(scopeSpecificValidation: Boolean) = Validator { metadata -> + if (!pathMatcher.matches(metadata.path)) { + return@Validator + } + isSchemaLocationCorrect.launchValidation(metadata) val ruleset = when { @@ -42,7 +46,7 @@ class RadarSchemaMetadataRules( metadata.scope == Scope.PASSIVE -> schemaRules.isPassiveSourceValid else -> schemaRules.isRecordValid } - isSchemaCorrect(ruleset).launchValidation(metadata) + ruleset.launchValidation(metadata.schema) } private fun isNamespaceSchemaLocationCorrect() = Validator { metadata -> @@ -66,12 +70,6 @@ class RadarSchemaMetadataRules( raise(metadata, "Record name should match file name. Expected record name is \"$expected\".") } } - - fun isSchemaCorrect(validator: Validator) = Validator { metadata -> - if (pathMatcher.matches(metadata.path)) { - validator.launchValidation(metadata.schema) - } - } } fun ValidationContext.raise(metadata: SchemaMetadata, text: String) { diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt similarity index 97% rename from java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt rename to java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt index 2c1ab78c..e8e9be76 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/RadarSchemaRules.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/SchemaRules.kt @@ -27,8 +27,8 @@ import org.radarbase.schema.validation.ValidationContext /** * Schema validation rules enforced for the RADAR-Schemas repository. */ -class RadarSchemaRules( - val fieldRules: RadarSchemaFieldRules = RadarSchemaFieldRules(), +class SchemaRules( + val fieldRules: SchemaFieldRules = SchemaFieldRules(), ) { val schemaStore: MutableMap = HashMap() @@ -159,9 +159,7 @@ class RadarSchemaRules( "Default validation can be applied only to an Avro RECORD, not to ${schema.type} of schema ${schema.fullName}.", ) schema.fields.isEmpty() -> raise("Schema ${schema.fullName} does not contain any fields.") - else -> schema.fields.forEach { field -> - validator.launchValidation(SchemaField(schema, field)) - } + else -> validator.validateAll(schema.fields.map { SchemaField(schema, it) }) } } diff --git a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt index 3f34fcea..f2485207 100644 --- a/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt +++ b/java-sdk/radar-schemas-core/src/main/java/org/radarbase/schema/validation/rules/Validator.kt @@ -16,35 +16,8 @@ package org.radarbase.schema.validation.rules import org.radarbase.schema.validation.ValidationContext -import java.nio.file.Path -import kotlin.io.path.extension -open class Validator( - private val validation: ValidationContext.(T) -> Unit, -) { - open fun ValidationContext.runValidation(value: T) { - this.validation(value) - } -} - -fun validator(predicate: (T) -> Boolean, message: String): Validator = - Validator { obj -> - if (!predicate(obj)) raise(message) - } - -fun validator(predicate: (T) -> Boolean, message: (T) -> String): Validator = - Validator { obj -> - if (!predicate(obj)) raise(message(obj)) - } - -fun all(vararg validators: Validator) = Validator { obj -> - validators.forEach { - it.launchValidation(obj) - } -} - -fun pathExtensionValidator(extension: String) = Validator { path -> - if (!path.extension.equals(extension, ignoreCase = true)) { - raise("Path $path does not have extension $extension") - } +/** Base validator type. */ +interface Validator { + fun ValidationContext.runValidation(value: T) } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt index a93aadbc..b1cf5d5b 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaMetadataRulesTest.kt @@ -32,12 +32,12 @@ import org.radarbase.schema.validation.validate import java.nio.file.Paths class RadarSchemaMetadataRulesTest { - private lateinit var validator: RadarSchemaMetadataRules + private lateinit var validator: SchemaMetadataRules @BeforeEach fun setUp() { val config = SchemaConfig() - validator = RadarSchemaMetadataRules( + validator = SchemaMetadataRules( SourceCatalogueValidationTest.BASE_PATH.resolve(ValidationHelper.COMMONS_PATH), config, ) } diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt index 5c183c10..a764c685 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaRulesTest.kt @@ -27,53 +27,53 @@ import org.junit.jupiter.api.Test import org.radarbase.schema.validation.validate class RadarSchemaRulesTest { - private lateinit var validator: RadarSchemaRules + private lateinit var validator: SchemaRules @BeforeEach fun setUp() { - validator = RadarSchemaRules() + validator = SchemaRules() } @Test fun nameSpaceRegex() { - assertTrue("org.radarcns".matches(RadarSchemaRules.NAMESPACE_PATTERN)) - assertFalse("Org.radarcns".matches(RadarSchemaRules.NAMESPACE_PATTERN)) - assertFalse("org.radarCns".matches(RadarSchemaRules.NAMESPACE_PATTERN)) - assertFalse(".org.radarcns".matches(RadarSchemaRules.NAMESPACE_PATTERN)) - assertFalse("org.radar-cns".matches(RadarSchemaRules.NAMESPACE_PATTERN)) - assertFalse("org.radarcns.empaticaE4".matches(RadarSchemaRules.NAMESPACE_PATTERN)) + assertTrue("org.radarcns".matches(SchemaRules.NAMESPACE_PATTERN)) + assertFalse("Org.radarcns".matches(SchemaRules.NAMESPACE_PATTERN)) + assertFalse("org.radarCns".matches(SchemaRules.NAMESPACE_PATTERN)) + assertFalse(".org.radarcns".matches(SchemaRules.NAMESPACE_PATTERN)) + assertFalse("org.radar-cns".matches(SchemaRules.NAMESPACE_PATTERN)) + assertFalse("org.radarcns.empaticaE4".matches(SchemaRules.NAMESPACE_PATTERN)) } @Test fun recordNameRegex() { - assertTrue("Questionnaire".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertTrue("EmpaticaE4Acceleration".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertTrue("Heart4Me".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertTrue("Heart4M".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertTrue("Heart4".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertFalse("Heart4me".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertTrue("Heart4ME".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertFalse("4Me".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertTrue("TTest".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertFalse("questionnaire".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertFalse("questionnaire4".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertFalse("questionnaire4Me".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertFalse("questionnaire4me".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertTrue("A4MM".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) - assertTrue("Aaaa4MMaa".matches(RadarSchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Questionnaire".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("EmpaticaE4Acceleration".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Heart4Me".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Heart4M".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Heart4".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertFalse("Heart4me".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Heart4ME".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertFalse("4Me".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("TTest".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertFalse("questionnaire".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertFalse("questionnaire4".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertFalse("questionnaire4Me".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertFalse("questionnaire4me".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("A4MM".matches(SchemaRules.RECORD_NAME_PATTERN)) + assertTrue("Aaaa4MMaa".matches(SchemaRules.RECORD_NAME_PATTERN)) } @Test fun enumerationRegex() { - assertTrue("PHQ8".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) - assertTrue("HELLO".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) - assertTrue("HELLOTHERE".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) - assertTrue("HELLO_THERE".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) - assertFalse("Hello".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) - assertFalse("hello".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) - assertFalse("HelloThere".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) - assertFalse("Hello_There".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) - assertFalse("HELLO.THERE".matches(RadarSchemaRules.ENUM_SYMBOL_PATTERN)) + assertTrue("PHQ8".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertTrue("HELLO".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertTrue("HELLOTHERE".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertTrue("HELLO_THERE".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("Hello".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("hello".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("HelloThere".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("Hello_There".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) + assertFalse("HELLO.THERE".matches(SchemaRules.ENUM_SYMBOL_PATTERN)) } @Test @@ -146,7 +146,7 @@ class RadarSchemaRulesTest { .builder("org.radarcns.time.test") .record(RECORD_NAME_MOCK) .fields() - .requiredDouble(RadarSchemaRules.TIME) + .requiredDouble(SchemaRules.TIME) .endRecord() result = validator.hasTime.validate(schema) Assertions.assertEquals(0, result.count()) diff --git a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/SchemaFieldRulesTest.kt similarity index 84% rename from java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt rename to java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/SchemaFieldRulesTest.kt index 98dea5f5..b4e0b1cf 100644 --- a/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/RadarSchemaFieldRulesTest.kt +++ b/java-sdk/radar-schemas-core/src/test/java/org/radarbase/schema/validation/rules/SchemaFieldRulesTest.kt @@ -29,14 +29,14 @@ import org.radarbase.schema.validation.toFormattedString import org.radarbase.schema.validation.validate import java.nio.file.Paths -class RadarSchemaFieldRulesTest { - private lateinit var validator: RadarSchemaFieldRules - private lateinit var schemaValidator: RadarSchemaRules +class SchemaFieldRulesTest { + private lateinit var validator: SchemaFieldRules + private lateinit var schemaValidator: SchemaRules @BeforeEach fun setUp() { - validator = RadarSchemaFieldRules() - schemaValidator = RadarSchemaRules(validator) + validator = SchemaFieldRules() + schemaValidator = SchemaRules(validator) } @Test @@ -53,13 +53,13 @@ class RadarSchemaFieldRulesTest { @Test fun fieldNameRegex() { - assertTrue("interBeatInterval".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) - assertTrue("x".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) - assertTrue(RadarSchemaRules.TIME.matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) - assertTrue("subjectId".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) - assertTrue("listOfSeveralThings".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) - assertFalse("Time".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) - assertFalse("E4Heart".matches(RadarSchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue("interBeatInterval".matches(SchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue("x".matches(SchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue(SchemaRules.TIME.matches(SchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue("subjectId".matches(SchemaFieldRules.FIELD_NAME_PATTERN)) + assertTrue("listOfSeveralThings".matches(SchemaFieldRules.FIELD_NAME_PATTERN)) + assertFalse("Time".matches(SchemaFieldRules.FIELD_NAME_PATTERN)) + assertFalse("E4Heart".matches(SchemaFieldRules.FIELD_NAME_PATTERN)) } @Test From e1f931f061a8a0f5fa2da28d73960c9f00ab758c Mon Sep 17 00:00:00 2001 From: Pauline Conde Date: Tue, 10 Oct 2023 16:26:51 +0100 Subject: [PATCH 11/27] Update snapshot version --- java-sdk/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java-sdk/build.gradle.kts b/java-sdk/build.gradle.kts index 6ae361c9..7798c2d2 100644 --- a/java-sdk/build.gradle.kts +++ b/java-sdk/build.gradle.kts @@ -9,7 +9,7 @@ plugins { } allprojects { - version = "0.8.5" + version = "0.8.6-SNAPSHOT" group = "org.radarbase" } From fc674c3f8cd0b5d0d6352f5a78718a9865952267 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 16 Oct 2023 14:40:23 +0200 Subject: [PATCH 12/27] Bump dev version --- java-sdk/buildSrc/src/main/kotlin/Versions.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java-sdk/buildSrc/src/main/kotlin/Versions.kt b/java-sdk/buildSrc/src/main/kotlin/Versions.kt index 4c5ea0a5..b43c3619 100644 --- a/java-sdk/buildSrc/src/main/kotlin/Versions.kt +++ b/java-sdk/buildSrc/src/main/kotlin/Versions.kt @@ -1,5 +1,5 @@ object Versions { - const val project = "0.8.5-SNAPSHOT" + const val project = "0.8.6-SNAPSHOT" const val kotlin = "1.9.10" const val java = 17 From ab2ce9febd5853636ed6394acc3dc678ef8f1349 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 16 Oct 2023 14:41:09 +0200 Subject: [PATCH 13/27] Added Fitbit Intraday HRV --- .../fitbit_intraday_heart_rate_variability.avsc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 commons/connector/fitbit/fitbit_intraday_heart_rate_variability.avsc diff --git a/commons/connector/fitbit/fitbit_intraday_heart_rate_variability.avsc b/commons/connector/fitbit/fitbit_intraday_heart_rate_variability.avsc new file mode 100644 index 00000000..a61171fa --- /dev/null +++ b/commons/connector/fitbit/fitbit_intraday_heart_rate_variability.avsc @@ -0,0 +1,14 @@ +{ + "namespace": "org.radarcns.connector.fitbit", + "type": "record", + "name": "FitbitIntradayHeartRateVariability", + "doc": "Intra day heart rate data from fitbit device.", + "fields": [ + { "name": "time", "type": "double", "doc": "Device timestamp in UTC (s)." }, + { "name": "timeReceived", "type": "double", "doc": "Time that the data was received from the Fitbit API (seconds since the Unix Epoch)." }, + { "name": "dailyRmssd", "type": "int", "doc": "The Root Mean Square of Successive Differences (RMSSD) between heart beats. It measures short-term variability in the user’s heart rate in milliseconds (ms)."}, + { "name": "coverage", "type": "int", "doc": "Data completeness in terms of the number of interbeat intervals (0-1)."}, + { "name": "highFrequency", "type": "int", "doc": "The power in interbeat interval fluctuations within the high frequency band (0.15 Hz - 0.4 Hz)."}, + { "name": "lowFrequency", "type": "int", "doc": "The power in interbeat interval fluctuations within the low frequency band (0.04 Hz - 0.15 Hz)."} + ] +} From 1f05d496b3f3ab4d89e0489852839dde1c9fa670 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 16 Oct 2023 14:47:00 +0200 Subject: [PATCH 14/27] Fix Fitbit HRV data type --- .../fitbit/fitbit_intraday_heart_rate_variability.avsc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/commons/connector/fitbit/fitbit_intraday_heart_rate_variability.avsc b/commons/connector/fitbit/fitbit_intraday_heart_rate_variability.avsc index a61171fa..750c3192 100644 --- a/commons/connector/fitbit/fitbit_intraday_heart_rate_variability.avsc +++ b/commons/connector/fitbit/fitbit_intraday_heart_rate_variability.avsc @@ -6,9 +6,9 @@ "fields": [ { "name": "time", "type": "double", "doc": "Device timestamp in UTC (s)." }, { "name": "timeReceived", "type": "double", "doc": "Time that the data was received from the Fitbit API (seconds since the Unix Epoch)." }, - { "name": "dailyRmssd", "type": "int", "doc": "The Root Mean Square of Successive Differences (RMSSD) between heart beats. It measures short-term variability in the user’s heart rate in milliseconds (ms)."}, - { "name": "coverage", "type": "int", "doc": "Data completeness in terms of the number of interbeat intervals (0-1)."}, - { "name": "highFrequency", "type": "int", "doc": "The power in interbeat interval fluctuations within the high frequency band (0.15 Hz - 0.4 Hz)."}, - { "name": "lowFrequency", "type": "int", "doc": "The power in interbeat interval fluctuations within the low frequency band (0.04 Hz - 0.15 Hz)."} + { "name": "dailyRmssd", "type": "float", "doc": "The Root Mean Square of Successive Differences (RMSSD) between heart beats. It measures short-term variability in the user’s heart rate in milliseconds (ms)."}, + { "name": "coverage", "type": "float", "doc": "Data completeness in terms of the number of interbeat intervals (0-1)."}, + { "name": "highFrequency", "type": "float", "doc": "The power in interbeat interval fluctuations within the high frequency band (0.15 Hz - 0.4 Hz)."}, + { "name": "lowFrequency", "type": "float", "doc": "The power in interbeat interval fluctuations within the low frequency band (0.04 Hz - 0.15 Hz)."} ] } From e10bb51af65d1c99c63e5d961a21bfc9c7d72499 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 16 Oct 2023 14:58:26 +0200 Subject: [PATCH 15/27] Added Fitbit Intraday HRV to spec --- specifications/connector/radar-fitbit-connector.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specifications/connector/radar-fitbit-connector.yml b/specifications/connector/radar-fitbit-connector.yml index eef2218f..387322d1 100644 --- a/specifications/connector/radar-fitbit-connector.yml +++ b/specifications/connector/radar-fitbit-connector.yml @@ -1,7 +1,7 @@ name: RADAR-FITBIT-CONNECTOR vendor: RADAR-base model: radar-connect-fitbit-source -version: 0.2.1 +version: 0.2.2 doc: Spec for Radar fitbit connector. Schemas should be registered in the connector. data: - doc: The intraday time series for heart rate. @@ -28,3 +28,6 @@ data: - doc: The Food Log for the day. topic: connect_fitbit_food_log value_schema: .connector.fitbit.FitbitFoodLog + - doc: Intraday heart rate variability + topic: connect_fitbit_intraday_heart_rate_variability + value_schema: .connector.fitbit.FitbitIntradayHeartRateVariability From 9ea8bfc490721ee9ce3157efa706a2429135ae50 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Oct 2023 15:28:33 +0200 Subject: [PATCH 16/27] Address PR comments --- .../fitbit/fitbit_intraday_heart_rate_variability.avsc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/commons/connector/fitbit/fitbit_intraday_heart_rate_variability.avsc b/commons/connector/fitbit/fitbit_intraday_heart_rate_variability.avsc index 750c3192..a9a9901f 100644 --- a/commons/connector/fitbit/fitbit_intraday_heart_rate_variability.avsc +++ b/commons/connector/fitbit/fitbit_intraday_heart_rate_variability.avsc @@ -2,11 +2,11 @@ "namespace": "org.radarcns.connector.fitbit", "type": "record", "name": "FitbitIntradayHeartRateVariability", - "doc": "Intra day heart rate data from fitbit device.", + "doc": "Intra day heart rate variability (HRV) data from fitbit device. HRV data applies specifically to a user’s “main sleep,” which is the longest single period of time asleep on a given date.", "fields": [ { "name": "time", "type": "double", "doc": "Device timestamp in UTC (s)." }, { "name": "timeReceived", "type": "double", "doc": "Time that the data was received from the Fitbit API (seconds since the Unix Epoch)." }, - { "name": "dailyRmssd", "type": "float", "doc": "The Root Mean Square of Successive Differences (RMSSD) between heart beats. It measures short-term variability in the user’s heart rate in milliseconds (ms)."}, + { "name": "rmssd", "type": "float", "doc": "The Root Mean Square of Successive Differences (RMSSD) between heart beats. It measures short-term variability in the user’s heart rate in milliseconds (ms)."}, { "name": "coverage", "type": "float", "doc": "Data completeness in terms of the number of interbeat intervals (0-1)."}, { "name": "highFrequency", "type": "float", "doc": "The power in interbeat interval fluctuations within the high frequency band (0.15 Hz - 0.4 Hz)."}, { "name": "lowFrequency", "type": "float", "doc": "The power in interbeat interval fluctuations within the low frequency band (0.04 Hz - 0.15 Hz)."} From e2d9b19c56cfde113c4f3ddcd71a50e1c9d7f50c Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Oct 2023 17:18:27 +0200 Subject: [PATCH 17/27] Fix snapshot check --- .github/workflows/publish_snapshots.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_snapshots.yml b/.github/workflows/publish_snapshots.yml index eaf3d802..ceebb9a0 100644 --- a/.github/workflows/publish_snapshots.yml +++ b/.github/workflows/publish_snapshots.yml @@ -23,7 +23,7 @@ jobs: - name: Has SNAPSHOT version id: is-snapshot - run: grep 'version = ".*-SNAPSHOT"' build.gradle.kts + run: grep 'const val project = ".*-SNAPSHOT"' buildSrc/src/main/kotlin/Versions.kt - uses: actions/setup-java@v3 with: From 3aad7e6b6d1918fb02426f6ec28abca178b1597e Mon Sep 17 00:00:00 2001 From: Peyman Mohtashami Date: Mon, 30 Oct 2023 11:24:51 +0100 Subject: [PATCH 18/27] Add Fitbit Skin Temperature & Breathing Rate specifications and avro schemas --- .../fitbit/fitbit_intraday_breathing_rate.avsc | 14 ++++++++++++++ .../connector/fitbit/fitbit_skin_temperature.avsc | 12 ++++++++++++ .../connector/radar-fitbit-connector.yml | 6 ++++++ 3 files changed, 32 insertions(+) create mode 100644 commons/connector/fitbit/fitbit_intraday_breathing_rate.avsc create mode 100644 commons/connector/fitbit/fitbit_skin_temperature.avsc diff --git a/commons/connector/fitbit/fitbit_intraday_breathing_rate.avsc b/commons/connector/fitbit/fitbit_intraday_breathing_rate.avsc new file mode 100644 index 00000000..126b3733 --- /dev/null +++ b/commons/connector/fitbit/fitbit_intraday_breathing_rate.avsc @@ -0,0 +1,14 @@ +{ + "namespace": "org.radarcns.connector.fitbit", + "type": "record", + "name": "FitbitIntradayBreathingRate", + "doc": "Intra day breathing rate (BR) data from fitbit device. BR measures the average breathing rate throughout the day and categories your breathing rate by sleep stage. Sleep stages vary between light sleep, deep sleep, REM sleep, and full sleep.", + "fields": [ + { "name": "time", "type": "double", "doc": "Device timestamp in UTC (s)." }, + { "name": "timeReceived", "type": "double", "doc": "Time that the data was received from the Fitbit API (seconds since the Unix Epoch)." }, + { "name": "lightSleepSummary", "type": "float", "doc": "Average number of breaths taken per minute when the user was in light sleep."}, + { "name": "deepSleepSummary", "type": "float", "doc": "Average number of breaths taken per minute when the user was in deep sleep."}, + { "name": "remSleepSummary", "type": "float", "doc": "Average number of breaths taken per minute when the user was in rem sleep."}, + { "name": "fullSleepSummary", "type": "float", "doc": "Average number of breaths taken per minute throughout the entire period of sleep which you can compare to the sleep stage-specific measurements."}, + ] +} diff --git a/commons/connector/fitbit/fitbit_skin_temperature.avsc b/commons/connector/fitbit/fitbit_skin_temperature.avsc new file mode 100644 index 00000000..e179ee71 --- /dev/null +++ b/commons/connector/fitbit/fitbit_skin_temperature.avsc @@ -0,0 +1,12 @@ +{ + "namespace": "org.radarcns.connector.fitbit", + "type": "record", + "name": "FitbitSkinTemperature", + "doc": "Skin temperature (tempSkin) data from fitbit device. tempSkin measures skin temperature data for a date range. It only returns a value for dates on which the Fitbit device was able to record Temperature (skin) data and the maximum date range cannot exceed 30 days. Temperature (Skin) data applies specifically to a user’s “main sleep,” which is the longest single period of time asleep on a given date.", + "fields": [ + { "name": "time", "type": "double", "doc": "Device timestamp in UTC (s)." }, + { "name": "timeReceived", "type": "double", "doc": "Time that the data was received from the Fitbit API (seconds since the Unix Epoch)." }, + { "name": "nightlyRelative", "type": "float", "doc": "The user's average temperature during a period of sleep. It is displayed to the user as a delta from their baseline temperature in degrees Celsius or Fahrenheit depending on the country specified in the Accept-Language header."}, + { "name": "logType", "type": { "name": "FitbitSkinTemperatureLogType", "type": "enum", "symbols": ["DEDICATED_TEMP_SENSOR", "OTHER_SENSORS", "UNKNOWN"], "doc": "The type of skin temperature log created."}, "doc": "The type of skin temperature log created."}, + ] +} diff --git a/specifications/connector/radar-fitbit-connector.yml b/specifications/connector/radar-fitbit-connector.yml index 387322d1..d10453e7 100644 --- a/specifications/connector/radar-fitbit-connector.yml +++ b/specifications/connector/radar-fitbit-connector.yml @@ -31,3 +31,9 @@ data: - doc: Intraday heart rate variability topic: connect_fitbit_intraday_heart_rate_variability value_schema: .connector.fitbit.FitbitIntradayHeartRateVariability + - doc: Intraday breathing rate + topic: connect_fitbit_intraday_breathing_rate + value_schema: .connector.fitbit.FitbitIntradayBreathingRate + - doc: Skin temperature + topic: connect_fitbit_skin_temperature + value_schema: .connector.fitbit.FitbitSkinTemperature From f3e34c1ecf5abb02c9f4b63b5bdf452e4f077c40 Mon Sep 17 00:00:00 2001 From: Peyman Mohtashami Date: Mon, 30 Oct 2023 12:40:02 +0100 Subject: [PATCH 19/27] Add default for skin temperature logType --- commons/connector/fitbit/fitbit_skin_temperature.avsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commons/connector/fitbit/fitbit_skin_temperature.avsc b/commons/connector/fitbit/fitbit_skin_temperature.avsc index e179ee71..4af58866 100644 --- a/commons/connector/fitbit/fitbit_skin_temperature.avsc +++ b/commons/connector/fitbit/fitbit_skin_temperature.avsc @@ -7,6 +7,6 @@ { "name": "time", "type": "double", "doc": "Device timestamp in UTC (s)." }, { "name": "timeReceived", "type": "double", "doc": "Time that the data was received from the Fitbit API (seconds since the Unix Epoch)." }, { "name": "nightlyRelative", "type": "float", "doc": "The user's average temperature during a period of sleep. It is displayed to the user as a delta from their baseline temperature in degrees Celsius or Fahrenheit depending on the country specified in the Accept-Language header."}, - { "name": "logType", "type": { "name": "FitbitSkinTemperatureLogType", "type": "enum", "symbols": ["DEDICATED_TEMP_SENSOR", "OTHER_SENSORS", "UNKNOWN"], "doc": "The type of skin temperature log created."}, "doc": "The type of skin temperature log created."}, + { "name": "logType", "type": { "name": "FitbitSkinTemperatureLogType", "type": "enum", "symbols": ["DEDICATED_TEMP_SENSOR", "OTHER_SENSORS", "UNKNOWN"], "doc": "The type of skin temperature log created."}, "doc": "The type of skin temperature log created.", "default": "UNKNOWN"}, ] } From 8db92590bcf8c87498cda820c711b0b8dc5367f8 Mon Sep 17 00:00:00 2001 From: Peyman Mohtashami Date: Mon, 30 Oct 2023 14:48:31 +0100 Subject: [PATCH 20/27] Modify BR name and fields and SkinTemperature relativeTemperature --- .../connector/fitbit/fitbit_breathing_rate.avsc | 14 ++++++++++++++ .../fitbit/fitbit_intraday_breathing_rate.avsc | 14 -------------- .../connector/fitbit/fitbit_skin_temperature.avsc | 2 +- .../connector/radar-fitbit-connector.yml | 6 +++--- 4 files changed, 18 insertions(+), 18 deletions(-) create mode 100644 commons/connector/fitbit/fitbit_breathing_rate.avsc delete mode 100644 commons/connector/fitbit/fitbit_intraday_breathing_rate.avsc diff --git a/commons/connector/fitbit/fitbit_breathing_rate.avsc b/commons/connector/fitbit/fitbit_breathing_rate.avsc new file mode 100644 index 00000000..4e37aeab --- /dev/null +++ b/commons/connector/fitbit/fitbit_breathing_rate.avsc @@ -0,0 +1,14 @@ +{ + "namespace": "org.radarcns.connector.fitbit", + "type": "record", + "name": "FitbitBreathingRate", + "doc": "Breathing rate (BR) data from fitbit device. BR measures the average breathing rate throughout the day and categories your breathing rate by sleep stage. Sleep stages vary between light sleep, deep sleep, REM sleep, and full sleep.", + "fields": [ + { "name": "time", "type": "double", "doc": "Device timestamp in UTC (s)." }, + { "name": "timeReceived", "type": "double", "doc": "Time that the data was received from the Fitbit API (seconds since the Unix Epoch)." }, + { "name": "lightSleep", "type": "float", "doc": "Average number of breaths taken per minute when the user was in light sleep."}, + { "name": "deepSleep", "type": "float", "doc": "Average number of breaths taken per minute when the user was in deep sleep."}, + { "name": "remSleep", "type": "float", "doc": "Average number of breaths taken per minute when the user was in rem sleep."}, + { "name": "fullSleep", "type": "float", "doc": "Average number of breaths taken per minute throughout the entire period of sleep which you can compare to the sleep stage-specific measurements."}, + ] +} diff --git a/commons/connector/fitbit/fitbit_intraday_breathing_rate.avsc b/commons/connector/fitbit/fitbit_intraday_breathing_rate.avsc deleted file mode 100644 index 126b3733..00000000 --- a/commons/connector/fitbit/fitbit_intraday_breathing_rate.avsc +++ /dev/null @@ -1,14 +0,0 @@ -{ - "namespace": "org.radarcns.connector.fitbit", - "type": "record", - "name": "FitbitIntradayBreathingRate", - "doc": "Intra day breathing rate (BR) data from fitbit device. BR measures the average breathing rate throughout the day and categories your breathing rate by sleep stage. Sleep stages vary between light sleep, deep sleep, REM sleep, and full sleep.", - "fields": [ - { "name": "time", "type": "double", "doc": "Device timestamp in UTC (s)." }, - { "name": "timeReceived", "type": "double", "doc": "Time that the data was received from the Fitbit API (seconds since the Unix Epoch)." }, - { "name": "lightSleepSummary", "type": "float", "doc": "Average number of breaths taken per minute when the user was in light sleep."}, - { "name": "deepSleepSummary", "type": "float", "doc": "Average number of breaths taken per minute when the user was in deep sleep."}, - { "name": "remSleepSummary", "type": "float", "doc": "Average number of breaths taken per minute when the user was in rem sleep."}, - { "name": "fullSleepSummary", "type": "float", "doc": "Average number of breaths taken per minute throughout the entire period of sleep which you can compare to the sleep stage-specific measurements."}, - ] -} diff --git a/commons/connector/fitbit/fitbit_skin_temperature.avsc b/commons/connector/fitbit/fitbit_skin_temperature.avsc index 4af58866..f1ec7446 100644 --- a/commons/connector/fitbit/fitbit_skin_temperature.avsc +++ b/commons/connector/fitbit/fitbit_skin_temperature.avsc @@ -6,7 +6,7 @@ "fields": [ { "name": "time", "type": "double", "doc": "Device timestamp in UTC (s)." }, { "name": "timeReceived", "type": "double", "doc": "Time that the data was received from the Fitbit API (seconds since the Unix Epoch)." }, - { "name": "nightlyRelative", "type": "float", "doc": "The user's average temperature during a period of sleep. It is displayed to the user as a delta from their baseline temperature in degrees Celsius or Fahrenheit depending on the country specified in the Accept-Language header."}, + { "name": "relativeTemperature", "type": "float", "doc": "The user's average temperature during a period of sleep. It is displayed to the user as a delta from their baseline temperature in degrees Celsius."}, { "name": "logType", "type": { "name": "FitbitSkinTemperatureLogType", "type": "enum", "symbols": ["DEDICATED_TEMP_SENSOR", "OTHER_SENSORS", "UNKNOWN"], "doc": "The type of skin temperature log created."}, "doc": "The type of skin temperature log created.", "default": "UNKNOWN"}, ] } diff --git a/specifications/connector/radar-fitbit-connector.yml b/specifications/connector/radar-fitbit-connector.yml index d10453e7..15328306 100644 --- a/specifications/connector/radar-fitbit-connector.yml +++ b/specifications/connector/radar-fitbit-connector.yml @@ -31,9 +31,9 @@ data: - doc: Intraday heart rate variability topic: connect_fitbit_intraday_heart_rate_variability value_schema: .connector.fitbit.FitbitIntradayHeartRateVariability - - doc: Intraday breathing rate - topic: connect_fitbit_intraday_breathing_rate - value_schema: .connector.fitbit.FitbitIntradayBreathingRate + - doc: Breathing rate + topic: connect_fitbit_breathing_rate + value_schema: .connector.fitbit.FitbitBreathingRate - doc: Skin temperature topic: connect_fitbit_skin_temperature value_schema: .connector.fitbit.FitbitSkinTemperature From 2949635072f7f571a4e042a010a7e6245d00ecc1 Mon Sep 17 00:00:00 2001 From: Peyman Mohtashami Date: Tue, 31 Oct 2023 11:45:18 +0100 Subject: [PATCH 21/27] Bump specification version --- specifications/connector/radar-fitbit-connector.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specifications/connector/radar-fitbit-connector.yml b/specifications/connector/radar-fitbit-connector.yml index 15328306..3825f4e0 100644 --- a/specifications/connector/radar-fitbit-connector.yml +++ b/specifications/connector/radar-fitbit-connector.yml @@ -1,7 +1,7 @@ name: RADAR-FITBIT-CONNECTOR vendor: RADAR-base model: radar-connect-fitbit-source -version: 0.2.2 +version: 0.2.3 doc: Spec for Radar fitbit connector. Schemas should be registered in the connector. data: - doc: The intraday time series for heart rate. From 9e9d46e12a0ab7911eb47c365ddd095a3588b8e1 Mon Sep 17 00:00:00 2001 From: Peyman Mohtashami Date: Tue, 31 Oct 2023 12:43:58 +0100 Subject: [PATCH 22/27] Fix json files - remove extra comma --- commons/connector/fitbit/fitbit_breathing_rate.avsc | 2 +- commons/connector/fitbit/fitbit_skin_temperature.avsc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/commons/connector/fitbit/fitbit_breathing_rate.avsc b/commons/connector/fitbit/fitbit_breathing_rate.avsc index 4e37aeab..9fa0eac2 100644 --- a/commons/connector/fitbit/fitbit_breathing_rate.avsc +++ b/commons/connector/fitbit/fitbit_breathing_rate.avsc @@ -9,6 +9,6 @@ { "name": "lightSleep", "type": "float", "doc": "Average number of breaths taken per minute when the user was in light sleep."}, { "name": "deepSleep", "type": "float", "doc": "Average number of breaths taken per minute when the user was in deep sleep."}, { "name": "remSleep", "type": "float", "doc": "Average number of breaths taken per minute when the user was in rem sleep."}, - { "name": "fullSleep", "type": "float", "doc": "Average number of breaths taken per minute throughout the entire period of sleep which you can compare to the sleep stage-specific measurements."}, + { "name": "fullSleep", "type": "float", "doc": "Average number of breaths taken per minute throughout the entire period of sleep which you can compare to the sleep stage-specific measurements."} ] } diff --git a/commons/connector/fitbit/fitbit_skin_temperature.avsc b/commons/connector/fitbit/fitbit_skin_temperature.avsc index f1ec7446..859c7070 100644 --- a/commons/connector/fitbit/fitbit_skin_temperature.avsc +++ b/commons/connector/fitbit/fitbit_skin_temperature.avsc @@ -7,6 +7,6 @@ { "name": "time", "type": "double", "doc": "Device timestamp in UTC (s)." }, { "name": "timeReceived", "type": "double", "doc": "Time that the data was received from the Fitbit API (seconds since the Unix Epoch)." }, { "name": "relativeTemperature", "type": "float", "doc": "The user's average temperature during a period of sleep. It is displayed to the user as a delta from their baseline temperature in degrees Celsius."}, - { "name": "logType", "type": { "name": "FitbitSkinTemperatureLogType", "type": "enum", "symbols": ["DEDICATED_TEMP_SENSOR", "OTHER_SENSORS", "UNKNOWN"], "doc": "The type of skin temperature log created."}, "doc": "The type of skin temperature log created.", "default": "UNKNOWN"}, + { "name": "logType", "type": { "name": "FitbitSkinTemperatureLogType", "type": "enum", "symbols": ["DEDICATED_TEMP_SENSOR", "OTHER_SENSORS", "UNKNOWN"], "doc": "The type of skin temperature log created."}, "doc": "The type of skin temperature log created.", "default": "UNKNOWN"} ] } From 18cd038d586feed63b8b4f7fcc39d87513402aaf Mon Sep 17 00:00:00 2001 From: Peyman Mohtashami Date: Wed, 1 Nov 2023 11:25:41 +0100 Subject: [PATCH 23/27] Bump dev version --- java-sdk/buildSrc/src/main/kotlin/Versions.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java-sdk/buildSrc/src/main/kotlin/Versions.kt b/java-sdk/buildSrc/src/main/kotlin/Versions.kt index b43c3619..cae1e66a 100644 --- a/java-sdk/buildSrc/src/main/kotlin/Versions.kt +++ b/java-sdk/buildSrc/src/main/kotlin/Versions.kt @@ -1,5 +1,5 @@ object Versions { - const val project = "0.8.6-SNAPSHOT" + const val project = "0.8.7-SNAPSHOT" const val kotlin = "1.9.10" const val java = 17 From b1a1fbe5d1ec6bfa9915a72caf169a1f149f6b91 Mon Sep 17 00:00:00 2001 From: Aditya Mishra Date: Wed, 8 Nov 2023 15:54:31 +0530 Subject: [PATCH 24/27] Upadted android-phone-1.0.0 to include PhoneBluetoothDeviceScanned --- specifications/passive/android_phone-1.0.0.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/specifications/passive/android_phone-1.0.0.yml b/specifications/passive/android_phone-1.0.0.yml index f5226e5d..bdbce9b5 100644 --- a/specifications/passive/android_phone-1.0.0.yml +++ b/specifications/passive/android_phone-1.0.0.yml @@ -134,6 +134,12 @@ data: sample_rate: interval: 3600 configurable: true + - type: PHONE_BLUETOOTH_DEVICE_SCANNED + app_provider: .phone.PhoneBluetoothService + unit: NON_DIMENSIONAL + processing_state: RAW + topic: android_phone_bluetooth_device_scanned + value_schema: .passive.phone.PhoneBluetoothDeviceScanned # Usage - type: USAGE_EVENT app_provider: .phone.PhoneUsageProvider From ff037ce15b5697162516557e1841236bba0ec6fd Mon Sep 17 00:00:00 2001 From: Aditya Mishra Date: Wed, 8 Nov 2023 15:55:07 +0530 Subject: [PATCH 25/27] Added schema PhoneBluetoothDeviceScanned --- .../phone/phone_bluetooth_device_scanned.avsc | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 commons/passive/phone/phone_bluetooth_device_scanned.avsc diff --git a/commons/passive/phone/phone_bluetooth_device_scanned.avsc b/commons/passive/phone/phone_bluetooth_device_scanned.avsc new file mode 100644 index 00000000..a4543050 --- /dev/null +++ b/commons/passive/phone/phone_bluetooth_device_scanned.avsc @@ -0,0 +1,13 @@ +{ + "namespace": "org.radarcns.passive.phone", + "type": "record", + "name": "PhoneBluetoothDeviceScanned", + "doc": "Phone Bluetooth device info.", + "fields": [ + {"name": "time", "type": "double", "doc": "Device timestamp in UTC (s)."}, + {"name": "timeReceived", "type": "double", "doc": "Device receiver timestamp in UTC (s)."}, + {"name": "macAddressHash", "type": ["null", "bytes"], "default": null, "doc":"Hash of Nearby Bluetooth device MAC address."}, + {"name": "hashSaltReference", "type": ["null", "int"], "doc": "Random identifier associated with the device or installation of the app. If the app gets reinstalled or installed on another device, it's clear during analysis that the mac addresses between iterations are not comparable.", "default": null}, + {"name": "isPaired", "type": "boolean", "doc": "Whether the bluetooth device is paired."} + ] +} From 788d3b06688a31d46e199f22b5d7cba4ca40e3ed Mon Sep 17 00:00:00 2001 From: Aditya Mishra Date: Wed, 8 Nov 2023 17:14:57 +0530 Subject: [PATCH 26/27] Updated default value of isPaired to null --- commons/passive/phone/phone_bluetooth_device_scanned.avsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commons/passive/phone/phone_bluetooth_device_scanned.avsc b/commons/passive/phone/phone_bluetooth_device_scanned.avsc index a4543050..80f52d35 100644 --- a/commons/passive/phone/phone_bluetooth_device_scanned.avsc +++ b/commons/passive/phone/phone_bluetooth_device_scanned.avsc @@ -8,6 +8,6 @@ {"name": "timeReceived", "type": "double", "doc": "Device receiver timestamp in UTC (s)."}, {"name": "macAddressHash", "type": ["null", "bytes"], "default": null, "doc":"Hash of Nearby Bluetooth device MAC address."}, {"name": "hashSaltReference", "type": ["null", "int"], "doc": "Random identifier associated with the device or installation of the app. If the app gets reinstalled or installed on another device, it's clear during analysis that the mac addresses between iterations are not comparable.", "default": null}, - {"name": "isPaired", "type": "boolean", "doc": "Whether the bluetooth device is paired."} + {"name": "isPaired", "type": ["null","boolean"], "doc": "Whether the bluetooth device is paired.", "default": null} ] } From c4ca60546cec95308f95ad80d575f445e8308fac Mon Sep 17 00:00:00 2001 From: Pauline Conde Date: Fri, 24 Nov 2023 16:44:18 -0400 Subject: [PATCH 27/27] Update versions --- java-sdk/buildSrc/src/main/kotlin/Versions.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java-sdk/buildSrc/src/main/kotlin/Versions.kt b/java-sdk/buildSrc/src/main/kotlin/Versions.kt index cae1e66a..460dd171 100644 --- a/java-sdk/buildSrc/src/main/kotlin/Versions.kt +++ b/java-sdk/buildSrc/src/main/kotlin/Versions.kt @@ -1,5 +1,5 @@ object Versions { - const val project = "0.8.7-SNAPSHOT" + const val project = "0.8.6" const val kotlin = "1.9.10" const val java = 17