diff --git a/NEWS.md b/NEWS.md index 05d569ec..3e3e0d1b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -78,6 +78,10 @@ This release includes: - The `outputFolder` parameter for the `executeDqChecks` function is now REQUIRED and no longer has a default value. **This may be a breaking change for users who have not specified this parameter in their script to run DQD.** +### Removal of measurement plausibility checks + + - Most plausibleValueLow and plausibleValueHigh measurement values were removed from the concept check threshold files, due to feedback from the community that many of these ranges included plausible values and as such were causing unexpected check failures. An initiative is planned to reinterrogate these ranges and add them back once the team has higher confidence that they will only flag legitimately implausible values + ### Integrated testing was also added and the package was refactored on the backend DataQualityDashboard 1.4.1 diff --git a/tests/testthat/setup.R b/tests/testthat/setup.R index 147aeb64..16ae8c13 100644 --- a/tests/testthat/setup.R +++ b/tests/testthat/setup.R @@ -20,46 +20,3 @@ remove_sql_comments <- function(sql) { sql4 <- gsub("@@@@ .+? @@@@", " ", sql3, ) # remove multi-line comments sql5 <- gsub("\\s+", " ", sql4) # remove multiple spaces } - -# dbms <- getOption("dbms", default = "sqlite") -# if (dbms == "sqlite") { -# connectionDetails <- Eunomia::getEunomiaConnectionDetails() -# cdmDatabaseSchema <- "main" -# resultsDatabaseSchema <- "main" -# cdmVersion <- 5 -# } -# if (dbms == "postgresql") { -# DatabaseConnector::downloadJdbcDrivers("postgresql", pathToDriver = jdbcDriverFolder) -# connectionDetails <- createConnectionDetails(dbms = "postgresql", -# user = Sys.getenv("CDM5_POSTGRESQL_USER"), -# password = URLdecode(Sys.getenv("CDM5_POSTGRESQL_PASSWORD")), -# server = Sys.getenv("CDM5_POSTGRESQL_SERVER"), -# pathToDriver = jdbcDriverFolder) -# -# cdmDatabaseSchema <- Sys.getenv("CDM5_POSTGRESQL_CDM_SCHEMA") -# cdmVersion <- 5 -# } -# if (dbms == "sql server") { -# DatabaseConnector::downloadJdbcDrivers("sql server", pathToDriver = jdbcDriverFolder) -# connectionDetails <- createConnectionDetails(dbms = "sql server", -# user = Sys.getenv("CDM5_SQL_SERVER_USER"), -# password = URLdecode(Sys.getenv("CDM5_SQL_SERVER_PASSWORD")), -# server = Sys.getenv("CDM5_SQL_SERVER_SERVER"), -# pathToDriver = jdbcDriverFolder) -# cdmDatabaseSchema <- Sys.getenv("CDM5_SQL_SERVER_CDM_SCHEMA") -# cdmVersion <- 5 -# } -# if (dbms == "oracle") { -# DatabaseConnector::downloadJdbcDrivers("oracle", pathToDriver = jdbcDriverFolder) -# connectionDetails <- createConnectionDetails(dbms = "oracle", -# user = Sys.getenv("CDM5_ORACLE_USER"), -# password = URLdecode(Sys.getenv("CDM5_ORACLE_PASSWORD")), -# server = Sys.getenv("CDM5_ORACLE_SERVER"), -# pathToDriver = jdbcDriverFolder) -# cdmDatabaseSchema <- Sys.getenv("CDM5_ORACLE_CDM_SCHEMA") -# -# # Restore temp schema setting after tests complete -# oldTempSchema <- getOption("sqlRenderTempEmulationSchema") -# options("sqlRenderTempEmulationSchema" = Sys.getenv("CDM5_ORACLE_OHDSI_SCHEMA")) -# cdmVersion <- 5 -# } diff --git a/tests/testthat/test-execute.R b/tests/testthat/test-executeDqChecks.R similarity index 84% rename from tests/testthat/test-execute.R rename to tests/testthat/test-executeDqChecks.R index 646d20cb..5a55fc8b 100644 --- a/tests/testthat/test-execute.R +++ b/tests/testthat/test-executeDqChecks.R @@ -1,11 +1,5 @@ library(testthat) -test_that("listDqChecks works", { - checks <- listDqChecks() - expect_equal(length(checks), 4) - expect_true(all(sapply(checks, is.data.frame))) -}) - test_that("Execute a single DQ check on Synthea/Eunomia", { outputFolder <- tempfile("dqd_") on.exit(unlink(outputFolder, recursive = TRUE)) @@ -43,7 +37,6 @@ test_that("Execute all TABLE checks on Synthea/Eunomia", { expect_true(nrow(results$CheckResults) > 0) }) - test_that("Execute FIELD checks on Synthea/Eunomia", { outputFolder <- tempfile("dqd_") on.exit(unlink(outputFolder, recursive = TRUE)) @@ -185,44 +178,6 @@ test_that("Check invalid cdm version", { ) }) -test_that("Write JSON results", { - outputFolder <- tempfile("dqd_") - on.exit(unlink(outputFolder, recursive = TRUE)) - - expect_warning( - results <- executeDqChecks( - connectionDetails = connectionDetailsEunomia, - cdmDatabaseSchema = cdmDatabaseSchemaEunomia, - resultsDatabaseSchema = resultsDatabaseSchemaEunomia, - cdmSourceName = "Eunomia", - checkNames = "measurePersonCompleteness", - outputFolder = outputFolder, - writeToTable = FALSE - ), - regexp = "^Missing check names.*" - ) - - jsonPath <- list.files(outputFolder, ".json", full.names = TRUE) - csvPath <- file.path(outputFolder, "results.csv") - writeJsonResultsToCsv( - jsonPath = jsonPath, - csvPath = csvPath - ) - expect_true(file.exists(csvPath)) - - DataQualityDashboard::writeJsonResultsToTable( - connectionDetails = connectionDetailsEunomia, - resultsDatabaseSchema = resultsDatabaseSchemaEunomia, - jsonFilePath = jsonPath, - writeTableName = "dqd_results" - ) - connection <- DatabaseConnector::connect(connectionDetailsEunomia) - on.exit(DatabaseConnector::disconnect(connection), add = TRUE) - tableNames <- DatabaseConnector::getTableNames(connection = connection, databaseSchema = resultsDatabaseSchemaEunomia) - expect_true("dqd_results" %in% tolower(tableNames)) - DatabaseConnector::renderTranslateExecuteSql(connection, "DROP TABLE @database_schema.dqd_results;", database_schema = resultsDatabaseSchemaEunomia) -}) - test_that("Execute DQ checks and write to table", { outputFolder <- tempfile("dqd_") on.exit(unlink(outputFolder, recursive = TRUE)) @@ -247,33 +202,6 @@ test_that("Execute DQ checks and write to table", { DatabaseConnector::renderTranslateExecuteSql(connection, "DROP TABLE @database_schema.dqd_results;", database_schema = resultsDatabaseSchemaEunomia) }) -test_that("Execute reEvaluateThresholds on Synthea/Eunomia", { - outputFolder <- tempfile("dqd_") - on.exit(unlink(outputFolder, recursive = TRUE)) - - expect_warning( - results <- executeDqChecks( - connectionDetails = connectionDetailsEunomia, - cdmDatabaseSchema = cdmDatabaseSchemaEunomia, - resultsDatabaseSchema = resultsDatabaseSchemaEunomia, - cdmSourceName = "Eunomia", - checkNames = "measurePersonCompleteness", - outputFolder = outputFolder, - writeToTable = F - ), - regexp = "^Missing check names.*" - ) - - jsonPath <- list.files(outputFolder, ".json", full.names = TRUE) - - results2 <- reEvaluateThresholds( - jsonFilePath = jsonPath, - outputFolder = outputFolder, - outputFile = "reEvaluated.txt" - ) - expect_is(results2, "list") -}) - test_that("Execute DQ checks using sqlOnly=TRUE and sqlOnlyUnionCount=4 and sqlOnlyIncrementalInsert=TRUE", { outputFolder <- tempfile("dqd_") on.exit(unlink(outputFolder, recursive = TRUE)) diff --git a/tests/testthat/test-listChecks.R b/tests/testthat/test-listChecks.R new file mode 100644 index 00000000..777127e5 --- /dev/null +++ b/tests/testthat/test-listChecks.R @@ -0,0 +1,7 @@ +library(testthat) + +test_that("listDqChecks works", { + checks <- listDqChecks() + expect_equal(length(checks), 4) + expect_true(all(sapply(checks, is.data.frame))) +}) \ No newline at end of file diff --git a/tests/testthat/test-reEvaluateThresholds.R b/tests/testthat/test-reEvaluateThresholds.R new file mode 100644 index 00000000..68d91cae --- /dev/null +++ b/tests/testthat/test-reEvaluateThresholds.R @@ -0,0 +1,28 @@ +library(testthat) + +test_that("Execute reEvaluateThresholds on Synthea/Eunomia", { + outputFolder <- tempfile("dqd_") + on.exit(unlink(outputFolder, recursive = TRUE)) + + expect_warning( + results <- executeDqChecks( + connectionDetails = connectionDetailsEunomia, + cdmDatabaseSchema = cdmDatabaseSchemaEunomia, + resultsDatabaseSchema = resultsDatabaseSchemaEunomia, + cdmSourceName = "Eunomia", + checkNames = "measurePersonCompleteness", + outputFolder = outputFolder, + writeToTable = F + ), + regexp = "^Missing check names.*" + ) + + jsonPath <- list.files(outputFolder, ".json", full.names = TRUE) + + results2 <- reEvaluateThresholds( + jsonFilePath = jsonPath, + outputFolder = outputFolder, + outputFile = "reEvaluated.txt" + ) + expect_is(results2, "list") +}) \ No newline at end of file diff --git a/tests/testthat/test-writeJsonResultsTo.R b/tests/testthat/test-writeJsonResultsTo.R new file mode 100644 index 00000000..b51b654e --- /dev/null +++ b/tests/testthat/test-writeJsonResultsTo.R @@ -0,0 +1,39 @@ +library(testthat) + +test_that("Write JSON results", { + outputFolder <- tempfile("dqd_") + on.exit(unlink(outputFolder, recursive = TRUE)) + + expect_warning( + results <- executeDqChecks( + connectionDetails = connectionDetailsEunomia, + cdmDatabaseSchema = cdmDatabaseSchemaEunomia, + resultsDatabaseSchema = resultsDatabaseSchemaEunomia, + cdmSourceName = "Eunomia", + checkNames = "measurePersonCompleteness", + outputFolder = outputFolder, + writeToTable = FALSE + ), + regexp = "^Missing check names.*" + ) + + jsonPath <- list.files(outputFolder, ".json", full.names = TRUE) + csvPath <- file.path(outputFolder, "results.csv") + writeJsonResultsToCsv( + jsonPath = jsonPath, + csvPath = csvPath + ) + expect_true(file.exists(csvPath)) + + DataQualityDashboard::writeJsonResultsToTable( + connectionDetails = connectionDetailsEunomia, + resultsDatabaseSchema = resultsDatabaseSchemaEunomia, + jsonFilePath = jsonPath, + writeTableName = "dqd_results" + ) + connection <- DatabaseConnector::connect(connectionDetailsEunomia) + on.exit(DatabaseConnector::disconnect(connection), add = TRUE) + tableNames <- DatabaseConnector::getTableNames(connection = connection, databaseSchema = resultsDatabaseSchemaEunomia) + expect_true("dqd_results" %in% tolower(tableNames)) + DatabaseConnector::renderTranslateExecuteSql(connection, "DROP TABLE @database_schema.dqd_results;", database_schema = resultsDatabaseSchemaEunomia) +}) \ No newline at end of file