From 55ef47503cd41faef725e7916cf75c6ed735e973 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Tue, 13 Feb 2024 16:59:28 +0100 Subject: [PATCH 01/43] refactor NA status calculation --- R/calculateNotApplicableStatus.R | 147 +++++++++++++++++++++++++++++++ R/evaluateThresholds.R | 114 +----------------------- 2 files changed, 148 insertions(+), 113 deletions(-) create mode 100644 R/calculateNotApplicableStatus.R diff --git a/R/calculateNotApplicableStatus.R b/R/calculateNotApplicableStatus.R new file mode 100644 index 00000000..2ee7fae4 --- /dev/null +++ b/R/calculateNotApplicableStatus.R @@ -0,0 +1,147 @@ +# Copyright 2023 Observational Health Data Sciences and Informatics +# +# This file is part of DataQualityDashboard +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#' Determines if check should be notApplicable +#' +#' @param checkResults A dataframe containing the results of the data quality checks +#' +#' @keywords internal + +.calculateNotApplicableStatus <- function(checkResults) { + # Look up missing tables and add variable tableIsMissing to checkResults + missingTables <- checkResults %>% + dplyr::filter( + .data$checkName == "cdmTable" + ) %>% + dplyr::mutate( + .data$cdmTableName, + tableIsMissing = .data$failed == 1, + .keep = "none" + ) + + # Look up missing fields and add variable fieldIsMissing to checkResults + missingFields <- checkResults %>% + dplyr::filter( + .data$checkName == "cdmField" + ) %>% + dplyr::mutate( + .data$cdmTableName, + .data$cdmFieldName, + fieldIsMissing = .data$failed == 1, + .keep = "none" + ) + + # Look up empty tables and add variable tableIsEmpty to checkResults + emptyTables <- checkResults %>% + dplyr::filter( + .data$checkName == "measureValueCompleteness" + ) %>% + dplyr::mutate( + .data$cdmTableName, + tableIsEmpty = .data$numDenominatorRows == 0, + .keep = "none" + ) %>% + dplyr::distinct() + + # Look up empty fields and add variable tableIsEmpty to checkResults + emptyFields <- checkResults %>% + dplyr::filter( + .data$checkName == "measureValueCompleteness" + ) %>% + dplyr::mutate( + .data$cdmTableName, + .data$cdmFieldName, + fieldIsEmpty = .data$numDenominatorRows == .data$numViolatedRows, + .keep = "none" + ) + + # Assign notApplicable status + checkResults <- checkResults %>% + dplyr::left_join( + missingTables, + by = "cdmTableName" + ) %>% + dplyr::left_join( + missingFields, + by = c("cdmTableName", "cdmFieldName") + ) %>% + dplyr::left_join( + emptyTables, + by = "cdmTableName" + ) %>% + dplyr::left_join( + emptyFields, + by = c("cdmTableName", "cdmFieldName") + ) %>% + dplyr::mutate( + conceptIsMissing = .data$checkLevel == "CONCEPT" & is.na(.data$unitConceptId) & .data$numDenominatorRows == 0, + conceptAndUnitAreMissing = .data$checkLevel == "CONCEPT" & !is.na(.data$unitConceptId) & .data$numDenominatorRows == 0, + fieldIsMissing = dplyr::coalesce(.data$fieldIsMissing, !is.na(.data$cdmFieldName)), + fieldIsEmpty = dplyr::coalesce(.data$fieldIsEmpty, !is.na(.data$cdmFieldName)), + ) + + for (i in seq_len(nrow(checkResults))) { + checkResults$notApplicable[i] <- .applyNotApplicable(checkResults[i, ]) + } + + checkResults <- checkResults %>% + dplyr::mutate( + notApplicableReason = ifelse( + .data$notApplicable == 1, + dplyr::case_when( + .data$tableIsMissing ~ sprintf("Table %s does not exist.", .data$cdmTableName), + .data$fieldIsMissing ~ sprintf("Field %s.%s does not exist.", .data$cdmTableName, .data$cdmFieldName), + .data$tableIsEmpty ~ sprintf("Table %s is empty.", .data$cdmTableName), + .data$fieldIsEmpty ~ sprintf("Field %s.%s is not populated.", .data$cdmTableName, .data$cdmFieldName), + .data$conceptIsMissing ~ sprintf("%s=%s is missing from the %s table.", .data$cdmFieldName, .data$conceptId, .data$cdmTableName), + .data$conceptAndUnitAreMissing ~ sprintf("Combination of %s=%s, unitConceptId=%s and VALUE_AS_NUMBER IS NOT NULL is missing from the %s table.", .data$cdmFieldName, .data$conceptId, .data$unitConceptId, .data$cdmTableName) + ), + NA + ), + failed = ifelse(.data$notApplicable == 1, 0, .data$failed), + passed = ifelse(.data$failed == 0 & .data$isError == 0 & .data$notApplicable == 0, 1, 0) + ) %>% + dplyr::select(-c("tableIsMissing", "fieldIsMissing", "tableIsEmpty", "fieldIsEmpty", "conceptIsMissing", "conceptAndUnitAreMissing")) + + return(checkResults) +} + +.applyNotApplicable <- function(x) { + # Errors precede all other statuses + if (x$isError == 1) { + return(0) + } + + # No NA status for cdmTable and cdmField if missing + if (x$checkName == "cdmTable" || x$checkName == "cdmField") { + return(0) + } + + if (any(x$tableIsMissing, x$fieldIsMissing, x$tableIsEmpty)) { + return(1) + } + + # No NA status for measureValueCompleteness if empty + if (x$checkName == "measureValueCompleteness") { + return(0) + } + + if (any(x$fieldIsEmpty, x$conceptIsMissing, x$conceptAndUnitAreMissing)) { + return(1) + } + + return(0) +} diff --git a/R/evaluateThresholds.R b/R/evaluateThresholds.R index 55b57891..4675d646 100644 --- a/R/evaluateThresholds.R +++ b/R/evaluateThresholds.R @@ -142,119 +142,7 @@ } } - missingTables <- dplyr::select( - dplyr::filter(checkResults, .data$checkName == "cdmTable" & .data$failed == 1), - "cdmTableName" - ) - if (nrow(missingTables) > 0) { - missingTables$tableIsMissing <- 1 - checkResults <- dplyr::mutate( - dplyr::left_join(checkResults, missingTables, by = "cdmTableName"), - tableIsMissing = ifelse(.data$checkName != "cdmTable" & .data$isError == 0, .data$tableIsMissing, NA) - ) - } else { - checkResults$tableIsMissing <- NA - } - - missingFields <- dplyr::select( - dplyr::filter(checkResults, .data$checkName == "cdmField" & .data$failed == 1 & is.na(.data$tableIsMissing)), - "cdmTableName", "cdmFieldName" - ) - if (nrow(missingFields) > 0) { - missingFields$fieldIsMissing <- 1 - checkResults <- dplyr::mutate( - dplyr::left_join(checkResults, missingFields, by = c("cdmTableName", "cdmFieldName")), - fieldIsMissing = ifelse(.data$checkName != "cdmField" & .data$isError == 0, .data$fieldIsMissing, NA) - ) - } else { - checkResults$fieldIsMissing <- NA - } - - emptyTables <- dplyr::distinct( - dplyr::select( - dplyr::filter(checkResults, .data$checkName == "measureValueCompleteness" & - .data$numDenominatorRows == 0 & - .data$isError == 0 & - is.na(.data$tableIsMissing) & - is.na(.data$fieldIsMissing)), - "cdmTableName" - ) - ) - if (nrow(emptyTables) > 0) { - emptyTables$tableIsEmpty <- 1 - checkResults <- dplyr::mutate( - dplyr::left_join(checkResults, emptyTables, by = c("cdmTableName")), - tableIsEmpty = ifelse(.data$checkName != "cdmField" & .data$checkName != "cdmTable" & .data$isError == 0, .data$tableIsEmpty, NA) - ) - } else { - checkResults$tableIsEmpty <- NA - } - - emptyFields <- - dplyr::select( - dplyr::filter(checkResults, .data$checkName == "measureValueCompleteness" & - .data$numDenominatorRows == .data$numViolatedRows & - is.na(.data$tableIsMissing) & is.na(.data$fieldIsMissing) & is.na(.data$tableIsEmpty)), - "cdmTableName", "cdmFieldName" - ) - if (nrow(emptyFields) > 0) { - emptyFields$fieldIsEmpty <- 1 - checkResults <- dplyr::mutate( - dplyr::left_join(checkResults, emptyFields, by = c("cdmTableName", "cdmFieldName")), - fieldIsEmpty = ifelse(.data$checkName != "measureValueCompleteness" & .data$checkName != "cdmField" & .data$checkName != "isRequired" & .data$isError == 0, .data$fieldIsEmpty, NA) - ) - } else { - checkResults$fieldIsEmpty <- NA - } - - checkResults <- dplyr::mutate( - checkResults, - conceptIsMissing = ifelse( - .data$isError == 0 & - is.na(.data$tableIsMissing) & - is.na(.data$fieldIsMissing) & - is.na(.data$tableIsEmpty) & - is.na(.data$fieldIsEmpty) & - .data$checkLevel == "CONCEPT" & - is.na(.data$unitConceptId) & - .data$numDenominatorRows == 0, - 1, - NA - ) - ) - - checkResults <- dplyr::mutate( - checkResults, - conceptAndUnitAreMissing = ifelse( - .data$isError == 0 & - is.na(.data$tableIsMissing) & - is.na(.data$fieldIsMissing) & - is.na(.data$tableIsEmpty) & - is.na(.data$fieldIsEmpty) & - .data$checkLevel == "CONCEPT" & - !is.na(.data$unitConceptId) & - .data$numDenominatorRows == 0, - 1, - NA - ) - ) - - checkResults <- dplyr::mutate( - checkResults, - notApplicable = dplyr::coalesce(.data$tableIsMissing, .data$fieldIsMissing, .data$tableIsEmpty, .data$fieldIsEmpty, .data$conceptIsMissing, .data$conceptAndUnitAreMissing, 0), - notApplicableReason = dplyr::case_when( - !is.na(.data$tableIsMissing) ~ sprintf("Table %s does not exist.", .data$cdmTableName), - !is.na(.data$fieldIsMissing) ~ sprintf("Field %s.%s does not exist.", .data$cdmTableName, .data$cdmFieldName), - !is.na(.data$tableIsEmpty) ~ sprintf("Table %s is empty.", .data$cdmTableName), - !is.na(.data$fieldIsEmpty) ~ sprintf("Field %s.%s is not populated.", .data$cdmTableName, .data$cdmFieldName), - !is.na(.data$conceptIsMissing) ~ sprintf("%s=%s is missing from the %s table.", .data$cdmFieldName, .data$conceptId, .data$cdmTableName), - !is.na(.data$conceptAndUnitAreMissing) ~ sprintf("Combination of %s=%s, unitConceptId=%s and VALUE_AS_NUMBER IS NOT NULL is missing from the %s table.", .data$cdmFieldName, .data$conceptId, .data$unitConceptId, .data$cdmTableName) - ) - ) - - checkResults <- dplyr::select(checkResults, -c("tableIsMissing", "fieldIsMissing", "tableIsEmpty", "fieldIsEmpty", "conceptIsMissing", "conceptAndUnitAreMissing")) - checkResults <- dplyr::mutate(checkResults, failed = ifelse(.data$notApplicable == 1, 0, .data$failed)) - checkResults <- dplyr::mutate(checkResults, passed = ifelse(.data$failed == 0 & .data$isError == 0 & .data$notApplicable == 0, 1, 0)) + checkResults <- .calculateNotApplicableStatus(checkResults) checkResults } From 0ceeb6ba59697f05ca916e81e0015b6b66492b70 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Tue, 13 Feb 2024 17:57:56 +0100 Subject: [PATCH 02/43] fixes #526 with an exception for NA status of measureConditionEraCompleteness --- R/calculateNotApplicableStatus.R | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/R/calculateNotApplicableStatus.R b/R/calculateNotApplicableStatus.R index 2ee7fae4..41536c43 100644 --- a/R/calculateNotApplicableStatus.R +++ b/R/calculateNotApplicableStatus.R @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -#' Determines if check should be notApplicable +#' Determines if check should be notApplicable and the notApplicableReason #' #' @param checkResults A dataframe containing the results of the data quality checks #' @@ -93,8 +93,31 @@ fieldIsEmpty = dplyr::coalesce(.data$fieldIsEmpty, !is.na(.data$cdmFieldName)), ) + checkResults$notApplicable <- NA + checkResults$notApplicableReason <- NA + + conditionOccurrenceIsEmpty <- emptyTables %>% dplyr::filter(.data$cdmTableName == "CONDITION_OCCURRENCE") %>% dplyr::pull(tableIsEmpty) + # drugExposureIsEmpty <- emptyTables %>% filter(.data$cdmTableName == "DRUG_EXPOSURE") %>% pull(tableIsEmpty) for (i in seq_len(nrow(checkResults))) { - checkResults$notApplicable[i] <- .applyNotApplicable(checkResults[i, ]) + if (checkResults[i, "checkName"] == "measureConditionEraCompleteness") { + # Special rule for measureConditionEraCompleteness, which should be notApplicable if CONDITION_OCCURRENCE is empty + if (conditionOccurrenceIsEmpty) { + checkResults$notApplicable[i] <- 1 + checkResults$notApplicableReason[i] <- "Table CONDITION_OCCURRENCE is empty." + } else { + checkResults$notApplicable[i] <- 0 + } + # } else if (checkResult[i, "checkName"] == "measureDrugEraCompleteness") { + # # Special rule for measureDrugEraCompleteness, which should be notApplicable if DRUG_EXPOSURE is empty + # if (drugExposureIsEmpty) { + # checkResults$notApplicable[i] <- 1 + # checkResults$notApplicableReason[i] <- "Table DRUG_EXPOSURE is empty." + # } else { + # checkResults$notApplicable[i] <- 0 + # } + } else { + checkResults$notApplicable[i] <- .applyNotApplicable(checkResults[i, ]) + } } checkResults <- checkResults %>% @@ -102,6 +125,7 @@ notApplicableReason = ifelse( .data$notApplicable == 1, dplyr::case_when( + !is.na(.data$notApplicableReason) ~ .data$notApplicableReason, .data$tableIsMissing ~ sprintf("Table %s does not exist.", .data$cdmTableName), .data$fieldIsMissing ~ sprintf("Field %s.%s does not exist.", .data$cdmTableName, .data$cdmFieldName), .data$tableIsEmpty ~ sprintf("Table %s is empty.", .data$cdmTableName), From c69d12c82463e9e594a53acdcffdbb1b8be4f392 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Tue, 13 Feb 2024 18:03:27 +0100 Subject: [PATCH 03/43] add rule for missing condition occurrence --- R/calculateNotApplicableStatus.R | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/R/calculateNotApplicableStatus.R b/R/calculateNotApplicableStatus.R index 41536c43..81f0c7c3 100644 --- a/R/calculateNotApplicableStatus.R +++ b/R/calculateNotApplicableStatus.R @@ -96,12 +96,14 @@ checkResults$notApplicable <- NA checkResults$notApplicableReason <- NA + conditionOccurrenceIsMissing <- missingTables %>% dplyr::filter(.data$cdmTableName == "CONDITION_OCCURRENCE") %>% dplyr::pull(tableIsMissing) conditionOccurrenceIsEmpty <- emptyTables %>% dplyr::filter(.data$cdmTableName == "CONDITION_OCCURRENCE") %>% dplyr::pull(tableIsEmpty) + # drugExposureIsMissing <- missingTables %>% filter(.data$cdmTableName == "DRUG_EXPOSURE") %>% pull(tableIsMissing) # drugExposureIsEmpty <- emptyTables %>% filter(.data$cdmTableName == "DRUG_EXPOSURE") %>% pull(tableIsEmpty) for (i in seq_len(nrow(checkResults))) { if (checkResults[i, "checkName"] == "measureConditionEraCompleteness") { # Special rule for measureConditionEraCompleteness, which should be notApplicable if CONDITION_OCCURRENCE is empty - if (conditionOccurrenceIsEmpty) { + if (conditionOccurrenceIsMissing || conditionOccurrenceIsEmpty) { checkResults$notApplicable[i] <- 1 checkResults$notApplicableReason[i] <- "Table CONDITION_OCCURRENCE is empty." } else { @@ -109,7 +111,7 @@ } # } else if (checkResult[i, "checkName"] == "measureDrugEraCompleteness") { # # Special rule for measureDrugEraCompleteness, which should be notApplicable if DRUG_EXPOSURE is empty - # if (drugExposureIsEmpty) { + # if (drugExposureIsMissing || drugExposureIsEmpty) { # checkResults$notApplicable[i] <- 1 # checkResults$notApplicableReason[i] <- "Table DRUG_EXPOSURE is empty." # } else { From e83563d6c335f9581b2fda46d3a0b49deaa084de Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Thu, 22 Feb 2024 08:51:14 +0100 Subject: [PATCH 04/43] fix failing test by checking if required checks are present --- R/calculateNotApplicableStatus.R | 41 ++++++++++++++++++++++---------- R/evaluateThresholds.R | 4 +++- R/executeDqChecks.R | 10 ++------ 3 files changed, 33 insertions(+), 22 deletions(-) diff --git a/R/calculateNotApplicableStatus.R b/R/calculateNotApplicableStatus.R index 81f0c7c3..ace2c329 100644 --- a/R/calculateNotApplicableStatus.R +++ b/R/calculateNotApplicableStatus.R @@ -14,6 +14,31 @@ # See the License for the specific language governing permissions and # limitations under the License. +#' Determines if all checks are present expected to calculate the 'Not Applicable' status +#' +#' @param checkResults A dataframe containing the results of the data quality checks +#' +#' @keywords internal +.hasNAchecks <- function(checkResults) { + checkNames <- unique(checkResults$checkName) + return(.containsNAchecks(checkNames)) +} + +#' Determines if all checks required for 'Not Applicable' status are in the checkNames +#' +#' @param checkNames A character vector of check names +#' +#' @keywords internal +.containsNAchecks <- function(checkNames) { + naCheckNames <- c("cdmTable", "cdmField", "measureValueCompleteness") + missingNAChecks <- !(naCheckNames %in% checkNames) + if (any(missingNAChecks)) { + missingNACheckNames <- paste(naCheckNames[missingNAChecks], collapse = ", ") + return(FALSE) + } + return(TRUE) +} + #' Determines if check should be notApplicable and the notApplicableReason #' #' @param checkResults A dataframe containing the results of the data quality checks @@ -98,25 +123,15 @@ conditionOccurrenceIsMissing <- missingTables %>% dplyr::filter(.data$cdmTableName == "CONDITION_OCCURRENCE") %>% dplyr::pull(tableIsMissing) conditionOccurrenceIsEmpty <- emptyTables %>% dplyr::filter(.data$cdmTableName == "CONDITION_OCCURRENCE") %>% dplyr::pull(tableIsEmpty) - # drugExposureIsMissing <- missingTables %>% filter(.data$cdmTableName == "DRUG_EXPOSURE") %>% pull(tableIsMissing) - # drugExposureIsEmpty <- emptyTables %>% filter(.data$cdmTableName == "DRUG_EXPOSURE") %>% pull(tableIsEmpty) for (i in seq_len(nrow(checkResults))) { + # Special rule for measureConditionEraCompleteness, which should be notApplicable if CONDITION_OCCURRENCE is empty if (checkResults[i, "checkName"] == "measureConditionEraCompleteness") { - # Special rule for measureConditionEraCompleteness, which should be notApplicable if CONDITION_OCCURRENCE is empty if (conditionOccurrenceIsMissing || conditionOccurrenceIsEmpty) { checkResults$notApplicable[i] <- 1 checkResults$notApplicableReason[i] <- "Table CONDITION_OCCURRENCE is empty." } else { checkResults$notApplicable[i] <- 0 } - # } else if (checkResult[i, "checkName"] == "measureDrugEraCompleteness") { - # # Special rule for measureDrugEraCompleteness, which should be notApplicable if DRUG_EXPOSURE is empty - # if (drugExposureIsMissing || drugExposureIsEmpty) { - # checkResults$notApplicable[i] <- 1 - # checkResults$notApplicableReason[i] <- "Table DRUG_EXPOSURE is empty." - # } else { - # checkResults$notApplicable[i] <- 0 - # } } else { checkResults$notApplicable[i] <- .applyNotApplicable(checkResults[i, ]) } @@ -156,7 +171,7 @@ return(0) } - if (any(x$tableIsMissing, x$fieldIsMissing, x$tableIsEmpty)) { + if (any(x$tableIsMissing, x$fieldIsMissing, x$tableIsEmpty, na.rm = TRUE)) { return(1) } @@ -165,7 +180,7 @@ return(0) } - if (any(x$fieldIsEmpty, x$conceptIsMissing, x$conceptAndUnitAreMissing)) { + if (any(x$fieldIsEmpty, x$conceptIsMissing, x$conceptAndUnitAreMissing, na.rm = TRUE)) { return(1) } diff --git a/R/evaluateThresholds.R b/R/evaluateThresholds.R index 4675d646..d72c143e 100644 --- a/R/evaluateThresholds.R +++ b/R/evaluateThresholds.R @@ -142,7 +142,9 @@ } } - checkResults <- .calculateNotApplicableStatus(checkResults) + if (.hasNAchecks(checkResults)) { + checkResults <- .calculateNotApplicableStatus(checkResults) + } checkResults } diff --git a/R/executeDqChecks.R b/R/executeDqChecks.R index 014a0199..5c824141 100644 --- a/R/executeDqChecks.R +++ b/R/executeDqChecks.R @@ -114,16 +114,10 @@ executeDqChecks <- function(connectionDetails, stopifnot(is.character(cdmVersion)) # Warning if check names for determining NA is missing - if (!length(checkNames) == 0) { - naCheckNames <- c("cdmTable", "cdmField", "measureValueCompleteness") - missingNAChecks <- !(naCheckNames %in% checkNames) - if (any(missingNAChecks)) { - missingNACheckNames <- paste(naCheckNames[missingNAChecks], collapse = ", ") - warning(sprintf("Missing check names to calculate the 'Not Applicable' status: %s", missingNACheckNames)) - } + if (length(checkNames) > 0 && !.containsNAchecks(checkNames)) { + warning("Missing check names to calculate the 'Not Applicable' status.") } - # temporary patch to work around vroom 1.6.4 bug readr::local_edition(1) From a75e984cbc0e7e70a93974d56b5cc48e2e3fe33d Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Thu, 22 Feb 2024 09:32:39 +0100 Subject: [PATCH 05/43] add tests for calculating Not Applicable status --- .../test-calculateNotApplicableStatus.R | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 tests/testthat/test-calculateNotApplicableStatus.R diff --git a/tests/testthat/test-calculateNotApplicableStatus.R b/tests/testthat/test-calculateNotApplicableStatus.R new file mode 100644 index 00000000..a70eef48 --- /dev/null +++ b/tests/testthat/test-calculateNotApplicableStatus.R @@ -0,0 +1,87 @@ +library(testthat) + +test_that("Not Applicable status Table Empty", { + outputFolder <- tempfile("dqd_") + on.exit(unlink(outputFolder, recursive = TRUE)) + + results <- executeDqChecks( + connectionDetails = connectionDetailsEunomia, + cdmDatabaseSchema = cdmDatabaseSchemaEunomia, + resultsDatabaseSchema = resultsDatabaseSchemaEunomia, + cdmSourceName = "Eunomia", + checkNames = c("cdmTable", "cdmField", "measureValueCompleteness"), + # Eunomia COST table has misspelled 'REVEUE_CODE_SOURCE_VALUE' + tablesToExclude = c("COST", "CONCEPT", "VOCABULARY", "CONCEPT_ANCESTOR", "CONCEPT_RELATIONSHIP", "CONCEPT_CLASS", "CONCEPT_SYNONYM", "RELATIONSHIP", "DOMAIN"), + outputFolder = outputFolder, + writeToTable = F + ) + + # Assumption that Eunomia has empty device_exposure table + r <- results$CheckResults[results$CheckResults$checkName == "measureValueCompleteness" & + results$CheckResults$tableName == "device_exposure", ] + expect_true(all(r$notApplicable == 1)) +}) + +test_that("measureConditionEraCompleteness Not Applicable if condition_occurrence empty", { + outputFolder <- tempfile("dqd_") + on.exit(unlink(outputFolder, recursive = TRUE)) + + # Remove records from Condition Occurrence + connection <- DatabaseConnector::connect(connectionDetailsEunomia) + DatabaseConnector::executeSql(connection, "CREATE TABLE CONDITION_OCCURRENCE_BACK AS SELECT * FROM CONDITION_OCCURRENCE;") + DatabaseConnector::executeSql(connection, "DELETE FROM CONDITION_OCCURRENCE;") + DatabaseConnector::disconnect(connection) + + results <- executeDqChecks( + connectionDetails = connectionDetailsEunomia, + cdmDatabaseSchema = cdmDatabaseSchemaEunomia, + resultsDatabaseSchema = resultsDatabaseSchemaEunomia, + cdmSourceName = "Eunomia", + checkNames = c("cdmTable", "cdmField", "measureValueCompleteness", "measureConditionEraCompleteness"), + # Eunomia COST table has misspelled 'REVEUE_CODE_SOURCE_VALUE' + tablesToExclude = c("COST", "CONCEPT", "VOCABULARY", "CONCEPT_ANCESTOR", "CONCEPT_RELATIONSHIP", "CONCEPT_CLASS", "CONCEPT_SYNONYM", "RELATIONSHIP", "DOMAIN"), + outputFolder = outputFolder, + writeToTable = F + ) + + # Reinstate Condition Occurrence + connection <- DatabaseConnector::connect(connectionDetailsEunomia) + DatabaseConnector::executeSql(connection, "INSERT INTO CONDITION_OCCURRENCE SELECT * FROM CONDITION_OCCURRENCE_BACK;") + DatabaseConnector::executeSql(connection, "DROP TABLE CONDITION_OCCURRENCE_BACK;") + disconnect(connection) + + r <- results$CheckResults[results$CheckResults$checkName == "measureConditionEraCompleteness", ] + expect_true(r$notApplicable == 1) +}) + +test_that("measureConditionEraCompleteness Fails if condition_era empty", { + outputFolder <- tempfile("dqd_") + on.exit(unlink(outputFolder, recursive = TRUE)) + + # Remove records from Condition Era + connection <- DatabaseConnector::connect(connectionDetailsEunomia) + DatabaseConnector::executeSql(connection, "CREATE TABLE CONDITION_ERA_BACK AS SELECT * FROM CONDITION_ERA;") + DatabaseConnector::executeSql(connection, "DELETE FROM CONDITION_ERA;") + DatabaseConnector::disconnect(connection) + + results <- executeDqChecks( + connectionDetails = connectionDetailsEunomia, + cdmDatabaseSchema = cdmDatabaseSchemaEunomia, + resultsDatabaseSchema = resultsDatabaseSchemaEunomia, + cdmSourceName = "Eunomia", + checkNames = c("cdmTable", "cdmField", "measureValueCompleteness", "measureConditionEraCompleteness"), + # Eunomia COST table has misspelled 'REVEUE_CODE_SOURCE_VALUE' + tablesToExclude = c("COST", "CONCEPT", "VOCABULARY", "CONCEPT_ANCESTOR", "CONCEPT_RELATIONSHIP", "CONCEPT_CLASS", "CONCEPT_SYNONYM", "RELATIONSHIP", "DOMAIN"), + outputFolder = outputFolder, + writeToTable = F + ) + + # Reinstate the Condition Era + connection <- DatabaseConnector::connect(connectionDetailsEunomia) + DatabaseConnector::executeSql(connection, "INSERT INTO CONDITION_ERA SELECT * FROM CONDITION_ERA_BACK;") + DatabaseConnector::executeSql(connection, "DROP TABLE CONDITION_ERA_BACK;") + DatabaseConnector::disconnect(connection) + + r <- results$CheckResults[results$CheckResults$checkName == "measureConditionEraCompleteness", ] + expect_true(r$failed == 1) +}) \ No newline at end of file From 022743817abad12a28ec65b0b705e32f191fd0d0 Mon Sep 17 00:00:00 2001 From: Nitesh Balakrishnan Date: Tue, 12 Mar 2024 18:50:33 +0000 Subject: [PATCH 06/43] New variable executionTimeSeconds added to store execution time in seconds --- R/executeDqChecks.R | 2 ++ 1 file changed, 2 insertions(+) diff --git a/R/executeDqChecks.R b/R/executeDqChecks.R index 4b0569f7..7bb40e75 100644 --- a/R/executeDqChecks.R +++ b/R/executeDqChecks.R @@ -333,6 +333,8 @@ executeDqChecks <- function(connectionDetails, startTimestamp = startTime, endTimestamp = endTime, executionTime = sprintf("%.0f %s", delta, attr(delta, "units")), + #new variable executionTimeSeconds added to store execution time in seconds + executionTimeSeconds = as.numeric(delta), CheckResults = checkResults, Metadata = metadata, Overview = overview From 68f3651997113ad0e5112a15f270a4a26204958c Mon Sep 17 00:00:00 2001 From: Nitesh Balakrishnan Date: Tue, 12 Mar 2024 19:11:56 +0000 Subject: [PATCH 07/43] Test case added for new variable executionTimeSeconds --- tests/testthat/test-executeDqChecks.R | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/testthat/test-executeDqChecks.R b/tests/testthat/test-executeDqChecks.R index 1cafeff2..5bb79306 100644 --- a/tests/testthat/test-executeDqChecks.R +++ b/tests/testthat/test-executeDqChecks.R @@ -355,3 +355,23 @@ test_that("Multiple cdm_source rows triggers warning.", { expect_true(nrow(results$CheckResults) > 1) }) + +test_that("Execute checks on Synthea/Eunomia to test new variable executionTimeSeconds", { + outputFolder <- tempfile("dqd_") + on.exit(unlink(outputFolder, recursive = TRUE)) + results <- executeDqChecks( + connectionDetails = connectionDetailsEunomia, + cdmDatabaseSchema = cdmDatabaseSchemaEunomia, + resultsDatabaseSchema = resultsDatabaseSchemaEunomia, + cdmSourceName = "Eunomia", + checkLevels = "CONCEPT", + conceptCheckThresholdLoc = system.file( + "csv", + "unittest_OMOP_CDMv5.3_Concept_Level.csv", + package = "DataQualityDashboard" + ), + outputFolder = outputFolder, + writeToTable = F + ) + expect_true(nrow(results$CheckResults) > 0) +}) \ No newline at end of file From ac3453dd974272d599cfc9ea66335278178c4289 Mon Sep 17 00:00:00 2001 From: Nitesh Balakrishnan Date: Thu, 28 Mar 2024 12:04:22 +0000 Subject: [PATCH 08/43] Test Case added to test new variable executionTimeSeconds --- tests/testthat/test-convertResultsCase.R | 7 +++++-- tests/testthat/test-executeDqChecks.R | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/testthat/test-convertResultsCase.R b/tests/testthat/test-convertResultsCase.R index 3e792dbd..d9a46f4f 100644 --- a/tests/testthat/test-convertResultsCase.R +++ b/tests/testthat/test-convertResultsCase.R @@ -25,8 +25,9 @@ test_that("Camel correctly converted to snake and back", { ) snakeResults <- convertJsonResultsFileCase(jsonFilePath, writeToFile = T, outputFolder, outputFile = "snake.json", targetCase = "snake") snakeNames <- c("NUM_VIOLATED_ROWS", "PCT_VIOLATED_ROWS", "NUM_DENOMINATOR_ROWS", "EXECUTION_TIME", "QUERY_TEXT", "CHECK_NAME", "CHECK_LEVEL", "CHECK_DESCRIPTION", "CDM_TABLE_NAME", "SQL_FILE", "CATEGORY", "CONTEXT", "checkId", "FAILED", "PASSED", "IS_ERROR", "NOT_APPLICABLE", "THRESHOLD_VALUE") + - expect_equal(length(snakeResults), 6) + expect_equal(length(snakeResults), 7) expect_true(setequal(names(snakeResults$CheckResults), snakeNames)) snakeFilePath <- file.path(outputFolder, "snake.json") @@ -37,8 +38,10 @@ test_that("Camel correctly converted to snake and back", { camelResults <- convertJsonResultsFileCase(snakeFilePath, writeToFile = T, outputFolder, targetCase = "camel") camelNames <- c("numViolatedRows", "pctViolatedRows", "numDenominatorRows", "executionTime", "queryText", "checkName", "checkLevel", "checkDescription", "cdmTableName", "sqlFile", "category", "context", "checkId", "failed", "passed", "isError", "notApplicable", "thresholdValue") camelFilePath <- file.path(outputFolder, "snake_camel.json") + - expect_equal(length(camelResults), 6) + + expect_equal(length(camelResults),7) expect_true(setequal(names(camelResults$CheckResults), camelNames)) expect_true(file.exists(camelFilePath)) diff --git a/tests/testthat/test-executeDqChecks.R b/tests/testthat/test-executeDqChecks.R index 5bb79306..8629ef73 100644 --- a/tests/testthat/test-executeDqChecks.R +++ b/tests/testthat/test-executeDqChecks.R @@ -373,5 +373,5 @@ test_that("Execute checks on Synthea/Eunomia to test new variable executionTimeS outputFolder = outputFolder, writeToTable = F ) - expect_true(nrow(results$CheckResults) > 0) + expect_true(is.numeric(results$executionTimeSeconds)) }) \ No newline at end of file From 035a24a363b3f4dccbf20b924985829e5e9e54b8 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Fri, 10 May 2024 13:51:59 +0200 Subject: [PATCH 09/43] add startBeforeEnd documentation --- vignettes/checks/plausibleAfterBirth.Rmd | 2 +- vignettes/checks/plausibleStartBeforeEnd.Rmd | 37 ++++++++++++++------ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/vignettes/checks/plausibleAfterBirth.Rmd b/vignettes/checks/plausibleAfterBirth.Rmd index eb399cb1..57ba0ced 100644 --- a/vignettes/checks/plausibleAfterBirth.Rmd +++ b/vignettes/checks/plausibleAfterBirth.Rmd @@ -20,7 +20,7 @@ output: The number and percent of records with a date value in the **cdmFieldName** field of the **cdmTableName** table that occurs prior to birth. ## Definition -This check verifies that events happen after birth. This check is only run on fields where the **PLAUSIBLE_AFTER_BIRTH** parameter is set to **Yes**. The birthdate is taken from the `person` table, either the `birth_datetime` or composed from `year_of_birth`, `month_of_birth`, `day_of_birth` (taking 1st month/1st day if missing). +This check verifies that events happen after birth. The birthdate is taken from the `person` table, either the `birth_datetime` or composed from `year_of_birth`, `month_of_birth`, `day_of_birth` (taking 1st month/1st day if missing). - *Numerator*: The number of records with a non-null date value that happen prior to birth - *Denominator*: The total number of records in the table with a non-null date value diff --git a/vignettes/checks/plausibleStartBeforeEnd.Rmd b/vignettes/checks/plausibleStartBeforeEnd.Rmd index 2d99c696..0607da43 100644 --- a/vignettes/checks/plausibleStartBeforeEnd.Rmd +++ b/vignettes/checks/plausibleStartBeforeEnd.Rmd @@ -14,33 +14,48 @@ output: **Context**: Verification\ **Category**: Plausibility\ **Subcategory**: Temporal\ -**Severity**: +**Severity**: CDM convention ⚠\ ## Description -The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName that occurs after the date in the @plausibleStartBeforeEndFieldName. +The number and percent of records with a value in the **cdmFieldName** field of the **cdmTableName** that occurs after the date in the **plausibleStartBeforeEndFieldName**. ## Definition +Most tables have a field for the start and a field for the end date for the event. This check verifies that the start date is not after the end date. The start date can be before the end date or equal to the end date. It is applied to the start date field and takes the end date field as a parameter. Both date and datetime fields are checked. -- *Numerator*: -- *Denominator*: -- *Related CDM Convention(s)*: -- *CDM Fields/Tables*: -- *Default Threshold Value*: +- *Numerator*: The number of records where date in **cdmFieldName** is after the date in **plausibleStartBeforeEndFieldName**. +- *Denominator*: The total number of records with a non-null start and non-null end date value +- *Related CDM Convention(s)*: -Not linked to a convention- +- *CDM Fields/Tables*: This check runs on all date and datetime fields that have a start and end date in the same table. It also runs on the `cdm_source` table, comparing `source_release_date` is before `cdm_release_date`. +- *Default Threshold Value*: 1% (except for vocabulary and cdm_source tables, where it is 0%) ## User Guidance - +If the start date is after the end date, it is likely that the data is incorrect or the dates are unreliable. ### Violated rows query ```sql - +SELECT + '@cdmTableName.@cdmFieldName' AS violating_field, + cdmTable.* +FROM @schema.@cdmTableName cdmTable +WHERE cdmTable.@cdmFieldName IS NOT NULL +AND cdmTable.@plausibleStartBeforeEndFieldName IS NOT NULL +AND cdmTable.@cdmFieldName > cdmTable.@plausibleStartBeforeEndFieldName ``` - ### ETL Developers +There main reason for this check to fail is often that the source data is incorrect. If the end date is derived from other data, the calculation might not take into account some edge cases. +Any violating checks should either be removed or corrected. In most cases this can be done by adjusting the end date: +- With a few exceptions, the end date is not mandatory and can be left empty. +- If the end date is mandatory (visit_occurrence and drug_exposure), the end date can be set to the start date if the event. Note tha +- If this check fails for the observation_period, it might signify a bigger underlying issue. Please investigate all records for this person in the CDM and source. +- If neither the start or end date can be trusted, pleaes remove the record from the CDM. -### Data Users +Make sure to clearly document the choices in your ETL specification. +### Data Users +An start date after the end date gives negative event durations, which might break analyses. +Especially take note if this check fails for the `observation_period` table. This means that there are persons with negative observation time. If these persons are included in a cohort, it will potentially skew e.g. survival analyses. From 5384e12e4cf48b55ba6d18a6f8eaf96ace93f545 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Fri, 10 May 2024 16:32:41 +0200 Subject: [PATCH 10/43] plausibleBeforeDeath docs --- vignettes/checkIndex.Rmd | 4 +- vignettes/checks/plausibleAfterBirth.Rmd | 1 + vignettes/checks/plausibleBeforeDeath.Rmd | 39 ++++++++++++++------ vignettes/checks/plausibleStartBeforeEnd.Rmd | 6 ++- 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/vignettes/checkIndex.Rmd b/vignettes/checkIndex.Rmd index e4fc563d..326f1812 100644 --- a/vignettes/checkIndex.Rmd +++ b/vignettes/checkIndex.Rmd @@ -47,7 +47,7 @@ above to navigate to the check's documentation page.\ - plausibleDuringLife (PAGE UNDER CONSTRUCTION) - withinVisitDates (PAGE UNDER CONSTRUCTION) - [plausibleAfterBirth](checks/plausibleAfterBirth.html) -- plausibleBeforeDeath (PAGE UNDER CONSTRUCTION) -- plausibleStartBeforeEnd (PAGE UNDER CONSTRUCTION) +- [plausibleBeforeDeath](checks/plausibleBeforeDeath.html) +- [plausibleStartBeforeEnd](checks/plausibleStartBeforeEnd.html) - plausibleGender (PAGE UNDER CONSTRUCTION) - plausibleUnitConceptIds (PAGE UNDER CONSTRUCTION) diff --git a/vignettes/checks/plausibleAfterBirth.Rmd b/vignettes/checks/plausibleAfterBirth.Rmd index 57ba0ced..6e0eeced 100644 --- a/vignettes/checks/plausibleAfterBirth.Rmd +++ b/vignettes/checks/plausibleAfterBirth.Rmd @@ -18,6 +18,7 @@ output: ## Description The number and percent of records with a date value in the **cdmFieldName** field of the **cdmTableName** table that occurs prior to birth. +Note that this check replaces the previous `plausibleTemporalAfter` check. ## Definition This check verifies that events happen after birth. The birthdate is taken from the `person` table, either the `birth_datetime` or composed from `year_of_birth`, `month_of_birth`, `day_of_birth` (taking 1st month/1st day if missing). diff --git a/vignettes/checks/plausibleBeforeDeath.Rmd b/vignettes/checks/plausibleBeforeDeath.Rmd index 91adf4c4..dfd20699 100644 --- a/vignettes/checks/plausibleBeforeDeath.Rmd +++ b/vignettes/checks/plausibleBeforeDeath.Rmd @@ -1,6 +1,6 @@ --- title: "plausibleBeforeDeath" -author: "" +author: "Maxim Moinat" date: "`r Sys.Date()`" output: html_document: @@ -14,33 +14,48 @@ output: **Context**: Verification\ **Category**: Plausibility\ **Subcategory**: Temporal\ -**Severity**: +**Severity**: Characterization ✔ ## Description -The number and percent of records with a date value in the @cdmFieldName field of the @cdmTableName table that occurs after death. +The number and percent of records with a date value in the **cdmFieldName** field of the **cdmTableName** table that occurs more than 60 days after death. +Note that this check replaces the previous `plausibleDuringLife` check. ## Definition +A record violates this check if the date is more than 60 days after the death date of the person, allowing administrative records directly after death. -- *Numerator*: -- *Denominator*: -- *Related CDM Convention(s)*: -- *CDM Fields/Tables*: -- *Default Threshold Value*: +- *Numerator*: The number of records where date in **cdmFieldName** is more than 60 days after the persons' death date. +- *Denominator*: Total number of records of persons with a death date, in the **cdmTableName**. +- *Related CDM Convention(s)*: -Not linked to a convention- +- *CDM Fields/Tables*: This check runs on all date and datetime fields. +- *Default Threshold Value*: 1% ## User Guidance - +Events are expected to occur between birth and death. The check `plausibleAfterbirth` checks for the former, this check for the latter. +The 60-day period is a conservative estimate of the time it takes for administrative records to be updated after a person's death. +By default, both start and end dates are checked. ### Violated rows query ```sql - +SELECT + '@cdmTableName.@cdmFieldName' AS violating_field, + cdmTable.* +FROM @cdmDatabaseSchema.@cdmTableName cdmTable +JOIN @cdmDatabaseSchema.death de + ON cdmTable.person_id = de.person_id +WHERE cdmTable.@cdmFieldName IS NOT NULL + AND CAST(cdmTable.@cdmFieldName AS DATE) > DATEADD(day, 60, de.death_date) ``` - ### ETL Developers +Start dates after death are likely to be source data issues, and failing this check should trigger investigation of the source data quality. +End dates after death can occur due to derivation logic. For example, a drug exposure can be prescribed as being continued long after death. +In such cases, it is recommended to update the logic to end the prescription at death. ### Data Users - +For most studies, a low number of violating records will have limited impact on data use as it could be caused by lagging administrative records. +However, it might signify a larger data quality issue. +Note that the percentage violating records reported is among records from death persons and such might be slightly inflated if comparing to the overall population. diff --git a/vignettes/checks/plausibleStartBeforeEnd.Rmd b/vignettes/checks/plausibleStartBeforeEnd.Rmd index 0607da43..d65bcfe3 100644 --- a/vignettes/checks/plausibleStartBeforeEnd.Rmd +++ b/vignettes/checks/plausibleStartBeforeEnd.Rmd @@ -1,6 +1,6 @@ --- title: "plausibleStartBeforeEnd" -author: "" +author: "Maxim Moinat" date: "`r Sys.Date()`" output: html_document: @@ -19,10 +19,12 @@ output: ## Description The number and percent of records with a value in the **cdmFieldName** field of the **cdmTableName** that occurs after the date in the **plausibleStartBeforeEndFieldName**. +Note that this check replaces the previous `plausibleTemporalAfter` check. ## Definition -Most tables have a field for the start and a field for the end date for the event. This check verifies that the start date is not after the end date. The start date can be before the end date or equal to the end date. It is applied to the start date field and takes the end date field as a parameter. Both date and datetime fields are checked. +This check is attempting to apply temporal rules within a table, specifically checking that all start dates are before the end dates. For example, in the VISIT_OCCURRENCE table it checks that the VISIT_OCCURRENCE_START_DATE is before VISIT_OCCURRENCE_END_DATE. +The start date can be before the end date or equal to the end date. It is applied to the start date field and takes the end date field as a parameter. Both date and datetime fields are checked. - *Numerator*: The number of records where date in **cdmFieldName** is after the date in **plausibleStartBeforeEndFieldName**. - *Denominator*: The total number of records with a non-null start and non-null end date value From 4586a77cb1303bb4e7bfd6860d961d25b4345759 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Mon, 13 May 2024 13:06:41 +0200 Subject: [PATCH 11/43] refactoring --- R/calculateNotApplicableStatus.R | 65 ++++++++++--------- .../test-calculateNotApplicableStatus.R | 12 ++-- 2 files changed, 42 insertions(+), 35 deletions(-) diff --git a/R/calculateNotApplicableStatus.R b/R/calculateNotApplicableStatus.R index ace2c329..eeafb203 100644 --- a/R/calculateNotApplicableStatus.R +++ b/R/calculateNotApplicableStatus.R @@ -33,18 +33,48 @@ naCheckNames <- c("cdmTable", "cdmField", "measureValueCompleteness") missingNAChecks <- !(naCheckNames %in% checkNames) if (any(missingNAChecks)) { - missingNACheckNames <- paste(naCheckNames[missingNAChecks], collapse = ", ") return(FALSE) } return(TRUE) } +#' Applies the 'Not Applicable' status to a single check +#' +#' @param x Results from a single check +#' +#' @keywords internal +.applyNotApplicable <- function(x) { + # Errors precede all other statuses + if (x$isError == 1) { + return(0) + } + + # No NA status for cdmTable and cdmField if missing + if (x$checkName == "cdmTable" || x$checkName == "cdmField") { + return(0) + } + + if (any(x$tableIsMissing, x$fieldIsMissing, x$tableIsEmpty, na.rm = TRUE)) { + return(1) + } + + # No NA status for measureValueCompleteness if empty + if (x$checkName == "measureValueCompleteness") { + return(0) + } + + if (any(x$fieldIsEmpty, x$conceptIsMissing, x$conceptAndUnitAreMissing, na.rm = TRUE)) { + return(1) + } + + return(0) +} + #' Determines if check should be notApplicable and the notApplicableReason #' #' @param checkResults A dataframe containing the results of the data quality checks #' #' @keywords internal - .calculateNotApplicableStatus <- function(checkResults) { # Look up missing tables and add variable tableIsMissing to checkResults missingTables <- checkResults %>% @@ -81,7 +111,7 @@ ) %>% dplyr::distinct() - # Look up empty fields and add variable tableIsEmpty to checkResults + # Look up empty fields and add variable fieldIsEmpty to checkResults emptyFields <- checkResults %>% dplyr::filter( .data$checkName == "measureValueCompleteness" @@ -148,7 +178,7 @@ .data$tableIsEmpty ~ sprintf("Table %s is empty.", .data$cdmTableName), .data$fieldIsEmpty ~ sprintf("Field %s.%s is not populated.", .data$cdmTableName, .data$cdmFieldName), .data$conceptIsMissing ~ sprintf("%s=%s is missing from the %s table.", .data$cdmFieldName, .data$conceptId, .data$cdmTableName), - .data$conceptAndUnitAreMissing ~ sprintf("Combination of %s=%s, unitConceptId=%s and VALUE_AS_NUMBER IS NOT NULL is missing from the %s table.", .data$cdmFieldName, .data$conceptId, .data$unitConceptId, .data$cdmTableName) + .data$conceptAndUnitAreMissing ~ sprintf("Combination of %s=%s, unitConceptId=%s and VALUE_AS_NUMBER IS NOT NULL is missing from the %s table.", .data$cdmFieldName, .data$conceptId, .data$unitConceptId, .data$cdmTableName) #nolint ), NA ), @@ -159,30 +189,3 @@ return(checkResults) } - -.applyNotApplicable <- function(x) { - # Errors precede all other statuses - if (x$isError == 1) { - return(0) - } - - # No NA status for cdmTable and cdmField if missing - if (x$checkName == "cdmTable" || x$checkName == "cdmField") { - return(0) - } - - if (any(x$tableIsMissing, x$fieldIsMissing, x$tableIsEmpty, na.rm = TRUE)) { - return(1) - } - - # No NA status for measureValueCompleteness if empty - if (x$checkName == "measureValueCompleteness") { - return(0) - } - - if (any(x$fieldIsEmpty, x$conceptIsMissing, x$conceptAndUnitAreMissing, na.rm = TRUE)) { - return(1) - } - - return(0) -} diff --git a/tests/testthat/test-calculateNotApplicableStatus.R b/tests/testthat/test-calculateNotApplicableStatus.R index a70eef48..6c19decd 100644 --- a/tests/testthat/test-calculateNotApplicableStatus.R +++ b/tests/testthat/test-calculateNotApplicableStatus.R @@ -4,6 +4,11 @@ test_that("Not Applicable status Table Empty", { outputFolder <- tempfile("dqd_") on.exit(unlink(outputFolder, recursive = TRUE)) + # Make sure the device exposure table is empty + connection <- DatabaseConnector::connect(connectionDetailsEunomia) + DatabaseConnector::executeSql(connection, "DELETE FROM DEVICE_EXPOSURE;") + DatabaseConnector::disconnect(connection) + results <- executeDqChecks( connectionDetails = connectionDetailsEunomia, cdmDatabaseSchema = cdmDatabaseSchemaEunomia, @@ -13,10 +18,9 @@ test_that("Not Applicable status Table Empty", { # Eunomia COST table has misspelled 'REVEUE_CODE_SOURCE_VALUE' tablesToExclude = c("COST", "CONCEPT", "VOCABULARY", "CONCEPT_ANCESTOR", "CONCEPT_RELATIONSHIP", "CONCEPT_CLASS", "CONCEPT_SYNONYM", "RELATIONSHIP", "DOMAIN"), outputFolder = outputFolder, - writeToTable = F + writeToTable = FALSE ) - # Assumption that Eunomia has empty device_exposure table r <- results$CheckResults[results$CheckResults$checkName == "measureValueCompleteness" & results$CheckResults$tableName == "device_exposure", ] expect_true(all(r$notApplicable == 1)) @@ -41,7 +45,7 @@ test_that("measureConditionEraCompleteness Not Applicable if condition_occurrenc # Eunomia COST table has misspelled 'REVEUE_CODE_SOURCE_VALUE' tablesToExclude = c("COST", "CONCEPT", "VOCABULARY", "CONCEPT_ANCESTOR", "CONCEPT_RELATIONSHIP", "CONCEPT_CLASS", "CONCEPT_SYNONYM", "RELATIONSHIP", "DOMAIN"), outputFolder = outputFolder, - writeToTable = F + writeToTable = FALSE ) # Reinstate Condition Occurrence @@ -73,7 +77,7 @@ test_that("measureConditionEraCompleteness Fails if condition_era empty", { # Eunomia COST table has misspelled 'REVEUE_CODE_SOURCE_VALUE' tablesToExclude = c("COST", "CONCEPT", "VOCABULARY", "CONCEPT_ANCESTOR", "CONCEPT_RELATIONSHIP", "CONCEPT_CLASS", "CONCEPT_SYNONYM", "RELATIONSHIP", "DOMAIN"), outputFolder = outputFolder, - writeToTable = F + writeToTable = FALSE ) # Reinstate the Condition Era From 3d1c35429b3852b778a6ff612e764ca3a0e057b5 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Mon, 13 May 2024 14:06:56 +0200 Subject: [PATCH 12/43] fixes #539 --- inst/csv/OMOP_CDMv5.4_Field_Level.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inst/csv/OMOP_CDMv5.4_Field_Level.csv b/inst/csv/OMOP_CDMv5.4_Field_Level.csv index 4362b702..3d30b554 100644 --- a/inst/csv/OMOP_CDMv5.4_Field_Level.csv +++ b/inst/csv/OMOP_CDMv5.4_Field_Level.csv @@ -7,14 +7,14 @@ CARE_SITE,cdm,place_of_service_concept_id,No,,,integer,0,,"This is a high-level CARE_SITE,cdm,place_of_service_source_value,No,,,varchar(50),0,,,Put the place of service of the care_site as it appears in the source data.,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,No,,, CDM_SOURCE,cdm,cdm_etl_reference,No,,,varchar(255),0,,,Put the link to the CDM version used.,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, CDM_SOURCE,cdm,cdm_holder,Yes,0,,varchar(255),0,,The holder of the CDM instance.,,No,,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -CDM_SOURCE,cdm,cdm_release_date,Yes,0,,date,0,,The release data of the CDM instance.,,No,,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,'20000101',1,,"DATEADD(dd,1,GETDATE())",1,,Yes,CDM_SOURCE,SOURCE_RELEASE_DATE,0,,,,,Yes,SOURCE_RELEASE_DATE,0,,,,,,,,,,, +CDM_SOURCE,cdm,cdm_release_date,Yes,0,,date,0,,The release data of the CDM instance.,,No,,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,'20000101',1,,"DATEADD(dd,1,GETDATE())",1,,Yes,CDM_SOURCE,SOURCE_RELEASE_DATE,0,,,,,,,,,,,,,,,,,, CDM_SOURCE,cdm,cdm_source_abbreviation,Yes,0,,varchar(25),0,,The abbreviation of the CDM instance.,,No,,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, CDM_SOURCE,cdm,cdm_source_name,Yes,0,,varchar(255),0,,The name of the CDM instance.,,No,,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, CDM_SOURCE,cdm,cdm_version,No,,,varchar(10),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, CDM_SOURCE,cdm,cdm_version_concept_id,Yes,0,,integer,0,,The Concept Id representing the version of the CDM.,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,Metadata,0,,,,,Yes,0,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, CDM_SOURCE,cdm,source_description,No,,,varchar(MAX),0,,The description of the CDM instance.,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, CDM_SOURCE,cdm,source_documentation_reference,No,,,varchar(255),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -CDM_SOURCE,cdm,source_release_date,Yes,0,,date,0,,The release date of the source data.,,No,,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,'20000101',1,,"DATEADD(dd,1,GETDATE())",1,,,,,,,,,,,,,,,,,,,,,,, +CDM_SOURCE,cdm,source_release_date,Yes,0,,date,0,,The release date of the source data.,,No,,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,'20000101',1,,"DATEADD(dd,1,GETDATE())",1,,,,,,,,,,Yes,SOURCE_RELEASE_DATE,0,,,,,,,,,,, CDM_SOURCE,cdm,vocabulary_version,Yes,0,,varchar(20),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, COHORT,cohort,cohort_definition_id,Yes,0,,integer,0,,,,No,,,No,,,,,,,,,,,No,,,No,0,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, COHORT,cohort,cohort_end_date,Yes,0,,date,0,,,,No,,,No,,,,,,,,,,,No,,,No,0,,No,,,No,,,No,,,,'19500101',1,,"DATEADD(dd,1,GETDATE())",1,,,,,,,,,,,,,,,,,,,,,,, From 8efebd445a4229ef096b599bfd424bfb6fb5f9fa Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Mon, 13 May 2024 14:17:53 +0200 Subject: [PATCH 13/43] update plausibleStartBeforeEnd documentation --- vignettes/checks/plausibleStartBeforeEnd.Rmd | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/vignettes/checks/plausibleStartBeforeEnd.Rmd b/vignettes/checks/plausibleStartBeforeEnd.Rmd index d65bcfe3..4095a7ae 100644 --- a/vignettes/checks/plausibleStartBeforeEnd.Rmd +++ b/vignettes/checks/plausibleStartBeforeEnd.Rmd @@ -29,8 +29,10 @@ The start date can be before the end date or equal to the end date. It is applie - *Numerator*: The number of records where date in **cdmFieldName** is after the date in **plausibleStartBeforeEndFieldName**. - *Denominator*: The total number of records with a non-null start and non-null end date value - *Related CDM Convention(s)*: -Not linked to a convention- -- *CDM Fields/Tables*: This check runs on all date and datetime fields that have a start and end date in the same table. It also runs on the `cdm_source` table, comparing `source_release_date` is before `cdm_release_date`. -- *Default Threshold Value*: 1% (except for vocabulary and cdm_source tables, where it is 0%) +- *CDM Fields/Tables*: This check runs on all start date/datetime fields with an end date/datetime in the same table. It also runs on the cdm_source table, comparing `source_release_date` is before `cdm_release_date`. +- *Default Threshold Value*: + - 0% for the observation_period, vocabulary (valid_start/end_date) and cdm_source tables. + - 1% for other tables with an end date. ## User Guidance @@ -52,7 +54,7 @@ There main reason for this check to fail is often that the source data is incorr Any violating checks should either be removed or corrected. In most cases this can be done by adjusting the end date: - With a few exceptions, the end date is not mandatory and can be left empty. -- If the end date is mandatory (visit_occurrence and drug_exposure), the end date can be set to the start date if the event. Note tha +- If the end date is mandatory (notably visit_occurrence and drug_exposure), the end date can be set to the start date if the event. Make sure to document this as it leads to loss of duration information. - If this check fails for the observation_period, it might signify a bigger underlying issue. Please investigate all records for this person in the CDM and source. - If neither the start or end date can be trusted, pleaes remove the record from the CDM. From 0e9a7029f05cf2c5021c37a7ae2a8334e85ed2d1 Mon Sep 17 00:00:00 2001 From: Katy Sadowski Date: Wed, 21 Feb 2024 21:31:29 -0600 Subject: [PATCH 14/43] Add missing changes to NEWS --- NEWS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS.md b/NEWS.md index 1e2cc31e..0c91c12c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -14,6 +14,10 @@ The 3 temporal plausibilty checks are intended to **replace** `plausibleTemporal For more information on the new checks, please check the [Check Type Definitions](https://ohdsi.github.io/DataQualityDashboard/articles/CheckTypeDescriptions.html) documentation page. If you'd like to disable the deprecated checks, please see the suggested check exclusion workflow in our Getting Started code [here](https://ohdsi.github.io/DataQualityDashboard/articles/DataQualityDashboard.html). +### Check Updates +- The number of measurements checked in `plausibleUnitConceptIds` has been reduced, and the lists of plausible units for those measurements have been re-reviewed and updated for accuracy. This change is intended to improve performance and reliablility of this check. Please file an issue if you would like to contribute additional measurements + plausible units to be checked in the future +- Some erroneous `plausibleValueLow` thresholds have been corrected to prevent false positive failures from occurring + ### New Documentation We have begun an initiative to add more comprehensive user documentation at the data quality check level. A dedicated documentation page is being created for each check type. Each check's page will include detailed information about how its result is generated and what to do if it fails. Guidance is provided for both ETL developers and data users. From f7016f4ccc1e69ffe54189499c08570934f4b402 Mon Sep 17 00:00:00 2001 From: Katy Sadowski Date: Wed, 21 Feb 2024 21:37:58 -0600 Subject: [PATCH 15/43] Update index.html --- docs/news/index.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/news/index.html b/docs/news/index.html index f49714d2..9d7f457c 100644 --- a/docs/news/index.html +++ b/docs/news/index.html @@ -139,6 +139,11 @@

New ChecksCheck Type Definitions documentation page. If you’d like to disable the deprecated checks, please see the suggested check exclusion workflow in our Getting Started code here.

+

Check Updates

+
  • The number of measurements checked in plausibleUnitConceptIds has been reduced, and the lists of plausible units for those measurements have been re-reviewed and updated for accuracy. This change is intended to improve performance and reliablility of this check. Please file an issue if you would like to contribute additional measurements + plausible units to be checked in the future
    • +
  • Some erroneous plausibleValueLow thresholds have been corrected to prevent false positive failures from occurring
    • +
+

New Documentation

We have begun an initiative to add more comprehensive user documentation at the data quality check level. A dedicated documentation page is being created for each check type. Each check’s page will include detailed information about how its result is generated and what to do if it fails. Guidance is provided for both ETL developers and data users.

9 pages have been added so far, and the rest will come in a future release. Check them out here and please reach out with feedback as we continue improving our documentation!

From 8ad5185714ed6b4ffc5eac369722c7352c675b70 Mon Sep 17 00:00:00 2001 From: Katy Sadowski Date: Wed, 21 Feb 2024 21:57:22 -0600 Subject: [PATCH 16/43] fix NEWS --- docs/404.html | 2 +- docs/LICENSE-text.html | 2 +- docs/articles/AddNewCheck.html | 2 +- docs/articles/CheckStatusDefinitions.html | 2 +- docs/articles/CheckTypeDescriptions.html | 2 +- docs/articles/DataQualityDashboard.html | 2 +- docs/articles/DqdForCohorts.html | 2 +- docs/articles/SqlOnly.html | 2 +- docs/articles/Thresholds.html | 2 +- docs/articles/checkIndex.html | 2 +- docs/articles/checks/cdmDatatype.html | 2 +- docs/articles/checks/cdmField.html | 2 +- docs/articles/checks/cdmTable.html | 2 +- docs/articles/checks/fkClass.html | 2 +- docs/articles/checks/fkDomain.html | 2 +- docs/articles/checks/isForeignKey.html | 2 +- docs/articles/checks/isPrimaryKey.html | 2 +- docs/articles/checks/isRequired.html | 2 +- docs/articles/checks/isStandardValidConcept.html | 2 +- docs/articles/checks/measureConditionEraCompleteness.html | 2 +- docs/articles/checks/measurePersonCompleteness.html | 2 +- docs/articles/checks/measureValueCompleteness.html | 2 +- docs/articles/checks/plausibleAfterBirth.html | 2 +- docs/articles/checks/plausibleBeforeDeath.html | 2 +- docs/articles/checks/plausibleDuringLife.html | 2 +- docs/articles/checks/plausibleGender.html | 2 +- docs/articles/checks/plausibleStartBeforeEnd.html | 2 +- docs/articles/checks/plausibleTemporalAfter.html | 2 +- docs/articles/checks/plausibleUnitConceptIds.html | 2 +- docs/articles/checks/plausibleValueHigh.html | 2 +- docs/articles/checks/plausibleValueLow.html | 2 +- docs/articles/checks/sourceConceptRecordCompleteness.html | 2 +- docs/articles/checks/sourceValueCompleteness.html | 2 +- docs/articles/checks/standardConceptRecordCompleteness.html | 2 +- docs/articles/checks/withinVisitDates.html | 2 +- docs/articles/index.html | 2 +- docs/authors.html | 2 +- docs/index.html | 2 +- docs/news/index.html | 2 +- docs/pkgdown.yml | 2 +- docs/pull_request_template.html | 2 +- docs/reference/convertJsonResultsFileCase.html | 2 +- docs/reference/dot-evaluateThresholds.html | 2 +- docs/reference/dot-getCheckId.html | 2 +- docs/reference/dot-processCheck.html | 2 +- docs/reference/dot-recordResult.html | 2 +- docs/reference/dot-runCheck.html | 2 +- docs/reference/dot-summarizeResults.html | 2 +- docs/reference/dot-writeResultsToCsv.html | 2 +- docs/reference/dot-writeResultsToJson.html | 2 +- docs/reference/dot-writeResultsToTable.html | 2 +- docs/reference/executeDqChecks.html | 2 +- docs/reference/index.html | 2 +- docs/reference/listDqChecks.html | 2 +- docs/reference/reEvaluateThresholds.html | 2 +- docs/reference/viewDqDashboard.html | 2 +- docs/reference/writeDBResultsToJson.html | 2 +- docs/reference/writeJsonResultsToCsv.html | 2 +- docs/reference/writeJsonResultsToTable.html | 2 +- 59 files changed, 59 insertions(+), 59 deletions(-) diff --git a/docs/404.html b/docs/404.html index 24d94dbd..bf6f0a21 100644 --- a/docs/404.html +++ b/docs/404.html @@ -122,7 +122,7 @@
  • @@ -166,7 +196,7 @@

    Page not found (404)

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/LICENSE-text.html b/docs/LICENSE-text.html index 742d45c2..d734fc77 100644 --- a/docs/LICENSE-text.html +++ b/docs/LICENSE-text.html @@ -1,5 +1,5 @@ -License • DataQualityDashboardLicense • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • Changelog @@ -338,7 +368,7 @@

    License

    • -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/AddNewCheck.html b/docs/articles/AddNewCheck.html index fe549430..11920ea1 100644 --- a/docs/articles/AddNewCheck.html +++ b/docs/articles/AddNewCheck.html @@ -6,7 +6,7 @@ Add a New Data Quality Check • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • @@ -147,7 +177,7 @@

    Add a New Data Quality Check

    Don Torok

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/AddNewCheck.rmd @@ -324,7 +354,7 @@

    Add Hooks to Field/Table/

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/CheckStatusDefinitions.html b/docs/articles/CheckStatusDefinitions.html index 58f0d310..34423e5e 100644 --- a/docs/articles/CheckStatusDefinitions.html +++ b/docs/articles/CheckStatusDefinitions.html @@ -6,7 +6,7 @@ Check Status Definitions • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +

  • @@ -148,7 +178,7 @@

    Check Status Definitions

    Dmitry Ilyn, Maxim Moinat

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/CheckStatusDefinitions.rmd @@ -264,7 +294,7 @@

    Not Applicable

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/CheckTypeDescriptions.html b/docs/articles/CheckTypeDescriptions.html index 442bbeb5..ced9cdd5 100644 --- a/docs/articles/CheckTypeDescriptions.html +++ b/docs/articles/CheckTypeDescriptions.html @@ -6,7 +6,7 @@ Data Quality Check Type Definitions • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +

  • @@ -148,7 +178,7 @@

    Data Quality Check Type Definitions

    Clair Blacketer

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/CheckTypeDescriptions.rmd @@ -683,7 +713,7 @@

    plausibleUnitConceptIds

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/DataQualityDashboard.html b/docs/articles/DataQualityDashboard.html index 011ad993..d370a9b3 100644 --- a/docs/articles/DataQualityDashboard.html +++ b/docs/articles/DataQualityDashboard.html @@ -6,7 +6,7 @@ Getting Started • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +

  • @@ -148,7 +178,7 @@

    Getting Started

    Clair Blacketer

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/DataQualityDashboard.rmd @@ -334,7 +364,7 @@

    View checks

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/DqdForCohorts.html b/docs/articles/DqdForCohorts.html index 695886ac..cf5cd2fb 100644 --- a/docs/articles/DqdForCohorts.html +++ b/docs/articles/DqdForCohorts.html @@ -6,7 +6,7 @@ Running the DQD on a Cohort • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +

  • @@ -148,7 +178,7 @@

    Running the DQD on a Cohort

    Clair Blacketer

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/DqdForCohorts.rmd @@ -209,7 +239,7 @@

    2024-02-21

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/SqlOnly.html b/docs/articles/SqlOnly.html index 83db2df8..68151ee8 100644 --- a/docs/articles/SqlOnly.html +++ b/docs/articles/SqlOnly.html @@ -6,7 +6,7 @@ Running the DQD in SqlOnly mode • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • @@ -148,7 +178,7 @@

    Running the DQD in SqlOnly mode

    Maxim Moinat

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/SqlOnly.rmd @@ -394,7 +424,7 @@

    (OPTIONAL) Execute queries

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/Thresholds.html b/docs/articles/Thresholds.html index a33a1685..d35f64c1 100644 --- a/docs/articles/Thresholds.html +++ b/docs/articles/Thresholds.html @@ -6,7 +6,7 @@ Failure Thresholds and How to Change Them • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +

  • @@ -148,7 +178,7 @@

    Failure Thresholds and How to Change Them

    Clair Blacketer

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/Thresholds.rmd @@ -346,7 +376,7 @@

    Step 3: Run the DQD Usi

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checkIndex.html b/docs/articles/checkIndex.html index 98f3fb0e..aa0590ac 100644 --- a/docs/articles/checkIndex.html +++ b/docs/articles/checkIndex.html @@ -6,7 +6,7 @@ Index • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +

  • @@ -148,7 +178,7 @@

    Index

    Katy Sadowski

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checkIndex.Rmd @@ -202,22 +232,20 @@

    Checks
  • isRequired
  • fkDomain
  • fkClass
    • -
  • measurePersonCompleteness (PAGE UNDER CONSTRUCTION)
    • +
  • measurePersonCompleteness
  • measureConditionEraCompleteness (PAGE UNDER CONSTRUCTION)
    • -
  • isStandardValidConcept (PAGE UNDER CONSTRUCTION)
    • -
  • measureValueCompleteness (PAGE UNDER CONSTRUCTION)
    • -
  • standardConceptRecordCompleteness (PAGE UNDER CONSTRUCTION)
    • -
  • sourceConceptRecordCompleteness (PAGE UNDER CONSTRUCTION)
    • -
  • sourceValueCompleteness (PAGE UNDER CONSTRUCTION)
    • -
  • plausibleValueLow (PAGE UNDER CONSTRUCTION)
    • -
  • plausibleValueHigh (PAGE UNDER CONSTRUCTION)
    • -
  • plausibleTemporalAfter (PAGE UNDER CONSTRUCTION)
    • -
  • plausibleDuringLife (PAGE UNDER CONSTRUCTION)
    • +
  • isStandardValidConcept
    • +
  • measureValueCompleteness
    • +
  • standardConceptRecordCompleteness
    • +
  • sourceConceptRecordCompleteness
    • +
  • sourceValueCompleteness
    • +
  • plausibleValueLow
    • +
  • plausibleValueHigh
  • withinVisitDates (PAGE UNDER CONSTRUCTION)
  • plausibleAfterBirth
    • -
  • plausibleBeforeDeath (PAGE UNDER CONSTRUCTION)
    • -
  • plausibleStartBeforeEnd (PAGE UNDER CONSTRUCTION)
    • -
  • plausibleGender (PAGE UNDER CONSTRUCTION)
    • +
  • plausibleBeforeDeath
    • +
  • plausibleStartBeforeEnd
    • +
  • plausibleGenderUseDescendants (PAGE UNDER CONSTRUCTION)
  • plausibleUnitConceptIds (PAGE UNDER CONSTRUCTION)
    • @@ -240,7 +268,7 @@

    Checks

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/cdmDatatype.html b/docs/articles/checks/cdmDatatype.html index 31f5f790..fb89677f 100644 --- a/docs/articles/checks/cdmDatatype.html +++ b/docs/articles/checks/cdmDatatype.html @@ -6,7 +6,7 @@ cdmDatatype • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +

  • @@ -148,7 +178,7 @@

    cdmDatatype

    Katy Sadowski

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/cdmDatatype.Rmd @@ -262,7 +292,7 @@

    Data User

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/cdmField.html b/docs/articles/checks/cdmField.html index c1fe7eb6..573aa1d3 100644 --- a/docs/articles/checks/cdmField.html +++ b/docs/articles/checks/cdmField.html @@ -6,7 +6,7 @@ cdmField • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +

  • @@ -148,7 +178,7 @@

    cdmField

    Heidi Schmidt, Katy Sadowski

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/cdmField.Rmd @@ -251,7 +281,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/cdmTable.html b/docs/articles/checks/cdmTable.html index c1c3df5d..1e3b523b 100644 --- a/docs/articles/checks/cdmTable.html +++ b/docs/articles/checks/cdmTable.html @@ -6,7 +6,7 @@ cdmTable • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +

  • @@ -148,7 +178,7 @@

    cdmTable

    John Gresh, Katy Sadowski

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/cdmTable.Rmd @@ -251,7 +281,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/fkClass.html b/docs/articles/checks/fkClass.html index 38018115..b133a025 100644 --- a/docs/articles/checks/fkClass.html +++ b/docs/articles/checks/fkClass.html @@ -6,7 +6,7 @@ fkClass • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +

  • @@ -148,7 +178,7 @@

    fkClass

    Clair Blacketer, Katy Sadowski

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/fkClass.Rmd @@ -273,7 +303,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/fkDomain.html b/docs/articles/checks/fkDomain.html index bdc8ede4..07bae9eb 100644 --- a/docs/articles/checks/fkDomain.html +++ b/docs/articles/checks/fkDomain.html @@ -6,7 +6,7 @@ fkDomain • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +

  • @@ -148,7 +178,7 @@

    fkDomain

    Clair Blacketer, Katy Sadowski

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/fkDomain.Rmd @@ -270,7 +300,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/isForeignKey.html b/docs/articles/checks/isForeignKey.html index c27c07eb..60484a17 100644 --- a/docs/articles/checks/isForeignKey.html +++ b/docs/articles/checks/isForeignKey.html @@ -6,7 +6,7 @@ isForeignKey • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +

  • @@ -148,7 +178,7 @@

    isForeignKey

    Dmytry Dymshyts, Katy Sadowski

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/isForeignKey.Rmd @@ -292,7 +322,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/isPrimaryKey.html b/docs/articles/checks/isPrimaryKey.html index 2f6cd952..726e0037 100644 --- a/docs/articles/checks/isPrimaryKey.html +++ b/docs/articles/checks/isPrimaryKey.html @@ -6,7 +6,7 @@ isPrimaryKey • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +

  • @@ -148,7 +178,7 @@

    isPrimaryKey

    John Gresh, Katy Sadowski

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/isPrimaryKey.Rmd @@ -262,7 +292,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/isRequired.html b/docs/articles/checks/isRequired.html index 694d065f..1e2f089e 100644 --- a/docs/articles/checks/isRequired.html +++ b/docs/articles/checks/isRequired.html @@ -6,7 +6,7 @@ isRequired • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +

  • @@ -148,7 +178,7 @@

    isRequired

    Katy Sadowski

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/isRequired.Rmd @@ -279,7 +309,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/isStandardValidConcept.html b/docs/articles/checks/isStandardValidConcept.html index 002d32ad..035c1c0f 100644 --- a/docs/articles/checks/isStandardValidConcept.html +++ b/docs/articles/checks/isStandardValidConcept.html @@ -6,7 +6,7 @@ isStandardValidConcept • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +

  • @@ -145,9 +175,10 @@

    isStandardValidConcept

    -

    +

    Katy +Sadowski

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/isStandardValidConcept.Rmd @@ -159,7 +190,7 @@

    2024-02-21

    Summary

    -

    Level: FIELD
    Context: Verification
    Category: Conformance
    Subcategory: Value
    Severity:

    +

    Level: FIELD
    Context: Verification
    Category: Conformance
    Subcategory: Value
    Severity: CDM convention ⚠

    Description @@ -173,32 +204,97 @@

    Definitionhttps://ohdsi.github.io/CommonDataModel/dataModelConventions.html#Mapping. +In other words, all non-zero values in the X_concept_id +column should exist in the concept_id column of the CONCEPT table and +have CONCEPT.standard_concept = ‘S’ and CONCEPT.invalid_reason = +NULL.

  • -CDM Fields/Tables:
    • +CDM Fields/Tables: All tables with an +X_concept_id column, and all X_concept_id +columns in those tables.
  • -Default Threshold Value:
    • +Default Threshold Value: 0

    User Guidance

    -
    -

    Violated rows query -

    -
    -
    +

    Failures of this check represent a violation of the fundamental CDM +convention requiring all concept IDs to belong to the OMOP standard +vocabulary. This is an essential convention in enabling standard +analytics. If source codes have not been properly mapped to OMOP +standard concepts in a CDM, studies designed using the OMOP standard +vocabulary will return inaccurate results for that database.

    ETL Developers

    +

    A failure of this check indicates an issue with the concept mapping +portion of your ETL, and must be resolved. Ensure that your ETL is only +mapping source codes to standard, valid concepts (via the ‘Maps to’ +relationship). Note as well that if no standard concept mapping exists +for a source code, you MUST populate its X_concept_id +column with 0. See the Book of OHDSI for additional guidance on the +concept mapping process: https://ohdsi.github.io/TheBookOfOhdsi/ExtractTransformLoad.html#step-2-create-the-code-mappings

    +

    You may inspect the failing rows using the following SQL:

    +
    SELECT  
+  '@cdmTableName.@cdmFieldName' AS violating_field,  
+  cdmTable.*  
+FROM @schema.@cdmTableName cdmTable 
+  JOIN @vocabDatabaseSchema.concept co ON cdmTable.@cdmFieldName = co.concept_id 
+WHERE co.concept_id != 0  
+  AND (co.standard_concept != 'S' OR co.invalid_reason IS NOT NULL)
    +

    You may build upon this query by joining the relevant +X_concept_id and X_source_concept_id columns +to the concept table and inspecting their names and vocabularies. If the +X_source_concept_id correctly represents the source code in +x_source_value, the fix will be a matter of ensuring your ETL is +correctly using the concept_relationship table to map the source concept +ID to a standard concept via the ‘Maps to’ relationship. If you are not +populating the X_source_concept_id column and/or are using +an intermediate concept mapping table, you may need to inspect the +mappings in your mapper table to ensure they’ve been generated correctly +using the ‘Maps to’ relationship for your CDM’s vocabulary version.

    Data Users

    +

    This check failure means that the failing rows will not be picked up +in a standard OHDSI analysis. It is highly recommended to work with your +ETL team or data provider, if possible, to resolve this issue.

    +

    However, you may work around it at your own risk by determining +whether or not the affected rows are relevant for your analysis. Here’s +an example query you could run to inspect failing rows in the +condition_occurrence table:

    +
    SELECT  
+  condition_concept_id AS violating_concept, 
+  c1.concept_name AS violating_concept_name, 
+  condition_source_concept_id AS source_concept, 
+  c2.concept_name AS source_concept_name, 
+  c2.vocabulary_id AS source_vocab, 
+  condition_source_value, 
+  COUNT(*) 
+FROM @cdmDatabaseSchema.condition_occurrence 
+  JOIN @vocabDatabaseSchema.concept c1 ON condition_occurrence.condition_concept_id = c1.concept_id 
+  LEFT JOIN @vocabDatabaseSchema.concept c2 ON condition_occurrence.condition_source_concept_id = c2.concept_id 
+WHERE c1.concept_id != 0  
+  AND (c1.standard_concept != 'S' OR c1.invalid_reason IS NOT NULL) 
+GROUP BY 1,2,3,4,5,6 
+ORDER BY 7 DESC
    +

    If you can confirm by inspecting the source concept and/or source +value that the affected rows are not relevant for your analysis, you can +proceed with your work and ignore the issue. However, especially if a +large number of rows are impacted it’s recommended to act upon these +failures as there could potentially be deeper issues with the ETL +concept mapping process that need to be fixed.

    @@ -220,7 +316,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/measureConditionEraCompleteness.html b/docs/articles/checks/measureConditionEraCompleteness.html index e158596f..618b8b80 100644 --- a/docs/articles/checks/measureConditionEraCompleteness.html +++ b/docs/articles/checks/measureConditionEraCompleteness.html @@ -6,7 +6,7 @@ measureConditionEraCompleteness • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • @@ -147,7 +177,7 @@

    measureConditionEraCompleteness

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/measureConditionEraCompleteness.Rmd @@ -219,7 +249,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/measurePersonCompleteness.html b/docs/articles/checks/measurePersonCompleteness.html index 57da0042..0cd476a6 100644 --- a/docs/articles/checks/measurePersonCompleteness.html +++ b/docs/articles/checks/measurePersonCompleteness.html @@ -6,7 +6,7 @@ measurePersonCompleteness • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +

  • @@ -145,9 +175,10 @@

    measurePersonCompleteness

    -

    +

    Katy +Sadowski

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/measurePersonCompleteness.Rmd @@ -159,46 +190,113 @@

    2024-02-21

    Summary

    -

    Level: TABLE
    Context: Validation
    Category: Completeness
    Subcategory:
    Severity:

    +

    Level: TABLE
    Context: Validation
    Category: Completeness
    Subcategory:
    Severity: CDM convention ⚠ Characterization ✔

    Description

    The number and percent of persons in the CDM that do not have at least one record in the @cdmTableName -table

    +table.

    Definition

    • -Numerator:
      • +Numerator: The number of persons with 0 rows in a given CDM +table.
    • -Denominator:
      • +Denominator: The total number of persons in the +person table.
    • -Related CDM Convention(s):
      • +Related CDM Convention(s): Each Person needs to have at +least one observation_period record. Otherwise, CDM +conventions do not dictate any rules for person completeness.
    • -CDM Fields/Tables:
      • +CDM Fields/Tables: By default, this check runs on all +tables with a foreign key to the person table.
    • -Default Threshold Value:
      • +Default Threshold Value: Set to 95 or 100 for most tables +but 0 for observation_period +

    User Guidance

    +

    For most tables, this check is a characterization of the completeness +of various data types in the source data. However, in the case of +observation_period, this check should actually be +considered a CDM convention check as it is used to enforce the +requirement that all persons have at least one observation period. A +failure of this check on the observation_period table is a +serious issue as persons without an observation_period +cannot be included in any standard OHDSI analysis.

    +

    Run the following query to obtain a list of persons who had no data +in a given table. From this list of person_ids you may join to other +tables in the CDM to understand trends in these individuals’ data which +may provide clues as to the root cause of the issue.

    Violated rows query

    -
    +
    SELECT 
+    cdmTable.* 
+FROM @cdmDatabaseSchema.person cdmTable
+    LEFT JOIN @schema.@cdmTableName cdmTable2 
+        ON cdmTable.person_id = cdmTable2.person_id
+WHERE cdmTable2.person_id IS NULL

    ETL Developers

    +
    +

    Observation period +

    +

    All persons in the CDM must have an observation period; OHDSI +analytics tools only operate on persons with observable time, as +represented by one or more observation periods. Persons missing +observation periods may represent a bug in the ETL code which generates +observation periods. Alternatively, some persons may have no observable +time in the source data. These persons should be removed from the +CDM.

    +
    +
    +

    All other tables +

    +

    Action on persons missing records in other clinical event tables will +depend on the characteristics of the source database. In certain cases, +missingness is expected – some persons may just not have a given type of +data available in the source. In others, various ETL issues may result +in persons missing records in a given event table: - Mis-mapping of +domains, resulting in the placement of records in the incorrect +table
    +- Incorrect parsing of source data, resulting in loss of valid records - +Failure of an ETL step, resulting in an empty table

    +

    If more persons than expected are missing data in a given table, run +the violated rows SQL snippet to retrieve these persons’ person_ids, and +inspect these persons’ other clinical event data in the CDM for trends. +You may also use person_source_value to trace back to these persons’ +source data to identify source data records potentially missed by the +ETL.

    +

    Note that in some cases, the failure threshold for this check may +need to be adjusted according to completeness expectations for a given +data source.

    +

    Data Users

    +

    Severe failures, such as unexpected nearly empty tables, must be +fixed by the ETL team before a dataset can be used. Note as well that +any person missing an observation_period will not be able +to be included in any analysis using OHDSI tools.

    +

    Failures with a result close to the specified failure threshold may +be accepted, at your own risk and only if the result matches your +understanding of the source data. The violated rows SQL may be used to +inspect the full records for persons missing data in a given table in +order to validate your expectations or point to potential issues in the +ETL which need to be resolved.

    @@ -220,7 +318,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/measureValueCompleteness.html b/docs/articles/checks/measureValueCompleteness.html index 00c3acf8..8af6bb34 100644 --- a/docs/articles/checks/measureValueCompleteness.html +++ b/docs/articles/checks/measureValueCompleteness.html @@ -6,7 +6,7 @@ measureValueCompleteness • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • @@ -145,9 +175,10 @@

    measureValueCompleteness

    -

    +

    Katy +Sadowski

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/measureValueCompleteness.Rmd @@ -159,7 +190,7 @@

    2024-02-21

    Summary

    -

    Level: FIELD
    Context: Verification
    Category: Completeness
    Subcategory:
    Severity:

    +

    Level: FIELD
    Context: Verification
    Category: Completeness
    Subcategory:
    Severity: Characterization ✔

    Description @@ -171,32 +202,94 @@

    Definition

    User Guidance

    +

    This check’s primary purpose is to characterize completeness of +non-required fields in the OMOP CDM. It is most useful when the failure +threshold for each non-required field is customized to expectations +based on the source data being transformed into OMOP. In this case, the +check can be used to catch unexpected missingness due to ETL errors. +However, in all cases, this check will serve as a useful +characterization to help data users understand if a CDM contains the +right data for a given analysis.

    +

    While the failure threshold is set to 0 for required fields, note +that this is duplicative with the isRequired check - and +fixing one failure will resolve the other!

    Violated rows query

    -
    +

    Use this SQL query to inspect rows with a missing value in a given +field:

    +
    SELECT  
+
+'@cdmTableName.@cdmFieldName' AS violating_field,  
+
+cdmTable.*  
+
+FROM @cdmDatabaseSchema.@cdmTableName cdmTable       
+
+WHERE cdmTable.@cdmFieldName IS NULL

    ETL Developers

    +

    Failures of this check on fields required in the CDM specification +are redundant with failures of isRequired. See isRequired documentation for more +information.

    +

    ETL developers have 2 main options for the use of this check on +non-required fields:

    +
      +
    • The check threshold may be set to 100% for non-required fields such +that the check will never fail. The check result can be used simply to +understand completeness for these fields
      • +
    • The check threshold may be set to an appropriate value corresponding +to completeness expectations for each field given what’s available in +the source data. The check may be disabled for fields known not to exist +in the source data. Other fields may be set to whichever threshold is +deemed worthy of investigation
      • +
    +

    Unexpectedly missing values should be investigated for a potential +root cause in the ETL. For expected missingness, rows that violate this +check in non-required fields are acceptable but should be clearly +communicated to data users so that they can know when and when not to +expect data to be present in each field. To avoid confusion for users, +however, thresholds should be modified to avoid check failures at +expected levels.

    Data Users

    +

    This check informs you of the level of missing data in each column of +the CDM. If data is missing in a required column, see the isRequired +documentation for more information.

    +

    The interpretation of a check failure on a non-required column will +depend on the context. In some cases, the threshold for this check will +have been very deliberately set, and any failure should be cause for +concern unless justified and explained by your ETL provider. In other +cases, even if the check fails it may not be worrisome if the check +result is in line with your expectations given the source of the data. +When in doubt, utilize the inspection query above to ensure you can +explain the missing values.

    +

    Of course, if there is a failure on a non-required field you know +that you will not need in your analysis (for example, missing drug +quantity in an analysis not utilizing drug data), the check failure may +be safely ignored.

    @@ -218,7 +311,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/plausibleAfterBirth.html b/docs/articles/checks/plausibleAfterBirth.html index adf2189c..7c68c5fb 100644 --- a/docs/articles/checks/plausibleAfterBirth.html +++ b/docs/articles/checks/plausibleAfterBirth.html @@ -6,7 +6,7 @@ plausibleAfterBirth • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • @@ -148,7 +178,7 @@

    plausibleAfterBirth

    Maxim Moinat, Katy Sadowski

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/plausibleAfterBirth.Rmd @@ -167,17 +197,17 @@

    Description

    Definition

    -

    This check verifies that events happen after birth. This check is -only run on fields where the PLAUSIBLE_AFTER_BIRTH -parameter is set to Yes. The birthdate is taken from -the person table, either the birth_datetime or -composed from year_of_birth, month_of_birth, -day_of_birth (taking 1st month/1st day if missing).

    +

    This check verifies that events happen after birth. The birthdate is +taken from the person table, either the +birth_datetime or composed from year_of_birth, +month_of_birth, day_of_birth (taking 1st +month/1st day if missing).

  • @@ -145,9 +175,10 @@

    plausibleBeforeDeath

    -

    +

    Maxim +Moinat

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/plausibleBeforeDeath.Rmd @@ -159,44 +190,79 @@

    2024-02-21

    Summary

    -

    Level: FIELD
    Context: Verification
    Category: Plausibility
    Subcategory: Temporal
    Severity:

    +

    Level: FIELD
    Context: Verification
    Category: Plausibility
    Subcategory: Temporal
    Severity: Characterization ✔

    Description

    -

    The number and percent of records with a date value in the @cdmFieldName field of the @cdmTableName table that occurs after death.

    +

    The number and percent of records with a date value in the +cdmFieldName field of the cdmTableName +table that occurs more than 60 days after death. Note that this check +replaces the previous plausibleDuringLife check.

    Definition

    +

    A record violates this check if the date is more than 60 days after +the death date of the person, allowing administrative records directly +after death.

    • -Numerator:
      • +Numerator: The number of records where date in +cdmFieldName is more than 60 days after the persons’ +death date.
    • -Denominator:
      • +Denominator: Total number of records of persons with a +death date, in the cdmTableName.
    • -Related CDM Convention(s):
      • +Related CDM Convention(s): -Not linked to a +convention-
    • -CDM Fields/Tables:
      • +CDM Fields/Tables: This check runs on all date and datetime +fields.
    • -Default Threshold Value:
      • +Default Threshold Value: 1%

    User Guidance

    +

    Events are expected to occur between birth and death. The check +plausibleAfterbirth checks for the former, this check for +the latter. The 60-day period is a conservative estimate of the time it +takes for administrative records to be updated after a person’s death. +By default, both start and end dates are checked.

    Violated rows query

    -
    +
    SELECT 
+    '@cdmTableName.@cdmFieldName' AS violating_field, 
+    cdmTable.*
+FROM @cdmDatabaseSchema.@cdmTableName cdmTable
+JOIN @cdmDatabaseSchema.death de 
+    ON cdmTable.person_id = de.person_id
+WHERE cdmTable.@cdmFieldName IS NOT NULL 
+    AND CAST(cdmTable.@cdmFieldName AS DATE) > DATEADD(day, 60, de.death_date)

    ETL Developers

    +

    Start dates after death are likely to be source data issues, and +failing this check should trigger investigation of the source data +quality. End dates after death can occur due to derivation logic. For +example, a drug exposure can be prescribed as being continued long after +death. In such cases, it is recommended to update the logic to end the +prescription at death.

    Data Users

    +

    For most studies, a low number of violating records will have limited +impact on data use as it could be caused by lagging administrative +records. However, it might signify a larger data quality issue. Note +that the percentage violating records reported is among records from +death persons and such might be slightly inflated if comparing to the +overall population.

    @@ -218,7 +284,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/plausibleDuringLife.html b/docs/articles/checks/plausibleDuringLife.html index 39ec53d6..bec0bd41 100644 --- a/docs/articles/checks/plausibleDuringLife.html +++ b/docs/articles/checks/plausibleDuringLife.html @@ -6,7 +6,7 @@ plausibleDuringLife • DataQualityDashboard - + @@ -112,6 +112,18 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • @@ -147,7 +159,7 @@

    plausibleDuringLife

    -

    2024-02-21

    +

    2024-06-13

    Source: vignettes/checks/plausibleDuringLife.Rmd @@ -219,7 +231,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/plausibleGender.html b/docs/articles/checks/plausibleGender.html index c7f28f82..a5e62088 100644 --- a/docs/articles/checks/plausibleGender.html +++ b/docs/articles/checks/plausibleGender.html @@ -6,7 +6,7 @@ plausibleGender • DataQualityDashboard - + @@ -112,6 +112,18 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • @@ -147,7 +159,7 @@

    plausibleGender

    -

    2024-02-21

    +

    2024-06-13

    Source: vignettes/checks/plausibleGender.Rmd @@ -220,7 +232,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/plausibleGenderUseDescendants.html b/docs/articles/checks/plausibleGenderUseDescendants.html new file mode 100644 index 00000000..c2765e30 --- /dev/null +++ b/docs/articles/checks/plausibleGenderUseDescendants.html @@ -0,0 +1,265 @@ + + + + + + + +plausibleGender • DataQualityDashboard + + + + + + + + + + + + +
    +
    + + + + +
    +
    +
    +

    plausibleGender

    +

    + +

    2024-06-26

    + + Source: vignettes/checks/plausibleGenderUseDescendants.Rmd + + +
    + + + +
    +

    Summary +

    +

    Level: CONCEPT
    Context: Validation
    Category: Plausibility
    Subcategory: Atemporal
    Severity:

    +
    +
    +

    Description +

    +

    For a CONCEPT_ID @conceptId (@conceptName), the number and percent of records +associated with patients with an implausible gender (correct gender = +@plausibleGender).

    +
    +
    +

    Definition +

    +
      +
    • +Numerator:
      • +
    • +Denominator:
      • +
    • +Related CDM Convention(s):
      • +
    • +CDM Fields/Tables:
      • +
    • +Default Threshold Value:
      • +
    +
    +
    +

    User Guidance +

    +
    +

    Violated rows query +

    +
    +
    +
    +

    ETL Developers +

    +
    +
    +

    Data Users +

    +
    +
    +
    + + + +
    + + + +
    +

    +

    Developed by Katy Sadowski, Clair Blacketer, Ajit Londhe, Anthony Sena, Anthony Molinaro, Frank DeFalco, Pavel Grafkin.

    +
    + +
    +

    +

    Site built with pkgdown 2.0.9.

    +
    + +
    +
    + + + + + + + + diff --git a/docs/articles/checks/plausibleStartBeforeEnd.html b/docs/articles/checks/plausibleStartBeforeEnd.html index 51c8de3f..cc874c5e 100644 --- a/docs/articles/checks/plausibleStartBeforeEnd.html +++ b/docs/articles/checks/plausibleStartBeforeEnd.html @@ -6,7 +6,7 @@ plausibleStartBeforeEnd • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • @@ -145,9 +175,10 @@

    plausibleStartBeforeEnd

    -

    +

    Maxim +Moinat

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/plausibleStartBeforeEnd.Rmd @@ -159,45 +190,97 @@

    2024-02-21

    Summary

    -

    Level: FIELD
    Context: Verification
    Category: Plausibility
    Subcategory: Temporal
    Severity:

    +

    Level: FIELD
    Context: Verification
    Category: Plausibility
    Subcategory: Temporal
    Severity: CDM convention ⚠

    Description

    -

    The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName that occurs after the date in the -@plausibleStartBeforeEndFieldName.

    +

    The number and percent of records with a value in the +cdmFieldName field of the cdmTableName +that occurs after the date in the +plausibleStartBeforeEndFieldName. Note that this check +replaces the previous plausibleTemporalAfter check.

    Definition

    +

    This check is attempting to apply temporal rules within a table, +specifically checking that all start dates are before the end dates. For +example, in the VISIT_OCCURRENCE table it checks that the +VISIT_OCCURRENCE_START_DATE is before VISIT_OCCURRENCE_END_DATE. The +start date can be before the end date or equal to the end date. It is +applied to the start date field and takes the end date field as a +parameter. Both date and datetime fields are checked.

    • -Numerator:
      • +Numerator: The number of records where date in +cdmFieldName is after the date in +plausibleStartBeforeEndFieldName.
    • -Denominator:
      • +Denominator: The total number of records with a non-null +start and non-null end date value
    • -Related CDM Convention(s):
      • +Related CDM Convention(s): -Not linked to a +convention-
    • -CDM Fields/Tables:
      • +CDM Fields/Tables: This check runs on all start +date/datetime fields with an end date/datetime in the same table. It +also runs on the cdm_source table, comparing +source_release_date is before +cdm_release_date.
    • -Default Threshold Value:
      • +Default Threshold Value: +
        +
      • 0% for the observation_period, vocabulary (valid_start/end_date) and +cdm_source tables.
        • +
      • 1% for other tables with an end date.
        • +
      +

    User Guidance

    +

    If the start date is after the end date, it is likely that the data +is incorrect or the dates are unreliable.

    Violated rows query

    -
    +
    SELECT 
+  '@cdmTableName.@cdmFieldName' AS violating_field, 
+  cdmTable.*
+FROM @schema.@cdmTableName cdmTable
+WHERE cdmTable.@cdmFieldName IS NOT NULL 
+AND cdmTable.@plausibleStartBeforeEndFieldName IS NOT NULL 
+AND cdmTable.@cdmFieldName > cdmTable.@plausibleStartBeforeEndFieldName

    ETL Developers

    +

    There main reason for this check to fail is often that the source +data is incorrect. If the end date is derived from other data, the +calculation might not take into account some edge cases.

    +

    Any violating checks should either be removed or corrected. In most +cases this can be done by adjusting the end date: - With a few +exceptions, the end date is not mandatory and can be left empty. - If +the end date is mandatory (notably visit_occurrence and drug_exposure), +the end date can be set to the start date if the event. Make sure to +document this as it leads to loss of duration information. - If this +check fails for the observation_period, it might signify a bigger +underlying issue. Please investigate all records for this person in the +CDM and source. - If neither the start or end date can be trusted, +pleaes remove the record from the CDM.

    +

    Make sure to clearly document the choices in your ETL +specification.

    Data Users

    +

    An start date after the end date gives negative event durations, +which might break analyses. Especially take note if this check fails for +the observation_period table. This means that there are +persons with negative observation time. If these persons are included in +a cohort, it will potentially skew e.g. survival analyses.

    @@ -219,7 +302,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/plausibleTemporalAfter.html b/docs/articles/checks/plausibleTemporalAfter.html index 82ff7137..00b0b159 100644 --- a/docs/articles/checks/plausibleTemporalAfter.html +++ b/docs/articles/checks/plausibleTemporalAfter.html @@ -6,7 +6,7 @@ plausibleTemporalAfter • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • @@ -147,7 +177,7 @@

    plausibleTemporalAfter

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/plausibleTemporalAfter.Rmd @@ -221,7 +251,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/plausibleUnitConceptIds.html b/docs/articles/checks/plausibleUnitConceptIds.html index d37e8a73..a6f1d972 100644 --- a/docs/articles/checks/plausibleUnitConceptIds.html +++ b/docs/articles/checks/plausibleUnitConceptIds.html @@ -6,7 +6,7 @@ plausibleUnitConceptIds • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +

  • @@ -147,7 +177,7 @@

    plausibleUnitConceptIds

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/plausibleUnitConceptIds.Rmd @@ -219,7 +249,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/plausibleValueHigh.html b/docs/articles/checks/plausibleValueHigh.html index 6d82ddf0..b937dbdf 100644 --- a/docs/articles/checks/plausibleValueHigh.html +++ b/docs/articles/checks/plausibleValueHigh.html @@ -6,7 +6,7 @@ plausibleValueHigh • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +

  • @@ -145,9 +175,10 @@

    plausibleValueHigh

    -

    +

    Dymytry +Dymshyts

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/plausibleValueHigh.Rmd @@ -159,7 +190,7 @@

    2024-02-21

    Summary

    -

    Level: FIELD
    Context: Verification
    Category: Plausibility
    Subcategory: Atemporal
    Severity:

    +

    Level: FIELD
    Context: Verification
    Category: Plausibility
    Subcategory: Atemporal
    Severity: Characterization ✔

    Description @@ -171,32 +202,107 @@

    Definition

    User Guidance

    +

    This check counts the number of records that have a value in the +specified field that is higher than some expected value. Failures of +this check might represent true data anomalies, but especially in the +case when the failure percentage is high, something may be afoot in the +ETL pipeline.

    +

    Use this query to inspect rows with an implausibly high value:

    Violated rows query

    -
    +
    SELECT 
+  '@cdmTableName.@cdmFieldName' AS violating_field,  
+  cdmTable.* 
+FROM @schema.@cdmTableName cdmTable 
+WHERE cdmTable.@cdmFieldName > @plausibleValueHigh

    ETL Developers

    +

    The investigation approach may differ by the field being checked. For +example, for CONDITION_OCURRENCE.condition_start_date you +might look how much it differs in average, to find a clue as to what +happened:

    +
    SELECT 
+  MEDIAN(DATEDIFF(day, condition_start_date, current_date)) 
+FROM condition_occurrence
+WHERE condition_start_date > current_date 
+;
    +

    Or the discrepancy be associated with specific attributes:

    +
    SELECT 
+  co.condition_concept_id, 
+  co.condition_type_concept_id, 
+  co.condition_status_concept_id, 
+  COUNT(1) 
+FROM condition_occurrence co 
+WHERE condition_start_date > current_date  
+GROUP BY co.condition_concept_id, co.condition_type_concept_id, co.condition_status_concept_id 
+ORDER BY COUNT(1) DESC 
+;
    +

    There might be several different causes of future dates: typos in the +source data, wrong data format used in the conversion, timezone issues +in the ETL environment and/or database, etc.

    +

    For the DRUG_EXPOSURE values, there might be be typos, +data processing bugs (for example, if days supply is calculated), or +rare true cases when a prescription deviated from standard industry +practices.

    +

    If the issue is determined to be related to ETL logic, it must be +fixed. If it’s a source data issue, work with your data partners and +users to determine the best remediation approach. PERSON +rows with invalid birth dates should be removed from the CDM, as any +analysis relying on age will be negatively impacted. Other implausible +values should be explainable based on your understanding of the source +data if they are to be retained. In some cases event rows may need to be +dropped from the CDM if the implausible value is unexplainable and could +cause downstream quality issues. Be sure to clearly document any data +removal logic in your ETL specification.

    Data Users

    +

    The implication of a failure of this check depends on the count of +errors and your need for the impacted columns. If it’s a small count, it +might just be noise in the data which will unlikely impact an analysis. +If the count is large, however, proceed carefully - events with future +dates will likely be excluded from your analysis, and drugs with +inflated supply values could throw off any analysis considering duration +or patterns of treatment.

    @@ -218,7 +324,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/plausibleValueLow.html b/docs/articles/checks/plausibleValueLow.html index f3791a30..303fb846 100644 --- a/docs/articles/checks/plausibleValueLow.html +++ b/docs/articles/checks/plausibleValueLow.html @@ -6,7 +6,7 @@ plausibleValueLow • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • @@ -145,9 +175,10 @@

    plausibleValueLow

    -

    +

    Dymytry +Dymshyts

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/plausibleValueLow.Rmd @@ -159,7 +190,7 @@

    2024-02-21

    Summary

    -

    Level: FIELD
    Context: Verification
    Category: Plausibility
    Subcategory: Atemporal
    Severity:

    +

    Level: FIELD
    Context: Verification
    Category: Plausibility
    Subcategory: Atemporal
    Severity: Characterization ✔

    Description @@ -171,32 +202,71 @@

    Definition

    User Guidance

    +

    This check counts the number of records that have a value in the +specified field that is lower than some expected value. Failures of this +check might represent true data anomalies, but especially in the case +when the failure percentage is high, something may be afoot in the ETL +pipeline.

    +

    Use this query to inspect rows with an implausibly high value:

    Violated rows query

    -
    -
    -
    -

    ETL Developers -

    -
    -
    -

    Data Users -

    +
    SELECT 
+  '@cdmTableName.@cdmFieldName' AS violating_field,  
+  cdmTable.* 
+FROM @schema.@cdmTableName cdmTable 
+WHERE cdmTable.@cdmFieldName < @plausibleValueHigh
    +

    See guidance for plausibleValueHigh for detailed +investigation instructions (swapping out “high” for “low” and “>” for +“<” where appropriate).

    @@ -218,7 +288,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/sourceConceptRecordCompleteness.html b/docs/articles/checks/sourceConceptRecordCompleteness.html index fc345659..54445850 100644 --- a/docs/articles/checks/sourceConceptRecordCompleteness.html +++ b/docs/articles/checks/sourceConceptRecordCompleteness.html @@ -6,7 +6,7 @@ sourceConceptRecordCompleteness • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • @@ -145,9 +175,10 @@

    sourceConceptRecordCompleteness

    -

    +

    Katy +Sadowski

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/sourceConceptRecordCompleteness.Rmd @@ -159,7 +190,7 @@

    2024-02-21

    Summary

    -

    Level: FIELD
    Context: Verification
    Category: Completeness
    Subcategory:
    Severity:

    +

    Level: FIELD
    Context: Verification
    Category: Completeness
    Subcategory:
    Severity: CDM convention ⚠

    Description @@ -172,32 +203,100 @@

    DefinitionSource +concept mapping +

  • -CDM Fields/Tables:
    • +CDM Fields/Tables: All source concept ID +(X_source_concept_id) columns in all event tables.
  • -Default Threshold Value:
    • +Default Threshold Value: 10 for primary source concept ID +columns in condition, drug, measurement, procedure, device, and +observation tables; 100 for all other source concept ID columns.

    User Guidance

    -
    -

    Violated rows query -

    -
    -
    +

    Source concept mapping is an important part of the OMOP concept +mapping process which allows data users insight into the provenance of +the data they are analyzing. It’s important to populate the source +concept ID field for all source values that exist in the OMOP +vocabulary. Failures of this check should be well-understood and +documented so that data users can plan accordingly in the case missing +data might impact their analysis.

    ETL Developers

    +

    Recall that the X_source_concept_id columns should +contain the OMOP concept representing the exact code used in the source +data for a given record: “If the is coded in the source +data using an OMOP supported vocabulary put the concept id representing +the source value here.”

    +

    A failure of this check usually indicates a failure to map a source +value to an OMOP concept. In some cases, such a failure can and should +be remediated in the concept-mapping step of the ETL. In other cases, it +may represent a mapping that currently is not possible to implement.

    +

    To investigate the failure, run the following query:

    +
    SELECT  
+  concept.concept_name AS standard_concept_name, 
+  cdmTable.X_concept_id, -- standard concept ID field for the table 
+  c2.concept_name AS source_value_concept_name, 
+  cdmTable.X_source_value, -- source value field for the table 
+  COUNT(*) 
+FROM @cdmDatabaseSchema.@cdmTableName cdmTable 
+LEFT JOIN @vocabDatabaseSchema.concept ON concept.concept_id = cdmTable.X_concept_id 
+-- WARNING this join may cause fanning if a source value exists in multiple vocabularies 
+LEFT JOIN @vocabDatabaseSchema.concept c2 ON concept.concept_code = cdmTable.X_source_value 
+AND c2.domain_id = <Domain of cdmTable> 
+WHERE cdmTable.@cdmFieldName = 0  
+-- AND cdmTable.value_as_number IS NOT NULL -- uncomment for unit_concept_id checks 
+GROUP BY 1,2,3 
+ORDER BY 4 DESC
    +

    The query results will give you a summary of the source codes which +failed to map to an OMOP concept. Inspecting this data should give you +an initial idea of what might be going on.

    +

    If source values return legitimate matches on concept_code, it’s +possible that there is an error in the concept mapping step of your ETL. +Please note that while the X_source_concept_id fields are +technically not required, it is highly recommended to populate them with +OMOP concepts whenever possible. This will greatly aid analysts in +understanding the provenance of the data.

    +

    If source values do NOT return matches on concept_code and you are +NOT handling concept mapping locally for a non-OMOP source vocabulary, +then you likely have a malformed source code or one that does not exist +in the OMOP vocabulary. Please see the documentation in the +standardConceptRecordCompleteness page for instructions on how to handle +this scenario.

    Data Users

    +

    Since most standard OHDSI analytic workflows rely on the standard +concept field and not the source concept field, failures of this check +will not necessarily impact your analysis. However, if your analysis +depends on understanding source coding practices or on codes you know +may not be fully mapped to OMOP standard concepts, then this will be a +critical check failure to understand.

    +

    Utilize the investigation queries above to understand the scope and +impact of the mapping failures on your specific analytic use case. If +none of the affected codes seem to be relevant for your analysis, it may +be acceptable to ignore the failure. However, since it is not always +possible to understand exactly what a given source value represents, you +should proceed with caution and confirm any findings with your ETL +provider if possible.

    @@ -219,7 +318,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/sourceValueCompleteness.html b/docs/articles/checks/sourceValueCompleteness.html index e0430a80..8baafec8 100644 --- a/docs/articles/checks/sourceValueCompleteness.html +++ b/docs/articles/checks/sourceValueCompleteness.html @@ -6,7 +6,7 @@ sourceValueCompleteness • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • @@ -145,9 +175,10 @@

    sourceValueCompleteness

    -

    +

    Jared +Houghtaling, Clair Blacketer

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/sourceValueCompleteness.Rmd @@ -159,7 +190,7 @@

    2024-02-21

    Summary

    -

    Level: FIELD
    Context: Verification
    Category: Completeness
    Subcategory:
    Severity:

    +

    Level: FIELD
    Context: Verification
    Category: Completeness
    Subcategory:
    Severity: CDM convention ⚠

    Description @@ -171,32 +202,85 @@

    DefinitionEHDEN CDM Inspection +Report package, as well as the AresIndexer package for +generating indices of unmapped codes.
    +

  • -CDM Fields/Tables:
    • +CDM Fields/Tables: Runs on all event tables that have +X_source_value fields.
  • -Default Threshold Value:
    • +Default Threshold Value: +
      +
    • 10 for critical event tables’ X_source_value fields +(condition, measurement, procedure, drug, visit)
      • +
    • 100 for all other fields - to be adjusted based on source-specific +expectations
      • +
    +

    User Guidance

    +

    This check will look at all distinct source values in the specified +field and calculate how many are mapped to a standard concept of 0. This +check should be used in conjunction with the standardConceptRecordCompleteness check +to identify potential mapping issues in the ETL.

    +

    This check is a good measure of the overall mapping rate within each +domain. For example, a table may have high +standardConceptRecordCompleteness (that is, a large percentage of +records with a non-zero standard concept) but a low score on this check. +This would indicate that the “long tail” of rarer codes have not been +mapped while more common codes have good mapping coverage. It is always +important to interrogate the results of these two checks together to +ensure complete understanding of vocabulary mapping in your CDM.

    +

    The following SQL can be used to summarize unmapped source values by +record count in a given CDM table:

    Violated rows query

    -
    +
    SELECT DISTINCT 
+  cdmTable.@cdmFieldName,
+  COUNT(*)
+FROM @cdmDatabaseSchema.@cdmTableName cdmTable
+WHERE cdmTable.@standardConceptFieldName = 0
+GROUP BY 1
+ORDER BY 2 DESC

    ETL Developers

    +

    Fails of this check are (most often) related directly to semantic +mapping. First, the ETL developer should investigate if a source +vocabulary is present in the native data that was not accounted for in +the ETL document and/or code. This is most likely if the unmapped source +values are codes rather than text values. Second, the +source-to-concept-map file or table should be updated to link the +unmapped source values with domain-appropriate concepts.

    Data Users

    +

    When this check fails, source data granularity is being lost; not all +of the information related to a particular event or modifier is being +captured in OMOP CDM format. Although the information about an event may +exist in the source value field, it cannot easily be used in downstream +analytics processes that rely on standard OMOP concepts.

    +

    Please see the standardConceptRecordCompleteness page +for a much more detailed overview of handling mapping quality issues in +your OMOP CDM.

    @@ -218,7 +302,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/standardConceptRecordCompleteness.html b/docs/articles/checks/standardConceptRecordCompleteness.html index 46684565..2981148a 100644 --- a/docs/articles/checks/standardConceptRecordCompleteness.html +++ b/docs/articles/checks/standardConceptRecordCompleteness.html @@ -6,7 +6,7 @@ standardConceptRecordCompleteness • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • @@ -145,9 +175,10 @@

    standardConceptRecordCompleteness

    -

    +

    Katy +Sadowski

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/standardConceptRecordCompleteness.Rmd @@ -159,7 +190,7 @@

    2024-02-21

    Summary

    -

    Level: FIELD
    Context: Verification
    Category: Completeness
    Subcategory:
    Severity:

    +

    Level: FIELD
    Context: Verification
    Category: Completeness
    Subcategory:
    Severity: CDM convention ⚠

    Description @@ -172,32 +203,168 @@

    DefinitionStandard +concept mapping
    +

  • -CDM Fields/Tables:
    • +CDM Fields/Tables: All standard concept ID +(X_concept_id) columns in all event tables.
  • -Default Threshold Value:
    • +Default Threshold Value: +
      +
    • 0 for type concept fields and standard concept fields in era +tables
      • +
    • 5 for most standard concept fields in clinical event tables
      • +
    • 100 for fields more susceptible to specific ETL implementation +context
      • +
    +

    User Guidance

    -
    -

    Violated rows query -

    -
    -
    +

    Standard concept mapping is one of the most fundamental conventions +of the OMOP CDM. It enables standardized analysis across diverse data +sources and allows users to abstract away the tedium of traversing +source vocabularies when building phenotypes. As such, it is highly +recommended to map as many concepts in your source as possible. Failures +of this check should be well-understood and documented so that data +users can plan accordingly in the case missing data might impact their +analysis.

    ETL Developers

    +

    A failure of this check usually indicates a failure to map a source +value to a standard OMOP concept. In some cases, such a failure can and +should be remediated in the concept-mapping step of the ETL. In other +cases, it may represent a mapping that currently is not possible to +implement.

    +

    To investigate the failure, run the following query:

    +
    SELECT  
+  concept_name, 
+  cdmTable.X_source_concept_id, -- source concept ID field for the table 
+  cdmTable.X_source_value, -- source value field for the table 
+  COUNT(*) 
+FROM @cdmDatabaseSchema.@cdmTableName cdmTable 
+LEFT JOIN @vocabDatabaseSchema.concept ON concept.concept_id = cdmTable.X_source_concept_id 
+WHERE cdmTable.@cdmFieldName = 0  
+-- AND cdmTable.value_as_number IS NOT NULL -- uncomment for unit_concept_id checks 
+GROUP BY 1,2,3 
+ORDER BY 4 DESC
    +

    This will give you a summary of the source codes which failed to map +to an OMOP standard concept. Inspecting this data should give you an +initial idea of what might be going on.

    +
      +
    • If the query returns a source value, source concept ID, and concept +name for a given code, run the following query to confirm that a +standard concept mapping exists for the source concept ID:
      • +
    +
    SELECT  
+  concept_id AS standard_concept_mapping 
+FROM @vocabDatabaseSchema.concept_relationship 
+JOIN @vocabDatabaseSchema.concept ON concept.concept_id = c oncept_relationship.concept_id_2 
+  AND relationship_id = ‘Maps to’ 
+WHERE concept_relationship.concept_id_1 = <source concept ID>
    +
      +

    • If no results are returned, consider whether the source concept +ID is part of the OMOP vocabularies. If it is, then there is likely a +vocabulary issue which should be reported. If it is not (i.e., it is a +local concept), then there is likely an issue with your local +source-to-concept mapping

      • +

    • If the investigation query returns a source value and source +concept ID but no concept name, this indicates the source concept ID +does not exist in your concept table. This may be expected if your ETL +includes local source-to-concept mappings. If not, then your ETL has +assigned a malformed source concept ID and will need to be +debugged

      • +

    • If the investigation query returns a source value but no source +concept ID (or a source concept ID of 0), run the following query to +search for the source value in the OMOP vocabulary (note that if your +ETL includes local mappings and the code in question is known not to +exist in OMOP, you should search your local mapping table/config +instead):

      • +
    +
    -- may return false positives if the same value exists in multiple vocabularies 
+-- only applicable in the case where the source value column is populated only with a vocabulary code 
+SELECT  
+  * 
+FROM @vocabDatabaseSchema.concept 
+WHERE concept_code = <source value>
    +
      +
    • If no result is returned, consider whether the source value may be a +malformed version of a legitimate code (for example, sometimes ICD10-CM +codes do not contain a “.” in source data). If you can confirm that the +code is properly formatted, then you have a source code which does not +exist in the OMOP vocabulary. If you believe the code was omitted from +the vocabulary in error, please report this issue to the vocabulary +team. Otherwise, the short-term course of action will be to generate a +mapping for the code locally and implement the mapping in your ETL. For +the longer term, the vocabulary team provides a workflow to submit new +vocabularies for inclusion in the OMOP vocabularies +
        +
      • Note that in some cases, you will find that no standard concept +exists to which to map your source code. In this case, the standard +concept ID field should be left as 0 in the short term; in the longer +term please work with the vocabulary team to address this gap as +recommended above
        • +
      +
      • +
    • Finally, if the investigation query returns no source value, you +must trace the relevant record(s) back to their source and confirm if +the missing value is expected. If not, identify and fix the related +issue in your ETL. If the record legitimately has no value/code in the +source data, then the standard concept ID may be left as 0. However, in +some cases these “code-less” records represent junk data which should be +filtered out in the ETL. The proper approach will be context-dependent +
        +
      • Note in the special case of unitless measurements/observations, the +unit_concept_id field should NOT be coded as 0 and rather should be left +NULL (the unit_concept_id fields are optional in the CDM spec)
        • +
      +
      • +
    +

    It is important to note that records with a 0 standard concept ID +field will be unusable in standard OHDSI analyses and thus should only +be preserved if there is truly no standard concept ID for a given +record. Depending on the significance of the records in question, one +should consider removing them from the dataset; however, this choice +will depend on a variety of context-specific factors and should be made +carefully. Either way, the presence/absence of these unmappable records +and an explanation for why they could not be mapped should be clearly +documented in the ETL documentation.

    Data Users

    +

    Since unmapped records will not be picked up in standard OHDSI +analytic workflows, this is an important check failure to understand. +Utilize the investigation queries above to understand the scope and +impact of the mapping failures on your specific analytic use case. If +none of the affected codes seem to be relevant for your analysis, it may +be acceptable to ignore the failure. However, since it is not always +possible to understand exactly what a given source value represents, you +should proceed with caution and confirm any findings with your ETL +provider if possible.

    +

    In the case where the source concept ID column is populated with a +legitimate OMOP concept, it will be possible to query this column +instead of the standard concept column in your analyses. However, doing +so will require building source concept sets and as such losing the +power of the OMOP standard vocabularies in defining comprehensive, +generalizable cohort definitions.

    @@ -219,7 +386,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/checks/withinVisitDates.html b/docs/articles/checks/withinVisitDates.html index 4173d562..9a19d0de 100644 --- a/docs/articles/checks/withinVisitDates.html +++ b/docs/articles/checks/withinVisitDates.html @@ -6,7 +6,7 @@ withinVisitDates • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • @@ -147,7 +177,7 @@

    withinVisitDates

    -

    2024-02-21

    +

    2024-06-26

    Source: vignettes/checks/withinVisitDates.Rmd @@ -219,7 +249,7 @@

    Data Users

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/articles/index.html b/docs/articles/index.html index 7555ec7b..c01a0cff 100644 --- a/docs/articles/index.html +++ b/docs/articles/index.html @@ -1,5 +1,5 @@ -Articles • DataQualityDashboardArticles • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +

  • Changelog @@ -126,25 +156,21 @@

    All vignettes

    Add a New Data Quality Check
    -
    Check Status Definitions
    -
    -
    Data Quality Check Type Definitions
    -
    -
    Getting Started
    -
    -
    Running the DQD on a Cohort
    +
    cdmDatatype
    -
    Running the DQD in SqlOnly mode
    +
    cdmField
    -
    Failure Thresholds and How to Change Them
    +
    cdmTable
    Index
    -
    cdmDatatype
    +
    Check Status Definitions
    -
    cdmField
    +
    Data Quality Check Type Definitions
    -
    cdmTable
    +
    Getting Started
    +
    +
    Running the DQD on a Cohort
    fkClass
    @@ -168,9 +194,7 @@

    All vignettes

    plausibleBeforeDeath
    -
    plausibleDuringLife
    -
    -
    plausibleGender
    +
    plausibleGender
    plausibleStartBeforeEnd
    @@ -186,8 +210,12 @@

    All vignettes

    sourceValueCompleteness
    +
    Running the DQD in SqlOnly mode
    +
    standardConceptRecordCompleteness
    +
    Failure Thresholds and How to Change Them
    +
    withinVisitDates
    @@ -200,7 +228,7 @@

    All vignettes

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/authors.html b/docs/authors.html index 5c0121e6..796cd4c8 100644 --- a/docs/authors.html +++ b/docs/authors.html @@ -1,5 +1,5 @@ -Authors and Citation • DataQualityDashboardAuthors and Citation • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • Changelog @@ -118,7 +148,7 @@
    -

    Authors

    +

    Authors and Citation

    @@ -186,7 +216,7 @@

    Citation

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/index.html b/docs/index.html index fa4bbb6f..b742927b 100644 --- a/docs/index.html +++ b/docs/index.html @@ -6,7 +6,7 @@ Execute and View Data Quality Checks on OMOP CDM Database • DataQualityDashboard - + @@ -112,9 +112,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • @@ -392,7 +422,7 @@

    Dev status

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/news/index.html b/docs/news/index.html index 1b145260..3d1f6026 100644 --- a/docs/news/index.html +++ b/docs/news/index.html @@ -1,5 +1,5 @@ -Changelog • DataQualityDashboardChangelog • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • Changelog @@ -327,7 +357,7 @@

    DataQua
    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/pkgdown.yml b/docs/pkgdown.yml index f5685aa7..4d9ad23c 100644 --- a/docs/pkgdown.yml +++ b/docs/pkgdown.yml @@ -1,18 +1,16 @@ -pandoc: 3.1.1 -pkgdown: 2.0.7 +pandoc: 3.1.11 +pkgdown: 2.0.9 pkgdown_sha: ~ articles: AddNewCheck: AddNewCheck.html + cdmDatatype: checks/cdmDatatype.html + cdmField: checks/cdmField.html + cdmTable: checks/cdmTable.html + checkIndex: checkIndex.html CheckStatusDefinitions: CheckStatusDefinitions.html CheckTypeDescriptions: CheckTypeDescriptions.html DataQualityDashboard: DataQualityDashboard.html DqdForCohorts: DqdForCohorts.html - SqlOnly: SqlOnly.html - Thresholds: Thresholds.html - checkIndex: checkIndex.html - cdmDatatype: checks/cdmDatatype.html - cdmField: checks/cdmField.html - cdmTable: checks/cdmTable.html fkClass: checks/fkClass.html fkDomain: checks/fkDomain.html isForeignKey: checks/isForeignKey.html @@ -24,8 +22,7 @@ articles: measureValueCompleteness: checks/measureValueCompleteness.html plausibleAfterBirth: checks/plausibleAfterBirth.html plausibleBeforeDeath: checks/plausibleBeforeDeath.html - plausibleDuringLife: checks/plausibleDuringLife.html - plausibleGender: checks/plausibleGender.html + plausibleGenderUseDescendants: checks/plausibleGenderUseDescendants.html plausibleStartBeforeEnd: checks/plausibleStartBeforeEnd.html plausibleTemporalAfter: checks/plausibleTemporalAfter.html plausibleUnitConceptIds: checks/plausibleUnitConceptIds.html @@ -33,9 +30,11 @@ articles: plausibleValueLow: checks/plausibleValueLow.html sourceConceptRecordCompleteness: checks/sourceConceptRecordCompleteness.html sourceValueCompleteness: checks/sourceValueCompleteness.html + SqlOnly: SqlOnly.html standardConceptRecordCompleteness: checks/standardConceptRecordCompleteness.html + Thresholds: Thresholds.html withinVisitDates: checks/withinVisitDates.html -last_built: 2024-02-22T03:55Z +last_built: 2024-06-26T15:29Z urls: reference: https://ohdsi.github.io/DataQualityDashboard/reference article: https://ohdsi.github.io/DataQualityDashboard/articles diff --git a/docs/reference/convertJsonResultsFileCase.html b/docs/reference/convertJsonResultsFileCase.html index 51c1a92c..47819d16 100644 --- a/docs/reference/convertJsonResultsFileCase.html +++ b/docs/reference/convertJsonResultsFileCase.html @@ -1,5 +1,5 @@ -Convert JSON results file case — convertJsonResultsFileCase • DataQualityDashboardConvert JSON results file case — convertJsonResultsFileCase • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +

  • Changelog @@ -177,7 +207,7 @@

    Value

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/reference/dot-evaluateThresholds.html b/docs/reference/dot-evaluateThresholds.html index 8f136d89..ba89215c 100644 --- a/docs/reference/dot-evaluateThresholds.html +++ b/docs/reference/dot-evaluateThresholds.html @@ -1,5 +1,5 @@ -Internal function to evaluate the data quality checks against given thresholds. — .evaluateThresholds • DataQualityDashboardInternal function to evaluate the data quality checks against given thresholds. — .evaluateThresholds • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • Changelog @@ -161,7 +191,7 @@

    Arguments

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/reference/dot-getCheckId.html b/docs/reference/dot-getCheckId.html index 1e0728d2..8f26d7fb 100644 --- a/docs/reference/dot-getCheckId.html +++ b/docs/reference/dot-getCheckId.html @@ -1,5 +1,5 @@ -Internal function to define the id of each check. — .getCheckId • DataQualityDashboardInternal function to define the id of each check. — .getCheckId • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • Changelog @@ -176,7 +206,7 @@

    Arguments

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/reference/dot-processCheck.html b/docs/reference/dot-processCheck.html index 1dfe2fc5..0431d9b5 100644 --- a/docs/reference/dot-processCheck.html +++ b/docs/reference/dot-processCheck.html @@ -1,5 +1,5 @@ -Internal function to send the fully qualified sql to the database and return the numerical result. — .processCheck • DataQualityDashboardInternal function to send the fully qualified sql to the database and return the numerical result. — .processCheck • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • Changelog @@ -176,7 +206,7 @@

    Arguments

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/reference/dot-recordResult.html b/docs/reference/dot-recordResult.html index a62c1345..0b3a1151 100644 --- a/docs/reference/dot-recordResult.html +++ b/docs/reference/dot-recordResult.html @@ -1,5 +1,5 @@ -Internal function to put the results of each quality check into a dataframe. — .recordResult • DataQualityDashboardInternal function to put the results of each quality check into a dataframe. — .recordResult • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • Changelog @@ -181,7 +211,7 @@

    Arguments

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/reference/dot-runCheck.html b/docs/reference/dot-runCheck.html index e70eb651..9407dceb 100644 --- a/docs/reference/dot-runCheck.html +++ b/docs/reference/dot-runCheck.html @@ -1,5 +1,5 @@ -Internal function to run and process each data quality check. — .runCheck • DataQualityDashboardInternal function to run and process each data quality check. — .runCheck • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • Changelog @@ -154,6 +184,10 @@

    Arguments

    The description of the data quality check

    +
    tableChecks
    +

    A dataframe containing the table checks

    + +
    fieldChecks

    A dataframe containing the field checks

    @@ -213,10 +247,6 @@

    Arguments

    sqlOnly

    Should the SQLs be executed (FALSE) or just returned (TRUE)?

    - -
    tablechecks
    -

    A dataframe containing the table checks

    - @@ -231,7 +261,7 @@

    Arguments

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/reference/dot-summarizeResults.html b/docs/reference/dot-summarizeResults.html index 5eda0e6d..52d47039 100644 --- a/docs/reference/dot-summarizeResults.html +++ b/docs/reference/dot-summarizeResults.html @@ -1,5 +1,5 @@ -Internal function to summarize the results of the DQD run. — .summarizeResults • DataQualityDashboardInternal function to summarize the results of the DQD run. — .summarizeResults • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • Changelog @@ -149,7 +179,7 @@

    Arguments

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/reference/dot-writeResultsToCsv.html b/docs/reference/dot-writeResultsToCsv.html index 1c6e017d..95bf0b43 100644 --- a/docs/reference/dot-writeResultsToCsv.html +++ b/docs/reference/dot-writeResultsToCsv.html @@ -1,5 +1,5 @@ -Internal function to write the check results to a csv file. — .writeResultsToCsv • DataQualityDashboardInternal function to write the check results to a csv file. — .writeResultsToCsv • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • Changelog @@ -170,7 +200,7 @@

    Arguments

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/reference/dot-writeResultsToJson.html b/docs/reference/dot-writeResultsToJson.html index a5d72902..fdfeaba6 100644 --- a/docs/reference/dot-writeResultsToJson.html +++ b/docs/reference/dot-writeResultsToJson.html @@ -1,5 +1,5 @@ -Write DQD results to json — .writeResultsToJson • DataQualityDashboardWrite DQD results to json — .writeResultsToJson • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • Changelog @@ -157,7 +187,7 @@

    Arguments

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/reference/dot-writeResultsToTable.html b/docs/reference/dot-writeResultsToTable.html index 32d33bf4..5004b036 100644 --- a/docs/reference/dot-writeResultsToTable.html +++ b/docs/reference/dot-writeResultsToTable.html @@ -1,5 +1,5 @@ -Internal function to write the check results to a table in the database. Requires write access to the database — .writeResultsToTable • DataQualityDashboardInternal function to write the check results to a table in the database. Requires write access to the database — .writeResultsToTable • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • Changelog @@ -172,7 +202,7 @@

    Arguments

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/reference/executeDqChecks.html b/docs/reference/executeDqChecks.html index 217941cd..4e42ef4a 100644 --- a/docs/reference/executeDqChecks.html +++ b/docs/reference/executeDqChecks.html @@ -1,5 +1,5 @@ -Execute DQ checks — executeDqChecks • DataQualityDashboardExecute DQ checks — executeDqChecks • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • Changelog @@ -284,7 +314,7 @@

    Value

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/reference/index.html b/docs/reference/index.html index e89c8bfb..be3c8cf5 100644 --- a/docs/reference/index.html +++ b/docs/reference/index.html @@ -1,5 +1,5 @@ -Function reference • DataQualityDashboardFunction reference • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • Changelog @@ -193,7 +223,7 @@

    Write database results to a JSON
    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/reference/listDqChecks.html b/docs/reference/listDqChecks.html index ed12b237..a3500e52 100644 --- a/docs/reference/listDqChecks.html +++ b/docs/reference/listDqChecks.html @@ -1,5 +1,5 @@ -List DQ checks — listDqChecks • DataQualityDashboardList DQ checks — listDqChecks • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +

  • Changelog @@ -166,7 +196,7 @@

    Arguments

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/reference/reEvaluateThresholds.html b/docs/reference/reEvaluateThresholds.html index 22ba8342..e9305b59 100644 --- a/docs/reference/reEvaluateThresholds.html +++ b/docs/reference/reEvaluateThresholds.html @@ -1,5 +1,5 @@ -Re-evaluate Thresholds — reEvaluateThresholds • DataQualityDashboardRe-evaluate Thresholds — reEvaluateThresholds • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • Changelog @@ -181,7 +211,7 @@

    Arguments

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/reference/viewDqDashboard.html b/docs/reference/viewDqDashboard.html index aad709b1..36efe47f 100644 --- a/docs/reference/viewDqDashboard.html +++ b/docs/reference/viewDqDashboard.html @@ -1,5 +1,5 @@ -View DQ Dashboard — viewDqDashboard • DataQualityDashboardView DQ Dashboard — viewDqDashboard • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • Changelog @@ -161,7 +191,7 @@

    Arguments

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/reference/writeDBResultsToJson.html b/docs/reference/writeDBResultsToJson.html index 128de643..090d5c45 100644 --- a/docs/reference/writeDBResultsToJson.html +++ b/docs/reference/writeDBResultsToJson.html @@ -1,5 +1,5 @@ -Write DQD results database table to json — writeDBResultsToJson • DataQualityDashboardWrite DQD results database table to json — writeDBResultsToJson • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • Changelog @@ -176,7 +206,7 @@

    Arguments

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/reference/writeJsonResultsToCsv.html b/docs/reference/writeJsonResultsToCsv.html index 9283686a..8ec3afb0 100644 --- a/docs/reference/writeJsonResultsToCsv.html +++ b/docs/reference/writeJsonResultsToCsv.html @@ -1,5 +1,5 @@ -Write JSON Results to CSV file — writeJsonResultsToCsv • DataQualityDashboardWrite JSON Results to CSV file — writeJsonResultsToCsv • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • Changelog @@ -170,7 +200,7 @@

    Arguments

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/reference/writeJsonResultsToTable.html b/docs/reference/writeJsonResultsToTable.html index f3e6009e..30c65c4a 100644 --- a/docs/reference/writeJsonResultsToTable.html +++ b/docs/reference/writeJsonResultsToTable.html @@ -1,5 +1,5 @@ -Write JSON Results to SQL Table — writeJsonResultsToTable • DataQualityDashboardWrite JSON Results to SQL Table — writeJsonResultsToTable • DataQualityDashboard @@ -92,9 +92,39 @@
  • fkClass
    • +
  • + measurePersonCompleteness +
    • +
  • + measureValueCompleteness +
    • +
  • + isStandardValidConcept +
    • +
  • + standardConceptRecordCompleteness +
    • +
  • + sourceConceptRecordCompleteness +
    • +
  • + sourceValueCompleteness +
  • plausibleAfterBirth
    • +
  • + plausibleBeforeDeath +
    • +
  • + plausibleStartBeforeEnd +
    • +
  • + plausibleValueHigh +
    • +
  • + plausibleValueLow +
  • Changelog @@ -171,7 +201,7 @@

    Arguments

    -

    Site built with pkgdown 2.0.7.

    +

    Site built with pkgdown 2.0.9.

    diff --git a/docs/sitemap.xml b/docs/sitemap.xml index 8b98e068..2cfecf95 100644 --- a/docs/sitemap.xml +++ b/docs/sitemap.xml @@ -78,6 +78,9 @@ https://ohdsi.github.io/DataQualityDashboard/articles/checks/plausibleGender.html + + https://ohdsi.github.io/DataQualityDashboard/articles/checks/plausibleGenderUseDescendants.html + https://ohdsi.github.io/DataQualityDashboard/articles/checks/plausibleStartBeforeEnd.html diff --git a/inst/csv/OMOP_CDMv5.2_Field_Level.csv b/inst/csv/OMOP_CDMv5.2_Field_Level.csv index b9f63fed..55e30219 100644 --- a/inst/csv/OMOP_CDMv5.2_Field_Level.csv +++ b/inst/csv/OMOP_CDMv5.2_Field_Level.csv @@ -38,7 +38,7 @@ VISIT_OCCURRENCE,CDM,visit_end_datetime,No,,,datetime,0,,,"If no time is given f VISIT_OCCURRENCE,CDM,visit_type_concept_id,Yes,0,,Integer,0,,"Use this field to understand the provenance of the visit record, or where the record comes from.","Populate this field based on the provenance of the visit record, as in whether it came from an EHR record or billing claim.",No,,,Yes,0,,CONCEPT,CONCEPT_ID,Type Concept,0,,,,,Yes,0,,Yes,0,,Yes,0,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, VISIT_OCCURRENCE,CDM,provider_id,No,,,integer,0,,,,No,,,Yes,0,,PROVIDER,PROVIDER_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, VISIT_OCCURRENCE,CDM,care_site_id,No,,,integer,0,,This field provides information about the care site where the visit took place.,There should only be one care site associated with a visit.,No,,,Yes,0,,CARE_SITE,CARE_SITE_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -VISIT_OCCURRENCE,CDM,visit_source_value,No,,,varchar(50),0,,"This field houses the verbatim value from the source data representing the kind of visit that took place (inpatient, outpatient, emergency, etc.)","If there is information about the kind of visit in the source data that value should be stored here. If a visit is an amalgamation of visits from the source then use a hierarchy to choose the visit source value, such as IP -> ER-> OP. This should line up with the logic chosen to determine how visits are created.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,VISIT_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +VISIT_OCCURRENCE,CDM,visit_source_value,No,,,varchar(50),0,,"This field houses the verbatim value from the source data representing the kind of visit that took place (inpatient, outpatient, emergency, etc.)","If there is information about the kind of visit in the source data that value should be stored here. If a visit is an amalgamation of visits from the source then use a hierarchy to choose the visit source value, such as IP -> ER-> OP. This should line up with the logic chosen to determine how visits are created.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,10,,VISIT_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, VISIT_OCCURRENCE,CDM,visit_source_concept_id,No,,,integer,0,,,If the visit source value is coded in the source data using an OMOP supported vocabulary put the concept id representing the source value here.,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,100,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, VISIT_OCCURRENCE,CDM,admitting_source_concept_id,No,,,integer,0,,"Use this field to determine where the patient was admitted from. This concept is part of the visit domain and can indicate if a patient was admitted to the hospital from a long-term care facility, for example.","If available, map the admitted_from_source_value to a standard concept in the visit domain.",No,,,Yes,0,,CONCEPT,CONCEPT_ID,Visit,0,,,,,Yes,0,,Yes,100,,Yes,100,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, VISIT_OCCURRENCE,CDM,admitting_source_value,No,,,varchar(50),0,,,"This information may be called something different in the source data but the field is meant to contain a value indicating where a person was admitted from. Typically this applies only to visits that have a length of stay, like inpatient visits or long-term care visits.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,ADMITTING_SOURCE_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, @@ -95,8 +95,8 @@ PROCEDURE_OCCURRENCE,CDM,modifier_concept_id,No,,,integer,0,,"These concepts are PROCEDURE_OCCURRENCE,CDM,quantity,No,,,integer,0,,"If the quantity value is omitted, a single procedure is assumed.","If a Procedure has a quantity of '0' in the source, this should default to '1' in the ETL. If there is a record in the source it can be assumed the exposure occurred at least once (THEMIS issue #26).",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,1,1,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, PROCEDURE_OCCURRENCE,CDM,provider_id,No,,,integer,0,,,,No,,,Yes,0,,PROVIDER,PROVIDER_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, PROCEDURE_OCCURRENCE,CDM,visit_occurrence_id,No,,,integer,0,,,,No,,,Yes,0,,VISIT_OCCURRENCE,VISIT_OCCURRENCE_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -PROCEDURE_OCCURRENCE,CDM,procedure_source_value,No,,,varchar(50),0,,,"This code is mapped to a standard procedure Concept in the Standardized Vocabularies and the original code is, stored here for reference. Procedure source codes are typically ICD-9-Proc, CPT-4, HCPCS or OPCS-4 codes.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,PROCEDURE_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -PROCEDURE_OCCURRENCE,CDM,procedure_source_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,100,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +PROCEDURE_OCCURRENCE,CDM,procedure_source_value,No,,,varchar(50),0,,,"This code is mapped to a standard procedure Concept in the Standardized Vocabularies and the original code is, stored here for reference. Procedure source codes are typically ICD-9-Proc, CPT-4, HCPCS or OPCS-4 codes.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,10,,PROCEDURE_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +PROCEDURE_OCCURRENCE,CDM,procedure_source_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,10,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, PROCEDURE_OCCURRENCE,CDM,qualifier_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,MODIFIER_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, DEVICE_EXPOSURE,CDM,device_exposure_id,Yes,0,,bigint,0,,,,Yes,0,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, DEVICE_EXPOSURE,CDM,person_id,Yes,0,,bigint,0,,,,No,,,Yes,0,,PERSON,PERSON_ID,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, @@ -111,7 +111,7 @@ DEVICE_EXPOSURE,CDM,quantity,No,,,integer,0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,N DEVICE_EXPOSURE,CDM,provider_id,No,,,integer,0,,,,No,,,Yes,0,,PROVIDER,PROVIDER_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, DEVICE_EXPOSURE,CDM,visit_occurrence_id,No,,,integer,0,,,,No,,,Yes,0,,VISIT_OCCURRENCE,VISIT_OCCURRENCE_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, DEVICE_EXPOSURE,CDM,device_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,DEVICE_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -DEVICE_EXPOSURE,CDM,device_source_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,100,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +DEVICE_EXPOSURE,CDM,device_source_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,10,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,CDM,measurement_id,Yes,0,,integer,0,,,,Yes,0,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,CDM,person_id,Yes,0,,integer,0,,,,No,,,Yes,0,,PERSON,PERSON_ID,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,CDM,measurement_concept_id,Yes,0,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,Measurement,0,,,,,Yes,0,,Yes,0,,Yes,5,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, @@ -134,8 +134,8 @@ MEASUREMENT,CDM,range_low,No,,,float,0,,Ranges have the same unit as the VALUE_A MEASUREMENT,CDM,range_high,No,,,float,0,,Ranges have the same unit as the VALUE_AS_NUMBER.,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,CDM,provider_id,No,,,integer,0,,,,No,,,Yes,0,,PROVIDER,PROVIDER_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,CDM,visit_occurrence_id,No,,,integer,0,,,,No,,,Yes,0,,VISIT_OCCURRENCE,VISIT_OCCURRENCE_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -MEASUREMENT,CDM,measurement_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,MEASUREMENT_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -MEASUREMENT,CDM,measurement_source_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,100,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +MEASUREMENT,CDM,measurement_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,10,,MEASUREMENT_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +MEASUREMENT,CDM,measurement_source_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,10,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,CDM,unit_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,UNIT_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,CDM,value_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, NOTE,CDM,note_id,Yes,0,,integer,0,,,,Yes,0,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, @@ -193,7 +193,7 @@ OBSERVATION,CDM,unit_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID OBSERVATION,CDM,provider_id,No,,,integer,0,,,,No,,,Yes,0,,PROVIDER,PROVIDER_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, OBSERVATION,CDM,visit_occurrence_id,No,,,integer,0,,,,No,,,Yes,0,,VISIT_OCCURRENCE,VISIT_OCCURRENCE_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, OBSERVATION,CDM,observation_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,OBSERVATION_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -OBSERVATION,CDM,observation_source_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,100,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +OBSERVATION,CDM,observation_source_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,10,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, OBSERVATION,CDM,unit_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,UNIT_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, OBSERVATION,CDM,qualifier_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,QUALIFIER_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, SPECIMEN,CDM,specimen_id,Yes,0,,integer,0,,,,Yes,0,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, @@ -249,7 +249,7 @@ PAYER_PLAN_PERIOD,CDM,payer_plan_period_start_date,Yes,0,,date,0,,,,No,,,No,,,,, PAYER_PLAN_PERIOD,CDM,payer_plan_period_end_date,Yes,0,,date,0,,,,No,,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,Yes,PAYER_PLAN_PERIOD,PAYER_PLAN_PERIOD_START_DATE,1,,No,,,,,,,Yes,1,,Yes,1,,Yes,,, PAYER_PLAN_PERIOD,CDM,payer_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,PAYER_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, PAYER_PLAN_PERIOD,CDM,plan_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,PLAN_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -PAYER_PLAN_PERIOD,CDM,family_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,0,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +PAYER_PLAN_PERIOD,CDM,family_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, COST,CDM,cost_id,Yes,0,,INTEGER,0,,,,Yes,0,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,No,,, COST,CDM,cost_event_id,Yes,0,,INTEGER,0,,,,No,,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,No,,, COST,CDM,cost_domain_id,Yes,0,,VARCHAR(20),0,,,,No,,,Yes,0,,DOMAIN,DOMAIN_ID,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,No,,, @@ -301,4 +301,4 @@ DEATH,cdm,cause_source_value,No,,,varchar(50),0,,,"If available, put the source DEATH,cdm,death_date,Yes,0,,date,0,,The date the person was deceased.,"If the precise date include day or month is not known or not allowed, December is used as the default month, and the last day of the month the default day.",No,,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,'19500101',1,,"DATEADD(dd,1,GETDATE())",1,,Yes,PERSON,BIRTH_DATETIME,1,,,,,,,,,Yes,1,,,,,Yes,,, DEATH,cdm,death_datetime,No,,,datetime,0,,,If not available set time to midnight (00:00:00),No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,'19500101',1,,"DATEADD(dd,1,GETDATE())",1,,Yes,PERSON,BIRTH_DATETIME,1,,,,,,,,,Yes,1,,,,,Yes,,, DEATH,cdm,death_type_concept_id,No,,,integer,0,,"This is the provenance of the death record, i.e., where it came from. It is possible that an administrative claims database would source death information from a government file so do not assume the Death Type is the same as the Visit Type, etc.",Use the type concept that be reflects the source of the death record. [Accepted Concepts](https://athena.ohdsi.org/search-terms/terms?domain=Type+Concept&standardConcept=Standard&page=1&pageSize=15&query=).,No,,,Yes,0,,CONCEPT,CONCEPT_ID,Type Concept,0,,,,,Yes,0,,Yes,100,,Yes,0,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,Yes,,, -DEATH,cdm,person_id,Yes,0,,integer,0,,,,No,,,Yes,0,,PERSON,PERSON_ID,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,Yes,,, +DEATH,cdm,person_id,Yes,0,,integer,0,,,,No,,,Yes,0,,PERSON,PERSON_ID,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,Yes,,, \ No newline at end of file diff --git a/inst/csv/OMOP_CDMv5.3_Field_Level.csv b/inst/csv/OMOP_CDMv5.3_Field_Level.csv index 9f86523c..c8a15f70 100644 --- a/inst/csv/OMOP_CDMv5.3_Field_Level.csv +++ b/inst/csv/OMOP_CDMv5.3_Field_Level.csv @@ -38,7 +38,7 @@ VISIT_OCCURRENCE,cdm,visit_end_datetime,No,,,datetime,0,,,"If no time is given f VISIT_OCCURRENCE,cdm,visit_type_concept_id,Yes,0,,Integer,0,,"Use this field to understand the provenance of the visit record, or where the record comes from.","Populate this field based on the provenance of the visit record, as in whether it came from an EHR record or billing claim.",No,,,Yes,0,,CONCEPT,CONCEPT_ID,Type Concept,0,,,,,Yes,0,,Yes,0,,Yes,0,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, VISIT_OCCURRENCE,cdm,provider_id,No,,,integer,0,,There will only be one provider per visit. If multiple providers are associated with a visit that information can be found in the VISIT_DETAIL table.,"If there are multiple providers associated with a visit, you will need to choose which one to put here. The additional providers can be stored in the visit_detail table.",No,,,Yes,0,,PROVIDER,PROVIDER_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, VISIT_OCCURRENCE,cdm,care_site_id,No,,,integer,0,,This field provides information about the care site where the visit took place.,There should only be one care site associated with a visit.,No,,,Yes,0,,CARE_SITE,CARE_SITE_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -VISIT_OCCURRENCE,cdm,visit_source_value,No,,,varchar(50),0,,"This field houses the verbatim value from the source data representing the kind of visit that took place (inpatient, outpatient, emergency, etc.)","If there is information about the kind of visit in the source data that value should be stored here. If a visit is an amalgamation of visits from the source then use a hierarchy to choose the visit source value, such as IP -> ER-> OP. This should line up with the logic chosen to determine how visits are created.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,VISIT_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +VISIT_OCCURRENCE,cdm,visit_source_value,No,,,varchar(50),0,,"This field houses the verbatim value from the source data representing the kind of visit that took place (inpatient, outpatient, emergency, etc.)","If there is information about the kind of visit in the source data that value should be stored here. If a visit is an amalgamation of visits from the source then use a hierarchy to choose the visit source value, such as IP -> ER-> OP. This should line up with the logic chosen to determine how visits are created.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,10,,VISIT_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, VISIT_OCCURRENCE,cdm,visit_source_concept_id,No,,,integer,0,,,If the visit source value is coded in the source data using an OMOP supported vocabulary put the concept id representing the source value here.,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,100,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, VISIT_OCCURRENCE,cdm,admitting_source_concept_id,No,,,integer,0,,"Use this field to determine where the patient was admitted from. This concept is part of the visit domain and can indicate if a patient was admitted to the hospital from a long-term care facility, for example.","If available, map the admitted_from_source_value to a standard concept in the visit domain.",No,,,Yes,0,,CONCEPT,CONCEPT_ID,Visit,0,,,,,Yes,0,,Yes,100,,Yes,100,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, VISIT_OCCURRENCE,cdm,admitting_source_value,No,,,varchar(50),0,,,"This information may be called something different in the source data but the field is meant to contain a value indicating where a person was admitted from. Typically this applies only to visits that have a length of stay, like inpatient visits or long-term care visits.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,ADMITTING_SOURCE_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, @@ -98,8 +98,8 @@ PROCEDURE_OCCURRENCE,cdm,quantity,No,,,integer,0,,"If the quantity value is omit PROCEDURE_OCCURRENCE,cdm,provider_id,No,,,integer,0,,,,No,,,Yes,0,,PROVIDER,PROVIDER_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, PROCEDURE_OCCURRENCE,cdm,visit_occurrence_id,No,,,integer,0,,,,No,,,Yes,0,,VISIT_OCCURRENCE,VISIT_OCCURRENCE_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, PROCEDURE_OCCURRENCE,cdm,visit_detail_id,No,,,integer,0,,,,No,,,Yes,0,,VISIT_DETAIL,VISIT_DETAIL_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -PROCEDURE_OCCURRENCE,cdm,procedure_source_value,No,,,varchar(50),0,,,"This code is mapped to a standard procedure Concept in the Standardized Vocabularies and the original code is, stored here for reference. Procedure source codes are typically ICD-9-Proc, CPT-4, HCPCS or OPCS-4 codes.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,PROCEDURE_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -PROCEDURE_OCCURRENCE,cdm,procedure_source_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,100,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +PROCEDURE_OCCURRENCE,cdm,procedure_source_value,No,,,varchar(50),0,,,"This code is mapped to a standard procedure Concept in the Standardized Vocabularies and the original code is, stored here for reference. Procedure source codes are typically ICD-9-Proc, CPT-4, HCPCS or OPCS-4 codes.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,10,,PROCEDURE_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +PROCEDURE_OCCURRENCE,cdm,procedure_source_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,10,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, PROCEDURE_OCCURRENCE,cdm,modifier_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,MODIFIER_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, DEVICE_EXPOSURE,cdm,device_exposure_id,Yes,0,,bigint,0,,,,Yes,0,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, DEVICE_EXPOSURE,cdm,person_id,Yes,0,,bigint,0,,,,No,,,Yes,0,,PERSON,PERSON_ID,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, @@ -115,7 +115,7 @@ DEVICE_EXPOSURE,cdm,provider_id,No,,,integer,0,,,,No,,,Yes,0,,PROVIDER,PROVIDER_ DEVICE_EXPOSURE,cdm,visit_occurrence_id,No,,,integer,0,,,,No,,,Yes,0,,VISIT_OCCURRENCE,VISIT_OCCURRENCE_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, DEVICE_EXPOSURE,cdm,visit_detail_id,No,,,integer,0,,,,No,,,Yes,0,,VISIT_DETAIL,VISIT_DETAIL_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, DEVICE_EXPOSURE,cdm,device_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,DEVICE_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -DEVICE_EXPOSURE,cdm,device_source_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,100,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +DEVICE_EXPOSURE,cdm,device_source_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,10,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,cdm,measurement_id,Yes,0,,integer,0,,,,Yes,0,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,cdm,person_id,Yes,0,,integer,0,,,,No,,,Yes,0,,PERSON,PERSON_ID,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,cdm,measurement_concept_id,Yes,0,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,Measurement,0,,,,,Yes,0,,Yes,0,,Yes,5,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, @@ -140,8 +140,8 @@ MEASUREMENT,cdm,range_high,No,,,float,0,,Ranges have the same unit as the VALUE_ MEASUREMENT,cdm,provider_id,No,,,integer,0,,,,No,,,Yes,0,,PROVIDER,PROVIDER_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,cdm,visit_occurrence_id,No,,,integer,0,,,,No,,,Yes,0,,VISIT_OCCURRENCE,VISIT_OCCURRENCE_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,cdm,visit_detail_id,No,,,integer,0,,,,No,,,Yes,0,,VISIT_DETAIL,VISIT_DETAIL_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -MEASUREMENT,cdm,measurement_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,MEASUREMENT_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -MEASUREMENT,cdm,measurement_source_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,100,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +MEASUREMENT,cdm,measurement_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,10,,MEASUREMENT_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +MEASUREMENT,cdm,measurement_source_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,10,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,cdm,unit_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,UNIT_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,cdm,value_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, VISIT_DETAIL,cdm,visit_detail_id,Yes,0,,integer,0,,,,Yes,0,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, @@ -220,7 +220,7 @@ OBSERVATION,cdm,provider_id,No,,,integer,0,,,,No,,,Yes,0,,PROVIDER,PROVIDER_ID,, OBSERVATION,cdm,visit_occurrence_id,No,,,integer,0,,,,No,,,Yes,0,,VISIT_OCCURRENCE,VISIT_OCCURRENCE_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, OBSERVATION,cdm,visit_detail_id,No,,,integer,0,,,,No,,,Yes,0,,VISIT_DETAIL,VISIT_DETAIL_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, OBSERVATION,cdm,observation_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,OBSERVATION_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -OBSERVATION,cdm,observation_source_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,100,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +OBSERVATION,cdm,observation_source_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,10,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, OBSERVATION,cdm,unit_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,UNIT_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, OBSERVATION,cdm,qualifier_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,QUALIFIER_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, SPECIMEN,cdm,specimen_id,Yes,0,,integer,0,,,,Yes,0,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, @@ -283,7 +283,7 @@ PAYER_PLAN_PERIOD,cdm,plan_source_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCE PAYER_PLAN_PERIOD,cdm,sponsor_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, PAYER_PLAN_PERIOD,cdm,sponsor_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,SPONSOR_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, PAYER_PLAN_PERIOD,cdm,sponsor_source_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -PAYER_PLAN_PERIOD,cdm,family_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,0,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +PAYER_PLAN_PERIOD,cdm,family_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, PAYER_PLAN_PERIOD,cdm,stop_reason_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, PAYER_PLAN_PERIOD,cdm,stop_reason_source_value,No,,,varchar(50),0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,STOP_REASON_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, PAYER_PLAN_PERIOD,cdm,stop_reason_source_concept_id,No,,,integer,0,,,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, @@ -338,4 +338,4 @@ DEATH,cdm,cause_source_value,No,,,varchar(50),0,,,"If available, put the source DEATH,cdm,death_date,Yes,0,,date,0,,The date the person was deceased.,"If the precise date include day or month is not known or not allowed, December is used as the default month, and the last day of the month the default day.",No,,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,'19500101',1,,"DATEADD(dd,1,GETDATE())",1,,Yes,PERSON,BIRTH_DATETIME,1,,,,,,,,,Yes,1,,,,,Yes,,, DEATH,cdm,death_datetime,No,,,datetime,0,,,If not available set time to midnight (00:00:00),No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,'19500101',1,,"DATEADD(dd,1,GETDATE())",1,,Yes,PERSON,BIRTH_DATETIME,1,,,,,,,,,Yes,1,,,,,Yes,,, DEATH,cdm,death_type_concept_id,No,,,integer,0,,"This is the provenance of the death record, i.e., where it came from. It is possible that an administrative claims database would source death information from a government file so do not assume the Death Type is the same as the Visit Type, etc.",Use the type concept that be reflects the source of the death record. [Accepted Concepts](https://athena.ohdsi.org/search-terms/terms?domain=Type+Concept&standardConcept=Standard&page=1&pageSize=15&query=).,No,,,Yes,0,,CONCEPT,CONCEPT_ID,Type Concept,0,,,,,Yes,0,,Yes,100,,Yes,0,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,Yes,,, -DEATH,cdm,person_id,Yes,0,,integer,0,,,,No,,,Yes,0,,PERSON,PERSON_ID,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,Yes,,, +DEATH,cdm,person_id,Yes,0,,integer,0,,,,No,,,Yes,0,,PERSON,PERSON_ID,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,Yes,,, \ No newline at end of file diff --git a/inst/csv/OMOP_CDMv5.4_Field_Level.csv b/inst/csv/OMOP_CDMv5.4_Field_Level.csv index 3d30b554..0af68bb2 100644 --- a/inst/csv/OMOP_CDMv5.4_Field_Level.csv +++ b/inst/csv/OMOP_CDMv5.4_Field_Level.csv @@ -166,8 +166,8 @@ DEVICE_EXPOSURE,cdm,provider_id,No,,,integer,0,,"The Provider associated with de DEVICE_EXPOSURE,cdm,quantity,No,,,integer,0,,,,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,1,1,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, DEVICE_EXPOSURE,cdm,unique_device_id,No,,,varchar(255),0,,"This is the Unique Device Identification (UDI-DI) number for devices regulated by the FDA, if given.","For medical devices that are regulated by the FDA, a Unique Device Identification (UDI) is provided if available in the data source and is recorded in the UNIQUE_DEVICE_ID field.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, DEVICE_EXPOSURE,cdm,unit_concept_id,No,,,integer,0,,UNIT_SOURCE_VALUES should be mapped to a Standard Concept in the Unit domain that best represents the unit as given in the source data.,"There is no standardization requirement for units associated with DEVICE_CONCEPT_IDs, however, it is the responsibility of the ETL to choose the most plausible unit. If there is no unit associated with a Device record, set to NULL.",No,,,Yes,0,,CONCEPT,CONCEPT_ID,Unit,0,,,,,Yes,0,,Yes,100,,Yes,5,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -DEVICE_EXPOSURE,cdm,unit_source_concept_id,No,,,integer,0,,"This is the concept representing the UNIT_SOURCE_VALUE and may not necessarily be standard. This field is discouraged from use in analysis because it is not required to contain Standard Concepts that are used across the OHDSI community, and should only be used when Standard Concepts do not adequately represent the source detail for the Unit necessary for a given analytic use case. Consider using UNIT_CONCEPT_ID instead to enable standardized analytics that can be consistent across the network.",If the UNIT_SOURCE_VALUE is coded in the source data using an OMOP supported vocabulary put the concept id representing the source value here.,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,10,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -DEVICE_EXPOSURE,cdm,unit_source_value,No,,,varchar(50),0,,"This field houses the verbatim value from the source data representing the unit of the Device. For example, blood transfusions are considered devices and can be given in mL quantities.","This code is mapped to a Standard Condition Concept in the Standardized Vocabularies and the original code is stored here for reference. Using the blood transfusion example, blood transfusion is represented by the DEVICE_CONCEPT_ID and the unit (mL) would be housed in the UNIT_SOURCE_VALUE and mapped to a standard concept in the unit domain.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,10,,UNIT_CONCEPT_ID,,,,,,,,,,,,,,,,,,,,,,,,,,,, +DEVICE_EXPOSURE,cdm,unit_source_concept_id,No,,,integer,0,,"This is the concept representing the UNIT_SOURCE_VALUE and may not necessarily be standard. This field is discouraged from use in analysis because it is not required to contain Standard Concepts that are used across the OHDSI community, and should only be used when Standard Concepts do not adequately represent the source detail for the Unit necessary for a given analytic use case. Consider using UNIT_CONCEPT_ID instead to enable standardized analytics that can be consistent across the network.",If the UNIT_SOURCE_VALUE is coded in the source data using an OMOP supported vocabulary put the concept id representing the source value here.,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,100,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +DEVICE_EXPOSURE,cdm,unit_source_value,No,,,varchar(50),0,,"This field houses the verbatim value from the source data representing the unit of the Device. For example, blood transfusions are considered devices and can be given in mL quantities.","This code is mapped to a Standard Condition Concept in the Standardized Vocabularies and the original code is stored here for reference. Using the blood transfusion example, blood transfusion is represented by the DEVICE_CONCEPT_ID and the unit (mL) would be housed in the UNIT_SOURCE_VALUE and mapped to a standard concept in the unit domain.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,UNIT_CONCEPT_ID,,,,,,,,,,,,,,,,,,,,,,,,,,,, DEVICE_EXPOSURE,cdm,visit_detail_id,No,,,integer,0,,The Visit Detail during which the device was prescribed or given.,To populate this field device exposures must be explicitly initiated in the visit detail record.,No,,,Yes,0,,VISIT_DETAIL,VISIT_DETAIL_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, DEVICE_EXPOSURE,cdm,visit_occurrence_id,No,,,integer,0,,The Visit during which the device was prescribed or given.,To populate this field device exposures must be explicitly initiated in the visit.,No,,,Yes,0,,VISIT_OCCURRENCE,VISIT_OCCURRENCE_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, DOMAIN,vocab,domain_concept_id,Yes,0,,integer,0,,A Concept representing the Domain Concept the DOMAIN record belongs to.,,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,No,0,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, @@ -269,8 +269,8 @@ MEASUREMENT,cdm,measurement_date,Yes,0,,date,0,,Use this date to determine the d MEASUREMENT,cdm,measurement_datetime,No,,,datetime,0,,,"This is not required, though it is in v6. If a source does not specify datetime the convention is to set the time to midnight (00:00:0000)",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,'19500101',1,,"DATEADD(dd,1,GETDATE())",1,,,,,,,No,,,,,,,Yes,1,,Yes,1,,Yes,,, MEASUREMENT,cdm,measurement_event_id,No,,,bigint,0,,"If the Measurement record is related to another record in the database, this field is the primary key of the linked record.","Put the primary key of the linked record, if applicable, here.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, MEASUREMENT,cdm,measurement_id,Yes,0,,integer,0,,The unique key given to a Measurement record for a Person. Refer to the ETL for how duplicate Measurements during the same Visit were handled.,"Each instance of a measurement present in the source data should be assigned this unique key. In some cases, a person can have multiple records of the same measurement within the same visit. It is valid to keep these duplicates and assign them individual, unique, MEASUREMENT_IDs, though it is up to the ETL how they should be handled.",Yes,0,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -MEASUREMENT,cdm,measurement_source_concept_id,No,,,integer,0,,"This is the concept representing the MEASUREMENT_SOURCE_VALUE and may not necessarily be standard. This field is discouraged from use in analysis because it is not required to contain Standard Concepts that are used across the OHDSI community, and should only be used when Standard Concepts do not adequately represent the source detail for the Measurement necessary for a given analytic use case. Consider using MEASUREMENT_CONCEPT_ID instead to enable standardized analytics that can be consistent across the network.",If the MEASUREMENT_SOURCE_VALUE is coded in the source data using an OMOP supported vocabulary put the concept id representing the source value here.,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,100,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -MEASUREMENT,cdm,measurement_source_value,No,,,varchar(50),0,,"This field houses the verbatim value from the source data representing the Measurement that occurred. For example, this could be an ICD10 or Read code.",This code is mapped to a Standard Measurement Concept in the Standardized Vocabularies and the original code is stored here for reference.,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,MEASUREMENT_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +MEASUREMENT,cdm,measurement_source_concept_id,No,,,integer,0,,"This is the concept representing the MEASUREMENT_SOURCE_VALUE and may not necessarily be standard. This field is discouraged from use in analysis because it is not required to contain Standard Concepts that are used across the OHDSI community, and should only be used when Standard Concepts do not adequately represent the source detail for the Measurement necessary for a given analytic use case. Consider using MEASUREMENT_CONCEPT_ID instead to enable standardized analytics that can be consistent across the network.",If the MEASUREMENT_SOURCE_VALUE is coded in the source data using an OMOP supported vocabulary put the concept id representing the source value here.,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,10,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +MEASUREMENT,cdm,measurement_source_value,No,,,varchar(50),0,,"This field houses the verbatim value from the source data representing the Measurement that occurred. For example, this could be an ICD10 or Read code.",This code is mapped to a Standard Measurement Concept in the Standardized Vocabularies and the original code is stored here for reference.,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,10,,MEASUREMENT_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,cdm,measurement_time,No,,,varchar(10),0,,,This is present for backwards compatibility and will be deprecated in an upcoming version.,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,cdm,measurement_type_concept_id,Yes,0,,integer,0,,"This field can be used to determine the provenance of the Measurement record, as in whether the measurement was from an EHR system, insurance claim, registry, or other sources.","Choose the MEASUREMENT_TYPE_CONCEPT_ID that best represents the provenance of the record, for example whether it came from an EHR record or billing claim. [Accepted Concepts](https://athena.ohdsi.org/search-terms/terms?domain=Type+Concept&standardConcept=Standard&page=1&pageSize=15&query=).",No,,,Yes,0,,CONCEPT,CONCEPT_ID,Type Concept,0,,,,,Yes,0,,Yes,0,,Yes,0,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,cdm,operator_concept_id,No,,,integer,0,,"The meaning of Concept [4172703](https://athena.ohdsi.org/search-terms/terms/4172703) for '=' is identical to omission of a OPERATOR_CONCEPT_ID value. Since the use of this field is rare, it's important when devising analyses to not to forget testing for the content of this field for values different from =.","Operators are <, <=, =, >=, > and these concepts belong to the 'Meas Value Operator' domain. [Accepted Concepts](https://athena.ohdsi.org/search-terms/terms?domain=Meas+Value+Operator&standardConcept=Standard&page=1&pageSize=15&query=).",No,,,Yes,0,,CONCEPT,CONCEPT_ID,Meas Value Operator,0,,,,,Yes,0,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, @@ -279,7 +279,7 @@ MEASUREMENT,cdm,provider_id,No,,,integer,0,,"The provider associated with measur MEASUREMENT,cdm,range_high,No,,,float,0,,Ranges have the same unit as the VALUE_AS_NUMBER. These ranges are provided by the source and should remain NULL if not given.,If reference ranges for upper and lower limit of normal as provided (typically by a laboratory) these are stored in the RANGE_HIGH and RANGE_LOW fields. This should be set to NULL if not provided.,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,cdm,range_low,No,,,float,0,,Ranges have the same unit as the VALUE_AS_NUMBER. These ranges are provided by the source and should remain NULL if not given.,If reference ranges for upper and lower limit of normal as provided (typically by a laboratory) these are stored in the RANGE_HIGH and RANGE_LOW fields. This should be set to NULL if not provided.,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,cdm,unit_concept_id,No,,,integer,0,,"There is currently no recommended unit for individual measurements, i.e. it is not mandatory to represent Hemoglobin a1C measurements as a percentage. UNIT_SOURCE_VALUES should be mapped to a Standard Concept in the Unit domain that best represents the unit as given in the source data.","There is no standardization requirement for units associated with MEASUREMENT_CONCEPT_IDs, however, it is the responsibility of the ETL to choose the most plausible unit.",No,,,Yes,0,,CONCEPT,CONCEPT_ID,Unit,0,,,,,Yes,0,,Yes,100,,Yes,5,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -MEASUREMENT,cdm,unit_source_concept_id,No,,,integer,0,,"""This is the concept representing the UNIT_SOURCE_VALUE and may not necessarily be standard. This field is discouraged from use in analysis because it is not required to contain Standard Concepts that are used across the OHDSI community, and should only be used when Standard Concepts do not adequately represent the source detail for the Measurement necessary for a given analytic use case. Consider using UNIT_CONCEPT_ID instead to enable standardized analytics that can be consistent across the network.""",If the UNIT_SOURCE_VALUE is coded in the source data using an OMOP supported vocabulary put the concept id representing the source value here.,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,50,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +MEASUREMENT,cdm,unit_source_concept_id,No,,,integer,0,,"""This is the concept representing the UNIT_SOURCE_VALUE and may not necessarily be standard. This field is discouraged from use in analysis because it is not required to contain Standard Concepts that are used across the OHDSI community, and should only be used when Standard Concepts do not adequately represent the source detail for the Measurement necessary for a given analytic use case. Consider using UNIT_CONCEPT_ID instead to enable standardized analytics that can be consistent across the network.""",If the UNIT_SOURCE_VALUE is coded in the source data using an OMOP supported vocabulary put the concept id representing the source value here.,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,100,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,cdm,unit_source_value,No,,,varchar(50),0,,This field houses the verbatim value from the source data representing the unit of the Measurement that occurred.,This code is mapped to a Standard Condition Concept in the Standardized Vocabularies and the original code is stored here for reference.,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,UNIT_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,cdm,value_as_concept_id,No,,,integer,0,,If the raw data gives a categorial result for measurements those values are captured and mapped to standard concepts in the 'Meas Value' domain.,"If the raw data provides categorial results as well as continuous results for measurements, it is a valid ETL choice to preserve both values. The continuous value should go in the VALUE_AS_NUMBER field and the categorical value should be mapped to a standard concept in the 'Meas Value' domain and put in the VALUE_AS_CONCEPT_ID field. This is also the destination for the 'Maps to value' relationship.",No,,,Yes,0,,CONCEPT,CONCEPT_ID,Meas Value,0,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, MEASUREMENT,cdm,value_as_number,No,,,float,0,,"This is the numerical value of the Result of the Measurement, if available. Note that measurements such as blood pressures will be split into their component parts i.e. one record for systolic, one record for diastolic.","If there is a negative value coming from the source, set the VALUE_AS_NUMBER to NULL, with the exception of the following Measurements (listed as LOINC codes):
    - [1925-7](https://athena.ohdsi.org/search-terms/terms/3003396) Base excess in Arterial blood by calculation - [1927-3](https://athena.ohdsi.org/search-terms/terms/3002032) Base excess in Venous blood by calculation - [8632-2](https://athena.ohdsi.org/search-terms/terms/3006277) QRS-Axis - [11555-0](https://athena.ohdsi.org/search-terms/terms/3012501) Base excess in Blood by calculation - [1926-5](https://athena.ohdsi.org/search-terms/terms/3003129) Base excess in Capillary blood by calculation - [28638-5](https://athena.ohdsi.org/search-terms/terms/3004959) Base excess in Arterial cord blood by calculation [28639-3](https://athena.ohdsi.org/search-terms/terms/3007435) Base excess in Venous cord blood by calculation",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, @@ -346,7 +346,7 @@ OBSERVATION,cdm,observation_date,Yes,0,,date,0,,"The date of the Observation. De OBSERVATION,cdm,observation_datetime,No,,,datetime,0,,,If no time is given set to midnight (00:00:00).,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,'19500101',1,,"DATEADD(dd,1,GETDATE())",1,,Yes,PERSON,BIRTH_DATETIME,1,,No,,,,,,,Yes,1,,Yes,1,,Yes,,, OBSERVATION,cdm,observation_event_id,No,,,bigint,0,,"If the Observation record is related to another record in the database, this field is the primary key of the linked record.","Put the primary key of the linked record, if applicable, here. See the [ETL Conventions for the OBSERVATION](https://ohdsi.github.io/CommonDataModel/cdm60.html#observation) table for more details.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, OBSERVATION,cdm,observation_id,Yes,0,,integer,0,,The unique key given to an Observation record for a Person. Refer to the ETL for how duplicate Observations during the same Visit were handled.,Each instance of an observation present in the source data should be assigned this unique key.,Yes,0,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -OBSERVATION,cdm,observation_source_concept_id,No,,,integer,0,,"This is the concept representing the OBSERVATION_SOURCE_VALUE and may not necessarily be standard. This field is discouraged from use in analysis because it is not required to contain Standard Concepts that are used across the OHDSI community, and should only be used when Standard Concepts do not adequately represent the source detail for the Observation necessary for a given analytic use case. Consider using OBSERVATION_CONCEPT_ID instead to enable standardized analytics that can be consistent across the network.",If the OBSERVATION_SOURCE_VALUE is coded in the source data using an OMOP supported vocabulary put the concept id representing the source value here.,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,100,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +OBSERVATION,cdm,observation_source_concept_id,No,,,integer,0,,"This is the concept representing the OBSERVATION_SOURCE_VALUE and may not necessarily be standard. This field is discouraged from use in analysis because it is not required to contain Standard Concepts that are used across the OHDSI community, and should only be used when Standard Concepts do not adequately represent the source detail for the Observation necessary for a given analytic use case. Consider using OBSERVATION_CONCEPT_ID instead to enable standardized analytics that can be consistent across the network.",If the OBSERVATION_SOURCE_VALUE is coded in the source data using an OMOP supported vocabulary put the concept id representing the source value here.,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,10,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, OBSERVATION,cdm,observation_source_value,No,,,varchar(50),0,,"This field houses the verbatim value from the source data representing the Observation that occurred. For example, this could be an ICD10 or Read code.",This code is mapped to a Standard Concept in the Standardized Vocabularies and the original code is stored here for reference.,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,OBSERVATION_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, OBSERVATION,cdm,observation_type_concept_id,Yes,0,,integer,0,,"This field can be used to determine the provenance of the Observation record, as in whether the measurement was from an EHR system, insurance claim, registry, or other sources.","Choose the OBSERVATION_TYPE_CONCEPT_ID that best represents the provenance of the record, for example whether it came from an EHR record or billing claim. [Accepted Concepts](https://athena.ohdsi.org/search-terms/terms?domain=Type+Concept&standardConcept=Standard&page=1&pageSize=15&query=).",No,,,Yes,0,,CONCEPT,CONCEPT_ID,Type Concept,0,,,,,Yes,0,,Yes,0,,Yes,0,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, OBSERVATION,cdm,person_id,Yes,0,,integer,0,,The PERSON_ID of the Person for whom the Observation is recorded. This may be a system generated code.,,No,,,Yes,0,,PERSON,PERSON_ID,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, @@ -366,7 +366,7 @@ OBSERVATION_PERIOD,cdm,observation_period_id,Yes,0,,integer,0,,A Person can have OBSERVATION_PERIOD,cdm,observation_period_start_date,Yes,0,,date,0,,Use this date to determine the start date of the Observation Period.,"It is often the case that the idea of Observation Periods does not exist in source data. In those cases, the observation_period_start_date can be inferred as the earliest Event date available for the Person. In insurance claim data, the Observation Period can be considered as the time period the Person is enrolled with a payer. If a Person switches plans but stays with the same payer, and therefore capturing of data continues, that change would be captured in [PAYER_PLAN_PERIOD](https://ohdsi.github.io/CommonDataModel/cdm531.html#payer_plan_period).",No,,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,'19500101',1,,"DATEADD(dd,1,GETDATE())",1,,Yes,PERSON,BIRTH_DATETIME,1,,Yes,1,,Yes,OBSERVATION_PERIOD_END_DATE,0,,Yes,1,,Yes,1,,Yes,,, OBSERVATION_PERIOD,cdm,period_type_concept_id,Yes,0,,integer,0,,"This field can be used to determine the provenance of the Observation Period as in whether the period was determined from an insurance enrollment file, EHR healthcare encounters, or other sources.",Choose the observation_period_type_concept_id that best represents how the period was determined. [Accepted Concepts](https://athena.ohdsi.org/search-terms/terms?domain=Type+Concept&standardConcept=Standard&page=1&pageSize=15&query=).,No,,,Yes,0,,CONCEPT,CONCEPT_ID,Type Concept,0,,,,,Yes,0,,Yes,0,,Yes,0,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, OBSERVATION_PERIOD,cdm,person_id,Yes,0,,integer,0,,The Person ID of the PERSON record for which the Observation Period is recorded.,,No,,,Yes,0,,PERSON,PERSON_ID,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -PAYER_PLAN_PERIOD,cdm,family_source_value,No,,,varchar(50),0,,The common identifier for all people (often a family) that covered by the same policy.,Often these are the common digits of the enrollment id of the policy members.,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,0,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +PAYER_PLAN_PERIOD,cdm,family_source_value,No,,,varchar(50),0,,The common identifier for all people (often a family) that covered by the same policy.,Often these are the common digits of the enrollment id of the policy members.,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,100,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, PAYER_PLAN_PERIOD,cdm,payer_concept_id,No,,,integer,0,,This field represents the organization who reimburses the provider which administers care to the Person.,"Map the Payer directly to a standard CONCEPT_ID. If one does not exists please contact the vocabulary team. There is no global controlled vocabulary available for this information. The point is to stratify on this information and identify if Persons have the same payer, though the name of the Payer is not necessary. [Accepted Concepts](http://athena.ohdsi.org/search-terms/terms?domain=Payer&standardConcept=Standard&page=1&pageSize=15&query=).",No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, PAYER_PLAN_PERIOD,cdm,payer_plan_period_end_date,Yes,0,,date,0,,End date of Plan coverage.,,No,,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,'19500101',1,,"DATEADD(dd,1,GETDATE())",1,,Yes,PAYER_PLAN_PERIOD,PAYER_PLAN_PERIOD_START_DATE,1,,No,,,,,,,Yes,1,,Yes,1,,Yes,,, PAYER_PLAN_PERIOD,cdm,payer_plan_period_id,Yes,0,,integer,0,,"A unique identifier for each unique combination of a Person, Payer, Plan, and Period of time.",,Yes,0,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, @@ -410,8 +410,8 @@ PROCEDURE_OCCURRENCE,cdm,procedure_datetime,No,,,datetime,0,,,"If the procedure PROCEDURE_OCCURRENCE,cdm,procedure_end_date,No,,,date,0,,Use this field to house the date that the procedure ended.,This is meant to be the end date of the procedure. It is not required and for most cases will be the same as the PROCEDURE_START_DATE.,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,'19500101',1,,"DATEADD(dd,1,GETDATE())",1,,Yes,PROCEDURE_OCCURRENCE,PROCEDURE_DATE,1,,Yes,1,,,,,,Yes,1,,Yes,1,,Yes,,, PROCEDURE_OCCURRENCE,cdm,procedure_end_datetime,No,,,datetime,0,,Use this field to house the datetime that the procedure ended.,This is meant to house the end datetime of the procedure and will most often be used in conjunction with the procedure_start_datetime to determine the length of the procedure.,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,'19500101',1,,"DATEADD(dd,1,GETDATE())",1,,Yes,PROCEDURE_OCCURRENCE,PROCEDURE_DATETIME,1,,Yes,1,,,,,,Yes,1,,Yes,1,,Yes,,, PROCEDURE_OCCURRENCE,cdm,procedure_occurrence_id,Yes,0,,integer,0,,The unique key given to a procedure record for a person. Refer to the ETL for how duplicate procedures during the same visit were handled.,"Each instance of a procedure occurrence in the source data should be assigned this unique key. In some cases, a person can have multiple records of the same procedure within the same visit. It is valid to keep these duplicates and assign them individual, unique, PROCEDURE_OCCURRENCE_IDs, though it is up to the ETL how they should be handled.",Yes,0,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -PROCEDURE_OCCURRENCE,cdm,procedure_source_concept_id,No,,,integer,0,,"This is the concept representing the procedure source value and may not necessarily be standard. This field is discouraged from use in analysis because it is not required to contain Standard Concepts that are used across the OHDSI community, and should only be used when Standard Concepts do not adequately represent the source detail for the Procedure necessary for a given analytic use case. Consider using PROCEDURE_CONCEPT_ID instead to enable standardized analytics that can be consistent across the network.",If the PROCEDURE_SOURCE_VALUE is coded in the source data using an OMOP supported vocabulary put the concept id representing the source value here.,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,100,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -PROCEDURE_OCCURRENCE,cdm,procedure_source_value,No,,,varchar(50),0,,"This field houses the verbatim value from the source data representing the procedure that occurred. For example, this could be an CPT4 or OPCS4 code.",Use this value to look up the source concept id and then map the source concept id to a standard concept id.,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,PROCEDURE_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +PROCEDURE_OCCURRENCE,cdm,procedure_source_concept_id,No,,,integer,0,,"This is the concept representing the procedure source value and may not necessarily be standard. This field is discouraged from use in analysis because it is not required to contain Standard Concepts that are used across the OHDSI community, and should only be used when Standard Concepts do not adequately represent the source detail for the Procedure necessary for a given analytic use case. Consider using PROCEDURE_CONCEPT_ID instead to enable standardized analytics that can be consistent across the network.",If the PROCEDURE_SOURCE_VALUE is coded in the source data using an OMOP supported vocabulary put the concept id representing the source value here.,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,10,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +PROCEDURE_OCCURRENCE,cdm,procedure_source_value,No,,,varchar(50),0,,"This field houses the verbatim value from the source data representing the procedure that occurred. For example, this could be an CPT4 or OPCS4 code.",Use this value to look up the source concept id and then map the source concept id to a standard concept id.,No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,10,,PROCEDURE_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, PROCEDURE_OCCURRENCE,cdm,procedure_type_concept_id,Yes,0,,integer,0,,"This field can be used to determine the provenance of the Procedure record, as in whether the procedure was from an EHR system, insurance claim, registry, or other sources.","Choose the PROCEDURE_TYPE_CONCEPT_ID that best represents the provenance of the record, for example whether it came from an EHR record or billing claim. If a procedure is recorded as an EHR encounter, the PROCEDURE_TYPE_CONCEPT would be 'EHR encounter record'. [Accepted Concepts](https://athena.ohdsi.org/search-terms/terms?domain=Type+Concept&standardConcept=Standard&page=1&pageSize=15&query=).",No,,,Yes,0,,CONCEPT,CONCEPT_ID,Type Concept,0,,,,,Yes,0,,Yes,0,,Yes,0,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, PROCEDURE_OCCURRENCE,cdm,provider_id,No,,,integer,0,,"The provider associated with the procedure record, e.g. the provider who performed the Procedure.","The ETL may need to make a choice as to which PROVIDER_ID to put here. Based on what is available this may or may not be different than the provider associated with the overall VISIT_OCCURRENCE record, for example the admitting vs attending physician on an EHR record.",No,,,Yes,0,,PROVIDER,PROVIDER_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, PROCEDURE_OCCURRENCE,cdm,quantity,No,,,integer,0,,"If the quantity value is omitted, a single procedure is assumed.","If a Procedure has a quantity of '0' in the source, this should default to '1' in the ETL. If there is a record in the source it can be assumed the exposure occurred at least once",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,1,1,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, @@ -489,10 +489,10 @@ SPECIMEN,cdm,specimen_type_concept_id,Yes,0,,integer,0,,,"Put the source of the SPECIMEN,cdm,unit_concept_id,No,,,integer,0,,The unit for the quantity of the specimen.,Map the UNIT_SOURCE_VALUE to a Standard Concept in the Unit domain. [Accepted Concepts](https://athena.ohdsi.org/search-terms/terms?domain=Unit&standardConcept=Standard&page=1&pageSize=15&query=),No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,Yes,0,,Yes,100,,Yes,5,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, SPECIMEN,cdm,unit_source_value,No,,,varchar(50),0,,,"This unit for the quantity of the specimen, as represented in the source.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,UNIT_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, VISIT_DETAIL,cdm,admitted_from_concept_id,No,,,Integer,0,,"Use this field to determine where the patient was admitted from. This concept is part of the visit domain and can indicate if a patient was admitted to the hospital from a long-term care facility, for example.","If available, map the admitted_from_source_value to a standard concept in the visit domain. [Accepted Concepts](https://athena.ohdsi.org/search-terms/terms?domain=Visit&standardConcept=Standard&page=1&pageSize=15&query=). If the person was admitted from home, set this to 0.",No,,,Yes,0,,CONCEPT,CONCEPT_ID,Visit,0,,,,,Yes,0,,Yes,100,,Yes,5,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -VISIT_DETAIL,cdm,admitted_from_source_value,No,,,varchar(50),0,,,"This information may be called something different in the source data but the field is meant to contain a value indicating where a person was admitted from. Typically this applies only to visits that have a length of stay, like inpatient visits or long-term care visits.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +VISIT_DETAIL,cdm,admitted_from_source_value,No,,,varchar(50),0,,,"This information may be called something different in the source data but the field is meant to contain a value indicating where a person was admitted from. Typically this applies only to visits that have a length of stay, like inpatient visits or long-term care visits.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, VISIT_DETAIL,cdm,care_site_id,No,,,integer,0,,This field provides information about the Care Site where the Visit Detail took place.,There should only be one Care Site associated with a Visit Detail.,No,,,Yes,0,,CARE_SITE,CARE_SITE_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, VISIT_DETAIL,cdm,discharged_to_concept_id,No,,,integer,0,,"Use this field to determine where the patient was discharged to after a visit. This concept is part of the visit domain and can indicate if a patient was transferred to another hospital or sent to a long-term care facility, for example. It is assumed that a person is discharged to home therefore there is not a standard concept id for ""home"". Use concept id = 0 when a person is discharged to home.","If available, map the DISCHARGE_TO_SOURCE_VALUE to a Standard Concept in the Visit domain. [Accepted Concepts](https://athena.ohdsi.org/search-terms/terms?domain=Visit&standardConcept=Standard&page=1&pageSize=15&query=).",No,,,Yes,0,,CONCEPT,CONCEPT_ID,Visit,0,,,,,Yes,0,,Yes,100,,Yes,5,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -VISIT_DETAIL,cdm,discharged_to_source_value,No,,,varchar(50),0,,,"This information may be called something different in the source data but the field is meant to contain a value indicating where a person was discharged to after a visit, as in they went home or were moved to long-term care. Typically this applies only to visits that have a length of stay of a day or more.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +VISIT_DETAIL,cdm,discharged_to_source_value,No,,,varchar(50),0,,,"This information may be called something different in the source data but the field is meant to contain a value indicating where a person was discharged to after a visit, as in they went home or were moved to long-term care. Typically this applies only to visits that have a length of stay of a day or more.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, VISIT_DETAIL,cdm,parent_visit_detail_id,No,,,integer,0,,Use this field to find the visit detail that subsumes the given visit detail record. This is used in the case that a visit detail record needs to be nested beyond the VISIT_OCCURRENCE/VISIT_DETAIL relationship.,"If there are multiple nested levels to how Visits are represented in the source, the VISIT_DETAIL_PARENT_ID can be used to record this relationship.",No,,,Yes,0,,VISIT_DETAIL,VISIT_DETAIL_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, VISIT_DETAIL,cdm,person_id,Yes,0,,integer,0,,,,No,,,Yes,0,,PERSON,PERSON_ID,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, VISIT_DETAIL,cdm,preceding_visit_detail_id,No,,,integer,0,,Use this field to find the visit detail that occurred for the person prior to the given visit detail record. There could be a few days or a few years in between.,"The PRECEDING_VISIT_DETAIL_ID can be used to link a visit immediately preceding the current Visit Detail. Note this is not symmetrical, and there is no such thing as a ""following_visit_id"".",No,,,Yes,0,,VISIT_DETAIL,VISIT_DETAIL_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, @@ -514,10 +514,10 @@ VISIT_DETAIL,cdm,visit_detail_start_datetime,No,,,datetime,0,,,"If no time is gi VISIT_DETAIL,cdm,visit_detail_type_concept_id,Yes,0,,integer,0,,"Use this field to understand the provenance of the visit detail record, or where the record comes from.","Populate this field based on the provenance of the visit detail record, as in whether it came from an EHR record or billing claim. [Accepted Concepts](https://athena.ohdsi.org/search-terms/terms?domain=Type+Concept&standardConcept=Standard&page=1&pageSize=15&query=).",No,,,Yes,0,,CONCEPT,CONCEPT_ID,Type Concept,0,,,,,Yes,0,,Yes,0,,Yes,0,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, VISIT_DETAIL,cdm,visit_occurrence_id,Yes,0,,integer,0,,Use this field to link the VISIT_DETAIL record to its VISIT_OCCURRENCE.,Put the VISIT_OCCURRENCE_ID that subsumes the VISIT_DETAIL record here.,No,,,Yes,0,,VISIT_OCCURRENCE,VISIT_OCCURRENCE_ID,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, VISIT_OCCURRENCE,cdm,admitted_from_concept_id,No,,,integer,0,,"Use this field to determine where the patient was admitted from. This concept is part of the visit domain and can indicate if a patient was admitted to the hospital from a long-term care facility, for example.","If available, map the admitted_from_source_value to a standard concept in the visit domain. [Accepted Concepts](https://athena.ohdsi.org/search-terms/terms?domain=Visit&standardConcept=Standard&page=1&pageSize=15&query=). If a person was admitted from home, set this to 0.",No,,,Yes,0,,CONCEPT,CONCEPT_ID,Visit,0,,,,,Yes,0,,Yes,100,,Yes,5,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -VISIT_OCCURRENCE,cdm,admitted_from_source_value,No,,,varchar(50),0,,,"This information may be called something different in the source data but the field is meant to contain a value indicating where a person was admitted from. Typically this applies only to visits that have a length of stay, like inpatient visits or long-term care visits.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +VISIT_OCCURRENCE,cdm,admitted_from_source_value,No,,,varchar(50),0,,,"This information may be called something different in the source data but the field is meant to contain a value indicating where a person was admitted from. Typically this applies only to visits that have a length of stay, like inpatient visits or long-term care visits.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, VISIT_OCCURRENCE,cdm,care_site_id,No,,,integer,0,,This field provides information about the Care Site where the Visit took place.,There should only be one Care Site associated with a Visit.,No,,,Yes,0,,CARE_SITE,CARE_SITE_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, VISIT_OCCURRENCE,cdm,discharged_to_concept_id,No,,,integer,0,,"Use this field to determine where the patient was discharged to after a visit. This concept is part of the visit domain and can indicate if a patient was transferred to another hospital or sent to a long-term care facility, for example. It is assumed that a person is discharged to home therefore there is not a standard concept id for ""home"". Use concept id = 0 when a person is discharged to home.","If available, map the discharged_to_source_value to a standard concept in the visit domain. [Accepted Concepts](https://athena.ohdsi.org/search-terms/terms?domain=Visit&standardConcept=Standard&page=1&pageSize=15&query=).",No,,,Yes,0,,CONCEPT,CONCEPT_ID,Visit,0,,,,,Yes,0,,Yes,100,,Yes,5,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -VISIT_OCCURRENCE,cdm,discharged_to_source_value,No,,,varchar(50),0,,,"This information may be called something different in the source data but the field is meant to contain a value indicating where a person was discharged to after a visit, as in they went home or were moved to long-term care. Typically this applies only to visits that have a length of stay of a day or more.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +VISIT_OCCURRENCE,cdm,discharged_to_source_value,No,,,varchar(50),0,,,"This information may be called something different in the source data but the field is meant to contain a value indicating where a person was discharged to after a visit, as in they went home or were moved to long-term care. Typically this applies only to visits that have a length of stay of a day or more.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, VISIT_OCCURRENCE,cdm,person_id,Yes,0,,integer,0,,,,No,,,Yes,0,,PERSON,PERSON_ID,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, VISIT_OCCURRENCE,cdm,preceding_visit_occurrence_id,No,,,integer,0,,Use this field to find the visit that occurred for the person prior to the given visit. There could be a few days or a few years in between.,"This field can be used to link a visit immediately preceding the current visit. Note this is not symmetrical, and there is no such thing as a ""following_visit_id"".",No,,,Yes,0,,VISIT_OCCURRENCE,VISIT_OCCURRENCE_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, VISIT_OCCURRENCE,cdm,provider_id,No,,,integer,0,,"There will only be one provider per visit record and the ETL document should clearly state how they were chosen (attending, admitting, etc.). If there are multiple providers associated with a visit in the source, this can be reflected in the event tables (CONDITION_OCCURRENCE, PROCEDURE_OCCURRENCE, etc.) or in the VISIT_DETAIL table.","If there are multiple providers associated with a visit, you will need to choose which one to put here. The additional providers can be stored in the [VISIT_DETAIL](https://ohdsi.github.io/CommonDataModel/cdm531.html#visit_detail) table.",No,,,Yes,0,,PROVIDER,PROVIDER_ID,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, @@ -532,7 +532,7 @@ For Inpatient Visits ongoing at the date of ETL, put date of processing the data VISIT_OCCURRENCE,cdm,visit_end_datetime,No,,,datetime,0,,"If a Person is still an inpatient in the hospital at the time of the data extract and does not have a visit_end_datetime, then set the visit_end_datetime to the datetime of the data pull.","If no time is given for the end date of a visit, set it to midnight (00:00:0000).",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,'19500101',1,,"DATEADD(dd,1,GETDATE())",1,,Yes,VISIT_OCCURRENCE,VISIT_START_DATETIME,1,,Yes,1,,,,,,Yes,1,,Yes,1,,Yes,,, VISIT_OCCURRENCE,cdm,visit_occurrence_id,Yes,0,,integer,0,,Use this to identify unique interactions between a person and the health care system. This identifier links across the other CDM event tables to associate events with a visit.,This should be populated by creating a unique identifier for each unique interaction between a person and the healthcare system where the person receives a medical good or service over a span of time.,Yes,0,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, VISIT_OCCURRENCE,cdm,visit_source_concept_id,No,,,integer,0,,,If the visit source value is coded in the source data using an OMOP supported vocabulary put the concept id representing the source value here.,No,,,Yes,0,,CONCEPT,CONCEPT_ID,,,,,,,No,,,Yes,100,,No,,,Yes,100,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, -VISIT_OCCURRENCE,cdm,visit_source_value,No,,,varchar(50),0,,"This field houses the verbatim value from the source data representing the kind of visit that took place (inpatient, outpatient, emergency, etc.)","If there is information about the kind of visit in the source data that value should be stored here. If a visit is an amalgamation of visits from the source then use a hierarchy to choose the visit source value, such as IP -> ER-> OP. This should line up with the logic chosen to determine how visits are created.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,100,,VISIT_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, +VISIT_OCCURRENCE,cdm,visit_source_value,No,,,varchar(50),0,,"This field houses the verbatim value from the source data representing the kind of visit that took place (inpatient, outpatient, emergency, etc.)","If there is information about the kind of visit in the source data that value should be stored here. If a visit is an amalgamation of visits from the source then use a hierarchy to choose the visit source value, such as IP -> ER-> OP. This should line up with the logic chosen to determine how visits are created.",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,Yes,10,,VISIT_CONCEPT_ID,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, VISIT_OCCURRENCE,cdm,visit_start_date,Yes,0,,date,0,,"For inpatient visits, the start date is typically the admission date. For outpatient visits the start date and end date will be the same.","When populating VISIT_START_DATE, you should think about the patient experience to make decisions on how to define visits. In the case of an inpatient visit this should be the date the patient was admitted to the hospital or institution. In all other cases this should be the date of the patient-provider interaction.",No,,,No,,,,,,,,,,,No,,,Yes,0,,No,,,No,,,No,,,,'19500101',1,,"DATEADD(dd,1,GETDATE())",1,,Yes,PERSON,BIRTH_DATETIME,1,,Yes,1,,Yes,VISIT_END_DATE,1,,Yes,1,,Yes,1,,Yes,,, VISIT_OCCURRENCE,cdm,visit_start_datetime,No,,,datetime,0,,,"If no time is given for the start date of a visit, set it to midnight (00:00:0000).",No,,,No,,,,,,,,,,,No,,,Yes,100,,No,,,No,,,No,,,,'19500101',1,,"DATEADD(dd,1,GETDATE())",1,,Yes,PERSON,BIRTH_DATETIME,1,,Yes,1,,Yes,VISIT_END_DATETIME,1,,Yes,1,,Yes,1,,Yes,,, VISIT_OCCURRENCE,cdm,visit_type_concept_id,Yes,0,,Integer,0,,"Use this field to understand the provenance of the visit record, or where the record comes from.","Populate this field based on the provenance of the visit record, as in whether it came from an EHR record or billing claim. [Accepted Concepts](https://athena.ohdsi.org/search-terms/terms?domain=Type+Concept&standardConcept=Standard&page=1&pageSize=15&query=).",No,,,Yes,0,,CONCEPT,CONCEPT_ID,Type Concept,0,,,,,Yes,0,,Yes,0,,Yes,0,,No,,,No,,,,,,,,,,,,,,,No,,,,,,,,,,,,,Yes,,, @@ -547,4 +547,4 @@ VOCABULARY,vocab,vocabulary_reference,No,,,varchar(255),0,,"External reference t available download of the about the vocabulary.",,No,,,No,,,,,,,,,,,No,,,No,100,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, VOCABULARY,vocab,vocabulary_version,No,,,varchar(255),0,,"Version of the Vocabulary as indicated in -the source.",,No,,,No,,,,,,,,,,,No,,,No,100,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +the source.",,No,,,No,,,,,,,,,,,No,,,No,100,,No,,,No,,,No,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, \ No newline at end of file diff --git a/vignettes/checkIndex.Rmd b/vignettes/checkIndex.Rmd index 326f1812..7f5c9ab2 100644 --- a/vignettes/checkIndex.Rmd +++ b/vignettes/checkIndex.Rmd @@ -34,20 +34,18 @@ above to navigate to the check's documentation page.\ - [isRequired](checks/isRequired.html) - [fkDomain](checks/fkDomain.html) - [fkClass](checks/fkClass.html) -- measurePersonCompleteness (PAGE UNDER CONSTRUCTION) +- [measurePersonCompleteness](checks/measurePersonCompleteness.html) - measureConditionEraCompleteness (PAGE UNDER CONSTRUCTION) -- isStandardValidConcept (PAGE UNDER CONSTRUCTION) -- measureValueCompleteness (PAGE UNDER CONSTRUCTION) -- standardConceptRecordCompleteness (PAGE UNDER CONSTRUCTION) -- sourceConceptRecordCompleteness (PAGE UNDER CONSTRUCTION) -- sourceValueCompleteness (PAGE UNDER CONSTRUCTION) -- plausibleValueLow (PAGE UNDER CONSTRUCTION) -- plausibleValueHigh (PAGE UNDER CONSTRUCTION) -- plausibleTemporalAfter (PAGE UNDER CONSTRUCTION) -- plausibleDuringLife (PAGE UNDER CONSTRUCTION) +- [isStandardValidConcept](checks/isStandardValidConcept.html) +- [measureValueCompleteness](checks/measureValueCompleteness.html) +- [standardConceptRecordCompleteness](checks/standardConceptRecordCompleteness.html) +- [sourceConceptRecordCompleteness](checks/sourceConceptRecordCompleteness.html) +- [sourceValueCompleteness](checks/sourceValueCompleteness.html) +- [plausibleValueLow](checks/plausibleValueLow.html) +- [plausibleValueHigh](checks/plausibleValueHigh.html) - withinVisitDates (PAGE UNDER CONSTRUCTION) - [plausibleAfterBirth](checks/plausibleAfterBirth.html) - [plausibleBeforeDeath](checks/plausibleBeforeDeath.html) - [plausibleStartBeforeEnd](checks/plausibleStartBeforeEnd.html) -- plausibleGender (PAGE UNDER CONSTRUCTION) +- plausibleGenderUseDescendants (PAGE UNDER CONSTRUCTION) - plausibleUnitConceptIds (PAGE UNDER CONSTRUCTION) diff --git a/vignettes/checks/isStandardValidConcept.Rmd b/vignettes/checks/isStandardValidConcept.Rmd index f0a8682d..0e1f596a 100644 --- a/vignettes/checks/isStandardValidConcept.Rmd +++ b/vignettes/checks/isStandardValidConcept.Rmd @@ -1,6 +1,6 @@ --- title: "isStandardValidConcept" -author: "" +author: "Katy Sadowski" date: "`r Sys.Date()`" output: html_document: @@ -14,7 +14,7 @@ output: **Context**: Verification\ **Category**: Conformance\ **Subcategory**: Value\ -**Severity**: +**Severity**: CDM convention ⚠\ ## Description @@ -23,24 +23,54 @@ The number and percent of records that do not have a standard, valid concept in ## Definition -- *Numerator*: -- *Denominator*: -- *Related CDM Convention(s)*: -- *CDM Fields/Tables*: -- *Default Threshold Value*: +- *Numerator*: The number of rows with an `X_concept_id` that exists in concept.concept_id but does not equal zero, and has a standard_concept != ‘S’ or a non-NULL invalid_reason. +- *Denominator*: The total number of rows in the table. +- *Related CDM Convention(s)*: All `X_concept_id` columns should contain a standard, valid concept, or 0: https://ohdsi.github.io/CommonDataModel/dataModelConventions.html#Mapping. In other words, all non-zero values in the `X_concept_id` column should exist in the concept_id column of the CONCEPT table and have CONCEPT.standard_concept = ‘S’ and CONCEPT.invalid_reason = NULL. +- *CDM Fields/Tables*: All tables with an `X_concept_id` column, and all `X_concept_id` columns in those tables. +- *Default Threshold Value*: 0 ## User Guidance +Failures of this check represent a violation of the fundamental CDM convention requiring all concept IDs to belong to the OMOP standard vocabulary. This is an essential convention in enabling standard analytics. If source codes have not been properly mapped to OMOP standard concepts in a CDM, studies designed using the OMOP standard vocabulary will return inaccurate results for that database. +### ETL Developers -### Violated rows query -```sql +A failure of this check indicates an issue with the concept mapping portion of your ETL, and must be resolved. Ensure that your ETL is only mapping source codes to standard, valid concepts (via the ‘Maps to’ relationship). Note as well that if no standard concept mapping exists for a source code, you MUST populate its `X_concept_id` column with 0. See the Book of OHDSI for additional guidance on the concept mapping process: https://ohdsi.github.io/TheBookOfOhdsi/ExtractTransformLoad.html#step-2-create-the-code-mappings -``` +You may inspect the failing rows using the following SQL: +```sql +SELECT + '@cdmTableName.@cdmFieldName' AS violating_field, + cdmTable.* +FROM @schema.@cdmTableName cdmTable + JOIN @vocabDatabaseSchema.concept co ON cdmTable.@cdmFieldName = co.concept_id +WHERE co.concept_id != 0 + AND (co.standard_concept != 'S' OR co.invalid_reason IS NOT NULL) +``` +You may build upon this query by joining the relevant `X_concept_id` and `X_source_concept_id` columns to the concept table and inspecting their names and vocabularies. If the `X_source_concept_id` correctly represents the source code in x_source_value, the fix will be a matter of ensuring your ETL is correctly using the concept_relationship table to map the source concept ID to a standard concept via the ‘Maps to’ relationship. If you are not populating the `X_source_concept_id` column and/or are using an intermediate concept mapping table, you may need to inspect the mappings in your mapper table to ensure they’ve been generated correctly using the ‘Maps to’ relationship for your CDM’s vocabulary version. -### ETL Developers +### Data Users +This check failure means that the failing rows will not be picked up in a standard OHDSI analysis. It is highly recommended to work with your ETL team or data provider, if possible, to resolve this issue. +However, you may work around it at your own risk by determining whether or not the affected rows are relevant for your analysis. Here’s an example query you could run to inspect failing rows in the condition_occurrence table: -### Data Users +```sql +SELECT + condition_concept_id AS violating_concept, + c1.concept_name AS violating_concept_name, + condition_source_concept_id AS source_concept, + c2.concept_name AS source_concept_name, + c2.vocabulary_id AS source_vocab, + condition_source_value, + COUNT(*) +FROM @cdmDatabaseSchema.condition_occurrence + JOIN @vocabDatabaseSchema.concept c1 ON condition_occurrence.condition_concept_id = c1.concept_id + LEFT JOIN @vocabDatabaseSchema.concept c2 ON condition_occurrence.condition_source_concept_id = c2.concept_id +WHERE c1.concept_id != 0 + AND (c1.standard_concept != 'S' OR c1.invalid_reason IS NOT NULL) +GROUP BY 1,2,3,4,5,6 +ORDER BY 7 DESC +``` +If you can confirm by inspecting the source concept and/or source value that the affected rows are not relevant for your analysis, you can proceed with your work and ignore the issue. However, especially if a large number of rows are impacted it’s recommended to act upon these failures as there could potentially be deeper issues with the ETL concept mapping process that need to be fixed. diff --git a/vignettes/checks/measurePersonCompleteness.Rmd b/vignettes/checks/measurePersonCompleteness.Rmd index 95b4a2cb..910c0d22 100644 --- a/vignettes/checks/measurePersonCompleteness.Rmd +++ b/vignettes/checks/measurePersonCompleteness.Rmd @@ -1,6 +1,6 @@ --- title: "measurePersonCompleteness" -author: "" +author: "Katy Sadowski" date: "`r Sys.Date()`" output: html_document: @@ -14,33 +14,59 @@ output: **Context**: Validation\ **Category**: Completeness\ **Subcategory**: \ -**Severity**: +**Severity**: CDM convention ⚠ Characterization ✔ \ ## Description -The number and percent of persons in the CDM that do not have at least one record in the @cdmTableName table +The number and percent of persons in the CDM that do not have at least one record in the @cdmTableName table. ## Definition -- *Numerator*: -- *Denominator*: -- *Related CDM Convention(s)*: -- *CDM Fields/Tables*: -- *Default Threshold Value*: +- *Numerator*: The number of persons with 0 rows in a given CDM table. +- *Denominator*: The total number of persons in the `person` table. +- *Related CDM Convention(s)*: Each Person needs to have at least one `observation_period` record. Otherwise, CDM conventions do not dictate any rules for person completeness. +- *CDM Fields/Tables*: By default, this check runs on all tables with a foreign key to the `person` table. +- *Default Threshold Value*: Set to 95 or 100 for most tables but 0 for `observation_period` ## User Guidance +For most tables, this check is a characterization of the completeness of various data types in the source data. However, in the case of `observation_period`, this check should actually be considered a CDM convention check as it is used to enforce the requirement that all persons have at least one observation period. +A failure of this check on the `observation_period` table is a serious issue as persons without an `observation_period` cannot be included in any standard OHDSI analysis. + +Run the following query to obtain a list of persons who had no data in a given table. From this list of person_ids you may join to other tables in the CDM to understand trends in these individuals' data which may provide clues as to the root cause of the issue. ### Violated rows query ```sql - +SELECT + cdmTable.* +FROM @cdmDatabaseSchema.person cdmTable + LEFT JOIN @schema.@cdmTableName cdmTable2 + ON cdmTable.person_id = cdmTable2.person_id +WHERE cdmTable2.person_id IS NULL ``` ### ETL Developers +#### Observation period +All persons in the CDM must have an observation period; OHDSI analytics tools only operate on persons with observable time, as represented by one or more observation periods. Persons missing observation periods may represent a bug in the ETL code which generates observation periods. Alternatively, some persons may have no observable time in the source data. These persons should be removed from the CDM. + +#### All other tables +Action on persons missing records in other clinical event tables will depend on the characteristics of the source database. In certain cases, missingness is expected – some persons may just not have a given type of data available in the source. In others, various ETL issues may result in persons missing records in a given event table: +- Mis-mapping of domains, resulting in the placement of records in the incorrect table +- Incorrect parsing of source data, resulting in loss of valid records +- Failure of an ETL step, resulting in an empty table + +If more persons than expected are missing data in a given table, run the violated rows SQL snippet to retrieve these persons’ person_ids, and inspect these persons’ other clinical event data in the CDM for trends. You may also use person_source_value to trace back to these persons’ source data to identify source data records potentially missed by the ETL. + +Note that in some cases, the failure threshold for this check may need to be adjusted according to completeness expectations for a given data source. + ### Data Users +Severe failures, such as unexpected nearly empty tables, must be fixed by the ETL team before a dataset can be used. Note as well that any person missing an `observation_period` will not be able to be included in any analysis using OHDSI tools. + +Failures with a result close to the specified failure threshold may be accepted, at your own risk and only if the result matches your understanding of the source data. The violated rows SQL may be used to inspect the full records for persons missing data in a given table in order to validate your expectations or point to potential issues in the ETL which need to be resolved. + diff --git a/vignettes/checks/measureValueCompleteness.Rmd b/vignettes/checks/measureValueCompleteness.Rmd index 470e285f..1defb0b1 100644 --- a/vignettes/checks/measureValueCompleteness.Rmd +++ b/vignettes/checks/measureValueCompleteness.Rmd @@ -1,6 +1,6 @@ --- title: "measureValueCompleteness" -author: "" +author: "Katy Sadowski" date: "`r Sys.Date()`" output: html_document: @@ -14,7 +14,7 @@ output: **Context**: Verification\ **Category**: Completeness\ **Subcategory**: \ -**Severity**: +**Severity**: Characterization ✔ ## Description @@ -23,24 +23,46 @@ The number and percent of records with a NULL value in the @cdmFieldName of the ## Definition -- *Numerator*: -- *Denominator*: -- *Related CDM Convention(s)*: -- *CDM Fields/Tables*: -- *Default Threshold Value*: +- *Numerator*: The number of rows with a NULL value in the field. +- *Denominator*: The total number of rows in the table. +- *Related CDM Convention(s)*: None. This check should be used to check local expectations about completeness of a field given characteristics of the source data. +- *CDM Fields/Tables*: All fields in all event tables. +- *Default Threshold Value*: 0 for required fields; 100 for all others ## User Guidance +This check’s primary purpose is to characterize completeness of non-required fields in the OMOP CDM. It is most useful when the failure threshold for each non-required field is customized to expectations based on the source data being transformed into OMOP. In this case, the check can be used to catch unexpected missingness due to ETL errors. However, in all cases, this check will serve as a useful characterization to help data users understand if a CDM contains the right data for a given analysis. +While the failure threshold is set to 0 for required fields, note that this is duplicative with the `isRequired` check - and fixing one failure will resolve the other! ### Violated rows query +Use this SQL query to inspect rows with a missing value in a given field: + ```sql +SELECT -``` +'@cdmTableName.@cdmFieldName' AS violating_field, + +cdmTable.* + +FROM @cdmDatabaseSchema.@cdmTableName cdmTable +WHERE cdmTable.@cdmFieldName IS NULL +``` ### ETL Developers +Failures of this check on fields required in the CDM specification are redundant with failures of `isRequired`. See [isRequired documentation](isRequired.html) for more information. + +ETL developers have 2 main options for the use of this check on non-required fields: +- The check threshold may be set to 100% for non-required fields such that the check will never fail. The check result can be used simply to understand completeness for these fields +- The check threshold may be set to an appropriate value corresponding to completeness expectations for each field given what’s available in the source data. The check may be disabled for fields known not to exist in the source data. Other fields may be set to whichever threshold is deemed worthy of investigation + +Unexpectedly missing values should be investigated for a potential root cause in the ETL. For expected missingness, rows that violate this check in non-required fields are acceptable but should be clearly communicated to data users so that they can know when and when not to expect data to be present in each field. To avoid confusion for users, however, thresholds should be modified to avoid check failures at expected levels. ### Data Users +This check informs you of the level of missing data in each column of the CDM. If data is missing in a required column, see the isRequired documentation for more information. + +The interpretation of a check failure on a non-required column will depend on the context. In some cases, the threshold for this check will have been very deliberately set, and any failure should be cause for concern unless justified and explained by your ETL provider. In other cases, even if the check fails it may not be worrisome if the check result is in line with your expectations given the source of the data. When in doubt, utilize the inspection query above to ensure you can explain the missing values. +Of course, if there is a failure on a non-required field you know that you will not need in your analysis (for example, missing drug quantity in an analysis not utilizing drug data), the check failure may be safely ignored. diff --git a/vignettes/checks/plausibleDuringLife.Rmd b/vignettes/checks/plausibleDuringLife.Rmd deleted file mode 100644 index 40326cd8..00000000 --- a/vignettes/checks/plausibleDuringLife.Rmd +++ /dev/null @@ -1,46 +0,0 @@ ---- -title: "plausibleDuringLife" -author: "" -date: "`r Sys.Date()`" -output: - html_document: - number_sections: yes - toc: yes ---- - -## Summary - -**Level**: FIELD\ -**Context**: Verification\ -**Category**: Plausibility\ -**Subcategory**: Temporal\ -**Severity**: - - -## Description -If yes, the number and percent of records with a date value in the @cdmFieldName field of the @cdmTableName table that occurs after death. - - -## Definition - -- *Numerator*: -- *Denominator*: -- *Related CDM Convention(s)*: -- *CDM Fields/Tables*: -- *Default Threshold Value*: - - -## User Guidance - - -### Violated rows query -```sql - -``` - - -### ETL Developers - - -### Data Users - diff --git a/vignettes/checks/plausibleGender.Rmd b/vignettes/checks/plausibleGenderUseDescendants.Rmd similarity index 100% rename from vignettes/checks/plausibleGender.Rmd rename to vignettes/checks/plausibleGenderUseDescendants.Rmd diff --git a/vignettes/checks/plausibleValueHigh.Rmd b/vignettes/checks/plausibleValueHigh.Rmd index cd5de2da..b6c810dc 100644 --- a/vignettes/checks/plausibleValueHigh.Rmd +++ b/vignettes/checks/plausibleValueHigh.Rmd @@ -1,6 +1,6 @@ --- title: "plausibleValueHigh" -author: "" +author: "Dymytry Dymshyts" date: "`r Sys.Date()`" output: html_document: @@ -14,7 +14,7 @@ output: **Context**: Verification\ **Category**: Plausibility\ **Subcategory**: Atemporal\ -**Severity**: +**Severity**: Characterization ✔ ## Description @@ -23,24 +23,66 @@ The number and percent of records with a value in the @cdmFieldName field of the ## Definition -- *Numerator*: -- *Denominator*: -- *Related CDM Convention(s)*: +- *Numerator*: The number of rows in a table where the checked field value is higher than some expected value. +- *Denominator*: The number of rows in a table where the checked field is not null. +- *Related CDM Convention(s)*: None. This check evaluates plausibility of values against common sense and known healthcare industry conventions. - *CDM Fields/Tables*: -- *Default Threshold Value*: + - All date and datetime fields (compared to today's date + 1 day) + - `PERSON.day_of_birth` (compared to 31) + - `PERSON.month_of_birth` (compared to 12) + - `PERSON.year_of_birth` (compared to this year + 1 year) + - `DRUG_EXPOSURE.refills` (compared to 24) + - `DRUG_EXPOSURE.days_supply` (compared to 365) + - `DRUG_EXPOSURE.quantity` (compared to 1095) +- *Default Threshold Value*: 1 ## User Guidance +This check counts the number of records that have a value in the specified field that is higher than some expected value. Failures of this check might represent true data anomalies, but especially in the case when the failure percentage is high, something may be afoot in the ETL pipeline. +Use this query to inspect rows with an implausibly high value: ### Violated rows query ```sql +SELECT + '@cdmTableName.@cdmFieldName' AS violating_field, + cdmTable.* +FROM @schema.@cdmTableName cdmTable +WHERE cdmTable.@cdmFieldName > @plausibleValueHigh +``` + +### ETL Developers + +The investigation approach may differ by the field being checked. For example, for `CONDITION_OCURRENCE.condition_start_date` you might look how much it differs in average, to find a clue as to what happened: + +```sql +SELECT + MEDIAN(DATEDIFF(day, condition_start_date, current_date)) +FROM condition_occurrence +WHERE condition_start_date > current_date +; ``` +Or the discrepancy be associated with specific attributes: +```sql +SELECT + co.condition_concept_id, + co.condition_type_concept_id, + co.condition_status_concept_id, + COUNT(1) +FROM condition_occurrence co +WHERE condition_start_date > current_date +GROUP BY co.condition_concept_id, co.condition_type_concept_id, co.condition_status_concept_id +ORDER BY COUNT(1) DESC +; +``` -### ETL Developers +There might be several different causes of future dates: typos in the source data, wrong data format used in the conversion, timezone issues in the ETL environment and/or database, etc. +For the `DRUG_EXPOSURE` values, there might be be typos, data processing bugs (for example, if days supply is calculated), or rare true cases when a prescription deviated from standard industry practices. -### Data Users +If the issue is determined to be related to ETL logic, it must be fixed. If it’s a source data issue, work with your data partners and users to determine the best remediation approach. `PERSON` rows with invalid birth dates should be removed from the CDM, as any analysis relying on age will be negatively impacted. Other implausible values should be explainable based on your understanding of the source data if they are to be retained. In some cases event rows may need to be dropped from the CDM if the implausible value is unexplainable and could cause downstream quality issues. Be sure to clearly document any data removal logic in your ETL specification. +### Data Users +The implication of a failure of this check depends on the count of errors and your need for the impacted columns. If it’s a small count, it might just be noise in the data which will unlikely impact an analysis. If the count is large, however, proceed carefully - events with future dates will likely be excluded from your analysis, and drugs with inflated supply values could throw off any analysis considering duration or patterns of treatment. \ No newline at end of file diff --git a/vignettes/checks/plausibleValueLow.Rmd b/vignettes/checks/plausibleValueLow.Rmd index 712a6391..455b7334 100644 --- a/vignettes/checks/plausibleValueLow.Rmd +++ b/vignettes/checks/plausibleValueLow.Rmd @@ -1,6 +1,6 @@ --- title: "plausibleValueLow" -author: "" +author: "Dymytry Dymshyts" date: "`r Sys.Date()`" output: html_document: @@ -14,7 +14,7 @@ output: **Context**: Verification\ **Category**: Plausibility\ **Subcategory**: Atemporal\ -**Severity**: +**Severity**: Characterization ✔ ## Description @@ -23,24 +23,36 @@ The number and percent of records with a value in the @cdmFieldName field of the ## Definition -- *Numerator*: -- *Denominator*: -- *Related CDM Convention(s)*: +- *Numerator*: The number of rows in a table where the checked field value is lower than some expected value. +- *Denominator*: The number of rows in a table where the checked field is not null. +- *Related CDM Convention(s)*: None. This check evaluates plausibility of values against common sense and known healthcare industry conventions. - *CDM Fields/Tables*: -- *Default Threshold Value*: + - All date and datetime fields (compared to 1/1/1950) + - `PERSON.day_of_birth` (compared to 1) + - `PERSON.month_of_birth` (compared to 1) + - `PERSON.year_of_birth` (compared to 1850) + - `PERSON.birth_datetime` (compared to 1/1/1850) + - `CDM_SOURCE.cdm_release_date`, `CDM_SOURCE.source_release_date` (compared to 1/1/2000) + - `DRUG_EXPOSURE.days_supply` (compared to 1) + - `DRUG_EXPOSURE.quantity` (compared to 0.0000001) + - `DRUG_EXPOSURE.refills` (compared to 0) + - `DEVICE_EXPOSURE.quantity`, `SPECIMEN.quantity`, `PROCEDURE_OCCURRENCE.quantity` (compared to 1) + - `DRUG_ERA.dose_value`, `DRUG_ERA.gap_days` (compared to 0) + - `DRUG_ERA.drug_exposure_count` (compared to 1) +- *Default Threshold Value*: 1 ## User Guidance +This check counts the number of records that have a value in the specified field that is lower than some expected value. Failures of this check might represent true data anomalies, but especially in the case when the failure percentage is high, something may be afoot in the ETL pipeline. +Use this query to inspect rows with an implausibly high value: ### Violated rows query ```sql - +SELECT + '@cdmTableName.@cdmFieldName' AS violating_field, + cdmTable.* +FROM @schema.@cdmTableName cdmTable +WHERE cdmTable.@cdmFieldName < @plausibleValueHigh ``` - - -### ETL Developers - - -### Data Users - +*See guidance for [plausibleValueHigh](plausibleValueHigh.html) for detailed investigation instructions (swapping out "high" for "low" and ">" for "<" where appropriate).* diff --git a/vignettes/checks/sourceConceptRecordCompleteness.Rmd b/vignettes/checks/sourceConceptRecordCompleteness.Rmd index d5efc4bc..bf7c0e01 100644 --- a/vignettes/checks/sourceConceptRecordCompleteness.Rmd +++ b/vignettes/checks/sourceConceptRecordCompleteness.Rmd @@ -1,6 +1,6 @@ --- title: "sourceConceptRecordCompleteness" -author: "" +author: "Katy Sadowski" date: "`r Sys.Date()`" output: html_document: @@ -14,7 +14,7 @@ output: **Context**: Verification\ **Category**: Completeness\ **Subcategory**: \ -**Severity**: +**Severity**: CDM convention ⚠\ ## Description @@ -23,24 +23,48 @@ The number and percent of records with a value of 0 in the source concept field ## Definition -- *Numerator*: -- *Denominator*: -- *Related CDM Convention(s)*: -- *CDM Fields/Tables*: -- *Default Threshold Value*: +- *Numerator*: The number of rows with a value of 0 in the `X_source_concept_id` source concept field. In the case of `MEASUREMENT.unit_source_concept_id` and `OBSERVATION.unit_source_concept_id`, the number of rows with a value of 0 in the `X_source_concept_id` source concept field AND a non-NULL value_as_number. +- *Denominator*: The total number of rows in the table. In the case of `MEASUREMENT.unit_source_concept_id` and `OBSERVATION.unit_source_concept_id`, the number of rows with non-NULL value_as_number. +- *Related CDM Convention(s)*: [Source concept mapping](https://ohdsi.github.io/CommonDataModel/dataModelConventions.html#Fields) +- *CDM Fields/Tables*: All source concept ID (`X_source_concept_id`) columns in all event tables. +- *Default Threshold Value*: 10 for primary source concept ID columns in condition, drug, measurement, procedure, device, and observation tables; 100 for all other source concept ID columns. ## User Guidance +Source concept mapping is an important part of the OMOP concept mapping process which allows data users insight into the provenance of the data they are analyzing. It’s important to populate the source concept ID field for all source values that exist in the OMOP vocabulary. Failures of this check should be well-understood and documented so that data users can plan accordingly in the case missing data might impact their analysis. +### ETL Developers +Recall that the `X_source_concept_id` columns should contain the OMOP concept representing the exact code used in the source data for a given record: “If the is coded in the source data using an OMOP supported vocabulary put the concept id representing the source value here.” -### Violated rows query -```sql +A failure of this check usually indicates a failure to map a source value to an OMOP concept. In some cases, such a failure can and should be remediated in the concept-mapping step of the ETL. In other cases, it may represent a mapping that currently is not possible to implement. + +To investigate the failure, run the following query: +```sql +SELECT + concept.concept_name AS standard_concept_name, + cdmTable.X_concept_id, -- standard concept ID field for the table + c2.concept_name AS source_value_concept_name, + cdmTable.X_source_value, -- source value field for the table + COUNT(*) +FROM @cdmDatabaseSchema.@cdmTableName cdmTable +LEFT JOIN @vocabDatabaseSchema.concept ON concept.concept_id = cdmTable.X_concept_id +-- WARNING this join may cause fanning if a source value exists in multiple vocabularies +LEFT JOIN @vocabDatabaseSchema.concept c2 ON concept.concept_code = cdmTable.X_source_value +AND c2.domain_id = +WHERE cdmTable.@cdmFieldName = 0 +-- AND cdmTable.value_as_number IS NOT NULL -- uncomment for unit_concept_id checks +GROUP BY 1,2,3 +ORDER BY 4 DESC ``` +The query results will give you a summary of the source codes which failed to map to an OMOP concept. Inspecting this data should give you an initial idea of what might be going on. -### ETL Developers +If source values return legitimate matches on concept_code, it’s possible that there is an error in the concept mapping step of your ETL. Please note that while the `X_source_concept_id` fields are technically not required, it is highly recommended to populate them with OMOP concepts whenever possible. This will greatly aid analysts in understanding the provenance of the data. +If source values do NOT return matches on concept_code and you are NOT handling concept mapping locally for a non-OMOP source vocabulary, then you likely have a malformed source code or one that does not exist in the OMOP vocabulary. Please see the documentation in the standardConceptRecordCompleteness page for instructions on how to handle this scenario. ### Data Users +Since most standard OHDSI analytic workflows rely on the standard concept field and not the source concept field, failures of this check will not necessarily impact your analysis. However, if your analysis depends on understanding source coding practices or on codes you know may not be fully mapped to OMOP standard concepts, then this will be a critical check failure to understand. +Utilize the investigation queries above to understand the scope and impact of the mapping failures on your specific analytic use case. If none of the affected codes seem to be relevant for your analysis, it may be acceptable to ignore the failure. However, since it is not always possible to understand exactly what a given source value represents, you should proceed with caution and confirm any findings with your ETL provider if possible. diff --git a/vignettes/checks/sourceValueCompleteness.Rmd b/vignettes/checks/sourceValueCompleteness.Rmd index 20c6706f..6dc5e3b2 100644 --- a/vignettes/checks/sourceValueCompleteness.Rmd +++ b/vignettes/checks/sourceValueCompleteness.Rmd @@ -1,6 +1,6 @@ --- title: "sourceValueCompleteness" -author: "" +author: "Jared Houghtaling, Clair Blacketer" date: "`r Sys.Date()`" output: html_document: @@ -14,7 +14,7 @@ output: **Context**: Verification\ **Category**: Completeness\ **Subcategory**: \ -**Severity**: +**Severity**: CDM convention ⚠ ## Description @@ -23,24 +23,38 @@ The number and percent of distinct source values in the @cdmFieldName field of t ## Definition -- *Numerator*: -- *Denominator*: -- *Related CDM Convention(s)*: -- *CDM Fields/Tables*: -- *Default Threshold Value*: +- *Numerator*: Distinct `X_source_value` entries where the corresponding standard x_concept_id field is 0. +- *Denominator*: Total distinct `X_source_value` entries, including NULL, in the respective event table. +- *Related CDM Convention(s)*: The OMOP Common Data Model specifies that codes that are present in a native database should be mapped to standard concepts using either the intrinsic mappings defined in the standard vocabularies or extrinsic mappings defined by the data owner or ETL development team. Note also that variations of this check logic are also used in the [EHDEN CDM Inspection Report](https://github.com/EHDEN/CdmInspection) package, as well as the [AresIndexer](https://github.com/OHDSI/AresIndexer) package for generating indices of unmapped codes. +- *CDM Fields/Tables*: Runs on all event tables that have `X_source_value` fields. +- *Default Threshold Value*: + - 10 for critical event tables' `X_source_value` fields (condition, measurement, procedure, drug, visit) + - 100 for all other fields - to be adjusted based on source-specific expectations ## User Guidance +This check will look at all distinct source values in the specified field and calculate how many are mapped to a standard concept of 0. This check should be used in conjunction with the [standardConceptRecordCompleteness](checks/standardConceptRecordCompleteness.html) check to identify potential mapping issues in the ETL. \ +This check is a good measure of the overall mapping rate within each domain. For example, a table may have high standardConceptRecordCompleteness (that is, a large percentage of records with a non-zero standard concept) but a low score on this check. This would indicate that the "long tail" of rarer codes have not been mapped while more common codes have good mapping coverage. It is always important to interrogate the results of these two checks together to ensure complete understanding of vocabulary mapping in your CDM. + +The following SQL can be used to summarize unmapped source values by record count in a given CDM table: ### Violated rows query ```sql - +SELECT DISTINCT + cdmTable.@cdmFieldName, + COUNT(*) +FROM @cdmDatabaseSchema.@cdmTableName cdmTable +WHERE cdmTable.@standardConceptFieldName = 0 +GROUP BY 1 +ORDER BY 2 DESC ``` ### ETL Developers - +Fails of this check are (most often) related directly to semantic mapping. First, the ETL developer should investigate if a source vocabulary is present in the native data that was not accounted for in the ETL document and/or code. This is most likely if the unmapped source values are codes rather than text values. Second, the source-to-concept-map file or table should be updated to link the unmapped source values with domain-appropriate concepts. ### Data Users +When this check fails, source data granularity is being lost; not all of the information related to a particular event or modifier is being captured in OMOP CDM format. Although the information about an event may exist in the source value field, it cannot easily be used in downstream analytics processes that rely on standard OMOP concepts. +**Please see the [standardConceptRecordCompleteness](standardConceptRecordCompleteness.html) page for a much more detailed overview of handling mapping quality issues in your OMOP CDM.** \ No newline at end of file diff --git a/vignettes/checks/standardConceptRecordCompleteness.Rmd b/vignettes/checks/standardConceptRecordCompleteness.Rmd index 2aa27587..6a67aab8 100644 --- a/vignettes/checks/standardConceptRecordCompleteness.Rmd +++ b/vignettes/checks/standardConceptRecordCompleteness.Rmd @@ -1,6 +1,6 @@ --- title: "standardConceptRecordCompleteness" -author: "" +author: "Katy Sadowski" date: "`r Sys.Date()`" output: html_document: @@ -14,7 +14,7 @@ output: **Context**: Verification\ **Category**: Completeness\ **Subcategory**: \ -**Severity**: +**Severity**: CDM convention ⚠\ ## Description @@ -23,24 +23,78 @@ The number and percent of records with a value of 0 in the standard concept fiel ## Definition -- *Numerator*: -- *Denominator*: -- *Related CDM Convention(s)*: -- *CDM Fields/Tables*: -- *Default Threshold Value*: +- *Numerator*: The number of rows with a value of 0 in the `X_concept_id` standard concept field. (**NOTE** in the case of `measurement.unit_concept_id` and `observation.unit_concept_id`, the number of rows with a value of 0 in the `X_concept_id` standard concept field AND a non-NULL value_as_number.) +- *Denominator*: The total number of rows in the table. (**NOTE** in the case of `measurement.unit_concept_id` and `observation.unit_concept_id`, the number of rows with non-NULL `value_as_number`.) +- *Related CDM Convention(s)*: [Standard concept mapping](https://ohdsi.github.io/CommonDataModel/dataModelConventions.html#Fields) +- *CDM Fields/Tables*: All standard concept ID (`X_concept_id`) columns in all event tables. +- *Default Threshold Value*: + - 0 for type concept fields and standard concept fields in era tables + - 5 for most standard concept fields in clinical event tables + - 100 for fields more susceptible to specific ETL implementation context ## User Guidance +Standard concept mapping is one of the most fundamental conventions of the OMOP CDM. It enables standardized analysis across diverse data sources and allows users to abstract away the tedium of traversing source vocabularies when building phenotypes. As such, it is highly recommended to map as many concepts in your source as possible. Failures of this check should be well-understood and documented so that data users can plan accordingly in the case missing data might impact their analysis. -### Violated rows query +### ETL Developers +A failure of this check usually indicates a failure to map a source value to a standard OMOP concept. In some cases, such a failure can and should be remediated in the concept-mapping step of the ETL. In other cases, it may represent a mapping that currently is not possible to implement. + +To investigate the failure, run the following query: + ```sql +SELECT + concept_name, + cdmTable.X_source_concept_id, -- source concept ID field for the table + cdmTable.X_source_value, -- source value field for the table + COUNT(*) +FROM @cdmDatabaseSchema.@cdmTableName cdmTable +LEFT JOIN @vocabDatabaseSchema.concept ON concept.concept_id = cdmTable.X_source_concept_id +WHERE cdmTable.@cdmFieldName = 0 +-- AND cdmTable.value_as_number IS NOT NULL -- uncomment for unit_concept_id checks +GROUP BY 1,2,3 +ORDER BY 4 DESC +``` + +This will give you a summary of the source codes which failed to map to an OMOP standard concept. Inspecting this data should give you an initial idea of what might be going on. + + - If the query returns a source value, source concept ID, and concept name for a given code, run the following query to confirm that a standard concept mapping exists for the source concept ID: +```sql +SELECT + concept_id AS standard_concept_mapping +FROM @vocabDatabaseSchema.concept_relationship +JOIN @vocabDatabaseSchema.concept ON concept.concept_id = c oncept_relationship.concept_id_2 + AND relationship_id = ‘Maps to’ +WHERE concept_relationship.concept_id_1 = ``` + - If no results are returned, consider whether the source concept ID is part of the OMOP vocabularies. If it is, then there is likely a vocabulary issue which should be reported. If it is not (i.e., it is a local concept), then there is likely an issue with your local source-to-concept mapping -### ETL Developers + - If the investigation query returns a source value and source concept ID but no concept name, this indicates the source concept ID does not exist in your concept table. This may be expected if your ETL includes local source-to-concept mappings. If not, then your ETL has assigned a malformed source concept ID and will need to be debugged + + - If the investigation query returns a source value but no source concept ID (or a source concept ID of 0), run the following query to search for the source value in the OMOP vocabulary (note that if your ETL includes local mappings and the code in question is known not to exist in OMOP, you should search your local mapping table/config instead): + +```sql +-- may return false positives if the same value exists in multiple vocabularies +-- only applicable in the case where the source value column is populated only with a vocabulary code +SELECT + * +FROM @vocabDatabaseSchema.concept +WHERE concept_code = +``` + + - If no result is returned, consider whether the source value may be a malformed version of a legitimate code (for example, sometimes ICD10-CM codes do not contain a “.” in source data). If you can confirm that the code is properly formatted, then you have a source code which does not exist in the OMOP vocabulary. If you believe the code was omitted from the vocabulary in error, please report this issue to the vocabulary team. Otherwise, the short-term course of action will be to generate a mapping for the code locally and implement the mapping in your ETL. For the longer term, the vocabulary team provides a workflow to submit new vocabularies for inclusion in the OMOP vocabularies + - Note that in some cases, you will find that no standard concept exists to which to map your source code. In this case, the standard concept ID field should be left as 0 in the short term; in the longer term please work with the vocabulary team to address this gap as recommended above + + - Finally, if the investigation query returns no source value, you must trace the relevant record(s) back to their source and confirm if the missing value is expected. If not, identify and fix the related issue in your ETL. If the record legitimately has no value/code in the source data, then the standard concept ID may be left as 0. However, in some cases these “code-less” records represent junk data which should be filtered out in the ETL. The proper approach will be context-dependent + - Note in the special case of unitless measurements/observations, the unit_concept_id field should NOT be coded as 0 and rather should be left NULL (the unit_concept_id fields are optional in the CDM spec) + +It is important to note that records with a 0 standard concept ID field will be unusable in standard OHDSI analyses and thus should only be preserved if there is truly no standard concept ID for a given record. Depending on the significance of the records in question, one should consider removing them from the dataset; however, this choice will depend on a variety of context-specific factors and should be made carefully. Either way, the presence/absence of these unmappable records and an explanation for why they could not be mapped should be clearly documented in the ETL documentation. ### Data Users +Since unmapped records will not be picked up in standard OHDSI analytic workflows, this is an important check failure to understand. Utilize the investigation queries above to understand the scope and impact of the mapping failures on your specific analytic use case. If none of the affected codes seem to be relevant for your analysis, it may be acceptable to ignore the failure. However, since it is not always possible to understand exactly what a given source value represents, you should proceed with caution and confirm any findings with your ETL provider if possible. + +In the case where the source concept ID column is populated with a legitimate OMOP concept, it will be possible to query this column instead of the standard concept column in your analyses. However, doing so will require building source concept sets and as such losing the power of the OMOP standard vocabularies in defining comprehensive, generalizable cohort definitions. From 45e38415bb3af40566c13e8f6f3312d885326655 Mon Sep 17 00:00:00 2001 From: Katy Sadowski Date: Sat, 29 Jun 2024 16:46:29 -0400 Subject: [PATCH 21/43] add withinvisitdates, fix typos --- _pkgdown.yml | 2 + docs/404.html | 3 + docs/LICENSE-text.html | 3 + docs/articles/AddNewCheck.html | 5 +- docs/articles/CheckStatusDefinitions.html | 5 +- docs/articles/CheckTypeDescriptions.html | 5 +- docs/articles/DataQualityDashboard.html | 5 +- docs/articles/DqdForCohorts.html | 5 +- docs/articles/SqlOnly.html | 5 +- docs/articles/Thresholds.html | 5 +- docs/articles/checkIndex.html | 7 +- docs/articles/checks/cdmDatatype.html | 5 +- docs/articles/checks/cdmField.html | 5 +- docs/articles/checks/cdmTable.html | 5 +- docs/articles/checks/fkClass.html | 5 +- docs/articles/checks/fkDomain.html | 5 +- docs/articles/checks/isForeignKey.html | 5 +- docs/articles/checks/isPrimaryKey.html | 5 +- docs/articles/checks/isRequired.html | 5 +- .../checks/isStandardValidConcept.html | 36 +++--- .../measureConditionEraCompleteness.html | 5 +- .../checks/measurePersonCompleteness.html | 43 ++++--- .../checks/measureValueCompleteness.html | 17 ++- docs/articles/checks/plausibleAfterBirth.html | 5 +- .../articles/checks/plausibleBeforeDeath.html | 5 +- .../checks/plausibleGenderUseDescendants.html | 5 +- .../checks/plausibleStartBeforeEnd.html | 5 +- .../checks/plausibleTemporalAfter.html | 5 +- .../checks/plausibleUnitConceptIds.html | 5 +- docs/articles/checks/plausibleValueHigh.html | 5 +- docs/articles/checks/plausibleValueLow.html | 5 +- .../sourceConceptRecordCompleteness.html | 24 ++-- .../checks/sourceValueCompleteness.html | 10 +- .../standardConceptRecordCompleteness.html | 22 ++-- docs/articles/checks/withinVisitDates.html | 107 ++++++++++++++++-- docs/articles/index.html | 3 + docs/authors.html | 3 + docs/index.html | 3 + docs/news/index.html | 3 + docs/pkgdown.yml | 2 +- .../reference/convertJsonResultsFileCase.html | 3 + docs/reference/dot-evaluateThresholds.html | 3 + docs/reference/dot-getCheckId.html | 3 + docs/reference/dot-processCheck.html | 3 + docs/reference/dot-recordResult.html | 3 + docs/reference/dot-runCheck.html | 3 + docs/reference/dot-summarizeResults.html | 3 + docs/reference/dot-writeResultsToCsv.html | 3 + docs/reference/dot-writeResultsToJson.html | 3 + docs/reference/dot-writeResultsToTable.html | 3 + docs/reference/executeDqChecks.html | 3 + docs/reference/index.html | 3 + docs/reference/listDqChecks.html | 3 + docs/reference/reEvaluateThresholds.html | 3 + docs/reference/viewDqDashboard.html | 3 + docs/reference/writeDBResultsToJson.html | 3 + docs/reference/writeJsonResultsToCsv.html | 3 + docs/reference/writeJsonResultsToTable.html | 3 + inst/csv/OMOP_CDMv5.2_Check_Descriptions.csv | 34 +++--- inst/csv/OMOP_CDMv5.3_Check_Descriptions.csv | 34 +++--- inst/csv/OMOP_CDMv5.4_Check_Descriptions.csv | 34 +++--- vignettes/checkIndex.Rmd | 2 +- vignettes/checks/isStandardValidConcept.Rmd | 9 +- .../checks/measurePersonCompleteness.Rmd | 17 +-- vignettes/checks/measureValueCompleteness.Rmd | 8 +- .../sourceConceptRecordCompleteness.Rmd | 10 +- vignettes/checks/sourceValueCompleteness.Rmd | 4 +- .../standardConceptRecordCompleteness.Rmd | 4 +- vignettes/checks/withinVisitDates.Rmd | 39 +++++-- 69 files changed, 464 insertions(+), 193 deletions(-) diff --git a/_pkgdown.yml b/_pkgdown.yml index acbe616b..822c5d5f 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -90,6 +90,8 @@ navbar: href: articles/checks/plausibleValueHigh.html - text: plausibleValueLow href: articles/checks/plausibleValueLow.html + - text: withinVisitDates + href: articles/checks/withinVisitDates.html hades: text: hadesLogo href: https://ohdsi.github.io/Hades diff --git a/docs/404.html b/docs/404.html index 75158e0f..7c89b76d 100644 --- a/docs/404.html +++ b/docs/404.html @@ -144,6 +144,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • diff --git a/docs/LICENSE-text.html b/docs/LICENSE-text.html index d734fc77..aaecd811 100644 --- a/docs/LICENSE-text.html +++ b/docs/LICENSE-text.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/docs/articles/AddNewCheck.html b/docs/articles/AddNewCheck.html index 11920ea1..bc027dea 100644 --- a/docs/articles/AddNewCheck.html +++ b/docs/articles/AddNewCheck.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -177,7 +180,7 @@

    Add a New Data Quality Check

    Don Torok

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/AddNewCheck.rmd diff --git a/docs/articles/CheckStatusDefinitions.html b/docs/articles/CheckStatusDefinitions.html index 34423e5e..919cfebc 100644 --- a/docs/articles/CheckStatusDefinitions.html +++ b/docs/articles/CheckStatusDefinitions.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -178,7 +181,7 @@

    Check Status Definitions

    Dmitry Ilyn, Maxim Moinat

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/CheckStatusDefinitions.rmd diff --git a/docs/articles/CheckTypeDescriptions.html b/docs/articles/CheckTypeDescriptions.html index ced9cdd5..6e89315a 100644 --- a/docs/articles/CheckTypeDescriptions.html +++ b/docs/articles/CheckTypeDescriptions.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -178,7 +181,7 @@

    Data Quality Check Type Definitions

    Clair Blacketer

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/CheckTypeDescriptions.rmd diff --git a/docs/articles/DataQualityDashboard.html b/docs/articles/DataQualityDashboard.html index d370a9b3..f1cf33d8 100644 --- a/docs/articles/DataQualityDashboard.html +++ b/docs/articles/DataQualityDashboard.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -178,7 +181,7 @@

    Getting Started

    Clair Blacketer

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/DataQualityDashboard.rmd diff --git a/docs/articles/DqdForCohorts.html b/docs/articles/DqdForCohorts.html index cf5cd2fb..8db9ac2c 100644 --- a/docs/articles/DqdForCohorts.html +++ b/docs/articles/DqdForCohorts.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -178,7 +181,7 @@

    Running the DQD on a Cohort

    Clair Blacketer

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/DqdForCohorts.rmd diff --git a/docs/articles/SqlOnly.html b/docs/articles/SqlOnly.html index 68151ee8..1c4a7d4c 100644 --- a/docs/articles/SqlOnly.html +++ b/docs/articles/SqlOnly.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -178,7 +181,7 @@

    Running the DQD in SqlOnly mode

    Maxim Moinat

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/SqlOnly.rmd diff --git a/docs/articles/Thresholds.html b/docs/articles/Thresholds.html index d35f64c1..1d94bcf7 100644 --- a/docs/articles/Thresholds.html +++ b/docs/articles/Thresholds.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -178,7 +181,7 @@

    Failure Thresholds and How to Change Them

    Clair Blacketer

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/Thresholds.rmd diff --git a/docs/articles/checkIndex.html b/docs/articles/checkIndex.html index aa0590ac..289565c4 100644 --- a/docs/articles/checkIndex.html +++ b/docs/articles/checkIndex.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -178,7 +181,7 @@

    Index

    Katy Sadowski

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checkIndex.Rmd @@ -241,7 +244,7 @@

    Checks
  • sourceValueCompleteness
  • plausibleValueLow
  • plausibleValueHigh
    • -
  • withinVisitDates (PAGE UNDER CONSTRUCTION)
    • +
  • withinVisitDates
  • plausibleAfterBirth
  • plausibleBeforeDeath
  • plausibleStartBeforeEnd
    • diff --git a/docs/articles/checks/cdmDatatype.html b/docs/articles/checks/cdmDatatype.html index fb89677f..c0b9d6c7 100644 --- a/docs/articles/checks/cdmDatatype.html +++ b/docs/articles/checks/cdmDatatype.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +

  • @@ -178,7 +181,7 @@

    cdmDatatype

    Katy Sadowski

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/cdmDatatype.Rmd diff --git a/docs/articles/checks/cdmField.html b/docs/articles/checks/cdmField.html index 573aa1d3..99406a82 100644 --- a/docs/articles/checks/cdmField.html +++ b/docs/articles/checks/cdmField.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -178,7 +181,7 @@

    cdmField

    Heidi Schmidt, Katy Sadowski

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/cdmField.Rmd diff --git a/docs/articles/checks/cdmTable.html b/docs/articles/checks/cdmTable.html index 1e3b523b..3150fd31 100644 --- a/docs/articles/checks/cdmTable.html +++ b/docs/articles/checks/cdmTable.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -178,7 +181,7 @@

    cdmTable

    John Gresh, Katy Sadowski

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/cdmTable.Rmd diff --git a/docs/articles/checks/fkClass.html b/docs/articles/checks/fkClass.html index b133a025..2904a2e3 100644 --- a/docs/articles/checks/fkClass.html +++ b/docs/articles/checks/fkClass.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -178,7 +181,7 @@

    fkClass

    Clair Blacketer, Katy Sadowski

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/fkClass.Rmd diff --git a/docs/articles/checks/fkDomain.html b/docs/articles/checks/fkDomain.html index 07bae9eb..d90b3fb0 100644 --- a/docs/articles/checks/fkDomain.html +++ b/docs/articles/checks/fkDomain.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -178,7 +181,7 @@

    fkDomain

    Clair Blacketer, Katy Sadowski

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/fkDomain.Rmd diff --git a/docs/articles/checks/isForeignKey.html b/docs/articles/checks/isForeignKey.html index 60484a17..3983b9fa 100644 --- a/docs/articles/checks/isForeignKey.html +++ b/docs/articles/checks/isForeignKey.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -178,7 +181,7 @@

    isForeignKey

    Dmytry Dymshyts, Katy Sadowski

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/isForeignKey.Rmd diff --git a/docs/articles/checks/isPrimaryKey.html b/docs/articles/checks/isPrimaryKey.html index 726e0037..8fdef03a 100644 --- a/docs/articles/checks/isPrimaryKey.html +++ b/docs/articles/checks/isPrimaryKey.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -178,7 +181,7 @@

    isPrimaryKey

    John Gresh, Katy Sadowski

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/isPrimaryKey.Rmd diff --git a/docs/articles/checks/isRequired.html b/docs/articles/checks/isRequired.html index 1e2f089e..994448e6 100644 --- a/docs/articles/checks/isRequired.html +++ b/docs/articles/checks/isRequired.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -178,7 +181,7 @@

    isRequired

    Katy Sadowski

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/isRequired.Rmd diff --git a/docs/articles/checks/isStandardValidConcept.html b/docs/articles/checks/isStandardValidConcept.html index 035c1c0f..6cbd6eca 100644 --- a/docs/articles/checks/isStandardValidConcept.html +++ b/docs/articles/checks/isStandardValidConcept.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -175,10 +178,10 @@

    isStandardValidConcept

    -

    Katy -Sadowski

    +

    Stephanie Hong, +Katy Sadowski

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/isStandardValidConcept.Rmd @@ -205,18 +208,14 @@

    Definitionhttps://ohdsi.github.io/CommonDataModel/dataModelConventions.html#Mapping. -In other words, all non-zero values in the X_concept_id -column should exist in the concept_id column of the CONCEPT table and -have CONCEPT.standard_concept = ‘S’ and CONCEPT.invalid_reason = -NULL.

    • +columns should contain a standard, valid concept, or 0: https://ohdsi.github.io/CommonDataModel/dataModelConventions.html#Mapping.
  • CDM Fields/Tables: All tables with an X_concept_id column, and all X_concept_id @@ -256,13 +255,14 @@

    ETL DevelopersX_concept_id and X_source_concept_id columns to the concept table and inspecting their names and vocabularies. If the X_source_concept_id correctly represents the source code in -x_source_value, the fix will be a matter of ensuring your ETL is -correctly using the concept_relationship table to map the source concept -ID to a standard concept via the ‘Maps to’ relationship. If you are not -populating the X_source_concept_id column and/or are using -an intermediate concept mapping table, you may need to inspect the -mappings in your mapper table to ensure they’ve been generated correctly -using the ‘Maps to’ relationship for your CDM’s vocabulary version.

    +X_source_value, the fix will be a matter of ensuring your +ETL is correctly using the concept_relationship table to map the source +concept ID to a standard concept via the ‘Maps to’ relationship. If you +are not populating the X_source_concept_id column and/or +are using an intermediate concept mapping table, you may need to inspect +the mappings in your mapper table to ensure they’ve been generated +correctly using the ‘Maps to’ relationship for your CDM’s vocabulary +version.

    Data Users diff --git a/docs/articles/checks/measureConditionEraCompleteness.html b/docs/articles/checks/measureConditionEraCompleteness.html index 618b8b80..e8ce2a9f 100644 --- a/docs/articles/checks/measureConditionEraCompleteness.html +++ b/docs/articles/checks/measureConditionEraCompleteness.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +

  • @@ -177,7 +180,7 @@

    measureConditionEraCompleteness

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/measureConditionEraCompleteness.Rmd diff --git a/docs/articles/checks/measurePersonCompleteness.html b/docs/articles/checks/measurePersonCompleteness.html index 0cd476a6..4e61a71d 100644 --- a/docs/articles/checks/measurePersonCompleteness.html +++ b/docs/articles/checks/measurePersonCompleteness.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -178,7 +181,7 @@

    measurePersonCompleteness

    Katy Sadowski

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/measurePersonCompleteness.Rmd @@ -208,17 +211,17 @@

    DefinitionUser GuidanceAll other tables -- Incorrect parsing of source data, resulting in loss of valid records - -Failure of an ETL step, resulting in an empty table

    +in persons missing records in a given event table:

    +
      +
    • Mis-mapping of domains, resulting in the placement of records in the +incorrect table
      +
      • +
    • Incorrect parsing of source data, resulting in loss of valid +records
      • +
    • Failure of an ETL step, resulting in an empty table
      • +

    If more persons than expected are missing data in a given table, run the violated rows SQL snippet to retrieve these persons’ person_ids, and inspect these persons’ other clinical event data in the CDM for trends. -You may also use person_source_value to trace back to these persons’ -source data to identify source data records potentially missed by the -ETL.

    +You may also use person_source_value to trace back to these +persons’ source data to identify source data records potentially missed +by the ETL.

    Note that in some cases, the failure threshold for this check may need to be adjusted according to completeness expectations for a given data source.

    @@ -289,8 +296,8 @@

    Data UsersplausibleValueLow

    • +
  • + withinVisitDates +
  • @@ -178,7 +181,7 @@

    measureValueCompleteness

    Katy Sadowski

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/measureValueCompleteness.Rmd @@ -237,14 +240,10 @@

    Violated rows query
    SELECT  
-
-'@cdmTableName.@cdmFieldName' AS violating_field,  
-
-cdmTable.*  
-
-FROM @cdmDatabaseSchema.@cdmTableName cdmTable       
-
-WHERE cdmTable.@cdmFieldName IS NULL
    + '@cdmTableName.@cdmFieldName' AS violating_field, + cdmTable.* +FROM @cdmDatabaseSchema.@cdmTableName cdmTable +WHERE cdmTable.@cdmFieldName IS NULL

    ETL Developers diff --git a/docs/articles/checks/plausibleAfterBirth.html b/docs/articles/checks/plausibleAfterBirth.html index 7c68c5fb..610bc1e4 100644 --- a/docs/articles/checks/plausibleAfterBirth.html +++ b/docs/articles/checks/plausibleAfterBirth.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +

  • @@ -178,7 +181,7 @@

    plausibleAfterBirth

    Maxim Moinat, Katy Sadowski

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/plausibleAfterBirth.Rmd diff --git a/docs/articles/checks/plausibleBeforeDeath.html b/docs/articles/checks/plausibleBeforeDeath.html index 34529a4e..2e91b839 100644 --- a/docs/articles/checks/plausibleBeforeDeath.html +++ b/docs/articles/checks/plausibleBeforeDeath.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -178,7 +181,7 @@

    plausibleBeforeDeath

    Maxim Moinat

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/plausibleBeforeDeath.Rmd diff --git a/docs/articles/checks/plausibleGenderUseDescendants.html b/docs/articles/checks/plausibleGenderUseDescendants.html index c2765e30..c4e8155a 100644 --- a/docs/articles/checks/plausibleGenderUseDescendants.html +++ b/docs/articles/checks/plausibleGenderUseDescendants.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -177,7 +180,7 @@

    plausibleGender

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/plausibleGenderUseDescendants.Rmd diff --git a/docs/articles/checks/plausibleStartBeforeEnd.html b/docs/articles/checks/plausibleStartBeforeEnd.html index cc874c5e..f80c0a9b 100644 --- a/docs/articles/checks/plausibleStartBeforeEnd.html +++ b/docs/articles/checks/plausibleStartBeforeEnd.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -178,7 +181,7 @@

    plausibleStartBeforeEnd

    Maxim Moinat

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/plausibleStartBeforeEnd.Rmd diff --git a/docs/articles/checks/plausibleTemporalAfter.html b/docs/articles/checks/plausibleTemporalAfter.html index 00b0b159..3a01db40 100644 --- a/docs/articles/checks/plausibleTemporalAfter.html +++ b/docs/articles/checks/plausibleTemporalAfter.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -177,7 +180,7 @@

    plausibleTemporalAfter

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/plausibleTemporalAfter.Rmd diff --git a/docs/articles/checks/plausibleUnitConceptIds.html b/docs/articles/checks/plausibleUnitConceptIds.html index a6f1d972..6855948c 100644 --- a/docs/articles/checks/plausibleUnitConceptIds.html +++ b/docs/articles/checks/plausibleUnitConceptIds.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -177,7 +180,7 @@

    plausibleUnitConceptIds

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/plausibleUnitConceptIds.Rmd diff --git a/docs/articles/checks/plausibleValueHigh.html b/docs/articles/checks/plausibleValueHigh.html index b937dbdf..799422ff 100644 --- a/docs/articles/checks/plausibleValueHigh.html +++ b/docs/articles/checks/plausibleValueHigh.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -178,7 +181,7 @@

    plausibleValueHigh

    Dymytry Dymshyts

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/plausibleValueHigh.Rmd diff --git a/docs/articles/checks/plausibleValueLow.html b/docs/articles/checks/plausibleValueLow.html index 303fb846..9cdbe4b0 100644 --- a/docs/articles/checks/plausibleValueLow.html +++ b/docs/articles/checks/plausibleValueLow.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -178,7 +181,7 @@

    plausibleValueLow

    Dymytry Dymshyts

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/plausibleValueLow.Rmd diff --git a/docs/articles/checks/sourceConceptRecordCompleteness.html b/docs/articles/checks/sourceConceptRecordCompleteness.html index 54445850..b5f59d0b 100644 --- a/docs/articles/checks/sourceConceptRecordCompleteness.html +++ b/docs/articles/checks/sourceConceptRecordCompleteness.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • @@ -178,7 +181,7 @@

    sourceConceptRecordCompleteness

    Katy Sadowski

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/sourceConceptRecordCompleteness.Rmd @@ -208,12 +211,12 @@

    DefinitionSource concept mapping @@ -222,9 +225,13 @@

    Definition @@ -277,9 +284,8 @@

    ETL DevelopersIf source values do NOT return matches on concept_code and you are NOT handling concept mapping locally for a non-OMOP source vocabulary, then you likely have a malformed source code or one that does not exist -in the OMOP vocabulary. Please see the documentation in the -standardConceptRecordCompleteness page for instructions on how to handle -this scenario.

    +in the OMOP vocabulary. Please see the documentation in the     standardConceptRecordCompleteness +page for instructions on how to handle this scenario.

    Data Users diff --git a/docs/articles/checks/sourceValueCompleteness.html b/docs/articles/checks/sourceValueCompleteness.html index 8baafec8..82e2d3af 100644 --- a/docs/articles/checks/sourceValueCompleteness.html +++ b/docs/articles/checks/sourceValueCompleteness.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +

  • @@ -178,7 +181,7 @@

    sourceValueCompleteness

    Jared Houghtaling, Clair Blacketer

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/sourceValueCompleteness.Rmd @@ -203,7 +206,8 @@

    DefinitionUser GuidancestandardConceptRecordCompleteness check +check should be used in conjunction with the standardConceptRecordCompleteness check to identify potential mapping issues in the ETL.

    This check is a good measure of the overall mapping rate within each domain. For example, a table may have high diff --git a/docs/articles/checks/standardConceptRecordCompleteness.html b/docs/articles/checks/standardConceptRecordCompleteness.html index 2981148a..d99d0ca9 100644 --- a/docs/articles/checks/standardConceptRecordCompleteness.html +++ b/docs/articles/checks/standardConceptRecordCompleteness.html @@ -145,6 +145,9 @@

  • plausibleValueLow
    • +
  • + withinVisitDates +

  • @@ -178,7 +181,7 @@

    standardConceptRecordCompleteness

    Katy Sadowski

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/standardConceptRecordCompleteness.Rmd @@ -204,17 +207,16 @@

    DefinitionStandard concept mapping
    diff --git a/docs/articles/checks/withinVisitDates.html b/docs/articles/checks/withinVisitDates.html index 9a19d0de..9ff9e5ba 100644 --- a/docs/articles/checks/withinVisitDates.html +++ b/docs/articles/checks/withinVisitDates.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +

  • @@ -175,9 +178,10 @@

    withinVisitDates

    -

    +

    Clair +Blacketer

    -

    2024-06-26

    +

    2024-06-29

    Source: vignettes/checks/withinVisitDates.Rmd @@ -189,45 +193,126 @@

    2024-06-26

    Summary

    -

    Level: FIELD
    Context: Verification
    Category: Conformance
    Subcategory:
    Severity:

    +

    Level: FIELD
    Context: Verification
    Category: Conformance
    Subcategory:
    Severity: Characterization ✔

    Description

    -

    The number and percent of records not within one week on either side -of the corresponding visit occurrence start and end date

    +

    The number and percent of records that occur one week before the +corresponding visit_start_date or one week after the +corresponding visit_end_date

    Definition

    • -Numerator:
      • +Numerator: The number of rows in a table where the event +date occurs more than 7 days prior to the corresponding visit start date +or more than 7 days after the corresponding visit end date.
    • -Denominator:
      • +Denominator: The total number of rows in the table with a +corresponding visit (linked through +visit_occurrence_id)
    • -Related CDM Convention(s):
      • +Related CDM Convention(s): There is no explicit convention +tied to this check. However, the CDM documentation describes the +visit_occurrence_id foreign key in the event tables as “The +visit during which the occurred.” The underlying assumption is +that if a record is tied to a visit, then the date of the record should +fall in some reasonable time period around the visit dates. This gives a +week of leeway on either side for physician notes or other activities +related to a visit to be recorded.
       +
    • -CDM Fields/Tables:
      • +CDM Fields/Tables: This check runs on all event tables: +CONDITION_OCCURRENCE, PROCEDURE_OCCURRENCE, +DRUG_EXPOSURE, DEVICE_EXPOSURE, +MEASUREMENT, NOTE, OBSERVATION, +and VISIT_DETAIL. It will check either the +X_date or X_start_date fields for alignment +with corresponding VISIT_OCCURRENCE dates by linking on the +visit_occurrence_id. (Note: For +VISIT_DETAIL it will check both the visit_detail_start_date and +visit_detail_end_date. The default threshold for these two checks is +1%.)
    • -Default Threshold Value:
      • +Default Threshold Value: 1% for VISIT_DETAIL, +5% for all other tables

    User Guidance

    +

    There is no explicit convention that describes how events should +align temporally with the visits they correspond to. This check is meant +to identify egregious mismatches in dates that could signify an +incorrect date field was used in the ETL or that the data should be used +with caution if there is no reason for the mismatch (history of a +condition, for example).

    +

    If this check fails the first action should be to investigate the +failing rows for any patterns. The main query to find failing rows is +below:

    Violated rows query

    -
    +
    SELECT 
+  '@cdmTableName.@cdmFieldName' AS violating_field,  
+  vo.visit_start_date, vo.visit_end_date, vo.person_id, 
+  cdmTable.* 
+FROM @cdmDatabaseSchema.@cdmTableName cdmTable 
+JOIN @cdmDatabaseSchema.visit_occurrence vo 
+  ON cdmTable.visit_occurrence_id = vo.visit_occurrence_id 
+WHERE cdmTable.@cdmFieldName < dateadd(day, -7, vo.visit_start_date) 
+  OR cdmTable.@cdmFieldName > dateadd(day, 7, vo.visit_end_date)

    ETL Developers

    +

    The first step is to investigate whether visit and event indeed +should be linked - e.g., do they belong to the same person; how far are +the dates apart; is it possible the event was recorded during the visit. +If they should be linked, then the next step is to investigate which of +the event date and visit date is accurate.

    +

    One suggestion would be to identify if all of the failures are due to +many events all having the same date. In some institutions there is a +default date given to events in the case where a date is not given. +Should this be the problem, the first step should be to identify if +there is a different date field in the native data that can be used. If +not, considerations should be made to determine if the rows should be +dropped. Without a correct date it is challenging to use such events in +health outcomes research.

    +

    Another reason for the discrepancy could be that the wrong date has +been used for events. For instance, in some systems a diagnosis could +have both an ‘observation date’ and an ‘administration date’. If the +physician is updating records at a later date, the administration date +can be later than the actual date the diagnosis was observed. In those +cases, the observation date has to be used. If there is only an +administration date, it is in some cases arguable to use the visit date +for the diagnosis date.

    +

    Another suggestion would be to investigate if the failures are +related to ‘History of’ conditions. It is often the case that a +patient’s history is recorded during a visit, in which case they may +report a diagnosis date prior to the given visit. In some cases it may +be appropriate to conserve these records; the decision to do so will +depend on the reliability of the recorded date in your source data.

    Data Users

    +

    If the failure percentage of withinVisitDates is high, a data user +should be careful with using the data. This check might indicate a +larger underlying conformance issue with either the event dates or +linkage with visits. At the same time, there might be a valid reason why +events do not happen within 7 days of the linked visit.

    +

    Make sure to understand why this check fails. Specifically, be +careful when using such data in outcomes research. Without specific +dates for an event, it is challenging to determine if an adverse event +occurred after a drug exposure, for example.

    +

    Note that this check specifically compares event dates to +VISIT_OCCURRENCE dates. There is no equivalent check for +VISIT_DETAIL that verifies whether the event date is within +(a week of) the visit detail start and end dates.

    diff --git a/docs/articles/index.html b/docs/articles/index.html index c01a0cff..a9c50ea9 100644 --- a/docs/articles/index.html +++ b/docs/articles/index.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/docs/authors.html b/docs/authors.html index 796cd4c8..89196518 100644 --- a/docs/authors.html +++ b/docs/authors.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/docs/index.html b/docs/index.html index b742927b..f76a3b99 100644 --- a/docs/index.html +++ b/docs/index.html @@ -145,6 +145,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • diff --git a/docs/news/index.html b/docs/news/index.html index 3d1f6026..83e23076 100644 --- a/docs/news/index.html +++ b/docs/news/index.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/docs/pkgdown.yml b/docs/pkgdown.yml index 4d9ad23c..a3657ed9 100644 --- a/docs/pkgdown.yml +++ b/docs/pkgdown.yml @@ -34,7 +34,7 @@ articles: standardConceptRecordCompleteness: checks/standardConceptRecordCompleteness.html Thresholds: Thresholds.html withinVisitDates: checks/withinVisitDates.html -last_built: 2024-06-26T15:29Z +last_built: 2024-06-29T20:33Z urls: reference: https://ohdsi.github.io/DataQualityDashboard/reference article: https://ohdsi.github.io/DataQualityDashboard/articles diff --git a/docs/reference/convertJsonResultsFileCase.html b/docs/reference/convertJsonResultsFileCase.html index 47819d16..0b2871f8 100644 --- a/docs/reference/convertJsonResultsFileCase.html +++ b/docs/reference/convertJsonResultsFileCase.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/docs/reference/dot-evaluateThresholds.html b/docs/reference/dot-evaluateThresholds.html index ba89215c..0e3a4c2c 100644 --- a/docs/reference/dot-evaluateThresholds.html +++ b/docs/reference/dot-evaluateThresholds.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/docs/reference/dot-getCheckId.html b/docs/reference/dot-getCheckId.html index 8f26d7fb..b631cb51 100644 --- a/docs/reference/dot-getCheckId.html +++ b/docs/reference/dot-getCheckId.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/docs/reference/dot-processCheck.html b/docs/reference/dot-processCheck.html index 0431d9b5..7f29c99a 100644 --- a/docs/reference/dot-processCheck.html +++ b/docs/reference/dot-processCheck.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/docs/reference/dot-recordResult.html b/docs/reference/dot-recordResult.html index 0b3a1151..96673dd9 100644 --- a/docs/reference/dot-recordResult.html +++ b/docs/reference/dot-recordResult.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/docs/reference/dot-runCheck.html b/docs/reference/dot-runCheck.html index 9407dceb..9b4c5b79 100644 --- a/docs/reference/dot-runCheck.html +++ b/docs/reference/dot-runCheck.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/docs/reference/dot-summarizeResults.html b/docs/reference/dot-summarizeResults.html index 52d47039..8d22ee63 100644 --- a/docs/reference/dot-summarizeResults.html +++ b/docs/reference/dot-summarizeResults.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/docs/reference/dot-writeResultsToCsv.html b/docs/reference/dot-writeResultsToCsv.html index 95bf0b43..1a209ffe 100644 --- a/docs/reference/dot-writeResultsToCsv.html +++ b/docs/reference/dot-writeResultsToCsv.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/docs/reference/dot-writeResultsToJson.html b/docs/reference/dot-writeResultsToJson.html index fdfeaba6..62410ef1 100644 --- a/docs/reference/dot-writeResultsToJson.html +++ b/docs/reference/dot-writeResultsToJson.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/docs/reference/dot-writeResultsToTable.html b/docs/reference/dot-writeResultsToTable.html index 5004b036..2e215376 100644 --- a/docs/reference/dot-writeResultsToTable.html +++ b/docs/reference/dot-writeResultsToTable.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/docs/reference/executeDqChecks.html b/docs/reference/executeDqChecks.html index 4e42ef4a..6382ed42 100644 --- a/docs/reference/executeDqChecks.html +++ b/docs/reference/executeDqChecks.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/docs/reference/index.html b/docs/reference/index.html index be3c8cf5..f9d83e64 100644 --- a/docs/reference/index.html +++ b/docs/reference/index.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/docs/reference/listDqChecks.html b/docs/reference/listDqChecks.html index a3500e52..606e2d87 100644 --- a/docs/reference/listDqChecks.html +++ b/docs/reference/listDqChecks.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/docs/reference/reEvaluateThresholds.html b/docs/reference/reEvaluateThresholds.html index e9305b59..dd74b726 100644 --- a/docs/reference/reEvaluateThresholds.html +++ b/docs/reference/reEvaluateThresholds.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/docs/reference/viewDqDashboard.html b/docs/reference/viewDqDashboard.html index 36efe47f..ece1e45c 100644 --- a/docs/reference/viewDqDashboard.html +++ b/docs/reference/viewDqDashboard.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/docs/reference/writeDBResultsToJson.html b/docs/reference/writeDBResultsToJson.html index 090d5c45..283d3e2a 100644 --- a/docs/reference/writeDBResultsToJson.html +++ b/docs/reference/writeDBResultsToJson.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/docs/reference/writeJsonResultsToCsv.html b/docs/reference/writeJsonResultsToCsv.html index 8ec3afb0..12a8f937 100644 --- a/docs/reference/writeJsonResultsToCsv.html +++ b/docs/reference/writeJsonResultsToCsv.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/docs/reference/writeJsonResultsToTable.html b/docs/reference/writeJsonResultsToTable.html index 30c65c4a..e944a846 100644 --- a/docs/reference/writeJsonResultsToTable.html +++ b/docs/reference/writeJsonResultsToTable.html @@ -125,6 +125,9 @@
  • plausibleValueLow
    • +
  • + withinVisitDates +
  • Changelog diff --git a/inst/csv/OMOP_CDMv5.2_Check_Descriptions.csv b/inst/csv/OMOP_CDMv5.2_Check_Descriptions.csv index 7932187a..63a4c7df 100644 --- a/inst/csv/OMOP_CDMv5.2_Check_Descriptions.csv +++ b/inst/csv/OMOP_CDMv5.2_Check_Descriptions.csv @@ -1,8 +1,8 @@ checkLevel,checkName,checkDescription,kahnContext,kahnCategory,kahnSubcategory,sqlFile,evaluationFilter,severity TABLE,cdmTable,A yes or no value indicating if @cdmTableName table is present as expected based on the specification. ,Verification,Conformance,Relational,table_cdm_table.sql,cdmTableName!='',fatal -TABLE,measurePersonCompleteness,The number and percent of persons in the CDM that do not have at least one record in the @cdmTableName table,Validation,Completeness,,table_person_completeness.sql,measurePersonCompleteness=='Yes', +TABLE,measurePersonCompleteness,The number and percent of persons in the CDM that do not have at least one record in the @cdmTableName table,Validation,Completeness,,table_person_completeness.sql,measurePersonCompleteness=='Yes',convention TABLE,measureConditionEraCompleteness,"The number and Percent of persons that does not have condition_era built successfully -for all persons in condition_occurrence",Validation,Completeness,,table_condition_era_completeness.sql,measureConditionEraCompleteness=='Yes', +for all persons in condition_occurrence",Validation,Completeness,,table_condition_era_completeness.sql,measureConditionEraCompleteness=='Yes',convention FIELD,cdmField,A yes or no value indicating if @cdmFieldName is present in the @cdmTableName table as expected based on the specification. ,Verification,Conformance,Relational,field_cdm_field.sql,cdmFieldName!='',fatal FIELD,isRequired,The number and percent of records with a NULL value in the @cdmFieldName of the @cdmTableName that is considered not nullable.,Validation,Conformance,Relational,field_is_not_nullable.sql,isRequired=='Yes',fatal FIELD,cdmDatatype,A yes or no value indicating if the @cdmFieldName in the @cdmTableName is the expected data type based on the specification. Only checks integer fields.,Verification,Conformance,Value,field_cdm_datatype.sql,cdmDatatype=='integer',fatal @@ -10,19 +10,19 @@ FIELD,isPrimaryKey,The number and percent of records that have a duplicate value FIELD,isForeignKey,The number and percent of records that have a value in the @cdmFieldName field in the @cdmTableName table that does not exist in the @fkTableName table.,Verification,Conformance,Relational,is_foreign_key.sql,isForeignKey=='Yes',fatal FIELD,fkDomain,The number and percent of records that have a value in the @cdmFieldName field in the @cdmTableName table that do not conform to the @fkDomain domain.,Verification,Conformance,Value,field_fk_domain.sql,isForeignKey=='Yes' & fkDomain!= '',convention FIELD,fkClass,The number and percent of records that have a value in the @cdmFieldName field in the @cdmTableName table that do not conform to the @fkClass class.,Verification,Conformance,Computational,field_fk_class.sql,isForeignKey=='Yes' & fkClass!='',convention -FIELD,isStandardValidConcept,"The number and percent of records that do not have a standard, valid concept in the @cdmFieldName field in the @cdmTableName table. ",Verification,Conformance,Value,field_is_standard_valid_concept.sql,isStandardValidConcept=='Yes', -FIELD,measureValueCompleteness,The number and percent of records with a NULL value in the @cdmFieldName of the @cdmTableName.,Verification,Completeness,,field_measure_value_completeness.sql,measureValueCompleteness=='Yes', -FIELD,standardConceptRecordCompleteness,The number and percent of records with a value of 0 in the standard concept field @cdmFieldName in the @cdmTableName table.,Verification,Completeness,,field_concept_record_completeness.sql,standardConceptRecordCompleteness=='Yes', -FIELD,sourceConceptRecordCompleteness,The number and percent of records with a value of 0 in the source concept field @cdmFieldName in the @cdmTableName table.,Verification,Completeness,,field_concept_record_completeness.sql,sourceConceptRecordCompleteness=='Yes', -FIELD,sourceValueCompleteness,The number and percent of distinct source values in the @cdmFieldName field of the @cdmTableName table mapped to 0.,Verification,Completeness,,field_source_value_completeness.sql,sourceValueCompleteness=='Yes', -FIELD,plausibleValueLow,The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName table less than @plausibleValueLow.,Verification,Plausibility,Atemporal,field_plausible_value_low.sql,plausibleValueLow!='', -FIELD,plausibleValueHigh,The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName table greater than @plausibleValueHigh.,Verification,Plausibility,Atemporal,field_plausible_value_high.sql,plausibleValueHigh!='', -FIELD,plausibleTemporalAfter,The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName that occurs prior to the date in the @plausibleTemporalAfterFieldName field of the @plausibleTemporalAfterTableName table.,Verification,Plausibility,Temporal,field_plausible_temporal_after.sql,plausibleTemporalAfter=='Yes', -FIELD,plausibleDuringLife,"If yes, the number and percent of records with a date value in the @cdmFieldName field of the @cdmTableName table that occurs after death.",Verification,Plausibility,Temporal,field_plausible_during_life.sql,plausibleDuringLife=='Yes', -FIELD,withinVisitDates,The number and percent of records not within one week on either side of the corresponding visit occurrence start and end date,Verification,Conformance,,field_within_visit_dates.sql,withinVisitDates=='Yes', +FIELD,isStandardValidConcept,"The number and percent of records that do not have a standard, valid concept in the @cdmFieldName field in the @cdmTableName table. ",Verification,Conformance,Value,field_is_standard_valid_concept.sql,isStandardValidConcept=='Yes',convention +FIELD,measureValueCompleteness,The number and percent of records with a NULL value in the @cdmFieldName of the @cdmTableName.,Verification,Completeness,,field_measure_value_completeness.sql,measureValueCompleteness=='Yes',characterization +FIELD,standardConceptRecordCompleteness,The number and percent of records with a value of 0 in the standard concept field @cdmFieldName in the @cdmTableName table.,Verification,Completeness,,field_concept_record_completeness.sql,standardConceptRecordCompleteness=='Yes',convention +FIELD,sourceConceptRecordCompleteness,The number and percent of records with a value of 0 in the source concept field @cdmFieldName in the @cdmTableName table.,Verification,Completeness,,field_concept_record_completeness.sql,sourceConceptRecordCompleteness=='Yes',convention +FIELD,sourceValueCompleteness,The number and percent of distinct source values in the @cdmFieldName field of the @cdmTableName table mapped to 0.,Verification,Completeness,,field_source_value_completeness.sql,sourceValueCompleteness=='Yes',convention +FIELD,plausibleValueLow,The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName table less than @plausibleValueLow.,Verification,Plausibility,Atemporal,field_plausible_value_low.sql,plausibleValueLow!='',characterization +FIELD,plausibleValueHigh,The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName table greater than @plausibleValueHigh.,Verification,Plausibility,Atemporal,field_plausible_value_high.sql,plausibleValueHigh!='',characterization +FIELD,plausibleTemporalAfter,The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName that occurs prior to the date in the @plausibleTemporalAfterFieldName field of the @plausibleTemporalAfterTableName table.,Verification,Plausibility,Temporal,field_plausible_temporal_after.sql,plausibleTemporalAfter=='Yes',characterization +FIELD,plausibleDuringLife,"If yes, the number and percent of records with a date value in the @cdmFieldName field of the @cdmTableName table that occurs after death.",Verification,Plausibility,Temporal,field_plausible_during_life.sql,plausibleDuringLife=='Yes',characterization +FIELD,withinVisitDates,The number and percent of records not within one week on either side of the corresponding visit occurrence start and end date,Verification,Conformance,,field_within_visit_dates.sql,withinVisitDates=='Yes',characterization FIELD,plausibleAfterBirth,"The number and percent of records with a date value in the @cdmFieldName field of the @cdmTableName table that occurs prior to birth.",Verification,Plausibility,Temporal,field_plausible_after_birth.sql,plausibleAfterBirth=='Yes',characterization -FIELD,plausibleBeforeDeath,"The number and percent of records with a date value in the @cdmFieldName field of the @cdmTableName table that occurs after death.",Verification,Plausibility,Temporal,field_plausible_before_death.sql,plausibleBeforeDeath=='Yes', -FIELD,plausibleStartBeforeEnd,"The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName that occurs after the date in the @plausibleStartBeforeEndFieldName.",Verification,Plausibility,Temporal,field_plausible_start_before_end.sql,plausibleStartBeforeEnd=='Yes', -CONCEPT,plausibleGender,"For a CONCEPT_ID @conceptId (@conceptName), the number and percent of records associated with patients with an implausible gender (correct gender = @plausibleGender).",Validation,Plausibility,Atemporal,concept_plausible_gender.sql,plausibleGender!='', -CONCEPT,plausibleGenderUseDescendants,"For descendants of CONCEPT_ID @conceptId (@conceptName), the number and percent of records associated with patients with an implausible gender (correct gender = @plausibleGenderUseDescendants).",Validation,Plausibility,Atemporal,concept_plausible_gender_use_descendants.sql,plausibleGenderUseDescendants!='', -CONCEPT,plausibleUnitConceptIds,"The number and percent of records for a given CONCEPT_ID @conceptId (@conceptName) with implausible units (i.e., UNIT_CONCEPT_ID NOT IN (@plausibleUnitConceptIds)).",Verification,Plausibility,Atemporal,concept_plausible_unit_concept_ids.sql,plausibleUnitConceptIdsThreshold!='', \ No newline at end of file +FIELD,plausibleBeforeDeath,"The number and percent of records with a date value in the @cdmFieldName field of the @cdmTableName table that occurs after death.",Verification,Plausibility,Temporal,field_plausible_before_death.sql,plausibleBeforeDeath=='Yes',characterization +FIELD,plausibleStartBeforeEnd,"The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName that occurs after the date in the @plausibleStartBeforeEndFieldName.",Verification,Plausibility,Temporal,field_plausible_start_before_end.sql,plausibleStartBeforeEnd=='Yes',characterization +CONCEPT,plausibleGender,"For a CONCEPT_ID @conceptId (@conceptName), the number and percent of records associated with patients with an implausible gender (correct gender = @plausibleGender).",Validation,Plausibility,Atemporal,concept_plausible_gender.sql,plausibleGender!='',characterization +CONCEPT,plausibleGenderUseDescendants,"For descendants of CONCEPT_ID @conceptId (@conceptName), the number and percent of records associated with patients with an implausible gender (correct gender = @plausibleGenderUseDescendants).",Validation,Plausibility,Atemporal,concept_plausible_gender_use_descendants.sql,plausibleGenderUseDescendants!='',characterization +CONCEPT,plausibleUnitConceptIds,"The number and percent of records for a given CONCEPT_ID @conceptId (@conceptName) with implausible units (i.e., UNIT_CONCEPT_ID NOT IN (@plausibleUnitConceptIds)).",Verification,Plausibility,Atemporal,concept_plausible_unit_concept_ids.sql,plausibleUnitConceptIdsThreshold!='',characterization \ No newline at end of file diff --git a/inst/csv/OMOP_CDMv5.3_Check_Descriptions.csv b/inst/csv/OMOP_CDMv5.3_Check_Descriptions.csv index 5a35f724..d0411ce5 100644 --- a/inst/csv/OMOP_CDMv5.3_Check_Descriptions.csv +++ b/inst/csv/OMOP_CDMv5.3_Check_Descriptions.csv @@ -1,8 +1,8 @@ checkLevel,checkName,checkDescription,kahnContext,kahnCategory,kahnSubcategory,sqlFile,evaluationFilter,severity TABLE,cdmTable,A yes or no value indicating if @cdmTableName table is present as expected based on the specification. ,Verification,Conformance,Relational,table_cdm_table.sql,cdmTableName!='',fatal -TABLE,measurePersonCompleteness,The number and percent of persons in the CDM that do not have at least one record in the @cdmTableName table,Validation,Completeness,,table_person_completeness.sql,measurePersonCompleteness=='Yes', +TABLE,measurePersonCompleteness,The number and percent of persons in the CDM that do not have at least one record in the @cdmTableName table,Validation,Completeness,,table_person_completeness.sql,measurePersonCompleteness=='Yes',convention TABLE,measureConditionEraCompleteness,"The number and Percent of persons that does not have condition_era built successfully -for all persons in condition_occurrence",Validation,Completeness,,table_condition_era_completeness.sql,measureConditionEraCompleteness=='Yes', +for all persons in condition_occurrence",Validation,Completeness,,table_condition_era_completeness.sql,measureConditionEraCompleteness=='Yes',convention FIELD,cdmField,A yes or no value indicating if @cdmFieldName is present in the @cdmTableName table as expected based on the specification. ,Verification,Conformance,Relational,field_cdm_field.sql,cdmFieldName!='',fatal FIELD,isRequired,The number and percent of records with a NULL value in the @cdmFieldName of the @cdmTableName that is considered not nullable.,Validation,Conformance,Relational,field_is_not_nullable.sql,isRequired=='Yes',fatal FIELD,cdmDatatype,A yes or no value indicating if the @cdmFieldName in the @cdmTableName is the expected data type based on the specification. Only checks integer fields.,Verification,Conformance,Value,field_cdm_datatype.sql,cdmDatatype=='integer',fatal @@ -10,19 +10,19 @@ FIELD,isPrimaryKey,The number and percent of records that have a duplicate value FIELD,isForeignKey,The number and percent of records that have a value in the @cdmFieldName field in the @cdmTableName table that does not exist in the @fkTableName table.,Verification,Conformance,Relational,is_foreign_key.sql,isForeignKey=='Yes',fatal FIELD,fkDomain,The number and percent of records that have a value in the @cdmFieldName field in the @cdmTableName table that do not conform to the @fkDomain domain.,Verification,Conformance,Value,field_fk_domain.sql,isForeignKey=='Yes' & fkDomain!= '',convention FIELD,fkClass,The number and percent of records that have a value in the @cdmFieldName field in the @cdmTableName table that do not conform to the @fkClass class.,Verification,Conformance,Computational,field_fk_class.sql,isForeignKey=='Yes' & fkClass!='',convention -FIELD,isStandardValidConcept,"The number and percent of records that do not have a standard, valid concept in the @cdmFieldName field in the @cdmTableName table. ",Verification,Conformance,Value,field_is_standard_valid_concept.sql,isStandardValidConcept=='Yes', -FIELD,measureValueCompleteness,The number and percent of records with a NULL value in the @cdmFieldName of the @cdmTableName.,Verification,Completeness,,field_measure_value_completeness.sql,measureValueCompleteness=='Yes', -FIELD,standardConceptRecordCompleteness,The number and percent of records with a value of 0 in the standard concept field @cdmFieldName in the @cdmTableName table.,Verification,Completeness,,field_concept_record_completeness.sql,standardConceptRecordCompleteness=='Yes', -FIELD,sourceConceptRecordCompleteness,The number and percent of records with a value of 0 in the source concept field @cdmFieldName in the @cdmTableName table.,Verification,Completeness,,field_concept_record_completeness.sql,sourceConceptRecordCompleteness=='Yes', -FIELD,sourceValueCompleteness,The number and percent of distinct source values in the @cdmFieldName field of the @cdmTableName table mapped to 0.,Verification,Completeness,,field_source_value_completeness.sql,sourceValueCompleteness=='Yes', -FIELD,plausibleValueLow,The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName table less than @plausibleValueLow.,Verification,Plausibility,Atemporal,field_plausible_value_low.sql,plausibleValueLow!='', -FIELD,plausibleValueHigh,The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName table greater than @plausibleValueHigh.,Verification,Plausibility,Atemporal,field_plausible_value_high.sql,plausibleValueHigh!='', -FIELD,plausibleTemporalAfter,The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName that occurs prior to the date in the @plausibleTemporalAfterFieldName field of the @plausibleTemporalAfterTableName table.,Verification,Plausibility,Temporal,field_plausible_temporal_after.sql,plausibleTemporalAfter=='Yes', -FIELD,plausibleDuringLife,"If yes, the number and percent of records with a date value in the @cdmFieldName field of the @cdmTableName table that occurs after death.",Verification,Plausibility,Temporal,field_plausible_during_life.sql,plausibleDuringLife=='Yes', -FIELD,withinVisitDates,The number and percent of records not within one week on either side of the corresponding visit occurrence start and end date,Verification,Conformance,,field_within_visit_dates.sql,withinVisitDates=='Yes', +FIELD,isStandardValidConcept,"The number and percent of records that do not have a standard, valid concept in the @cdmFieldName field in the @cdmTableName table. ",Verification,Conformance,Value,field_is_standard_valid_concept.sql,isStandardValidConcept=='Yes',convention +FIELD,measureValueCompleteness,The number and percent of records with a NULL value in the @cdmFieldName of the @cdmTableName.,Verification,Completeness,,field_measure_value_completeness.sql,measureValueCompleteness=='Yes',characterization +FIELD,standardConceptRecordCompleteness,The number and percent of records with a value of 0 in the standard concept field @cdmFieldName in the @cdmTableName table.,Verification,Completeness,,field_concept_record_completeness.sql,standardConceptRecordCompleteness=='Yes',convention +FIELD,sourceConceptRecordCompleteness,The number and percent of records with a value of 0 in the source concept field @cdmFieldName in the @cdmTableName table.,Verification,Completeness,,field_concept_record_completeness.sql,sourceConceptRecordCompleteness=='Yes',convention +FIELD,sourceValueCompleteness,The number and percent of distinct source values in the @cdmFieldName field of the @cdmTableName table mapped to 0.,Verification,Completeness,,field_source_value_completeness.sql,sourceValueCompleteness=='Yes',convention +FIELD,plausibleValueLow,The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName table less than @plausibleValueLow.,Verification,Plausibility,Atemporal,field_plausible_value_low.sql,plausibleValueLow!='',characterization +FIELD,plausibleValueHigh,The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName table greater than @plausibleValueHigh.,Verification,Plausibility,Atemporal,field_plausible_value_high.sql,plausibleValueHigh!='',characterization +FIELD,plausibleTemporalAfter,The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName that occurs prior to the date in the @plausibleTemporalAfterFieldName field of the @plausibleTemporalAfterTableName table.,Verification,Plausibility,Temporal,field_plausible_temporal_after.sql,plausibleTemporalAfter=='Yes',characterization +FIELD,plausibleDuringLife,"If yes, the number and percent of records with a date value in the @cdmFieldName field of the @cdmTableName table that occurs after death.",Verification,Plausibility,Temporal,field_plausible_during_life.sql,plausibleDuringLife=='Yes',characterization +FIELD,withinVisitDates,The number and percent of records not within one week on either side of the corresponding visit occurrence start and end date,Verification,Conformance,,field_within_visit_dates.sql,withinVisitDates=='Yes',characterization FIELD,plausibleAfterBirth,"The number and percent of records with a date value in the @cdmFieldName field of the @cdmTableName table that occurs prior to birth.",Verification,Plausibility,Temporal,field_plausible_after_birth.sql,plausibleAfterBirth=='Yes',characterization -FIELD,plausibleBeforeDeath,"The number and percent of records with a date value in the @cdmFieldName field of the @cdmTableName table that occurs after death.",Verification,Plausibility,Temporal,field_plausible_before_death.sql,plausibleBeforeDeath=='Yes', -FIELD,plausibleStartBeforeEnd,"The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName that occurs after the date in the @plausibleStartBeforeEndFieldName.",Verification,Plausibility,Temporal,field_plausible_start_before_end.sql,plausibleStartBeforeEnd=='Yes', -CONCEPT,plausibleGender,"For a CONCEPT_ID @conceptId (@conceptName), the number and percent of records associated with patients with an implausible gender (correct gender = @plausibleGender).",Validation,Plausibility,Atemporal,concept_plausible_gender.sql,plausibleGender!='', -CONCEPT,plausibleGenderUseDescendants,"For descendants of CONCEPT_ID @conceptId (@conceptName), the number and percent of records associated with patients with an implausible gender (correct gender = @plausibleGenderUseDescendants).",Validation,Plausibility,Atemporal,concept_plausible_gender_use_descendants.sql,plausibleGenderUseDescendants!='', -CONCEPT,plausibleUnitConceptIds,"The number and percent of records for a given CONCEPT_ID @conceptId (@conceptName) with implausible units (i.e., UNIT_CONCEPT_ID NOT IN (@plausibleUnitConceptIds)).",Verification,Plausibility,Atemporal,concept_plausible_unit_concept_ids.sql,plausibleUnitConceptIdsThreshold!='', \ No newline at end of file +FIELD,plausibleBeforeDeath,"The number and percent of records with a date value in the @cdmFieldName field of the @cdmTableName table that occurs after death.",Verification,Plausibility,Temporal,field_plausible_before_death.sql,plausibleBeforeDeath=='Yes',characterization +FIELD,plausibleStartBeforeEnd,"The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName that occurs after the date in the @plausibleStartBeforeEndFieldName.",Verification,Plausibility,Temporal,field_plausible_start_before_end.sql,plausibleStartBeforeEnd=='Yes',characterization +CONCEPT,plausibleGender,"For a CONCEPT_ID @conceptId (@conceptName), the number and percent of records associated with patients with an implausible gender (correct gender = @plausibleGender).",Validation,Plausibility,Atemporal,concept_plausible_gender.sql,plausibleGender!='',characterization +CONCEPT,plausibleGenderUseDescendants,"For descendants of CONCEPT_ID @conceptId (@conceptName), the number and percent of records associated with patients with an implausible gender (correct gender = @plausibleGenderUseDescendants).",Validation,Plausibility,Atemporal,concept_plausible_gender_use_descendants.sql,plausibleGenderUseDescendants!='',characterization +CONCEPT,plausibleUnitConceptIds,"The number and percent of records for a given CONCEPT_ID @conceptId (@conceptName) with implausible units (i.e., UNIT_CONCEPT_ID NOT IN (@plausibleUnitConceptIds)).",Verification,Plausibility,Atemporal,concept_plausible_unit_concept_ids.sql,plausibleUnitConceptIdsThreshold!='',characterization \ No newline at end of file diff --git a/inst/csv/OMOP_CDMv5.4_Check_Descriptions.csv b/inst/csv/OMOP_CDMv5.4_Check_Descriptions.csv index 7a8e9227..a9244366 100644 --- a/inst/csv/OMOP_CDMv5.4_Check_Descriptions.csv +++ b/inst/csv/OMOP_CDMv5.4_Check_Descriptions.csv @@ -1,8 +1,8 @@ checkLevel,checkName,checkDescription,kahnContext,kahnCategory,kahnSubcategory,sqlFile,evaluationFilter,severity TABLE,cdmTable,A yes or no value indicating if @cdmTableName table is present as expected based on the specification. ,Verification,Conformance,Relational,table_cdm_table.sql,cdmTableName!='',fatal -TABLE,measurePersonCompleteness,The number and percent of persons in the CDM that do not have at least one record in the @cdmTableName table,Validation,Completeness,,table_person_completeness.sql,measurePersonCompleteness=='Yes', +TABLE,measurePersonCompleteness,The number and percent of persons in the CDM that do not have at least one record in the @cdmTableName table,Validation,Completeness,,table_person_completeness.sql,measurePersonCompleteness=='Yes',convention TABLE,measureConditionEraCompleteness,"The number and Percent of persons that does not have condition_era built successfully -for all persons in condition_occurrence",Validation,Completeness,,table_condition_era_completeness.sql,measureConditionEraCompleteness=='Yes', +for all persons in condition_occurrence",Validation,Completeness,,table_condition_era_completeness.sql,measureConditionEraCompleteness=='Yes',convention FIELD,cdmField,A yes or no value indicating if @cdmFieldName is present in the @cdmTableName table as expected based on the specification. ,Verification,Conformance,Relational,field_cdm_field.sql,cdmFieldName!='',fatal FIELD,isRequired,The number and percent of records with a NULL value in the @cdmFieldName of the @cdmTableName that is considered not nullable.,Validation,Conformance,Relational,field_is_not_nullable.sql,isRequired=='Yes',fatal FIELD,cdmDatatype,A yes or no value indicating if the @cdmFieldName in the @cdmTableName is the expected data type based on the specification. Only checks integer fields.,Verification,Conformance,Value,field_cdm_datatype.sql,cdmDatatype=='integer',fatal @@ -10,19 +10,19 @@ FIELD,isPrimaryKey,The number and percent of records that have a duplicate value FIELD,isForeignKey,The number and percent of records that have a value in the @cdmFieldName field in the @cdmTableName table that does not exist in the @fkTableName table.,Verification,Conformance,Relational,is_foreign_key.sql,isForeignKey=='Yes',fatal FIELD,fkDomain,The number and percent of records that have a value in the @cdmFieldName field in the @cdmTableName table that do not conform to the @fkDomain domain.,Verification,Conformance,Value,field_fk_domain.sql,isForeignKey=='Yes' & fkDomain!= '',convention FIELD,fkClass,The number and percent of records that have a value in the @cdmFieldName field in the @cdmTableName table that do not conform to the @fkClass class.,Verification,Conformance,Computational,field_fk_class.sql,isForeignKey=='Yes' & fkClass!='',convention -FIELD,isStandardValidConcept,"The number and percent of records that do not have a standard, valid concept in the @cdmFieldName field in the @cdmTableName table. ",Verification,Conformance,Value,field_is_standard_valid_concept.sql,isStandardValidConcept=='Yes', -FIELD,measureValueCompleteness,The number and percent of records with a NULL value in the @cdmFieldName of the @cdmTableName.,Verification,Completeness,,field_measure_value_completeness.sql,measureValueCompleteness=='Yes', -FIELD,standardConceptRecordCompleteness,The number and percent of records with a value of 0 in the standard concept field @cdmFieldName in the @cdmTableName table.,Verification,Completeness,,field_concept_record_completeness.sql,standardConceptRecordCompleteness=='Yes', -FIELD,sourceConceptRecordCompleteness,The number and percent of records with a value of 0 in the source concept field @cdmFieldName in the @cdmTableName table.,Verification,Completeness,,field_concept_record_completeness.sql,sourceConceptRecordCompleteness=='Yes', -FIELD,sourceValueCompleteness,The number and percent of distinct source values in the @cdmFieldName field of the @cdmTableName table mapped to 0.,Verification,Completeness,,field_source_value_completeness.sql,sourceValueCompleteness=='Yes', -FIELD,plausibleValueLow,The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName table less than @plausibleValueLow.,Verification,Plausibility,Atemporal,field_plausible_value_low.sql,plausibleValueLow!='', -FIELD,plausibleValueHigh,The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName table greater than @plausibleValueHigh.,Verification,Plausibility,Atemporal,field_plausible_value_high.sql,plausibleValueHigh!='', -FIELD,plausibleTemporalAfter,The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName that occurs prior to the date in the @plausibleTemporalAfterFieldName field of the @plausibleTemporalAfterTableName table.,Verification,Plausibility,Temporal,field_plausible_temporal_after.sql,plausibleTemporalAfter=='Yes', -FIELD,plausibleDuringLife,"If yes, the number and percent of records with a date value in the @cdmFieldName field of the @cdmTableName table that occurs after death.",Verification,Plausibility,Temporal,field_plausible_during_life.sql,plausibleDuringLife=='Yes', -FIELD,withinVisitDates,The number and percent of records not within one week on either side of the corresponding visit occurrence start and end date,Verification,Conformance,,field_within_visit_dates.sql,withinVisitDates=='Yes', +FIELD,isStandardValidConcept,"The number and percent of records that do not have a standard, valid concept in the @cdmFieldName field in the @cdmTableName table. ",Verification,Conformance,Value,field_is_standard_valid_concept.sql,isStandardValidConcept=='Yes',convention +FIELD,measureValueCompleteness,The number and percent of records with a NULL value in the @cdmFieldName of the @cdmTableName.,Verification,Completeness,,field_measure_value_completeness.sql,measureValueCompleteness=='Yes',characterization +FIELD,standardConceptRecordCompleteness,The number and percent of records with a value of 0 in the standard concept field @cdmFieldName in the @cdmTableName table.,Verification,Completeness,,field_concept_record_completeness.sql,standardConceptRecordCompleteness=='Yes',convention +FIELD,sourceConceptRecordCompleteness,The number and percent of records with a value of 0 in the source concept field @cdmFieldName in the @cdmTableName table.,Verification,Completeness,,field_concept_record_completeness.sql,sourceConceptRecordCompleteness=='Yes',convention +FIELD,sourceValueCompleteness,The number and percent of distinct source values in the @cdmFieldName field of the @cdmTableName table mapped to 0.,Verification,Completeness,,field_source_value_completeness.sql,sourceValueCompleteness=='Yes',convention +FIELD,plausibleValueLow,The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName table less than @plausibleValueLow.,Verification,Plausibility,Atemporal,field_plausible_value_low.sql,plausibleValueLow!='',characterization +FIELD,plausibleValueHigh,The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName table greater than @plausibleValueHigh.,Verification,Plausibility,Atemporal,field_plausible_value_high.sql,plausibleValueHigh!='',characterization +FIELD,plausibleTemporalAfter,The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName that occurs prior to the date in the @plausibleTemporalAfterFieldName field of the @plausibleTemporalAfterTableName table.,Verification,Plausibility,Temporal,field_plausible_temporal_after.sql,plausibleTemporalAfter=='Yes',characterization +FIELD,plausibleDuringLife,"If yes, the number and percent of records with a date value in the @cdmFieldName field of the @cdmTableName table that occurs after death.",Verification,Plausibility,Temporal,field_plausible_during_life.sql,plausibleDuringLife=='Yes',characterization +FIELD,withinVisitDates,The number and percent of records not within one week on either side of the corresponding visit occurrence start and end date,Verification,Conformance,,field_within_visit_dates.sql,withinVisitDates=='Yes',characterization FIELD,plausibleAfterBirth,"The number and percent of records with a date value in the @cdmFieldName field of the @cdmTableName table that occurs prior to birth.",Verification,Plausibility,Temporal,field_plausible_after_birth.sql,plausibleAfterBirth=='Yes',characterization -FIELD,plausibleBeforeDeath,"The number and percent of records with a date value in the @cdmFieldName field of the @cdmTableName table that occurs after death.",Verification,Plausibility,Temporal,field_plausible_before_death.sql,plausibleBeforeDeath=='Yes', -FIELD,plausibleStartBeforeEnd,"The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName that occurs after the date in the @plausibleStartBeforeEndFieldName.",Verification,Plausibility,Temporal,field_plausible_start_before_end.sql,plausibleStartBeforeEnd=='Yes', -CONCEPT,plausibleGender,"For a CONCEPT_ID @conceptId (@conceptName), the number and percent of records associated with patients with an implausible gender (correct gender = @plausibleGender).",Validation,Plausibility,Atemporal,concept_plausible_gender.sql,plausibleGender!='', -CONCEPT,plausibleGenderUseDescendants,"For descendants of CONCEPT_ID @conceptId (@conceptName), the number and percent of records associated with patients with an implausible gender (correct gender = @plausibleGenderUseDescendants).",Validation,Plausibility,Atemporal,concept_plausible_gender_use_descendants.sql,plausibleGenderUseDescendants!='', -CONCEPT,plausibleUnitConceptIds,"The number and percent of records for a given CONCEPT_ID @conceptId (@conceptName) with implausible units (i.e., UNIT_CONCEPT_ID NOT IN (@plausibleUnitConceptIds)).",Verification,Plausibility,Atemporal,concept_plausible_unit_concept_ids.sql,plausibleUnitConceptIdsThreshold!='', \ No newline at end of file +FIELD,plausibleBeforeDeath,"The number and percent of records with a date value in the @cdmFieldName field of the @cdmTableName table that occurs after death.",Verification,Plausibility,Temporal,field_plausible_before_death.sql,plausibleBeforeDeath=='Yes',characterization +FIELD,plausibleStartBeforeEnd,"The number and percent of records with a value in the @cdmFieldName field of the @cdmTableName that occurs after the date in the @plausibleStartBeforeEndFieldName.",Verification,Plausibility,Temporal,field_plausible_start_before_end.sql,plausibleStartBeforeEnd=='Yes',characterization +CONCEPT,plausibleGender,"For a CONCEPT_ID @conceptId (@conceptName), the number and percent of records associated with patients with an implausible gender (correct gender = @plausibleGender).",Validation,Plausibility,Atemporal,concept_plausible_gender.sql,plausibleGender!='',characterization +CONCEPT,plausibleGenderUseDescendants,"For descendants of CONCEPT_ID @conceptId (@conceptName), the number and percent of records associated with patients with an implausible gender (correct gender = @plausibleGenderUseDescendants).",Validation,Plausibility,Atemporal,concept_plausible_gender_use_descendants.sql,plausibleGenderUseDescendants!='',characterization +CONCEPT,plausibleUnitConceptIds,"The number and percent of records for a given CONCEPT_ID @conceptId (@conceptName) with implausible units (i.e., UNIT_CONCEPT_ID NOT IN (@plausibleUnitConceptIds)).",Verification,Plausibility,Atemporal,concept_plausible_unit_concept_ids.sql,plausibleUnitConceptIdsThreshold!='',characterization \ No newline at end of file diff --git a/vignettes/checkIndex.Rmd b/vignettes/checkIndex.Rmd index 7f5c9ab2..4e93a905 100644 --- a/vignettes/checkIndex.Rmd +++ b/vignettes/checkIndex.Rmd @@ -43,7 +43,7 @@ above to navigate to the check's documentation page.\ - [sourceValueCompleteness](checks/sourceValueCompleteness.html) - [plausibleValueLow](checks/plausibleValueLow.html) - [plausibleValueHigh](checks/plausibleValueHigh.html) -- withinVisitDates (PAGE UNDER CONSTRUCTION) +- [withinVisitDates](checks/withinVisitDates.html) - [plausibleAfterBirth](checks/plausibleAfterBirth.html) - [plausibleBeforeDeath](checks/plausibleBeforeDeath.html) - [plausibleStartBeforeEnd](checks/plausibleStartBeforeEnd.html) diff --git a/vignettes/checks/isStandardValidConcept.Rmd b/vignettes/checks/isStandardValidConcept.Rmd index 0e1f596a..d1001aea 100644 --- a/vignettes/checks/isStandardValidConcept.Rmd +++ b/vignettes/checks/isStandardValidConcept.Rmd @@ -1,6 +1,6 @@ --- title: "isStandardValidConcept" -author: "Katy Sadowski" +author: "Stephanie Hong, Katy Sadowski" date: "`r Sys.Date()`" output: html_document: @@ -23,9 +23,9 @@ The number and percent of records that do not have a standard, valid concept in ## Definition -- *Numerator*: The number of rows with an `X_concept_id` that exists in concept.concept_id but does not equal zero, and has a standard_concept != ‘S’ or a non-NULL invalid_reason. +- *Numerator*: The number of rows with an `X_concept_id` that exists in `CONCEPT.concept_id` but does not equal zero, and has `CONCEPT.standard_concept` != ‘S’ or non-NULL `CONCEPT.invalid_reason`. - *Denominator*: The total number of rows in the table. -- *Related CDM Convention(s)*: All `X_concept_id` columns should contain a standard, valid concept, or 0: https://ohdsi.github.io/CommonDataModel/dataModelConventions.html#Mapping. In other words, all non-zero values in the `X_concept_id` column should exist in the concept_id column of the CONCEPT table and have CONCEPT.standard_concept = ‘S’ and CONCEPT.invalid_reason = NULL. +- *Related CDM Convention(s)*: All `X_concept_id` columns should contain a standard, valid concept, or 0: https://ohdsi.github.io/CommonDataModel/dataModelConventions.html#Mapping. - *CDM Fields/Tables*: All tables with an `X_concept_id` column, and all `X_concept_id` columns in those tables. - *Default Threshold Value*: 0 @@ -48,7 +48,8 @@ FROM @schema.@cdmTableName cdmTable WHERE co.concept_id != 0 AND (co.standard_concept != 'S' OR co.invalid_reason IS NOT NULL) ``` -You may build upon this query by joining the relevant `X_concept_id` and `X_source_concept_id` columns to the concept table and inspecting their names and vocabularies. If the `X_source_concept_id` correctly represents the source code in x_source_value, the fix will be a matter of ensuring your ETL is correctly using the concept_relationship table to map the source concept ID to a standard concept via the ‘Maps to’ relationship. If you are not populating the `X_source_concept_id` column and/or are using an intermediate concept mapping table, you may need to inspect the mappings in your mapper table to ensure they’ve been generated correctly using the ‘Maps to’ relationship for your CDM’s vocabulary version. + +You may build upon this query by joining the relevant `X_concept_id` and `X_source_concept_id` columns to the concept table and inspecting their names and vocabularies. If the `X_source_concept_id` correctly represents the source code in `X_source_value`, the fix will be a matter of ensuring your ETL is correctly using the concept_relationship table to map the source concept ID to a standard concept via the ‘Maps to’ relationship. If you are not populating the `X_source_concept_id` column and/or are using an intermediate concept mapping table, you may need to inspect the mappings in your mapper table to ensure they’ve been generated correctly using the ‘Maps to’ relationship for your CDM’s vocabulary version. ### Data Users This check failure means that the failing rows will not be picked up in a standard OHDSI analysis. It is highly recommended to work with your ETL team or data provider, if possible, to resolve this issue. diff --git a/vignettes/checks/measurePersonCompleteness.Rmd b/vignettes/checks/measurePersonCompleteness.Rmd index 910c0d22..c1a346d0 100644 --- a/vignettes/checks/measurePersonCompleteness.Rmd +++ b/vignettes/checks/measurePersonCompleteness.Rmd @@ -24,16 +24,16 @@ The number and percent of persons in the CDM that do not have at least one recor ## Definition - *Numerator*: The number of persons with 0 rows in a given CDM table. -- *Denominator*: The total number of persons in the `person` table. -- *Related CDM Convention(s)*: Each Person needs to have at least one `observation_period` record. Otherwise, CDM conventions do not dictate any rules for person completeness. -- *CDM Fields/Tables*: By default, this check runs on all tables with a foreign key to the `person` table. -- *Default Threshold Value*: Set to 95 or 100 for most tables but 0 for `observation_period` +- *Denominator*: The total number of persons in the `PERSON` table. +- *Related CDM Convention(s)*: Each Person needs to have at least one `OBSERVATION_PERIOD` record. Otherwise, CDM conventions do not dictate any rules for person completeness. +- *CDM Fields/Tables*: By default, this check runs on all tables with a foreign key to the `PERSON` table. +- *Default Threshold Value*: Set to 95 or 100 for most tables but 0 for `OBSERVATION_PERIOD` ## User Guidance -For most tables, this check is a characterization of the completeness of various data types in the source data. However, in the case of `observation_period`, this check should actually be considered a CDM convention check as it is used to enforce the requirement that all persons have at least one observation period. -A failure of this check on the `observation_period` table is a serious issue as persons without an `observation_period` cannot be included in any standard OHDSI analysis. +For most tables, this check is a characterization of the completeness of various data types in the source data. However, in the case of `OBSERVATION_PERIOD`, this check should actually be considered a CDM convention check as it is used to enforce the requirement that all persons have at least one observation period. +A failure of this check on the `OBSERVATION_PERIOD` table is a serious issue as persons without an `OBSERVATION_PERIOD` cannot be included in any standard OHDSI analysis. Run the following query to obtain a list of persons who had no data in a given table. From this list of person_ids you may join to other tables in the CDM to understand trends in these individuals' data which may provide clues as to the root cause of the issue. @@ -55,18 +55,19 @@ All persons in the CDM must have an observation period; OHDSI analytics tools on #### All other tables Action on persons missing records in other clinical event tables will depend on the characteristics of the source database. In certain cases, missingness is expected – some persons may just not have a given type of data available in the source. In others, various ETL issues may result in persons missing records in a given event table: + - Mis-mapping of domains, resulting in the placement of records in the incorrect table - Incorrect parsing of source data, resulting in loss of valid records - Failure of an ETL step, resulting in an empty table -If more persons than expected are missing data in a given table, run the violated rows SQL snippet to retrieve these persons’ person_ids, and inspect these persons’ other clinical event data in the CDM for trends. You may also use person_source_value to trace back to these persons’ source data to identify source data records potentially missed by the ETL. +If more persons than expected are missing data in a given table, run the violated rows SQL snippet to retrieve these persons’ person_ids, and inspect these persons’ other clinical event data in the CDM for trends. You may also use `person_source_value` to trace back to these persons’ source data to identify source data records potentially missed by the ETL. Note that in some cases, the failure threshold for this check may need to be adjusted according to completeness expectations for a given data source. ### Data Users -Severe failures, such as unexpected nearly empty tables, must be fixed by the ETL team before a dataset can be used. Note as well that any person missing an `observation_period` will not be able to be included in any analysis using OHDSI tools. +Severe failures, such as unexpected nearly empty tables, must be fixed by the ETL team before a dataset can be used. Note as well that any person missing an observation period will not be able to be included in any analysis using OHDSI tools. Failures with a result close to the specified failure threshold may be accepted, at your own risk and only if the result matches your understanding of the source data. The violated rows SQL may be used to inspect the full records for persons missing data in a given table in order to validate your expectations or point to potential issues in the ETL which need to be resolved. diff --git a/vignettes/checks/measureValueCompleteness.Rmd b/vignettes/checks/measureValueCompleteness.Rmd index 1defb0b1..045837a5 100644 --- a/vignettes/checks/measureValueCompleteness.Rmd +++ b/vignettes/checks/measureValueCompleteness.Rmd @@ -40,13 +40,9 @@ Use this SQL query to inspect rows with a missing value in a given field: ```sql SELECT - -'@cdmTableName.@cdmFieldName' AS violating_field, - -cdmTable.* - + '@cdmTableName.@cdmFieldName' AS violating_field, + cdmTable.* FROM @cdmDatabaseSchema.@cdmTableName cdmTable - WHERE cdmTable.@cdmFieldName IS NULL ``` diff --git a/vignettes/checks/sourceConceptRecordCompleteness.Rmd b/vignettes/checks/sourceConceptRecordCompleteness.Rmd index bf7c0e01..0827a1ab 100644 --- a/vignettes/checks/sourceConceptRecordCompleteness.Rmd +++ b/vignettes/checks/sourceConceptRecordCompleteness.Rmd @@ -23,11 +23,13 @@ The number and percent of records with a value of 0 in the source concept field ## Definition -- *Numerator*: The number of rows with a value of 0 in the `X_source_concept_id` source concept field. In the case of `MEASUREMENT.unit_source_concept_id` and `OBSERVATION.unit_source_concept_id`, the number of rows with a value of 0 in the `X_source_concept_id` source concept field AND a non-NULL value_as_number. -- *Denominator*: The total number of rows in the table. In the case of `MEASUREMENT.unit_source_concept_id` and `OBSERVATION.unit_source_concept_id`, the number of rows with non-NULL value_as_number. +- *Numerator*: The number of rows with a value of 0 in the `X_source_concept_id` source concept field. In the case of `MEASUREMENT.unit_source_concept_id` and `OBSERVATION.unit_source_concept_id`, the number of rows with a value of 0 in the `X_source_concept_id` source concept field AND a non-NULL `value_as_number`. +- *Denominator*: The total number of rows in the table. In the case of `MEASUREMENT.unit_source_concept_id` and `OBSERVATION.unit_source_concept_id`, the number of rows with non-NULL `value_as_number`. - *Related CDM Convention(s)*: [Source concept mapping](https://ohdsi.github.io/CommonDataModel/dataModelConventions.html#Fields) - *CDM Fields/Tables*: All source concept ID (`X_source_concept_id`) columns in all event tables. -- *Default Threshold Value*: 10 for primary source concept ID columns in condition, drug, measurement, procedure, device, and observation tables; 100 for all other source concept ID columns. +- *Default Threshold Value*: + - 10 for primary source concept ID columns in condition, drug, measurement, procedure, device, and observation tables + - 100 for all other source concept ID columns ## User Guidance @@ -62,7 +64,7 @@ The query results will give you a summary of the source codes which failed to ma If source values return legitimate matches on concept_code, it’s possible that there is an error in the concept mapping step of your ETL. Please note that while the `X_source_concept_id` fields are technically not required, it is highly recommended to populate them with OMOP concepts whenever possible. This will greatly aid analysts in understanding the provenance of the data. -If source values do NOT return matches on concept_code and you are NOT handling concept mapping locally for a non-OMOP source vocabulary, then you likely have a malformed source code or one that does not exist in the OMOP vocabulary. Please see the documentation in the standardConceptRecordCompleteness page for instructions on how to handle this scenario. +If source values do NOT return matches on concept_code and you are NOT handling concept mapping locally for a non-OMOP source vocabulary, then you likely have a malformed source code or one that does not exist in the OMOP vocabulary. Please see the documentation in the [standardConceptRecordCompleteness](standardConceptRecordCompleteness.html) page for instructions on how to handle this scenario. ### Data Users Since most standard OHDSI analytic workflows rely on the standard concept field and not the source concept field, failures of this check will not necessarily impact your analysis. However, if your analysis depends on understanding source coding practices or on codes you know may not be fully mapped to OMOP standard concepts, then this will be a critical check failure to understand. diff --git a/vignettes/checks/sourceValueCompleteness.Rmd b/vignettes/checks/sourceValueCompleteness.Rmd index 6dc5e3b2..9e39cfba 100644 --- a/vignettes/checks/sourceValueCompleteness.Rmd +++ b/vignettes/checks/sourceValueCompleteness.Rmd @@ -23,7 +23,7 @@ The number and percent of distinct source values in the @cdmFieldName field of t ## Definition -- *Numerator*: Distinct `X_source_value` entries where the corresponding standard x_concept_id field is 0. +- *Numerator*: Distinct `X_source_value` entries where the corresponding standard `X_concept_id` field is 0. - *Denominator*: Total distinct `X_source_value` entries, including NULL, in the respective event table. - *Related CDM Convention(s)*: The OMOP Common Data Model specifies that codes that are present in a native database should be mapped to standard concepts using either the intrinsic mappings defined in the standard vocabularies or extrinsic mappings defined by the data owner or ETL development team. Note also that variations of this check logic are also used in the [EHDEN CDM Inspection Report](https://github.com/EHDEN/CdmInspection) package, as well as the [AresIndexer](https://github.com/OHDSI/AresIndexer) package for generating indices of unmapped codes. - *CDM Fields/Tables*: Runs on all event tables that have `X_source_value` fields. @@ -33,7 +33,7 @@ The number and percent of distinct source values in the @cdmFieldName field of t ## User Guidance -This check will look at all distinct source values in the specified field and calculate how many are mapped to a standard concept of 0. This check should be used in conjunction with the [standardConceptRecordCompleteness](checks/standardConceptRecordCompleteness.html) check to identify potential mapping issues in the ETL. \ +This check will look at all distinct source values in the specified field and calculate how many are mapped to a standard concept of 0. This check should be used in conjunction with the [standardConceptRecordCompleteness](standardConceptRecordCompleteness.html) check to identify potential mapping issues in the ETL. \ This check is a good measure of the overall mapping rate within each domain. For example, a table may have high standardConceptRecordCompleteness (that is, a large percentage of records with a non-zero standard concept) but a low score on this check. This would indicate that the "long tail" of rarer codes have not been mapped while more common codes have good mapping coverage. It is always important to interrogate the results of these two checks together to ensure complete understanding of vocabulary mapping in your CDM. diff --git a/vignettes/checks/standardConceptRecordCompleteness.Rmd b/vignettes/checks/standardConceptRecordCompleteness.Rmd index 6a67aab8..e745a21f 100644 --- a/vignettes/checks/standardConceptRecordCompleteness.Rmd +++ b/vignettes/checks/standardConceptRecordCompleteness.Rmd @@ -23,8 +23,8 @@ The number and percent of records with a value of 0 in the standard concept fiel ## Definition -- *Numerator*: The number of rows with a value of 0 in the `X_concept_id` standard concept field. (**NOTE** in the case of `measurement.unit_concept_id` and `observation.unit_concept_id`, the number of rows with a value of 0 in the `X_concept_id` standard concept field AND a non-NULL value_as_number.) -- *Denominator*: The total number of rows in the table. (**NOTE** in the case of `measurement.unit_concept_id` and `observation.unit_concept_id`, the number of rows with non-NULL `value_as_number`.) +- *Numerator*: The number of rows with a value of 0 in the `X_concept_id` standard concept field. In the case of `MEASUREMENT.unit_concept_id` and `OBSERVATION.unit_concept_id`, the number of rows with a value of 0 in the `X_concept_id` standard concept field AND a non-NULL `value_as_number`. +- *Denominator*: The total number of rows in the table. In the case of `MEASUREMENT.unit_concept_id` and `OBSERVATION.unit_concept_id`, the number of rows with a non-NULL `value_as_number`. - *Related CDM Convention(s)*: [Standard concept mapping](https://ohdsi.github.io/CommonDataModel/dataModelConventions.html#Fields) - *CDM Fields/Tables*: All standard concept ID (`X_concept_id`) columns in all event tables. - *Default Threshold Value*: diff --git a/vignettes/checks/withinVisitDates.Rmd b/vignettes/checks/withinVisitDates.Rmd index b6c9ec1e..6bd7aaa8 100644 --- a/vignettes/checks/withinVisitDates.Rmd +++ b/vignettes/checks/withinVisitDates.Rmd @@ -1,6 +1,6 @@ --- title: "withinVisitDates" -author: "" +author: "Clair Blacketer" date: "`r Sys.Date()`" output: html_document: @@ -14,33 +14,52 @@ output: **Context**: Verification\ **Category**: Conformance\ **Subcategory**: \ -**Severity**: +**Severity**: Characterization ✔ ## Description -The number and percent of records not within one week on either side of the corresponding visit occurrence start and end date +The number and percent of records that occur one week before the corresponding `visit_start_date` or one week after the corresponding `visit_end_date` ## Definition -- *Numerator*: -- *Denominator*: -- *Related CDM Convention(s)*: -- *CDM Fields/Tables*: -- *Default Threshold Value*: +- *Numerator*: The number of rows in a table where the event date occurs more than 7 days prior to the corresponding visit start date or more than 7 days after the corresponding visit end date. +- *Denominator*: The total number of rows in the table with a corresponding visit (linked through `visit_occurrence_id`) +- *Related CDM Convention(s)*: There is no explicit convention tied to this check. However, the CDM documentation describes the `visit_occurrence_id` foreign key in the event tables as “The visit during which the occurred.” The underlying assumption is that if a record is tied to a visit, then the date of the record should fall in some reasonable time period around the visit dates. This gives a week of leeway on either side for physician notes or other activities related to a visit to be recorded. +- *CDM Fields/Tables*: This check runs on all event tables: `CONDITION_OCCURRENCE`, `PROCEDURE_OCCURRENCE`, `DRUG_EXPOSURE`, `DEVICE_EXPOSURE`, `MEASUREMENT`, `NOTE`, `OBSERVATION`, and `VISIT_DETAIL`. It will check either the `X_date` or `X_start_date` fields for alignment with corresponding `VISIT_OCCURRENCE` dates by linking on the `visit_occurrence_id`. (**Note:** For VISIT_DETAIL it will check both the visit_detail_start_date and visit_detail_end_date. The default threshold for these two checks is 1%.) +- *Default Threshold Value*: 1% for `VISIT_DETAIL`, 5% for all other tables ## User Guidance +There is no explicit convention that describes how events should align temporally with the visits they correspond to. This check is meant to identify egregious mismatches in dates that could signify an incorrect date field was used in the ETL or that the data should be used with caution if there is no reason for the mismatch (history of a condition, for example). +If this check fails the first action should be to investigate the failing rows for any patterns. The main query to find failing rows is below: ### Violated rows query ```sql - +SELECT + '@cdmTableName.@cdmFieldName' AS violating_field, + vo.visit_start_date, vo.visit_end_date, vo.person_id, + cdmTable.* +FROM @cdmDatabaseSchema.@cdmTableName cdmTable +JOIN @cdmDatabaseSchema.visit_occurrence vo + ON cdmTable.visit_occurrence_id = vo.visit_occurrence_id +WHERE cdmTable.@cdmFieldName < dateadd(day, -7, vo.visit_start_date) + OR cdmTable.@cdmFieldName > dateadd(day, 7, vo.visit_end_date) ``` - ### ETL Developers +The first step is to investigate whether visit and event indeed should be linked - e.g., do they belong to the same person; how far are the dates apart; is it possible the event was recorded during the visit. If they should be linked, then the next step is to investigate which of the event date and visit date is accurate. + +One suggestion would be to identify if all of the failures are due to many events all having the same date. In some institutions there is a default date given to events in the case where a date is not given. Should this be the problem, the first step should be to identify if there is a different date field in the native data that can be used. If not, considerations should be made to determine if the rows should be dropped. Without a correct date it is challenging to use such events in health outcomes research. +Another reason for the discrepancy could be that the wrong date has been used for events. For instance, in some systems a diagnosis could have both an ‘observation date’ and an ‘administration date’. If the physician is updating records at a later date, the administration date can be later than the actual date the diagnosis was observed. In those cases, the observation date has to be used. If there is only an administration date, it is in some cases arguable to use the visit date for the diagnosis date. + +Another suggestion would be to investigate if the failures are related to ‘History of’ conditions. It is often the case that a patient’s history is recorded during a visit, in which case they may report a diagnosis date prior to the given visit. In some cases it may be appropriate to conserve these records; the decision to do so will depend on the reliability of the recorded date in your source data. ### Data Users +If the failure percentage of withinVisitDates is high, a data user should be careful with using the data. This check might indicate a larger underlying conformance issue with either the event dates or linkage with visits. At the same time, there might be a valid reason why events do not happen within 7 days of the linked visit. + +Make sure to understand why this check fails. Specifically, be careful when using such data in outcomes research. Without specific dates for an event, it is challenging to determine if an adverse event occurred after a drug exposure, for example. +Note that this check specifically compares event dates to `VISIT_OCCURRENCE` dates. There is no equivalent check for `VISIT_DETAIL` that verifies whether the event date is within (a week of) the visit detail start and end dates. From cee587af8c222cab62ddff030a813a0b741942c3 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Sat, 6 Jul 2024 14:28:16 +0200 Subject: [PATCH 22/43] separate connectionDetails for NA Checks --- tests/testthat/setup.R | 3 +++ tests/testthat/test-calculateNotApplicableStatus.R | 14 +++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/testthat/setup.R b/tests/testthat/setup.R index f1aa8636..761fdeab 100644 --- a/tests/testthat/setup.R +++ b/tests/testthat/setup.R @@ -12,3 +12,6 @@ if (Sys.getenv("DONT_DOWNLOAD_JDBC_DRIVERS", "") == "TRUE") { connectionDetailsEunomia <- Eunomia::getEunomiaConnectionDetails() cdmDatabaseSchemaEunomia <- "main" resultsDatabaseSchemaEunomia <- "main" + +# Separate connection details for NA tests, as this requires removing records +connectionDetailsEunomiaNaChecks <- Eunomia::getEunomiaConnectionDetails() \ No newline at end of file diff --git a/tests/testthat/test-calculateNotApplicableStatus.R b/tests/testthat/test-calculateNotApplicableStatus.R index 6c19decd..321dc6cf 100644 --- a/tests/testthat/test-calculateNotApplicableStatus.R +++ b/tests/testthat/test-calculateNotApplicableStatus.R @@ -10,7 +10,7 @@ test_that("Not Applicable status Table Empty", { DatabaseConnector::disconnect(connection) results <- executeDqChecks( - connectionDetails = connectionDetailsEunomia, + connectionDetails = connectionDetailsEunomiaNaChecks, cdmDatabaseSchema = cdmDatabaseSchemaEunomia, resultsDatabaseSchema = resultsDatabaseSchemaEunomia, cdmSourceName = "Eunomia", @@ -31,13 +31,13 @@ test_that("measureConditionEraCompleteness Not Applicable if condition_occurrenc on.exit(unlink(outputFolder, recursive = TRUE)) # Remove records from Condition Occurrence - connection <- DatabaseConnector::connect(connectionDetailsEunomia) + connection <- DatabaseConnector::connect(connectionDetailsEunomiaNaChecks) DatabaseConnector::executeSql(connection, "CREATE TABLE CONDITION_OCCURRENCE_BACK AS SELECT * FROM CONDITION_OCCURRENCE;") DatabaseConnector::executeSql(connection, "DELETE FROM CONDITION_OCCURRENCE;") DatabaseConnector::disconnect(connection) results <- executeDqChecks( - connectionDetails = connectionDetailsEunomia, + connectionDetails = connectionDetailsEunomiaNaChecks, cdmDatabaseSchema = cdmDatabaseSchemaEunomia, resultsDatabaseSchema = resultsDatabaseSchemaEunomia, cdmSourceName = "Eunomia", @@ -49,7 +49,7 @@ test_that("measureConditionEraCompleteness Not Applicable if condition_occurrenc ) # Reinstate Condition Occurrence - connection <- DatabaseConnector::connect(connectionDetailsEunomia) + connection <- DatabaseConnector::connect(connectionDetailsEunomiaNaChecks) DatabaseConnector::executeSql(connection, "INSERT INTO CONDITION_OCCURRENCE SELECT * FROM CONDITION_OCCURRENCE_BACK;") DatabaseConnector::executeSql(connection, "DROP TABLE CONDITION_OCCURRENCE_BACK;") disconnect(connection) @@ -63,13 +63,13 @@ test_that("measureConditionEraCompleteness Fails if condition_era empty", { on.exit(unlink(outputFolder, recursive = TRUE)) # Remove records from Condition Era - connection <- DatabaseConnector::connect(connectionDetailsEunomia) + connection <- DatabaseConnector::connect(connectionDetailsEunomiaNaChecks) DatabaseConnector::executeSql(connection, "CREATE TABLE CONDITION_ERA_BACK AS SELECT * FROM CONDITION_ERA;") DatabaseConnector::executeSql(connection, "DELETE FROM CONDITION_ERA;") DatabaseConnector::disconnect(connection) results <- executeDqChecks( - connectionDetails = connectionDetailsEunomia, + connectionDetails = connectionDetailsEunomiaNaChecks, cdmDatabaseSchema = cdmDatabaseSchemaEunomia, resultsDatabaseSchema = resultsDatabaseSchemaEunomia, cdmSourceName = "Eunomia", @@ -81,7 +81,7 @@ test_that("measureConditionEraCompleteness Fails if condition_era empty", { ) # Reinstate the Condition Era - connection <- DatabaseConnector::connect(connectionDetailsEunomia) + connection <- DatabaseConnector::connect(connectionDetailsEunomiaNaChecks) DatabaseConnector::executeSql(connection, "INSERT INTO CONDITION_ERA SELECT * FROM CONDITION_ERA_BACK;") DatabaseConnector::executeSql(connection, "DROP TABLE CONDITION_ERA_BACK;") DatabaseConnector::disconnect(connection) From 327870de3725fadb5fd9d60ce40e8c63b2b1d19b Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Sat, 6 Jul 2024 22:18:37 +0200 Subject: [PATCH 23/43] add description for conditionEraCompleteness --- .../measureConditionEraCompleteness.Rmd | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/vignettes/checks/measureConditionEraCompleteness.Rmd b/vignettes/checks/measureConditionEraCompleteness.Rmd index 1eaf2b86..f2ffdb46 100644 --- a/vignettes/checks/measureConditionEraCompleteness.Rmd +++ b/vignettes/checks/measureConditionEraCompleteness.Rmd @@ -14,34 +14,44 @@ output: **Context**: Validation\ **Category**: Completeness\ **Subcategory**: \ -**Severity**: +**Severity**: CDM convention ⚠ ## Description -The number and Percent of persons that does not have condition_era built successfully, -for all persons in condition_occurrence +The number and percent of persons that does not have condition_era built successfully, +for all persons in condition_occurrence. ## Definition -- *Numerator*: -- *Denominator*: -- *Related CDM Convention(s)*: -- *CDM Fields/Tables*: -- *Default Threshold Value*: +- *Numerator*: Number of unique person_ids that exist in the condition_occurrence table but not in the condition_era table. +- *Denominator*: Number of unique person_ids in the condition_occurrence table. +- *Related CDM Convention(s)*: Condition Era is derived from Condition Occurrence. +- *CDM Fields/Tables*: condition_era +- *Default Threshold Value*: 0% ## User Guidance +The [Condition Era CDM documentation](https://ohdsi.github.io/CommonDataModel/cdm54.html#condition_era) states that the condition era should be derived by combining condition occurrences. This implies that each person with a condition occurrence should have at least a condition era. +It does NOT clearly state that the condition_era table is required when there are condition occurrences. Still, it is has always been a common convention in the OHDSI community to derive condition era. +There is no THEMIS convention on condition era. ### Violated rows query ```sql - +SELECT DISTINCT + co.person_id +FROM @cdmDatabaseSchema.condition_occurrence co +LEFT JOIN @cdmDatabaseSchema.condition_era cdmTable + ON co.person_id = cdmTable.person_id +WHERE cdmTable.person_id IS NULL ``` - ### ETL Developers +If this check fails, it is likely that there is an issue with the condition era derivation script. Please review the ETL execution log. It might be that this script was not executed and the condition era table is empty, or it had issues running and the condition era table has been partially populated. +If no issues with the ETL run found, the condition era derivation script might have bugs. Please review the code. An example script can be found on [the CDM Documentation page](https://ohdsi.github.io/CommonDataModel/sqlScripts.html#Condition_Eras). +In both cases it is advised to truncate the condition era table and rerun the derivation script. ### Data Users - +The condition era might seem redundant, as for most uses the condition occurrence table can be used. However, tools like FeatureExtraction use condition era to build some covariates and network studies might use cohorts that are based on condition era. It is therefore important that the condition era table is fully populated and captures the same persons as in condition occurrence. From a1f9a183c30956b76617efce35d738603e435a4c Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Sun, 7 Jul 2024 23:21:34 +0200 Subject: [PATCH 24/43] consistent use of % for threshold values --- vignettes/checks/isStandardValidConcept.Rmd | 2 +- vignettes/checks/measurePersonCompleteness.Rmd | 4 +++- vignettes/checks/measureValueCompleteness.Rmd | 6 ++++-- vignettes/checks/plausibleValueHigh.Rmd | 2 +- vignettes/checks/plausibleValueLow.Rmd | 2 +- vignettes/checks/sourceConceptRecordCompleteness.Rmd | 4 ++-- vignettes/checks/sourceValueCompleteness.Rmd | 4 ++-- vignettes/checks/standardConceptRecordCompleteness.Rmd | 6 +++--- vignettes/checks/withinVisitDates.Rmd | 4 +++- 9 files changed, 20 insertions(+), 14 deletions(-) diff --git a/vignettes/checks/isStandardValidConcept.Rmd b/vignettes/checks/isStandardValidConcept.Rmd index d1001aea..75226264 100644 --- a/vignettes/checks/isStandardValidConcept.Rmd +++ b/vignettes/checks/isStandardValidConcept.Rmd @@ -27,7 +27,7 @@ The number and percent of records that do not have a standard, valid concept in - *Denominator*: The total number of rows in the table. - *Related CDM Convention(s)*: All `X_concept_id` columns should contain a standard, valid concept, or 0: https://ohdsi.github.io/CommonDataModel/dataModelConventions.html#Mapping. - *CDM Fields/Tables*: All tables with an `X_concept_id` column, and all `X_concept_id` columns in those tables. -- *Default Threshold Value*: 0 +- *Default Threshold Value*: 0% ## User Guidance diff --git a/vignettes/checks/measurePersonCompleteness.Rmd b/vignettes/checks/measurePersonCompleteness.Rmd index c1a346d0..402fe48f 100644 --- a/vignettes/checks/measurePersonCompleteness.Rmd +++ b/vignettes/checks/measurePersonCompleteness.Rmd @@ -27,7 +27,9 @@ The number and percent of persons in the CDM that do not have at least one recor - *Denominator*: The total number of persons in the `PERSON` table. - *Related CDM Convention(s)*: Each Person needs to have at least one `OBSERVATION_PERIOD` record. Otherwise, CDM conventions do not dictate any rules for person completeness. - *CDM Fields/Tables*: By default, this check runs on all tables with a foreign key to the `PERSON` table. -- *Default Threshold Value*: Set to 95 or 100 for most tables but 0 for `OBSERVATION_PERIOD` +- *Default Threshold Value*: + - 0% for `OBSERVATION_PERIOD` + - 95% or 100% for other tables ## User Guidance diff --git a/vignettes/checks/measureValueCompleteness.Rmd b/vignettes/checks/measureValueCompleteness.Rmd index 045837a5..1ac6771c 100644 --- a/vignettes/checks/measureValueCompleteness.Rmd +++ b/vignettes/checks/measureValueCompleteness.Rmd @@ -27,7 +27,9 @@ The number and percent of records with a NULL value in the @cdmFieldName of the - *Denominator*: The total number of rows in the table. - *Related CDM Convention(s)*: None. This check should be used to check local expectations about completeness of a field given characteristics of the source data. - *CDM Fields/Tables*: All fields in all event tables. -- *Default Threshold Value*: 0 for required fields; 100 for all others +- *Default Threshold Value*: + - 0% for required fields + - 100% for all others ## User Guidance @@ -57,7 +59,7 @@ ETL developers have 2 main options for the use of this check on non-required fie Unexpectedly missing values should be investigated for a potential root cause in the ETL. For expected missingness, rows that violate this check in non-required fields are acceptable but should be clearly communicated to data users so that they can know when and when not to expect data to be present in each field. To avoid confusion for users, however, thresholds should be modified to avoid check failures at expected levels. ### Data Users -This check informs you of the level of missing data in each column of the CDM. If data is missing in a required column, see the isRequired documentation for more information. +This check informs you of the level of missing data in each column of the CDM. If data is missing in a required column, see the `isRequired` documentation for more information. The interpretation of a check failure on a non-required column will depend on the context. In some cases, the threshold for this check will have been very deliberately set, and any failure should be cause for concern unless justified and explained by your ETL provider. In other cases, even if the check fails it may not be worrisome if the check result is in line with your expectations given the source of the data. When in doubt, utilize the inspection query above to ensure you can explain the missing values. diff --git a/vignettes/checks/plausibleValueHigh.Rmd b/vignettes/checks/plausibleValueHigh.Rmd index b6c810dc..1c02484a 100644 --- a/vignettes/checks/plausibleValueHigh.Rmd +++ b/vignettes/checks/plausibleValueHigh.Rmd @@ -34,7 +34,7 @@ The number and percent of records with a value in the @cdmFieldName field of the - `DRUG_EXPOSURE.refills` (compared to 24) - `DRUG_EXPOSURE.days_supply` (compared to 365) - `DRUG_EXPOSURE.quantity` (compared to 1095) -- *Default Threshold Value*: 1 +- *Default Threshold Value*: 1% ## User Guidance diff --git a/vignettes/checks/plausibleValueLow.Rmd b/vignettes/checks/plausibleValueLow.Rmd index 455b7334..e17d9d23 100644 --- a/vignettes/checks/plausibleValueLow.Rmd +++ b/vignettes/checks/plausibleValueLow.Rmd @@ -39,7 +39,7 @@ The number and percent of records with a value in the @cdmFieldName field of the - `DEVICE_EXPOSURE.quantity`, `SPECIMEN.quantity`, `PROCEDURE_OCCURRENCE.quantity` (compared to 1) - `DRUG_ERA.dose_value`, `DRUG_ERA.gap_days` (compared to 0) - `DRUG_ERA.drug_exposure_count` (compared to 1) -- *Default Threshold Value*: 1 +- *Default Threshold Value*: 1% ## User Guidance diff --git a/vignettes/checks/sourceConceptRecordCompleteness.Rmd b/vignettes/checks/sourceConceptRecordCompleteness.Rmd index 0827a1ab..25ac71aa 100644 --- a/vignettes/checks/sourceConceptRecordCompleteness.Rmd +++ b/vignettes/checks/sourceConceptRecordCompleteness.Rmd @@ -28,8 +28,8 @@ The number and percent of records with a value of 0 in the source concept field - *Related CDM Convention(s)*: [Source concept mapping](https://ohdsi.github.io/CommonDataModel/dataModelConventions.html#Fields) - *CDM Fields/Tables*: All source concept ID (`X_source_concept_id`) columns in all event tables. - *Default Threshold Value*: - - 10 for primary source concept ID columns in condition, drug, measurement, procedure, device, and observation tables - - 100 for all other source concept ID columns + - 10% for source concept ID columns in condition, drug, measurement, procedure, device, and observation tables + - 100% for all other source concept ID columns ## User Guidance diff --git a/vignettes/checks/sourceValueCompleteness.Rmd b/vignettes/checks/sourceValueCompleteness.Rmd index 9e39cfba..06c4f1ca 100644 --- a/vignettes/checks/sourceValueCompleteness.Rmd +++ b/vignettes/checks/sourceValueCompleteness.Rmd @@ -28,8 +28,8 @@ The number and percent of distinct source values in the @cdmFieldName field of t - *Related CDM Convention(s)*: The OMOP Common Data Model specifies that codes that are present in a native database should be mapped to standard concepts using either the intrinsic mappings defined in the standard vocabularies or extrinsic mappings defined by the data owner or ETL development team. Note also that variations of this check logic are also used in the [EHDEN CDM Inspection Report](https://github.com/EHDEN/CdmInspection) package, as well as the [AresIndexer](https://github.com/OHDSI/AresIndexer) package for generating indices of unmapped codes. - *CDM Fields/Tables*: Runs on all event tables that have `X_source_value` fields. - *Default Threshold Value*: - - 10 for critical event tables' `X_source_value` fields (condition, measurement, procedure, drug, visit) - - 100 for all other fields - to be adjusted based on source-specific expectations + - 10% for `_source_value` fields in condition, measurement, procedure, drug, visit. + - 100% for all other fields ## User Guidance diff --git a/vignettes/checks/standardConceptRecordCompleteness.Rmd b/vignettes/checks/standardConceptRecordCompleteness.Rmd index e745a21f..3f037c8a 100644 --- a/vignettes/checks/standardConceptRecordCompleteness.Rmd +++ b/vignettes/checks/standardConceptRecordCompleteness.Rmd @@ -28,9 +28,9 @@ The number and percent of records with a value of 0 in the standard concept fiel - *Related CDM Convention(s)*: [Standard concept mapping](https://ohdsi.github.io/CommonDataModel/dataModelConventions.html#Fields) - *CDM Fields/Tables*: All standard concept ID (`X_concept_id`) columns in all event tables. - *Default Threshold Value*: - - 0 for type concept fields and standard concept fields in era tables - - 5 for most standard concept fields in clinical event tables - - 100 for fields more susceptible to specific ETL implementation context + - 0% for type concept fields and standard concept fields in era tables + - 5% for most standard concept fields in clinical event tables + - 100% for fields more susceptible to specific ETL implementation context ## User Guidance diff --git a/vignettes/checks/withinVisitDates.Rmd b/vignettes/checks/withinVisitDates.Rmd index 6bd7aaa8..f15d3a4c 100644 --- a/vignettes/checks/withinVisitDates.Rmd +++ b/vignettes/checks/withinVisitDates.Rmd @@ -27,7 +27,9 @@ The number and percent of records that occur one week before the corresponding ` - *Denominator*: The total number of rows in the table with a corresponding visit (linked through `visit_occurrence_id`) - *Related CDM Convention(s)*: There is no explicit convention tied to this check. However, the CDM documentation describes the `visit_occurrence_id` foreign key in the event tables as “The visit during which the occurred.” The underlying assumption is that if a record is tied to a visit, then the date of the record should fall in some reasonable time period around the visit dates. This gives a week of leeway on either side for physician notes or other activities related to a visit to be recorded. - *CDM Fields/Tables*: This check runs on all event tables: `CONDITION_OCCURRENCE`, `PROCEDURE_OCCURRENCE`, `DRUG_EXPOSURE`, `DEVICE_EXPOSURE`, `MEASUREMENT`, `NOTE`, `OBSERVATION`, and `VISIT_DETAIL`. It will check either the `X_date` or `X_start_date` fields for alignment with corresponding `VISIT_OCCURRENCE` dates by linking on the `visit_occurrence_id`. (**Note:** For VISIT_DETAIL it will check both the visit_detail_start_date and visit_detail_end_date. The default threshold for these two checks is 1%.) -- *Default Threshold Value*: 1% for `VISIT_DETAIL`, 5% for all other tables +- *Default Threshold Value*: + - 1% for `VISIT_DETAIL` + - 5% for all other tables ## User Guidance From 63d4126053f4613292bce4eec48be557861a2ae5 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Sun, 7 Jul 2024 23:27:52 +0200 Subject: [PATCH 25/43] table syntax --- .../measureConditionEraCompleteness.Rmd | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/vignettes/checks/measureConditionEraCompleteness.Rmd b/vignettes/checks/measureConditionEraCompleteness.Rmd index f2ffdb46..9655ef69 100644 --- a/vignettes/checks/measureConditionEraCompleteness.Rmd +++ b/vignettes/checks/measureConditionEraCompleteness.Rmd @@ -19,22 +19,22 @@ output: ## Description The number and percent of persons that does not have condition_era built successfully, -for all persons in condition_occurrence. +for all persons in `CONDITION_OCCURRENCE`. ## Definition -- *Numerator*: Number of unique person_ids that exist in the condition_occurrence table but not in the condition_era table. -- *Denominator*: Number of unique person_ids in the condition_occurrence table. -- *Related CDM Convention(s)*: Condition Era is derived from Condition Occurrence. -- *CDM Fields/Tables*: condition_era +- *Numerator*: Number of unique person_ids that exist in the `CONDITION_OCCURRENCE` table but not in the `CONDITION_ERA` table. +- *Denominator*: Number of unique person_ids in the `CONDITION_OCCURRENCE` table. +- *Related CDM Convention(s)*: Condition Era's are directly derived from Condition Occurrence. +- *CDM Fields/Tables*: `CONDITION_ERA` - *Default Threshold Value*: 0% ## User Guidance -The [Condition Era CDM documentation](https://ohdsi.github.io/CommonDataModel/cdm54.html#condition_era) states that the condition era should be derived by combining condition occurrences. This implies that each person with a condition occurrence should have at least a condition era. -It does NOT clearly state that the condition_era table is required when there are condition occurrences. Still, it is has always been a common convention in the OHDSI community to derive condition era. -There is no THEMIS convention on condition era. +The [Condition Era CDM documentation](https://ohdsi.github.io/CommonDataModel/cdm54.html#condition_era) states that the condition era's should be derived by combining condition occurrences. This implies that each person with a condition occurrence should have at least a condition era. +It does NOT clearly state that the `CONDITION_ERA` table is required when there are condition occurrences. Still, it is has always been a common convention in the OHDSI community to derive condition era. +There is currently no THEMIS convention on condition eras. ### Violated rows query @@ -42,16 +42,16 @@ There is no THEMIS convention on condition era. SELECT DISTINCT co.person_id FROM @cdmDatabaseSchema.condition_occurrence co -LEFT JOIN @cdmDatabaseSchema.condition_era cdmTable - ON co.person_id = cdmTable.person_id + LEFT JOIN @cdmDatabaseSchema.condition_era cdmTable + ON co.person_id = cdmTable.person_id WHERE cdmTable.person_id IS NULL ``` ### ETL Developers If this check fails, it is likely that there is an issue with the condition era derivation script. Please review the ETL execution log. It might be that this script was not executed and the condition era table is empty, or it had issues running and the condition era table has been partially populated. If no issues with the ETL run found, the condition era derivation script might have bugs. Please review the code. An example script can be found on [the CDM Documentation page](https://ohdsi.github.io/CommonDataModel/sqlScripts.html#Condition_Eras). -In both cases it is advised to truncate the condition era table and rerun the derivation script. +In both cases it is advised to truncate the `CONDITION_ERA` table and rerun the derivation script. ### Data Users -The condition era might seem redundant, as for most uses the condition occurrence table can be used. However, tools like FeatureExtraction use condition era to build some covariates and network studies might use cohorts that are based on condition era. It is therefore important that the condition era table is fully populated and captures the same persons as in condition occurrence. +The `CONDITION_ERA` table might seem to contain redundant information, as for most uses the `CONDITION_OCCURRENCE` table can be used. However, tools like FeatureExtraction use condition eras to build some covariates and network studies might use cohorts that are based on condition eras. It is therefore important that the `CONDITION_ERA` table is fully populated and captures the same persons as in condition occurrence. From 7da80fe321218de537d804eb652b55cdc32e84e4 Mon Sep 17 00:00:00 2001 From: Katy Sadowski Date: Tue, 9 Jul 2024 08:55:55 -0400 Subject: [PATCH 26/43] Apply suggestions from Maxim's code review Co-authored-by: Maxim Moinat --- vignettes/checks/isStandardValidConcept.Rmd | 4 ++-- vignettes/checks/measurePersonCompleteness.Rmd | 3 +-- vignettes/checks/measureValueCompleteness.Rmd | 4 ++-- vignettes/checks/sourceConceptRecordCompleteness.Rmd | 2 +- vignettes/checks/standardConceptRecordCompleteness.Rmd | 2 +- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/vignettes/checks/isStandardValidConcept.Rmd b/vignettes/checks/isStandardValidConcept.Rmd index 75226264..bce6180c 100644 --- a/vignettes/checks/isStandardValidConcept.Rmd +++ b/vignettes/checks/isStandardValidConcept.Rmd @@ -26,7 +26,7 @@ The number and percent of records that do not have a standard, valid concept in - *Numerator*: The number of rows with an `X_concept_id` that exists in `CONCEPT.concept_id` but does not equal zero, and has `CONCEPT.standard_concept` != ‘S’ or non-NULL `CONCEPT.invalid_reason`. - *Denominator*: The total number of rows in the table. - *Related CDM Convention(s)*: All `X_concept_id` columns should contain a standard, valid concept, or 0: https://ohdsi.github.io/CommonDataModel/dataModelConventions.html#Mapping. -- *CDM Fields/Tables*: All tables with an `X_concept_id` column, and all `X_concept_id` columns in those tables. +- *CDM Fields/Tables*: All standard concept ID (`X_concept_id`) columns in all event tables. - *Default Threshold Value*: 0% @@ -52,7 +52,7 @@ WHERE co.concept_id != 0 You may build upon this query by joining the relevant `X_concept_id` and `X_source_concept_id` columns to the concept table and inspecting their names and vocabularies. If the `X_source_concept_id` correctly represents the source code in `X_source_value`, the fix will be a matter of ensuring your ETL is correctly using the concept_relationship table to map the source concept ID to a standard concept via the ‘Maps to’ relationship. If you are not populating the `X_source_concept_id` column and/or are using an intermediate concept mapping table, you may need to inspect the mappings in your mapper table to ensure they’ve been generated correctly using the ‘Maps to’ relationship for your CDM’s vocabulary version. ### Data Users -This check failure means that the failing rows will not be picked up in a standard OHDSI analysis. It is highly recommended to work with your ETL team or data provider, if possible, to resolve this issue. +This check failure means that the failing rows will not be picked up in a standard OHDSI analysis. Especially when participating in network research, where only standard concepts are used, this might result in invalid results. It is highly recommended to work with your ETL team or data provider, if possible, to resolve this issue. However, you may work around it at your own risk by determining whether or not the affected rows are relevant for your analysis. Here’s an example query you could run to inspect failing rows in the condition_occurrence table: diff --git a/vignettes/checks/measurePersonCompleteness.Rmd b/vignettes/checks/measurePersonCompleteness.Rmd index 402fe48f..7c7e9735 100644 --- a/vignettes/checks/measurePersonCompleteness.Rmd +++ b/vignettes/checks/measurePersonCompleteness.Rmd @@ -14,7 +14,7 @@ output: **Context**: Validation\ **Category**: Completeness\ **Subcategory**: \ -**Severity**: CDM convention ⚠ Characterization ✔ \ +**Severity**: CDM convention ⚠ (for observation period), Characterization ✔ \ (for all other tables) ## Description @@ -64,7 +64,6 @@ Action on persons missing records in other clinical event tables will depend on If more persons than expected are missing data in a given table, run the violated rows SQL snippet to retrieve these persons’ person_ids, and inspect these persons’ other clinical event data in the CDM for trends. You may also use `person_source_value` to trace back to these persons’ source data to identify source data records potentially missed by the ETL. -Note that in some cases, the failure threshold for this check may need to be adjusted according to completeness expectations for a given data source. ### Data Users diff --git a/vignettes/checks/measureValueCompleteness.Rmd b/vignettes/checks/measureValueCompleteness.Rmd index 1ac6771c..1983d3dd 100644 --- a/vignettes/checks/measureValueCompleteness.Rmd +++ b/vignettes/checks/measureValueCompleteness.Rmd @@ -49,11 +49,11 @@ WHERE cdmTable.@cdmFieldName IS NULL ``` ### ETL Developers -Failures of this check on fields required in the CDM specification are redundant with failures of `isRequired`. See [isRequired documentation](isRequired.html) for more information. +Failures of this check on required fields are redundant with failures of `isRequired`. See [isRequired documentation](isRequired.html) for more information. ETL developers have 2 main options for the use of this check on non-required fields: -- The check threshold may be set to 100% for non-required fields such that the check will never fail. The check result can be used simply to understand completeness for these fields +- The check threshold may be left on 100% for non-required fields such that the check will never fail. The check result can be used simply to understand completeness for these fields - The check threshold may be set to an appropriate value corresponding to completeness expectations for each field given what’s available in the source data. The check may be disabled for fields known not to exist in the source data. Other fields may be set to whichever threshold is deemed worthy of investigation Unexpectedly missing values should be investigated for a potential root cause in the ETL. For expected missingness, rows that violate this check in non-required fields are acceptable but should be clearly communicated to data users so that they can know when and when not to expect data to be present in each field. To avoid confusion for users, however, thresholds should be modified to avoid check failures at expected levels. diff --git a/vignettes/checks/sourceConceptRecordCompleteness.Rmd b/vignettes/checks/sourceConceptRecordCompleteness.Rmd index 25ac71aa..51627af4 100644 --- a/vignettes/checks/sourceConceptRecordCompleteness.Rmd +++ b/vignettes/checks/sourceConceptRecordCompleteness.Rmd @@ -67,6 +67,6 @@ If source values return legitimate matches on concept_code, it’s possible that If source values do NOT return matches on concept_code and you are NOT handling concept mapping locally for a non-OMOP source vocabulary, then you likely have a malformed source code or one that does not exist in the OMOP vocabulary. Please see the documentation in the [standardConceptRecordCompleteness](standardConceptRecordCompleteness.html) page for instructions on how to handle this scenario. ### Data Users -Since most standard OHDSI analytic workflows rely on the standard concept field and not the source concept field, failures of this check will not necessarily impact your analysis. However, if your analysis depends on understanding source coding practices or on codes you know may not be fully mapped to OMOP standard concepts, then this will be a critical check failure to understand. +Since most standard OHDSI analytic workflows rely on the standard concept field and not the source concept field, failures of this check will not necessarily impact your analysis. However, having the source concept will give you a better understanding of the provenance of the code and highlight potential issues where meaning is lost due to mapping to a standard concept. Utilize the investigation queries above to understand the scope and impact of the mapping failures on your specific analytic use case. If none of the affected codes seem to be relevant for your analysis, it may be acceptable to ignore the failure. However, since it is not always possible to understand exactly what a given source value represents, you should proceed with caution and confirm any findings with your ETL provider if possible. diff --git a/vignettes/checks/standardConceptRecordCompleteness.Rmd b/vignettes/checks/standardConceptRecordCompleteness.Rmd index 3f037c8a..80f77bf7 100644 --- a/vignettes/checks/standardConceptRecordCompleteness.Rmd +++ b/vignettes/checks/standardConceptRecordCompleteness.Rmd @@ -87,7 +87,7 @@ WHERE concept_code = - If no result is returned, consider whether the source value may be a malformed version of a legitimate code (for example, sometimes ICD10-CM codes do not contain a “.” in source data). If you can confirm that the code is properly formatted, then you have a source code which does not exist in the OMOP vocabulary. If you believe the code was omitted from the vocabulary in error, please report this issue to the vocabulary team. Otherwise, the short-term course of action will be to generate a mapping for the code locally and implement the mapping in your ETL. For the longer term, the vocabulary team provides a workflow to submit new vocabularies for inclusion in the OMOP vocabularies - Note that in some cases, you will find that no standard concept exists to which to map your source code. In this case, the standard concept ID field should be left as 0 in the short term; in the longer term please work with the vocabulary team to address this gap as recommended above - - Finally, if the investigation query returns no source value, you must trace the relevant record(s) back to their source and confirm if the missing value is expected. If not, identify and fix the related issue in your ETL. If the record legitimately has no value/code in the source data, then the standard concept ID may be left as 0. However, in some cases these “code-less” records represent junk data which should be filtered out in the ETL. The proper approach will be context-dependent + - Finally, if the investigation query returns no source value, you must trace the relevant record(s) back to their source and confirm if the missing source value is expected. If not, identify and fix the related issue in your ETL. If the record legitimately has no value/code in the source data, then the standard concept ID may be left as 0. However, in some cases these “code-less” records represent junk data which should be filtered out in the ETL. The proper approach will be context-dependent - Note in the special case of unitless measurements/observations, the unit_concept_id field should NOT be coded as 0 and rather should be left NULL (the unit_concept_id fields are optional in the CDM spec) It is important to note that records with a 0 standard concept ID field will be unusable in standard OHDSI analyses and thus should only be preserved if there is truly no standard concept ID for a given record. Depending on the significance of the records in question, one should consider removing them from the dataset; however, this choice will depend on a variety of context-specific factors and should be made carefully. Either way, the presence/absence of these unmappable records and an explanation for why they could not be mapped should be clearly documented in the ETL documentation. From 9e451d33ff504eab248c6fe0292c77bd2edaab15 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Thu, 11 Jul 2024 14:48:53 +0200 Subject: [PATCH 27/43] fix na status test --- tests/testthat/test-calculateNotApplicableStatus.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test-calculateNotApplicableStatus.R b/tests/testthat/test-calculateNotApplicableStatus.R index 321dc6cf..0f3bc64c 100644 --- a/tests/testthat/test-calculateNotApplicableStatus.R +++ b/tests/testthat/test-calculateNotApplicableStatus.R @@ -5,7 +5,7 @@ test_that("Not Applicable status Table Empty", { on.exit(unlink(outputFolder, recursive = TRUE)) # Make sure the device exposure table is empty - connection <- DatabaseConnector::connect(connectionDetailsEunomia) + connection <- DatabaseConnector::connect(connectionDetailsEunomiaNaChecks) DatabaseConnector::executeSql(connection, "DELETE FROM DEVICE_EXPOSURE;") DatabaseConnector::disconnect(connection) From 7f0f85f8573dbf66311bf9b12b8a2a5b07cda318 Mon Sep 17 00:00:00 2001 From: Katy Sadowski Date: Thu, 11 Jul 2024 21:47:20 -0400 Subject: [PATCH 28/43] updates from PR review --- docs/articles/AddNewCheck.html | 2 +- docs/articles/CheckStatusDefinitions.html | 2 +- docs/articles/CheckTypeDescriptions.html | 2 +- docs/articles/DataQualityDashboard.html | 2 +- docs/articles/DqdForCohorts.html | 2 +- docs/articles/SqlOnly.html | 2 +- docs/articles/Thresholds.html | 2 +- docs/articles/checkIndex.html | 2 +- docs/articles/checks/cdmDatatype.html | 2 +- docs/articles/checks/cdmField.html | 2 +- docs/articles/checks/cdmTable.html | 2 +- docs/articles/checks/fkClass.html | 2 +- docs/articles/checks/fkDomain.html | 2 +- docs/articles/checks/isForeignKey.html | 8 +-- docs/articles/checks/isPrimaryKey.html | 2 +- docs/articles/checks/isRequired.html | 8 +-- .../checks/isStandardValidConcept.html | 64 ++++++++++--------- .../measureConditionEraCompleteness.html | 2 +- .../checks/measurePersonCompleteness.html | 24 ++++--- .../checks/measureValueCompleteness.html | 32 +++++----- docs/articles/checks/plausibleAfterBirth.html | 2 +- .../articles/checks/plausibleBeforeDeath.html | 2 +- .../checks/plausibleGenderUseDescendants.html | 2 +- .../checks/plausibleStartBeforeEnd.html | 2 +- .../checks/plausibleTemporalAfter.html | 2 +- .../checks/plausibleUnitConceptIds.html | 2 +- docs/articles/checks/plausibleValueHigh.html | 4 +- docs/articles/checks/plausibleValueLow.html | 4 +- .../sourceConceptRecordCompleteness.html | 53 +++++++-------- .../checks/sourceValueCompleteness.html | 17 +++-- .../standardConceptRecordCompleteness.html | 34 +++++----- docs/articles/checks/withinVisitDates.html | 11 +++- docs/pkgdown.yml | 2 +- vignettes/checks/isForeignKey.Rmd | 6 +- vignettes/checks/isRequired.Rmd | 2 +- vignettes/checks/isStandardValidConcept.Rmd | 15 +++-- .../checks/measurePersonCompleteness.Rmd | 4 +- vignettes/checks/measureValueCompleteness.Rmd | 2 +- .../sourceConceptRecordCompleteness.Rmd | 19 +++--- vignettes/checks/sourceValueCompleteness.Rmd | 6 +- .../standardConceptRecordCompleteness.Rmd | 12 ++-- 41 files changed, 193 insertions(+), 176 deletions(-) diff --git a/docs/articles/AddNewCheck.html b/docs/articles/AddNewCheck.html index bc027dea..0e7ab7f1 100644 --- a/docs/articles/AddNewCheck.html +++ b/docs/articles/AddNewCheck.html @@ -180,7 +180,7 @@

    Add a New Data Quality Check

    Don Torok

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/AddNewCheck.rmd diff --git a/docs/articles/CheckStatusDefinitions.html b/docs/articles/CheckStatusDefinitions.html index 919cfebc..fe8df052 100644 --- a/docs/articles/CheckStatusDefinitions.html +++ b/docs/articles/CheckStatusDefinitions.html @@ -181,7 +181,7 @@

    Check Status Definitions

    Dmitry Ilyn, Maxim Moinat

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/CheckStatusDefinitions.rmd diff --git a/docs/articles/CheckTypeDescriptions.html b/docs/articles/CheckTypeDescriptions.html index 6e89315a..a58ea494 100644 --- a/docs/articles/CheckTypeDescriptions.html +++ b/docs/articles/CheckTypeDescriptions.html @@ -181,7 +181,7 @@

    Data Quality Check Type Definitions

    Clair Blacketer

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/CheckTypeDescriptions.rmd diff --git a/docs/articles/DataQualityDashboard.html b/docs/articles/DataQualityDashboard.html index f1cf33d8..d22f577a 100644 --- a/docs/articles/DataQualityDashboard.html +++ b/docs/articles/DataQualityDashboard.html @@ -181,7 +181,7 @@

    Getting Started

    Clair Blacketer

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/DataQualityDashboard.rmd diff --git a/docs/articles/DqdForCohorts.html b/docs/articles/DqdForCohorts.html index 8db9ac2c..d9530035 100644 --- a/docs/articles/DqdForCohorts.html +++ b/docs/articles/DqdForCohorts.html @@ -181,7 +181,7 @@

    Running the DQD on a Cohort

    Clair Blacketer

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/DqdForCohorts.rmd diff --git a/docs/articles/SqlOnly.html b/docs/articles/SqlOnly.html index 1c4a7d4c..71653b00 100644 --- a/docs/articles/SqlOnly.html +++ b/docs/articles/SqlOnly.html @@ -181,7 +181,7 @@

    Running the DQD in SqlOnly mode

    Maxim Moinat

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/SqlOnly.rmd diff --git a/docs/articles/Thresholds.html b/docs/articles/Thresholds.html index 1d94bcf7..7bcce30c 100644 --- a/docs/articles/Thresholds.html +++ b/docs/articles/Thresholds.html @@ -181,7 +181,7 @@

    Failure Thresholds and How to Change Them

    Clair Blacketer

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/Thresholds.rmd diff --git a/docs/articles/checkIndex.html b/docs/articles/checkIndex.html index 289565c4..9289ad6a 100644 --- a/docs/articles/checkIndex.html +++ b/docs/articles/checkIndex.html @@ -181,7 +181,7 @@

    Index

    Katy Sadowski

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checkIndex.Rmd diff --git a/docs/articles/checks/cdmDatatype.html b/docs/articles/checks/cdmDatatype.html index c0b9d6c7..bd765aeb 100644 --- a/docs/articles/checks/cdmDatatype.html +++ b/docs/articles/checks/cdmDatatype.html @@ -181,7 +181,7 @@

    cdmDatatype

    Katy Sadowski

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/cdmDatatype.Rmd diff --git a/docs/articles/checks/cdmField.html b/docs/articles/checks/cdmField.html index 99406a82..3464b260 100644 --- a/docs/articles/checks/cdmField.html +++ b/docs/articles/checks/cdmField.html @@ -181,7 +181,7 @@

    cdmField

    Heidi Schmidt, Katy Sadowski

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/cdmField.Rmd diff --git a/docs/articles/checks/cdmTable.html b/docs/articles/checks/cdmTable.html index 3150fd31..03d2cd40 100644 --- a/docs/articles/checks/cdmTable.html +++ b/docs/articles/checks/cdmTable.html @@ -181,7 +181,7 @@

    cdmTable

    John Gresh, Katy Sadowski

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/cdmTable.Rmd diff --git a/docs/articles/checks/fkClass.html b/docs/articles/checks/fkClass.html index 2904a2e3..bbfbd439 100644 --- a/docs/articles/checks/fkClass.html +++ b/docs/articles/checks/fkClass.html @@ -181,7 +181,7 @@

    fkClass

    Clair Blacketer, Katy Sadowski

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/fkClass.Rmd diff --git a/docs/articles/checks/fkDomain.html b/docs/articles/checks/fkDomain.html index d90b3fb0..bc151de1 100644 --- a/docs/articles/checks/fkDomain.html +++ b/docs/articles/checks/fkDomain.html @@ -181,7 +181,7 @@

    fkDomain

    Clair Blacketer, Katy Sadowski

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/fkDomain.Rmd diff --git a/docs/articles/checks/isForeignKey.html b/docs/articles/checks/isForeignKey.html index 3983b9fa..49b25eeb 100644 --- a/docs/articles/checks/isForeignKey.html +++ b/docs/articles/checks/isForeignKey.html @@ -181,7 +181,7 @@

    isForeignKey

    Dmytry Dymshyts, Katy Sadowski

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/isForeignKey.Rmd @@ -246,7 +246,7 @@

    User GuidanceViolated rows query
    -- @cdmTableName.@cdmFieldName is the x_concept_id or x_source_concept_id field in a CDM table
--- Inspect the contents of the x_source_value field to investigate the source of the error
+
    -- @cdmTableName.@cdmFieldName is the _concept_id or _source_concept_id field in a CDM table
+-- Inspect the contents of the _source_value field to investigate the source of the error
 
 SELECT 
   '@cdmTableName.@cdmFieldName' AS violating_field,  
diff --git a/docs/articles/checks/isPrimaryKey.html b/docs/articles/checks/isPrimaryKey.html
index 8fdef03a..cc5d3275 100644
--- a/docs/articles/checks/isPrimaryKey.html
+++ b/docs/articles/checks/isPrimaryKey.html
@@ -181,7 +181,7 @@ 
    isPrimaryKey
    
                         
    John Gresh,
 Katy Sadowski
    
             
-            
    2024-06-29
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/isPrimaryKey.Rmd
       
diff --git a/docs/articles/checks/isRequired.html b/docs/articles/checks/isRequired.html
index 994448e6..3da6dfaa 100644
--- a/docs/articles/checks/isRequired.html
+++ b/docs/articles/checks/isRequired.html
@@ -181,7 +181,7 @@ 
    isRequired
    
                         
    Katy
 Sadowski
    
             
-            
    2024-06-29
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/isRequired.Rmd
       
@@ -253,9 +253,9 @@ 
    ETL DevelopersFill in the missing values:
 
 
    
 
    
@@ -240,36 +239,43 @@ 
    ETL DevelopersX_concept_id
-column with 0. See the Book of OHDSI for additional guidance on the
-concept mapping process: https://ohdsi.github.io/TheBookOfOhdsi/ExtractTransformLoad.html#step-2-create-the-code-mappings
    
+for a source code, you MUST populate its _concept_id column
+with 0. See the Book of OHDSI for additional guidance on the concept
+mapping process: https://ohdsi.github.io/TheBookOfOhdsi/ExtractTransformLoad.html#step-2-create-the-code-mappings
    
 
    You may inspect the failing rows using the following SQL:
    
 
    SELECT  
   '@cdmTableName.@cdmFieldName' AS violating_field,  
-  cdmTable.*  
-FROM @schema.@cdmTableName cdmTable 
-  JOIN @vocabDatabaseSchema.concept co ON cdmTable.@cdmFieldName = co.concept_id 
-WHERE co.concept_id != 0  
-  AND (co.standard_concept != 'S' OR co.invalid_reason IS NOT NULL) 
    
-
    You may build upon this query by joining the relevant
-X_concept_id and X_source_concept_id columns
-to the concept table and inspecting their names and vocabularies. If the
-X_source_concept_id correctly represents the source code in
-X_source_value, the fix will be a matter of ensuring your
-ETL is correctly using the concept_relationship table to map the source
-concept ID to a standard concept via the ‘Maps to’ relationship. If you
-are not populating the X_source_concept_id column and/or
-are using an intermediate concept mapping table, you may need to inspect
-the mappings in your mapper table to ensure they’ve been generated
-correctly using the ‘Maps to’ relationship for your CDM’s vocabulary
-version.
    
+  cdmTable.*,
+  co.*
+FROM @schema.@cdmTableName cdmTable 
+  JOIN @vocabDatabaseSchema.concept co ON cdmTable.@cdmFieldName = co.concept_id 
+WHERE co.concept_id != 0  
+  AND (co.standard_concept != 'S' OR co.invalid_reason IS NOT NULL)
    +

    You may build upon this query by joining the +_source_concept_id column to the concept table and +inspecting the source concepts from which the failing non-standard +concepts were mapped. If the _source_concept_id correctly +represents the source code in _source_value, the fix will +be a matter of ensuring your ETL is correctly using the +concept_relationship table to map the source concept ID to a standard +concept via the ‘Maps to’ relationship. If you are not populating the +_source_concept_id column and/or are using an intermediate +concept mapping table, you may need to inspect the mappings in your +mapper table to ensure they’ve been generated correctly using the ‘Maps +to’ relationship for your CDM’s vocabulary version.

    +

    Also note that when updating the OMOP vocabularies, previously +standard concepts could have been become non-standard and need +remapping. Often this remapping can be done programatically, by +following the ‘Maps to’ relationship to the new standard concept.

    Data Users

    This check failure means that the failing rows will not be picked up -in a standard OHDSI analysis. It is highly recommended to work with your -ETL team or data provider, if possible, to resolve this issue.

    +in a standard OHDSI analysis. Especially when participating in network +research, where only standard concepts are used, this might result in +invalid results. It is highly recommended to work with your ETL team or +data provider, if possible, to resolve this issue.

    However, you may work around it at your own risk by determining whether or not the affected rows are relevant for your analysis. Here’s an example query you could run to inspect failing rows in the diff --git a/docs/articles/checks/measureConditionEraCompleteness.html b/docs/articles/checks/measureConditionEraCompleteness.html index e8ce2a9f..2689e33c 100644 --- a/docs/articles/checks/measureConditionEraCompleteness.html +++ b/docs/articles/checks/measureConditionEraCompleteness.html @@ -180,7 +180,7 @@

    measureConditionEraCompleteness

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/measureConditionEraCompleteness.Rmd diff --git a/docs/articles/checks/measurePersonCompleteness.html b/docs/articles/checks/measurePersonCompleteness.html index 4e61a71d..3d68ce34 100644 --- a/docs/articles/checks/measurePersonCompleteness.html +++ b/docs/articles/checks/measurePersonCompleteness.html @@ -181,7 +181,7 @@

    measurePersonCompleteness

    Katy Sadowski

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/measurePersonCompleteness.Rmd @@ -193,7 +193,8 @@

    2024-06-29

    Summary

    -

    Level: TABLE
    Context: Validation
    Category: Completeness
    Subcategory:
    Severity: CDM convention ⚠ Characterization ✔

    +

    Level: TABLE
    Context: Validation
    Category: Completeness
    Subcategory:
    Severity: CDM convention ⚠ (for observation period), +Characterization ✔  (for all other tables)

    Description @@ -220,8 +221,12 @@

    DefinitionAll other tablesAction on persons missing records in other clinical event tables will depend on the characteristics of the source database. In certain cases, missingness is expected – some persons may just not have a given type of -data available in the source. In others, various ETL issues may result -in persons missing records in a given event table:

    +data available in the source. For instance, in most data sources, one +would expect most patients to have at least one visit, diagnosis, and +drug, while one would not expect every single patient to have +had a medical device.

    +

    Various ETL issues may result in persons missing records in a given +event table:

    diff --git a/docs/articles/checks/measureValueCompleteness.html b/docs/articles/checks/measureValueCompleteness.html index f0be5e07..114f34ad 100644 --- a/docs/articles/checks/measureValueCompleteness.html +++ b/docs/articles/checks/measureValueCompleteness.html @@ -181,7 +181,7 @@

    measureValueCompleteness

    Katy Sadowski

    -

    2024-06-29

    +

    2024-07-11

    Source:     vignettes/checks/measureValueCompleteness.Rmd @@ -216,8 +216,12 @@

    Definition @@ -248,13 +252,13 @@

    Violated rows query

    ETL Developers

    -

    Failures of this check on fields required in the CDM specification -are redundant with failures of isRequired. See isRequired documentation for more -information.

    +

    Failures of this check on required fields are redundant with failures +of isRequired. See isRequired +documentation for more information.

    ETL developers have 2 main options for the use of this check on non-required fields:

      -
    • The check threshold may be set to 100% for non-required fields such +
    • The check threshold may be left on 100% for non-required fields such that the check will never fail. The check result can be used simply to understand completeness for these fields
    • The check threshold may be set to an appropriate value corresponding @@ -264,19 +268,17 @@

      ETL Developers

    Unexpectedly missing values should be investigated for a potential -root cause in the ETL. For expected missingness, rows that violate this -check in non-required fields are acceptable but should be clearly -communicated to data users so that they can know when and when not to -expect data to be present in each field. To avoid confusion for users, -however, thresholds should be modified to avoid check failures at -expected levels.

    +root cause in the ETL. If a threshold has been adjusted to account for +expected missingness, this should be clearly communicated to data users +so that they can know when and when not to expect data to be present in +each field.

    Data Users

    This check informs you of the level of missing data in each column of -the CDM. If data is missing in a required column, see the isRequired -documentation for more information.

    +the CDM. If data is missing in a required column, see the +isRequired documentation for more information.

    The interpretation of a check failure on a non-required column will depend on the context. In some cases, the threshold for this check will have been very deliberately set, and any failure should be cause for diff --git a/docs/articles/checks/plausibleAfterBirth.html b/docs/articles/checks/plausibleAfterBirth.html index 610bc1e4..cacfbb0a 100644 --- a/docs/articles/checks/plausibleAfterBirth.html +++ b/docs/articles/checks/plausibleAfterBirth.html @@ -181,7 +181,7 @@

    plausibleAfterBirth

    Maxim Moinat, Katy Sadowski

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/plausibleAfterBirth.Rmd diff --git a/docs/articles/checks/plausibleBeforeDeath.html b/docs/articles/checks/plausibleBeforeDeath.html index 2e91b839..a2d11704 100644 --- a/docs/articles/checks/plausibleBeforeDeath.html +++ b/docs/articles/checks/plausibleBeforeDeath.html @@ -181,7 +181,7 @@

    plausibleBeforeDeath

    Maxim Moinat

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/plausibleBeforeDeath.Rmd diff --git a/docs/articles/checks/plausibleGenderUseDescendants.html b/docs/articles/checks/plausibleGenderUseDescendants.html index c4e8155a..2e92d60d 100644 --- a/docs/articles/checks/plausibleGenderUseDescendants.html +++ b/docs/articles/checks/plausibleGenderUseDescendants.html @@ -180,7 +180,7 @@

    plausibleGender

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/plausibleGenderUseDescendants.Rmd diff --git a/docs/articles/checks/plausibleStartBeforeEnd.html b/docs/articles/checks/plausibleStartBeforeEnd.html index f80c0a9b..489a615d 100644 --- a/docs/articles/checks/plausibleStartBeforeEnd.html +++ b/docs/articles/checks/plausibleStartBeforeEnd.html @@ -181,7 +181,7 @@

    plausibleStartBeforeEnd

    Maxim Moinat

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/plausibleStartBeforeEnd.Rmd diff --git a/docs/articles/checks/plausibleTemporalAfter.html b/docs/articles/checks/plausibleTemporalAfter.html index 3a01db40..2c11f327 100644 --- a/docs/articles/checks/plausibleTemporalAfter.html +++ b/docs/articles/checks/plausibleTemporalAfter.html @@ -180,7 +180,7 @@

    plausibleTemporalAfter

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/plausibleTemporalAfter.Rmd diff --git a/docs/articles/checks/plausibleUnitConceptIds.html b/docs/articles/checks/plausibleUnitConceptIds.html index 6855948c..cf0df5fb 100644 --- a/docs/articles/checks/plausibleUnitConceptIds.html +++ b/docs/articles/checks/plausibleUnitConceptIds.html @@ -180,7 +180,7 @@

    plausibleUnitConceptIds

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/plausibleUnitConceptIds.Rmd diff --git a/docs/articles/checks/plausibleValueHigh.html b/docs/articles/checks/plausibleValueHigh.html index 799422ff..7879501f 100644 --- a/docs/articles/checks/plausibleValueHigh.html +++ b/docs/articles/checks/plausibleValueHigh.html @@ -181,7 +181,7 @@

    plausibleValueHigh

    Dymytry Dymshyts

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/plausibleValueHigh.Rmd @@ -234,7 +234,7 @@

    Definition diff --git a/docs/articles/checks/plausibleValueLow.html b/docs/articles/checks/plausibleValueLow.html index 9cdbe4b0..40d10b8a 100644 --- a/docs/articles/checks/plausibleValueLow.html +++ b/docs/articles/checks/plausibleValueLow.html @@ -181,7 +181,7 @@

    plausibleValueLow

    Dymytry Dymshyts

    -

    2024-06-29

    +

    2024-07-11

    Source:     vignettes/checks/plausibleValueLow.Rmd @@ -247,7 +247,7 @@

    Definition diff --git a/docs/articles/checks/sourceConceptRecordCompleteness.html b/docs/articles/checks/sourceConceptRecordCompleteness.html index b5f59d0b..2d4effb4 100644 --- a/docs/articles/checks/sourceConceptRecordCompleteness.html +++ b/docs/articles/checks/sourceConceptRecordCompleteness.html @@ -181,7 +181,7 @@

    sourceConceptRecordCompleteness

    Katy Sadowski

    -

    2024-06-29

    +

    2024-07-11

    Source:     vignettes/checks/sourceConceptRecordCompleteness.Rmd @@ -207,29 +207,23 @@

    DefinitionSource concept mapping

  • CDM Fields/Tables: All source concept ID -(X_source_concept_id) columns in all event tables.
    • +(_source_concept_id) columns in all event tables.
  • Default Threshold Value:
      -
    • 10 for primary source concept ID columns in condition, drug, -measurement, procedure, device, and observation tables
      • -
    • 100 for all other source concept ID columns
      • +
    • 10% for source concept ID columns in condition, drug, measurement, +procedure, device, and observation tables
      • +
    • 100% for all other source concept ID columns
    • @@ -247,11 +241,11 @@

    User Guidance

    ETL Developers

    -

    Recall that the X_source_concept_id columns should +

    Recall that the _source_concept_id columns should contain the OMOP concept representing the exact code used in the source -data for a given record: “If the is coded in the source -data using an OMOP supported vocabulary put the concept id representing -the source value here.”

    +data for a given record: “If the <_source_value> is coded in the +source data using an OMOP supported vocabulary put the concept id +representing the source value here.”

    A failure of this check usually indicates a failure to map a source value to an OMOP concept. In some cases, such a failure can and should be remediated in the concept-mapping step of the ETL. In other cases, it @@ -259,25 +253,24 @@

    ETL DevelopersTo investigate the failure, run the following query:

    SELECT  
   concept.concept_name AS standard_concept_name, 
-  cdmTable.X_concept_id, -- standard concept ID field for the table 
+  cdmTable._concept_id, -- standard concept ID field for the table 
   c2.concept_name AS source_value_concept_name, 
-  cdmTable.X_source_value, -- source value field for the table 
+  cdmTable._source_value, -- source value field for the table 
   COUNT(*) 
 FROM @cdmDatabaseSchema.@cdmTableName cdmTable 
-LEFT JOIN @vocabDatabaseSchema.concept ON concept.concept_id = cdmTable.X_concept_id 
+LEFT JOIN @vocabDatabaseSchema.concept ON concept.concept_id = cdmTable._concept_id 
 -- WARNING this join may cause fanning if a source value exists in multiple vocabularies 
-LEFT JOIN @vocabDatabaseSchema.concept c2 ON concept.concept_code = cdmTable.X_source_value 
+LEFT JOIN @vocabDatabaseSchema.concept c2 ON concept.concept_code = cdmTable._source_value 
 AND c2.domain_id = <Domain of cdmTable> 
 WHERE cdmTable.@cdmFieldName = 0  
--- AND cdmTable.value_as_number IS NOT NULL -- uncomment for unit_concept_id checks 
-GROUP BY 1,2,3 
-ORDER BY 4 DESC
    +GROUP BY 1,2,3 +ORDER BY 4 DESC

    The query results will give you a summary of the source codes which failed to map to an OMOP concept. Inspecting this data should give you an initial idea of what might be going on.

    If source values return legitimate matches on concept_code, it’s possible that there is an error in the concept mapping step of your ETL. -Please note that while the X_source_concept_id fields are +Please note that while the _source_concept_id fields are technically not required, it is highly recommended to populate them with OMOP concepts whenever possible. This will greatly aid analysts in understanding the provenance of the data.

    @@ -292,10 +285,10 @@

    Data UsersJared Houghtaling, Clair Blacketer

    -

    2024-06-29

    +

    2024-07-11

    Source:     vignettes/checks/sourceValueCompleteness.Rmd @@ -205,11 +205,11 @@

    DefinitionDefinitionKaty Sadowski

    -

    2024-06-29

    +

    2024-07-11

    Source:     vignettes/checks/standardConceptRecordCompleteness.Rmd @@ -207,10 +207,10 @@

    DefinitionDefinitionETL DevelopersTo investigate the failure, run the following query:

    SELECT  
   concept_name, 
-  cdmTable.X_source_concept_id, -- source concept ID field for the table 
-  cdmTable.X_source_value, -- source value field for the table 
+  cdmTable._source_concept_id, -- source concept ID field for the table 
+  cdmTable._source_value, -- source value field for the table 
   COUNT(*) 
 FROM @cdmDatabaseSchema.@cdmTableName cdmTable 
-LEFT JOIN @vocabDatabaseSchema.concept ON concept.concept_id = cdmTable.X_source_concept_id 
+LEFT JOIN @vocabDatabaseSchema.concept ON concept.concept_id = cdmTable._source_concept_id 
 WHERE cdmTable.@cdmFieldName = 0  
 -- AND cdmTable.value_as_number IS NOT NULL -- uncomment for unit_concept_id checks 
 GROUP BY 1,2,3 
@@ -327,11 +328,12 @@ 
    ETL Developers
 
  • Finally, if the investigation query returns no source value, you
 must trace the relevant record(s) back to their source and confirm if
-the missing value is expected. If not, identify and fix the related
-issue in your ETL. If the record legitimately has no value/code in the
-source data, then the standard concept ID may be left as 0. However, in
-some cases these “code-less” records represent junk data which should be
-filtered out in the ETL. The proper approach will be context-dependent
+the missing source value is expected. If not, identify and fix the
+related issue in your ETL. If the record legitimately has no value/code
+in the source data, then the standard concept ID may be left as 0.
+However, in some cases these “code-less” records represent junk data
+which should be filtered out in the ETL. The proper approach will be
+context-dependent
    • +

    You may build upon this query by joining the +_source_concept_id column to the concept table and +inspecting the source concepts from which the failing non-standard +concepts were mapped. If the _source_concept_id correctly +represents the source code in _source_value, the fix will +be a matter of ensuring your ETL is correctly using the +concept_relationship table to map the source concept ID to a standard +concept via the ‘Maps to’ relationship. If you are not populating the +_source_concept_id column and/or are using an intermediate +concept mapping table, you may need to inspect the mappings in your +mapper table to ensure they’ve been generated correctly using the ‘Maps +to’ relationship for your CDM’s vocabulary version.

    +

    Also note that when updating the OMOP vocabularies, previously +standard concepts could have been become non-standard and need +remapping. Often this remapping can be done programatically, by +following the ‘Maps to’ relationship to the new standard concept.

    Data Users

    This check failure means that the failing rows will not be picked up -in a standard OHDSI analysis. It is highly recommended to work with your -ETL team or data provider, if possible, to resolve this issue.

    +in a standard OHDSI analysis. Especially when participating in network +research, where only standard concepts are used, this might result in +invalid results. It is highly recommended to work with your ETL team or +data provider, if possible, to resolve this issue.

    However, you may work around it at your own risk by determining whether or not the affected rows are relevant for your analysis. Here’s an example query you could run to inspect failing rows in the diff --git a/docs/articles/checks/measureConditionEraCompleteness.html b/docs/articles/checks/measureConditionEraCompleteness.html index e8ce2a9f..2689e33c 100644 --- a/docs/articles/checks/measureConditionEraCompleteness.html +++ b/docs/articles/checks/measureConditionEraCompleteness.html @@ -180,7 +180,7 @@

    measureConditionEraCompleteness

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/measureConditionEraCompleteness.Rmd diff --git a/docs/articles/checks/measurePersonCompleteness.html b/docs/articles/checks/measurePersonCompleteness.html index 4e61a71d..3d68ce34 100644 --- a/docs/articles/checks/measurePersonCompleteness.html +++ b/docs/articles/checks/measurePersonCompleteness.html @@ -181,7 +181,7 @@

    measurePersonCompleteness

    Katy Sadowski

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/measurePersonCompleteness.Rmd @@ -193,7 +193,8 @@

    2024-06-29

    Summary

    -

    Level: TABLE
    Context: Validation
    Category: Completeness
    Subcategory:
    Severity: CDM convention ⚠ Characterization ✔

    +

    Level: TABLE
    Context: Validation
    Category: Completeness
    Subcategory:
    Severity: CDM convention ⚠ (for observation period), +Characterization ✔  (for all other tables)

    Description @@ -220,8 +221,12 @@

    DefinitionAll other tablesAction on persons missing records in other clinical event tables will depend on the characteristics of the source database. In certain cases, missingness is expected – some persons may just not have a given type of -data available in the source. In others, various ETL issues may result -in persons missing records in a given event table:

    +data available in the source. For instance, in most data sources, one +would expect most patients to have at least one visit, diagnosis, and +drug, while one would not expect every single patient to have +had a medical device.

    +

    Various ETL issues may result in persons missing records in a given +event table:

    diff --git a/docs/articles/checks/measureValueCompleteness.html b/docs/articles/checks/measureValueCompleteness.html index f0be5e07..114f34ad 100644 --- a/docs/articles/checks/measureValueCompleteness.html +++ b/docs/articles/checks/measureValueCompleteness.html @@ -181,7 +181,7 @@

    measureValueCompleteness

    Katy Sadowski

    -

    2024-06-29

    +

    2024-07-11

    Source:     vignettes/checks/measureValueCompleteness.Rmd @@ -216,8 +216,12 @@

    Definition @@ -248,13 +252,13 @@

    Violated rows query

    ETL Developers

    -

    Failures of this check on fields required in the CDM specification -are redundant with failures of isRequired. See isRequired documentation for more -information.

    +

    Failures of this check on required fields are redundant with failures +of isRequired. See isRequired +documentation for more information.

    ETL developers have 2 main options for the use of this check on non-required fields:

      -
    • The check threshold may be set to 100% for non-required fields such +
    • The check threshold may be left on 100% for non-required fields such that the check will never fail. The check result can be used simply to understand completeness for these fields
    • The check threshold may be set to an appropriate value corresponding @@ -264,19 +268,17 @@

      ETL Developers

    Unexpectedly missing values should be investigated for a potential -root cause in the ETL. For expected missingness, rows that violate this -check in non-required fields are acceptable but should be clearly -communicated to data users so that they can know when and when not to -expect data to be present in each field. To avoid confusion for users, -however, thresholds should be modified to avoid check failures at -expected levels.

    +root cause in the ETL. If a threshold has been adjusted to account for +expected missingness, this should be clearly communicated to data users +so that they can know when and when not to expect data to be present in +each field.

    Data Users

    This check informs you of the level of missing data in each column of -the CDM. If data is missing in a required column, see the isRequired -documentation for more information.

    +the CDM. If data is missing in a required column, see the +isRequired documentation for more information.

    The interpretation of a check failure on a non-required column will depend on the context. In some cases, the threshold for this check will have been very deliberately set, and any failure should be cause for diff --git a/docs/articles/checks/plausibleAfterBirth.html b/docs/articles/checks/plausibleAfterBirth.html index 610bc1e4..cacfbb0a 100644 --- a/docs/articles/checks/plausibleAfterBirth.html +++ b/docs/articles/checks/plausibleAfterBirth.html @@ -181,7 +181,7 @@

    plausibleAfterBirth

    Maxim Moinat, Katy Sadowski

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/plausibleAfterBirth.Rmd diff --git a/docs/articles/checks/plausibleBeforeDeath.html b/docs/articles/checks/plausibleBeforeDeath.html index 2e91b839..a2d11704 100644 --- a/docs/articles/checks/plausibleBeforeDeath.html +++ b/docs/articles/checks/plausibleBeforeDeath.html @@ -181,7 +181,7 @@

    plausibleBeforeDeath

    Maxim Moinat

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/plausibleBeforeDeath.Rmd diff --git a/docs/articles/checks/plausibleGenderUseDescendants.html b/docs/articles/checks/plausibleGenderUseDescendants.html index c4e8155a..2e92d60d 100644 --- a/docs/articles/checks/plausibleGenderUseDescendants.html +++ b/docs/articles/checks/plausibleGenderUseDescendants.html @@ -180,7 +180,7 @@

    plausibleGender

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/plausibleGenderUseDescendants.Rmd diff --git a/docs/articles/checks/plausibleStartBeforeEnd.html b/docs/articles/checks/plausibleStartBeforeEnd.html index f80c0a9b..489a615d 100644 --- a/docs/articles/checks/plausibleStartBeforeEnd.html +++ b/docs/articles/checks/plausibleStartBeforeEnd.html @@ -181,7 +181,7 @@

    plausibleStartBeforeEnd

    Maxim Moinat

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/plausibleStartBeforeEnd.Rmd diff --git a/docs/articles/checks/plausibleTemporalAfter.html b/docs/articles/checks/plausibleTemporalAfter.html index 3a01db40..2c11f327 100644 --- a/docs/articles/checks/plausibleTemporalAfter.html +++ b/docs/articles/checks/plausibleTemporalAfter.html @@ -180,7 +180,7 @@

    plausibleTemporalAfter

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/plausibleTemporalAfter.Rmd diff --git a/docs/articles/checks/plausibleUnitConceptIds.html b/docs/articles/checks/plausibleUnitConceptIds.html index 6855948c..cf0df5fb 100644 --- a/docs/articles/checks/plausibleUnitConceptIds.html +++ b/docs/articles/checks/plausibleUnitConceptIds.html @@ -180,7 +180,7 @@

    plausibleUnitConceptIds

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/plausibleUnitConceptIds.Rmd diff --git a/docs/articles/checks/plausibleValueHigh.html b/docs/articles/checks/plausibleValueHigh.html index 799422ff..7879501f 100644 --- a/docs/articles/checks/plausibleValueHigh.html +++ b/docs/articles/checks/plausibleValueHigh.html @@ -181,7 +181,7 @@

    plausibleValueHigh

    Dymytry Dymshyts

    -

    2024-06-29

    +

    2024-07-11

    Source: vignettes/checks/plausibleValueHigh.Rmd @@ -234,7 +234,7 @@

    Definition diff --git a/docs/articles/checks/plausibleValueLow.html b/docs/articles/checks/plausibleValueLow.html index 9cdbe4b0..40d10b8a 100644 --- a/docs/articles/checks/plausibleValueLow.html +++ b/docs/articles/checks/plausibleValueLow.html @@ -181,7 +181,7 @@

    plausibleValueLow

    Dymytry Dymshyts

    -

    2024-06-29

    +

    2024-07-11

    Source:     vignettes/checks/plausibleValueLow.Rmd @@ -247,7 +247,7 @@

    Definition diff --git a/docs/articles/checks/sourceConceptRecordCompleteness.html b/docs/articles/checks/sourceConceptRecordCompleteness.html index b5f59d0b..2d4effb4 100644 --- a/docs/articles/checks/sourceConceptRecordCompleteness.html +++ b/docs/articles/checks/sourceConceptRecordCompleteness.html @@ -181,7 +181,7 @@

    sourceConceptRecordCompleteness

    Katy Sadowski

    -

    2024-06-29

    +

    2024-07-11

    Source:     vignettes/checks/sourceConceptRecordCompleteness.Rmd @@ -207,29 +207,23 @@

    DefinitionSource concept mapping
  • CDM Fields/Tables: All source concept ID -(X_source_concept_id) columns in all event tables.
    • +(_source_concept_id) columns in all event tables.
  • Default Threshold Value:
      -
    • 10 for primary source concept ID columns in condition, drug, -measurement, procedure, device, and observation tables
      • -
    • 100 for all other source concept ID columns
      • +
    • 10% for source concept ID columns in condition, drug, measurement, +procedure, device, and observation tables
      • +
    • 100% for all other source concept ID columns
    • @@ -247,11 +241,11 @@

    User Guidance

    ETL Developers

    -

    Recall that the X_source_concept_id columns should +

    Recall that the _source_concept_id columns should contain the OMOP concept representing the exact code used in the source -data for a given record: “If the is coded in the source -data using an OMOP supported vocabulary put the concept id representing -the source value here.”

    +data for a given record: “If the <_source_value> is coded in the +source data using an OMOP supported vocabulary put the concept id +representing the source value here.”

    A failure of this check usually indicates a failure to map a source value to an OMOP concept. In some cases, such a failure can and should be remediated in the concept-mapping step of the ETL. In other cases, it @@ -259,25 +253,24 @@

    ETL DevelopersTo investigate the failure, run the following query:

    SELECT  
   concept.concept_name AS standard_concept_name, 
-  cdmTable.X_concept_id, -- standard concept ID field for the table 
+  cdmTable._concept_id, -- standard concept ID field for the table 
   c2.concept_name AS source_value_concept_name, 
-  cdmTable.X_source_value, -- source value field for the table 
+  cdmTable._source_value, -- source value field for the table 
   COUNT(*) 
 FROM @cdmDatabaseSchema.@cdmTableName cdmTable 
-LEFT JOIN @vocabDatabaseSchema.concept ON concept.concept_id = cdmTable.X_concept_id 
+LEFT JOIN @vocabDatabaseSchema.concept ON concept.concept_id = cdmTable._concept_id 
 -- WARNING this join may cause fanning if a source value exists in multiple vocabularies 
-LEFT JOIN @vocabDatabaseSchema.concept c2 ON concept.concept_code = cdmTable.X_source_value 
+LEFT JOIN @vocabDatabaseSchema.concept c2 ON concept.concept_code = cdmTable._source_value 
 AND c2.domain_id = <Domain of cdmTable> 
 WHERE cdmTable.@cdmFieldName = 0  
--- AND cdmTable.value_as_number IS NOT NULL -- uncomment for unit_concept_id checks 
-GROUP BY 1,2,3 
-ORDER BY 4 DESC
    +GROUP BY 1,2,3 +ORDER BY 4 DESC

    The query results will give you a summary of the source codes which failed to map to an OMOP concept. Inspecting this data should give you an initial idea of what might be going on.

    If source values return legitimate matches on concept_code, it’s possible that there is an error in the concept mapping step of your ETL. -Please note that while the X_source_concept_id fields are +Please note that while the _source_concept_id fields are technically not required, it is highly recommended to populate them with OMOP concepts whenever possible. This will greatly aid analysts in understanding the provenance of the data.

    @@ -292,10 +285,10 @@

    Data UsersJared Houghtaling, Clair Blacketer

    -

    2024-06-29

    +

    2024-07-11

    Source:     vignettes/checks/sourceValueCompleteness.Rmd @@ -205,11 +205,11 @@

    DefinitionDefinitionKaty Sadowski

    -

    2024-06-29

    +

    2024-07-11

    Source:     vignettes/checks/standardConceptRecordCompleteness.Rmd @@ -207,10 +207,10 @@

    DefinitionDefinitionETL DevelopersTo investigate the failure, run the following query:

    SELECT  
   concept_name, 
-  cdmTable.X_source_concept_id, -- source concept ID field for the table 
-  cdmTable.X_source_value, -- source value field for the table 
+  cdmTable._source_concept_id, -- source concept ID field for the table 
+  cdmTable._source_value, -- source value field for the table 
   COUNT(*) 
 FROM @cdmDatabaseSchema.@cdmTableName cdmTable 
-LEFT JOIN @vocabDatabaseSchema.concept ON concept.concept_id = cdmTable.X_source_concept_id 
+LEFT JOIN @vocabDatabaseSchema.concept ON concept.concept_id = cdmTable._source_concept_id 
 WHERE cdmTable.@cdmFieldName = 0  
 -- AND cdmTable.value_as_number IS NOT NULL -- uncomment for unit_concept_id checks 
 GROUP BY 1,2,3 
@@ -327,11 +328,12 @@ 
    ETL Developers
 
  • Finally, if the investigation query returns no source value, you
 must trace the relevant record(s) back to their source and confirm if
-the missing value is expected. If not, identify and fix the related
-issue in your ETL. If the record legitimately has no value/code in the
-source data, then the standard concept ID may be left as 0. However, in
-some cases these “code-less” records represent junk data which should be
-filtered out in the ETL. The proper approach will be context-dependent
+the missing source value is expected. If not, identify and fix the
+related issue in your ETL. If the record legitimately has no value/code
+in the source data, then the standard concept ID may be left as 0.
+However, in some cases these “code-less” records represent junk data
+which should be filtered out in the ETL. The proper approach will be
+context-dependent
 
 
    • 
 
  • 
diff --git a/docs/LICENSE-text.html b/docs/LICENSE-text.html
index aaecd811..36a3d443 100644
--- a/docs/LICENSE-text.html
+++ b/docs/LICENSE-text.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/docs/articles/AddNewCheck.html b/docs/articles/AddNewCheck.html
index 0e7ab7f1..83f74dc5 100644
--- a/docs/articles/AddNewCheck.html
+++ b/docs/articles/AddNewCheck.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -180,7 +183,7 @@
       
    Add a New Data Quality Check
    
                         
    Don Torok
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/AddNewCheck.rmd
       
diff --git a/docs/articles/CheckStatusDefinitions.html b/docs/articles/CheckStatusDefinitions.html
index fe8df052..a081dc68 100644
--- a/docs/articles/CheckStatusDefinitions.html
+++ b/docs/articles/CheckStatusDefinitions.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    Check Status Definitions
    
                         
    Dmitry Ilyn,
 Maxim Moinat
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/CheckStatusDefinitions.rmd
       
diff --git a/docs/articles/CheckTypeDescriptions.html b/docs/articles/CheckTypeDescriptions.html
index a58ea494..45cb4259 100644
--- a/docs/articles/CheckTypeDescriptions.html
+++ b/docs/articles/CheckTypeDescriptions.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    Data Quality Check Type Definitions
    
                         
    Clair
 Blacketer
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/CheckTypeDescriptions.rmd
       
diff --git a/docs/articles/DataQualityDashboard.html b/docs/articles/DataQualityDashboard.html
index d22f577a..9b4e2174 100644
--- a/docs/articles/DataQualityDashboard.html
+++ b/docs/articles/DataQualityDashboard.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    Getting Started
    
                         
    Clair
 Blacketer
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/DataQualityDashboard.rmd
       
diff --git a/docs/articles/DqdForCohorts.html b/docs/articles/DqdForCohorts.html
index d9530035..7077e6a3 100644
--- a/docs/articles/DqdForCohorts.html
+++ b/docs/articles/DqdForCohorts.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    Running the DQD on a Cohort
    
                         
    Clair
 Blacketer
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/DqdForCohorts.rmd
       
diff --git a/docs/articles/SqlOnly.html b/docs/articles/SqlOnly.html
index 71653b00..b40aea46 100644
--- a/docs/articles/SqlOnly.html
+++ b/docs/articles/SqlOnly.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    Running the DQD in SqlOnly mode
    
                         
    Maxim
 Moinat
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/SqlOnly.rmd
       
diff --git a/docs/articles/Thresholds.html b/docs/articles/Thresholds.html
index 7bcce30c..e6796eed 100644
--- a/docs/articles/Thresholds.html
+++ b/docs/articles/Thresholds.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    Failure Thresholds and How to Change Them
    
                         
    Clair
 Blacketer
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/Thresholds.rmd
       
diff --git a/docs/articles/checkIndex.html b/docs/articles/checkIndex.html
index 9289ad6a..d75f7b67 100644
--- a/docs/articles/checkIndex.html
+++ b/docs/articles/checkIndex.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    Index
    
                         
    Katy
 Sadowski
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checkIndex.Rmd
       
@@ -236,7 +239,7 @@ 
    Checks
 
  • fkDomain
    • 
 
  • fkClass
    • 
 
  • measurePersonCompleteness
    • 
-
  • measureConditionEraCompleteness (PAGE UNDER CONSTRUCTION)
    • 
+
  • measureConditionEraCompleteness
    • 
 
  • isStandardValidConcept
    • 
 
  • measureValueCompleteness
    • 
 
  • standardConceptRecordCompleteness
    • 
diff --git a/docs/articles/checks/cdmDatatype.html b/docs/articles/checks/cdmDatatype.html
index bd765aeb..0d0020a1 100644
--- a/docs/articles/checks/cdmDatatype.html
+++ b/docs/articles/checks/cdmDatatype.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
    • 
 
  • 
@@ -181,7 +184,7 @@ 
    cdmDatatype
    
                         
    Katy
 Sadowski
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/cdmDatatype.Rmd
       
diff --git a/docs/articles/checks/cdmField.html b/docs/articles/checks/cdmField.html
index 3464b260..64d70791 100644
--- a/docs/articles/checks/cdmField.html
+++ b/docs/articles/checks/cdmField.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    cdmField
    
                         
    Heidi Schmidt,
 Katy Sadowski
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/cdmField.Rmd
       
diff --git a/docs/articles/checks/cdmTable.html b/docs/articles/checks/cdmTable.html
index 03d2cd40..e5c80c29 100644
--- a/docs/articles/checks/cdmTable.html
+++ b/docs/articles/checks/cdmTable.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    cdmTable
    
                         
    John Gresh,
 Katy Sadowski
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/cdmTable.Rmd
       
diff --git a/docs/articles/checks/fkClass.html b/docs/articles/checks/fkClass.html
index bbfbd439..5c6f9d9e 100644
--- a/docs/articles/checks/fkClass.html
+++ b/docs/articles/checks/fkClass.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    fkClass
    
                         
    Clair
 Blacketer, Katy Sadowski
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/fkClass.Rmd
       
diff --git a/docs/articles/checks/fkDomain.html b/docs/articles/checks/fkDomain.html
index bc151de1..8335bb08 100644
--- a/docs/articles/checks/fkDomain.html
+++ b/docs/articles/checks/fkDomain.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    fkDomain
    
                         
    Clair
 Blacketer, Katy Sadowski
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/fkDomain.Rmd
       
diff --git a/docs/articles/checks/isForeignKey.html b/docs/articles/checks/isForeignKey.html
index 49b25eeb..0e24b288 100644
--- a/docs/articles/checks/isForeignKey.html
+++ b/docs/articles/checks/isForeignKey.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    isForeignKey
    
                         
    Dmytry
 Dymshyts, Katy Sadowski
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/isForeignKey.Rmd
       
diff --git a/docs/articles/checks/isPrimaryKey.html b/docs/articles/checks/isPrimaryKey.html
index cc5d3275..89b8a58d 100644
--- a/docs/articles/checks/isPrimaryKey.html
+++ b/docs/articles/checks/isPrimaryKey.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    isPrimaryKey
    
                         
    John Gresh,
 Katy Sadowski
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/isPrimaryKey.Rmd
       
diff --git a/docs/articles/checks/isRequired.html b/docs/articles/checks/isRequired.html
index 3da6dfaa..3dab8f3c 100644
--- a/docs/articles/checks/isRequired.html
+++ b/docs/articles/checks/isRequired.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    isRequired
    
                         
    Katy
 Sadowski
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/isRequired.Rmd
       
diff --git a/docs/articles/checks/isStandardValidConcept.html b/docs/articles/checks/isStandardValidConcept.html
index 7f40000a..b80a0b53 100644
--- a/docs/articles/checks/isStandardValidConcept.html
+++ b/docs/articles/checks/isStandardValidConcept.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    isStandardValidConcept
    
                         
    Stephanie Hong,
 Katy Sadowski
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/isStandardValidConcept.Rmd
       
diff --git a/docs/articles/checks/measureConditionEraCompleteness.html b/docs/articles/checks/measureConditionEraCompleteness.html
index 2689e33c..feb1d1af 100644
--- a/docs/articles/checks/measureConditionEraCompleteness.html
+++ b/docs/articles/checks/measureConditionEraCompleteness.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -178,9 +181,10 @@
   
    
     
    
       
    measureConditionEraCompleteness
    
-                        
    
+                        
    Maxim
+Moinat
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/measureConditionEraCompleteness.Rmd
       
@@ -192,45 +196,80 @@ 
    2024-07-11
    
 
    
 
    Summary
 
    
-
    Level: TABLE
    Context: Validation
    Category: Completeness
    Subcategory:
    Severity:
    
+
    Level: TABLE
    Context: Validation
    Category: Completeness
    Subcategory:
    Severity: CDM convention ⚠
    
 
    
 
    
 
    Description
 
    
-
    The number and Percent of persons that does not have condition_era
-built successfully, for all persons in condition_occurrence
    
+
    The number and percent of persons that does not have condition_era
+built successfully, for all persons in
+CONDITION_OCCURRENCE.
    
 
    
 
    
 
    Definition
 
    
 
      
 
    • 
-Numerator:
      • 
+Numerator: Number of unique person_ids that exist in the
+CONDITION_OCCURRENCE table but not in the
+CONDITION_ERA table.
 
    • 
-Denominator:
      • 
+Denominator: Number of unique person_ids in the
+CONDITION_OCCURRENCE table.
 
    • 
-Related CDM Convention(s):
      • 
+Related CDM Convention(s): Condition Era’s are directly
+derived from Condition Occurrence.
 
    • 
-CDM Fields/Tables:
      • 
+CDM Fields/Tables: CONDITION_ERA
+
 
    • 
-Default Threshold Value:
      • 
+Default Threshold Value: 0%
 
    
 
    
 
    
 
    User Guidance
 
    
+
    The Condition
+Era CDM documentation states that the condition era’s should be
+derived by combining condition occurrences. This implies that each
+person with a condition occurrence should have at least a condition era.
+It does NOT clearly state that the CONDITION_ERA table is
+required when there are condition occurrences. Still, it is has always
+been a common convention in the OHDSI community to derive condition era.
+There is currently no THEMIS convention on condition eras.
    
 
    
 
    Violated rows query
 
    
-
    
+
    SELECT DISTINCT 
+  co.person_id
+FROM @cdmDatabaseSchema.condition_occurrence co
+  LEFT JOIN @cdmDatabaseSchema.condition_era cdmTable 
+    ON co.person_id = cdmTable.person_id
+WHERE cdmTable.person_id IS NULL
    
 
    
 
    
 
    ETL Developers
 
    
+
    If this check fails, it is likely that there is an issue with the
+condition era derivation script. Please review the ETL execution log. It
+might be that this script was not executed and the condition era table
+is empty, or it had issues running and the condition era table has been
+partially populated. If no issues with the ETL run found, the condition
+era derivation script might have bugs. Please review the code. An
+example script can be found on the
+CDM Documentation page. In both cases it is advised to truncate the
+CONDITION_ERA table and rerun the derivation script.
    
 
    
 
    
 
    Data Users
 
    
+
    The CONDITION_ERA table might seem to contain redundant
+information, as for most uses the CONDITION_OCCURRENCE
+table can be used. However, tools like FeatureExtraction use condition
+eras to build some covariates and network studies might use cohorts that
+are based on condition eras. It is therefore important that the
+CONDITION_ERA table is fully populated and captures the
+same persons as in condition occurrence.
    
 
    
 
    
   
    
diff --git a/docs/articles/checks/measurePersonCompleteness.html b/docs/articles/checks/measurePersonCompleteness.html
index 3d68ce34..b0317098 100644
--- a/docs/articles/checks/measurePersonCompleteness.html
+++ b/docs/articles/checks/measurePersonCompleteness.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    measurePersonCompleteness
    
                         
    Katy
 Sadowski
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/measurePersonCompleteness.Rmd
       
diff --git a/docs/articles/checks/measureValueCompleteness.html b/docs/articles/checks/measureValueCompleteness.html
index 114f34ad..dbb2554e 100644
--- a/docs/articles/checks/measureValueCompleteness.html
+++ b/docs/articles/checks/measureValueCompleteness.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    measureValueCompleteness
    
                         
    Katy
 Sadowski
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/measureValueCompleteness.Rmd
       
diff --git a/docs/articles/checks/plausibleAfterBirth.html b/docs/articles/checks/plausibleAfterBirth.html
index cacfbb0a..a8e0f1d7 100644
--- a/docs/articles/checks/plausibleAfterBirth.html
+++ b/docs/articles/checks/plausibleAfterBirth.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    plausibleAfterBirth
    
                         
    Maxim Moinat,
 Katy Sadowski
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/plausibleAfterBirth.Rmd
       
diff --git a/docs/articles/checks/plausibleBeforeDeath.html b/docs/articles/checks/plausibleBeforeDeath.html
index a2d11704..771b3883 100644
--- a/docs/articles/checks/plausibleBeforeDeath.html
+++ b/docs/articles/checks/plausibleBeforeDeath.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    plausibleBeforeDeath
    
                         
    Maxim
 Moinat
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/plausibleBeforeDeath.Rmd
       
diff --git a/docs/articles/checks/plausibleGenderUseDescendants.html b/docs/articles/checks/plausibleGenderUseDescendants.html
index 2e92d60d..23ca231b 100644
--- a/docs/articles/checks/plausibleGenderUseDescendants.html
+++ b/docs/articles/checks/plausibleGenderUseDescendants.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -180,7 +183,7 @@
       
    plausibleGender
    
                         
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/plausibleGenderUseDescendants.Rmd
       
diff --git a/docs/articles/checks/plausibleStartBeforeEnd.html b/docs/articles/checks/plausibleStartBeforeEnd.html
index 489a615d..66ae6eb9 100644
--- a/docs/articles/checks/plausibleStartBeforeEnd.html
+++ b/docs/articles/checks/plausibleStartBeforeEnd.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    plausibleStartBeforeEnd
    
                         
    Maxim
 Moinat
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/plausibleStartBeforeEnd.Rmd
       
diff --git a/docs/articles/checks/plausibleTemporalAfter.html b/docs/articles/checks/plausibleTemporalAfter.html
index 2c11f327..7c3cf881 100644
--- a/docs/articles/checks/plausibleTemporalAfter.html
+++ b/docs/articles/checks/plausibleTemporalAfter.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -180,7 +183,7 @@
       
    plausibleTemporalAfter
    
                         
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/plausibleTemporalAfter.Rmd
       
diff --git a/docs/articles/checks/plausibleUnitConceptIds.html b/docs/articles/checks/plausibleUnitConceptIds.html
index cf0df5fb..71f25bfe 100644
--- a/docs/articles/checks/plausibleUnitConceptIds.html
+++ b/docs/articles/checks/plausibleUnitConceptIds.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -180,7 +183,7 @@
       
    plausibleUnitConceptIds
    
                         
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/plausibleUnitConceptIds.Rmd
       
diff --git a/docs/articles/checks/plausibleValueHigh.html b/docs/articles/checks/plausibleValueHigh.html
index 7879501f..fdcdc415 100644
--- a/docs/articles/checks/plausibleValueHigh.html
+++ b/docs/articles/checks/plausibleValueHigh.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    plausibleValueHigh
    
                         
    Dymytry
 Dymshyts
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/plausibleValueHigh.Rmd
       
diff --git a/docs/articles/checks/plausibleValueLow.html b/docs/articles/checks/plausibleValueLow.html
index 40d10b8a..86b7bf72 100644
--- a/docs/articles/checks/plausibleValueLow.html
+++ b/docs/articles/checks/plausibleValueLow.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    plausibleValueLow
    
                         
    Dymytry
 Dymshyts
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/plausibleValueLow.Rmd
       
diff --git a/docs/articles/checks/sourceConceptRecordCompleteness.html b/docs/articles/checks/sourceConceptRecordCompleteness.html
index 2d4effb4..7dba571d 100644
--- a/docs/articles/checks/sourceConceptRecordCompleteness.html
+++ b/docs/articles/checks/sourceConceptRecordCompleteness.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    sourceConceptRecordCompleteness
    
                         
    Katy
 Sadowski
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/sourceConceptRecordCompleteness.Rmd
       
diff --git a/docs/articles/checks/sourceValueCompleteness.html b/docs/articles/checks/sourceValueCompleteness.html
index e02e771f..f52bd772 100644
--- a/docs/articles/checks/sourceValueCompleteness.html
+++ b/docs/articles/checks/sourceValueCompleteness.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    sourceValueCompleteness
    
                         
    Jared
 Houghtaling, Clair Blacketer
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/sourceValueCompleteness.Rmd
       
diff --git a/docs/articles/checks/standardConceptRecordCompleteness.html b/docs/articles/checks/standardConceptRecordCompleteness.html
index ee2794ce..4b1e3cbf 100644
--- a/docs/articles/checks/standardConceptRecordCompleteness.html
+++ b/docs/articles/checks/standardConceptRecordCompleteness.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    standardConceptRecordCompleteness
    
                         
    Katy
 Sadowski
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/standardConceptRecordCompleteness.Rmd
       
diff --git a/docs/articles/checks/withinVisitDates.html b/docs/articles/checks/withinVisitDates.html
index 1ca9c9b6..3089b657 100644
--- a/docs/articles/checks/withinVisitDates.html
+++ b/docs/articles/checks/withinVisitDates.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
@@ -181,7 +184,7 @@ 
    withinVisitDates
    
                         
    Clair
 Blacketer
    
             
-            
    2024-07-11
    
+            
    2024-07-12
    
       
       Source: vignettes/checks/withinVisitDates.Rmd
       
diff --git a/docs/articles/index.html b/docs/articles/index.html
index a9c50ea9..1eadbf9d 100644
--- a/docs/articles/index.html
+++ b/docs/articles/index.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/docs/authors.html b/docs/authors.html
index 89196518..d68b7afa 100644
--- a/docs/authors.html
+++ b/docs/authors.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/docs/index.html b/docs/index.html
index f76a3b99..3aa90c26 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -148,6 +148,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
 
  • 
diff --git a/docs/news/index.html b/docs/news/index.html
index 83e23076..272d626f 100644
--- a/docs/news/index.html
+++ b/docs/news/index.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/docs/pkgdown.yml b/docs/pkgdown.yml
index ed074627..deb660db 100644
--- a/docs/pkgdown.yml
+++ b/docs/pkgdown.yml
@@ -34,7 +34,7 @@ articles:
   standardConceptRecordCompleteness: checks/standardConceptRecordCompleteness.html
   Thresholds: Thresholds.html
   withinVisitDates: checks/withinVisitDates.html
-last_built: 2024-07-12T01:44Z
+last_built: 2024-07-13T01:28Z
 urls:
   reference: https://ohdsi.github.io/DataQualityDashboard/reference
   article: https://ohdsi.github.io/DataQualityDashboard/articles
diff --git a/docs/reference/convertJsonResultsFileCase.html b/docs/reference/convertJsonResultsFileCase.html
index 0b2871f8..00d09faa 100644
--- a/docs/reference/convertJsonResultsFileCase.html
+++ b/docs/reference/convertJsonResultsFileCase.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/docs/reference/dot-evaluateThresholds.html b/docs/reference/dot-evaluateThresholds.html
index 0e3a4c2c..bfe86dfa 100644
--- a/docs/reference/dot-evaluateThresholds.html
+++ b/docs/reference/dot-evaluateThresholds.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/docs/reference/dot-getCheckId.html b/docs/reference/dot-getCheckId.html
index b631cb51..2a1aaa17 100644
--- a/docs/reference/dot-getCheckId.html
+++ b/docs/reference/dot-getCheckId.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/docs/reference/dot-processCheck.html b/docs/reference/dot-processCheck.html
index 7f29c99a..aa5e208e 100644
--- a/docs/reference/dot-processCheck.html
+++ b/docs/reference/dot-processCheck.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/docs/reference/dot-recordResult.html b/docs/reference/dot-recordResult.html
index 96673dd9..66ad9690 100644
--- a/docs/reference/dot-recordResult.html
+++ b/docs/reference/dot-recordResult.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/docs/reference/dot-runCheck.html b/docs/reference/dot-runCheck.html
index 9b4c5b79..2a33c3c0 100644
--- a/docs/reference/dot-runCheck.html
+++ b/docs/reference/dot-runCheck.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/docs/reference/dot-summarizeResults.html b/docs/reference/dot-summarizeResults.html
index 8d22ee63..b158e832 100644
--- a/docs/reference/dot-summarizeResults.html
+++ b/docs/reference/dot-summarizeResults.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/docs/reference/dot-writeResultsToCsv.html b/docs/reference/dot-writeResultsToCsv.html
index 1a209ffe..7449327b 100644
--- a/docs/reference/dot-writeResultsToCsv.html
+++ b/docs/reference/dot-writeResultsToCsv.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/docs/reference/dot-writeResultsToJson.html b/docs/reference/dot-writeResultsToJson.html
index 62410ef1..105d5a9c 100644
--- a/docs/reference/dot-writeResultsToJson.html
+++ b/docs/reference/dot-writeResultsToJson.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/docs/reference/dot-writeResultsToTable.html b/docs/reference/dot-writeResultsToTable.html
index 2e215376..b235d9f2 100644
--- a/docs/reference/dot-writeResultsToTable.html
+++ b/docs/reference/dot-writeResultsToTable.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/docs/reference/executeDqChecks.html b/docs/reference/executeDqChecks.html
index 6382ed42..8306931c 100644
--- a/docs/reference/executeDqChecks.html
+++ b/docs/reference/executeDqChecks.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/docs/reference/index.html b/docs/reference/index.html
index f9d83e64..cc610a6d 100644
--- a/docs/reference/index.html
+++ b/docs/reference/index.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/docs/reference/listDqChecks.html b/docs/reference/listDqChecks.html
index 606e2d87..3b478b14 100644
--- a/docs/reference/listDqChecks.html
+++ b/docs/reference/listDqChecks.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/docs/reference/reEvaluateThresholds.html b/docs/reference/reEvaluateThresholds.html
index dd74b726..f2d1bbc4 100644
--- a/docs/reference/reEvaluateThresholds.html
+++ b/docs/reference/reEvaluateThresholds.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/docs/reference/viewDqDashboard.html b/docs/reference/viewDqDashboard.html
index ece1e45c..2173fbd9 100644
--- a/docs/reference/viewDqDashboard.html
+++ b/docs/reference/viewDqDashboard.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/docs/reference/writeDBResultsToJson.html b/docs/reference/writeDBResultsToJson.html
index 283d3e2a..26b34df4 100644
--- a/docs/reference/writeDBResultsToJson.html
+++ b/docs/reference/writeDBResultsToJson.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/docs/reference/writeJsonResultsToCsv.html b/docs/reference/writeJsonResultsToCsv.html
index 12a8f937..bbc1d1a6 100644
--- a/docs/reference/writeJsonResultsToCsv.html
+++ b/docs/reference/writeJsonResultsToCsv.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/docs/reference/writeJsonResultsToTable.html b/docs/reference/writeJsonResultsToTable.html
index e944a846..556aff0f 100644
--- a/docs/reference/writeJsonResultsToTable.html
+++ b/docs/reference/writeJsonResultsToTable.html
@@ -128,6 +128,9 @@
     
  • 
       withinVisitDates
     
    • 
+    
  • 
+      measureConditionEraCompleteness
+    
    • 
   
 
  • 
   Changelog
diff --git a/vignettes/checkIndex.Rmd b/vignettes/checkIndex.Rmd
index 4e93a905..af436761 100644
--- a/vignettes/checkIndex.Rmd
+++ b/vignettes/checkIndex.Rmd
@@ -35,7 +35,7 @@ above to navigate to the check's documentation page.\
 - [fkDomain](checks/fkDomain.html)
 - [fkClass](checks/fkClass.html)
 - [measurePersonCompleteness](checks/measurePersonCompleteness.html)
-- measureConditionEraCompleteness (PAGE UNDER CONSTRUCTION)
+- [measureConditionEraCompleteness](checks/measureConditionEraCompleteness.html)
 - [isStandardValidConcept](checks/isStandardValidConcept.html)
 - [measureValueCompleteness](checks/measureValueCompleteness.html)
 - [standardConceptRecordCompleteness](checks/standardConceptRecordCompleteness.html)

From a923eb263b1c12f14ca02ed422193bac5989f884 Mon Sep 17 00:00:00 2001
From: Katy Sadowski 
Date: Fri, 12 Jul 2024 21:37:44 -0400
Subject: [PATCH 42/43] catch up develop

---
 docs/articles/AddNewCheck.html                |   5 +-
 docs/articles/CheckStatusDefinitions.html     |   5 +-
 docs/articles/CheckTypeDescriptions.html      |   5 +-
 docs/articles/DataQualityDashboard.html       |   5 +-
 docs/articles/DqdForCohorts.html              |   5 +-
 docs/articles/SqlOnly.html                    |   5 +-
 docs/articles/Thresholds.html                 |   5 +-
 docs/articles/checkIndex.html                 |   7 +-
 docs/articles/checks/cdmDatatype.html         |   5 +-
 docs/articles/checks/cdmField.html            |   5 +-
 docs/articles/checks/cdmTable.html            |   5 +-
 docs/articles/checks/fkClass.html             |   5 +-
 docs/articles/checks/fkDomain.html            |   5 +-
 docs/articles/checks/isForeignKey.html        |   5 +-
 docs/articles/checks/isPrimaryKey.html        |   5 +-
 docs/articles/checks/isRequired.html          |   5 +-
 .../checks/isStandardValidConcept.html        |   5 +-
 .../checks/measurePersonCompleteness.html     |   5 +-
 .../checks/measureValueCompleteness.html      |   5 +-
 docs/articles/checks/plausibleAfterBirth.html |   5 +-
 .../articles/checks/plausibleBeforeDeath.html |   5 +-
 docs/articles/checks/plausibleDuringLife.html | 246 -----------------
 docs/articles/checks/plausibleGender.html     | 247 ------------------
 .../checks/plausibleGenderUseDescendants.html |   5 +-
 .../checks/plausibleStartBeforeEnd.html       |   5 +-
 .../checks/plausibleTemporalAfter.html        |   5 +-
 .../checks/plausibleUnitConceptIds.html       |   5 +-
 docs/articles/checks/plausibleValueHigh.html  |   5 +-
 docs/articles/checks/plausibleValueLow.html   |   5 +-
 .../sourceConceptRecordCompleteness.html      |   5 +-
 .../checks/sourceValueCompleteness.html       |   5 +-
 .../standardConceptRecordCompleteness.html    |   5 +-
 docs/articles/checks/withinVisitDates.html    |   5 +-
 docs/articles/index.html                      |   3 -
 34 files changed, 32 insertions(+), 621 deletions(-)
 delete mode 100644 docs/articles/checks/plausibleDuringLife.html
 delete mode 100644 docs/articles/checks/plausibleGender.html

diff --git a/docs/articles/AddNewCheck.html b/docs/articles/AddNewCheck.html
index 83f74dc5..0e7ab7f1 100644
--- a/docs/articles/AddNewCheck.html
+++ b/docs/articles/AddNewCheck.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -183,7 +180,7 @@
       
    Add a New Data Quality Check
    
                         
    Don Torok
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/AddNewCheck.rmd
       
diff --git a/docs/articles/CheckStatusDefinitions.html b/docs/articles/CheckStatusDefinitions.html
index a081dc68..fe8df052 100644
--- a/docs/articles/CheckStatusDefinitions.html
+++ b/docs/articles/CheckStatusDefinitions.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    Check Status Definitions
    
                         
    Dmitry Ilyn,
 Maxim Moinat
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/CheckStatusDefinitions.rmd
       
diff --git a/docs/articles/CheckTypeDescriptions.html b/docs/articles/CheckTypeDescriptions.html
index 45cb4259..a58ea494 100644
--- a/docs/articles/CheckTypeDescriptions.html
+++ b/docs/articles/CheckTypeDescriptions.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    Data Quality Check Type Definitions
    
                         
    Clair
 Blacketer
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/CheckTypeDescriptions.rmd
       
diff --git a/docs/articles/DataQualityDashboard.html b/docs/articles/DataQualityDashboard.html
index 9b4e2174..d22f577a 100644
--- a/docs/articles/DataQualityDashboard.html
+++ b/docs/articles/DataQualityDashboard.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    Getting Started
    
                         
    Clair
 Blacketer
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/DataQualityDashboard.rmd
       
diff --git a/docs/articles/DqdForCohorts.html b/docs/articles/DqdForCohorts.html
index 7077e6a3..d9530035 100644
--- a/docs/articles/DqdForCohorts.html
+++ b/docs/articles/DqdForCohorts.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    Running the DQD on a Cohort
    
                         
    Clair
 Blacketer
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/DqdForCohorts.rmd
       
diff --git a/docs/articles/SqlOnly.html b/docs/articles/SqlOnly.html
index b40aea46..71653b00 100644
--- a/docs/articles/SqlOnly.html
+++ b/docs/articles/SqlOnly.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    Running the DQD in SqlOnly mode
    
                         
    Maxim
 Moinat
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/SqlOnly.rmd
       
diff --git a/docs/articles/Thresholds.html b/docs/articles/Thresholds.html
index e6796eed..7bcce30c 100644
--- a/docs/articles/Thresholds.html
+++ b/docs/articles/Thresholds.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    Failure Thresholds and How to Change Them
    
                         
    Clair
 Blacketer
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/Thresholds.rmd
       
diff --git a/docs/articles/checkIndex.html b/docs/articles/checkIndex.html
index d75f7b67..9289ad6a 100644
--- a/docs/articles/checkIndex.html
+++ b/docs/articles/checkIndex.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    Index
    
                         
    Katy
 Sadowski
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checkIndex.Rmd
       
@@ -239,7 +236,7 @@ 
    Checks
 
  • fkDomain
    • 
 
  • fkClass
    • 
 
  • measurePersonCompleteness
    • 
-
  • measureConditionEraCompleteness
    • 
+
  • measureConditionEraCompleteness (PAGE UNDER CONSTRUCTION)
    • 
 
  • isStandardValidConcept
    • 
 
  • measureValueCompleteness
    • 
 
  • standardConceptRecordCompleteness
    • 
diff --git a/docs/articles/checks/cdmDatatype.html b/docs/articles/checks/cdmDatatype.html
index 0d0020a1..bd765aeb 100644
--- a/docs/articles/checks/cdmDatatype.html
+++ b/docs/articles/checks/cdmDatatype.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
    • 
 
  • 
@@ -184,7 +181,7 @@ 
    cdmDatatype
    
                         
    Katy
 Sadowski
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/cdmDatatype.Rmd
       
diff --git a/docs/articles/checks/cdmField.html b/docs/articles/checks/cdmField.html
index 64d70791..3464b260 100644
--- a/docs/articles/checks/cdmField.html
+++ b/docs/articles/checks/cdmField.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    cdmField
    
                         
    Heidi Schmidt,
 Katy Sadowski
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/cdmField.Rmd
       
diff --git a/docs/articles/checks/cdmTable.html b/docs/articles/checks/cdmTable.html
index e5c80c29..03d2cd40 100644
--- a/docs/articles/checks/cdmTable.html
+++ b/docs/articles/checks/cdmTable.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    cdmTable
    
                         
    John Gresh,
 Katy Sadowski
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/cdmTable.Rmd
       
diff --git a/docs/articles/checks/fkClass.html b/docs/articles/checks/fkClass.html
index 5c6f9d9e..bbfbd439 100644
--- a/docs/articles/checks/fkClass.html
+++ b/docs/articles/checks/fkClass.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    fkClass
    
                         
    Clair
 Blacketer, Katy Sadowski
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/fkClass.Rmd
       
diff --git a/docs/articles/checks/fkDomain.html b/docs/articles/checks/fkDomain.html
index 8335bb08..bc151de1 100644
--- a/docs/articles/checks/fkDomain.html
+++ b/docs/articles/checks/fkDomain.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    fkDomain
    
                         
    Clair
 Blacketer, Katy Sadowski
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/fkDomain.Rmd
       
diff --git a/docs/articles/checks/isForeignKey.html b/docs/articles/checks/isForeignKey.html
index 0e24b288..49b25eeb 100644
--- a/docs/articles/checks/isForeignKey.html
+++ b/docs/articles/checks/isForeignKey.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    isForeignKey
    
                         
    Dmytry
 Dymshyts, Katy Sadowski
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/isForeignKey.Rmd
       
diff --git a/docs/articles/checks/isPrimaryKey.html b/docs/articles/checks/isPrimaryKey.html
index 89b8a58d..cc5d3275 100644
--- a/docs/articles/checks/isPrimaryKey.html
+++ b/docs/articles/checks/isPrimaryKey.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    isPrimaryKey
    
                         
    John Gresh,
 Katy Sadowski
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/isPrimaryKey.Rmd
       
diff --git a/docs/articles/checks/isRequired.html b/docs/articles/checks/isRequired.html
index 3dab8f3c..3da6dfaa 100644
--- a/docs/articles/checks/isRequired.html
+++ b/docs/articles/checks/isRequired.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    isRequired
    
                         
    Katy
 Sadowski
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/isRequired.Rmd
       
diff --git a/docs/articles/checks/isStandardValidConcept.html b/docs/articles/checks/isStandardValidConcept.html
index b80a0b53..7f40000a 100644
--- a/docs/articles/checks/isStandardValidConcept.html
+++ b/docs/articles/checks/isStandardValidConcept.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    isStandardValidConcept
    
                         
    Stephanie Hong,
 Katy Sadowski
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/isStandardValidConcept.Rmd
       
diff --git a/docs/articles/checks/measurePersonCompleteness.html b/docs/articles/checks/measurePersonCompleteness.html
index b0317098..3d68ce34 100644
--- a/docs/articles/checks/measurePersonCompleteness.html
+++ b/docs/articles/checks/measurePersonCompleteness.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    measurePersonCompleteness
    
                         
    Katy
 Sadowski
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/measurePersonCompleteness.Rmd
       
diff --git a/docs/articles/checks/measureValueCompleteness.html b/docs/articles/checks/measureValueCompleteness.html
index dbb2554e..114f34ad 100644
--- a/docs/articles/checks/measureValueCompleteness.html
+++ b/docs/articles/checks/measureValueCompleteness.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    measureValueCompleteness
    
                         
    Katy
 Sadowski
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/measureValueCompleteness.Rmd
       
diff --git a/docs/articles/checks/plausibleAfterBirth.html b/docs/articles/checks/plausibleAfterBirth.html
index a8e0f1d7..cacfbb0a 100644
--- a/docs/articles/checks/plausibleAfterBirth.html
+++ b/docs/articles/checks/plausibleAfterBirth.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    plausibleAfterBirth
    
                         
    Maxim Moinat,
 Katy Sadowski
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/plausibleAfterBirth.Rmd
       
diff --git a/docs/articles/checks/plausibleBeforeDeath.html b/docs/articles/checks/plausibleBeforeDeath.html
index 771b3883..a2d11704 100644
--- a/docs/articles/checks/plausibleBeforeDeath.html
+++ b/docs/articles/checks/plausibleBeforeDeath.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    plausibleBeforeDeath
    
                         
    Maxim
 Moinat
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/plausibleBeforeDeath.Rmd
       
diff --git a/docs/articles/checks/plausibleDuringLife.html b/docs/articles/checks/plausibleDuringLife.html
deleted file mode 100644
index bec0bd41..00000000
--- a/docs/articles/checks/plausibleDuringLife.html
+++ /dev/null
@@ -1,246 +0,0 @@
-
-
-
-
-
-
-
-plausibleDuringLife • DataQualityDashboard
-
-
-
-
-
-
-
-
-
-
-    
-
-    
    
-      
    
-
-
-      
-
-      
    
-  
    
-    
    
-      
    plausibleDuringLife
    
-                        
    
-            
-            
    2024-06-13
    
-      
-      Source: vignettes/checks/plausibleDuringLife.Rmd
-      
-
-    
    
-
-    
-    
-
    
-
    Summary
-
    
-
    Level: FIELD
    Context: Verification
    Category: Plausibility
    Subcategory: Temporal
    Severity:
    
-
    
-
    
-
    Description
-
    
-
    If yes, the number and percent of records with a date value in the
-@cdmFieldName field of the @cdmTableName table that occurs after death.
    
-
    
-
    
-
    Definition
-
    
-
      
-
    • 
-Numerator:
      • 
-
    • 
-Denominator:
      • 
-
    • 
-Related CDM Convention(s):
      • 
-
    • 
-CDM Fields/Tables:
      • 
-
    • 
-Default Threshold Value:
      • 
-
    
-
    
-
    
-
    User Guidance
-
    
-
    
-
    Violated rows query
-
    
-
    
-
    
-
    
-
    ETL Developers
-
    
-
    
-
    
-
    Data Users
-
    
-
    
-
    
-  
    
-
-  
-
-
    
-
-
-
-      
    
-  
    
-
    Developed by Katy Sadowski, Clair Blacketer, Ajit Londhe, Anthony Sena, Anthony Molinaro, Frank DeFalco, Pavel Grafkin.
    
-
    
-
-
    
-  
    
-
    Site built with pkgdown 2.0.9.
    
-
    
-
-      
    
-
    
-
-  
-
-
-  
-
-  
-
diff --git a/docs/articles/checks/plausibleGender.html b/docs/articles/checks/plausibleGender.html
deleted file mode 100644
index a5e62088..00000000
--- a/docs/articles/checks/plausibleGender.html
+++ /dev/null
@@ -1,247 +0,0 @@
-
-
-
-
-
-
-
-plausibleGender • DataQualityDashboard
-
-
-
-
-
-
-
-
-
-
-    
-
-    
    
-      
    
-
-
-      
-
-      
    
-  
    
-    
    
-      
    plausibleGender
    
-                        
    
-            
-            
    2024-06-13
    
-      
-      Source: vignettes/checks/plausibleGender.Rmd
-      
-
-    
    
-
-    
-    
-
    
-
    Summary
-
    
-
    Level: CONCEPT
    Context: Validation
    Category: Plausibility
    Subcategory: Atemporal
    Severity:
    
-
    
-
    
-
    Description
-
    
-
    For a CONCEPT_ID @conceptId (@conceptName), the number and percent of records
-associated with patients with an implausible gender (correct gender =
-@plausibleGender).
    
-
    
-
    
-
    Definition
-
    
-
      
-
    • 
-Numerator:
      • 
-
    • 
-Denominator:
      • 
-
    • 
-Related CDM Convention(s):
      • 
-
    • 
-CDM Fields/Tables:
      • 
-
    • 
-Default Threshold Value:
      • 
-
    
-
    
-
    
-
    User Guidance
-
    
-
    
-
    Violated rows query
-
    
-
    
-
    
-
    
-
    ETL Developers
-
    
-
    
-
    
-
    Data Users
-
    
-
    
-
    
-  
    
-
-  
-
-
    
-
-
-
-      
    
-  
    
-
    Developed by Katy Sadowski, Clair Blacketer, Ajit Londhe, Anthony Sena, Anthony Molinaro, Frank DeFalco, Pavel Grafkin.
    
-
    
-
-
    
-  
    
-
    Site built with pkgdown 2.0.9.
    
-
    
-
-      
    
-
    
-
-  
-
-
-  
-
-  
-
diff --git a/docs/articles/checks/plausibleGenderUseDescendants.html b/docs/articles/checks/plausibleGenderUseDescendants.html
index 23ca231b..2e92d60d 100644
--- a/docs/articles/checks/plausibleGenderUseDescendants.html
+++ b/docs/articles/checks/plausibleGenderUseDescendants.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -183,7 +180,7 @@
       
    plausibleGender
    
                         
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/plausibleGenderUseDescendants.Rmd
       
diff --git a/docs/articles/checks/plausibleStartBeforeEnd.html b/docs/articles/checks/plausibleStartBeforeEnd.html
index 66ae6eb9..489a615d 100644
--- a/docs/articles/checks/plausibleStartBeforeEnd.html
+++ b/docs/articles/checks/plausibleStartBeforeEnd.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    plausibleStartBeforeEnd
    
                         
    Maxim
 Moinat
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/plausibleStartBeforeEnd.Rmd
       
diff --git a/docs/articles/checks/plausibleTemporalAfter.html b/docs/articles/checks/plausibleTemporalAfter.html
index 7c3cf881..2c11f327 100644
--- a/docs/articles/checks/plausibleTemporalAfter.html
+++ b/docs/articles/checks/plausibleTemporalAfter.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -183,7 +180,7 @@
       
    plausibleTemporalAfter
    
                         
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/plausibleTemporalAfter.Rmd
       
diff --git a/docs/articles/checks/plausibleUnitConceptIds.html b/docs/articles/checks/plausibleUnitConceptIds.html
index 71f25bfe..cf0df5fb 100644
--- a/docs/articles/checks/plausibleUnitConceptIds.html
+++ b/docs/articles/checks/plausibleUnitConceptIds.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -183,7 +180,7 @@
       
    plausibleUnitConceptIds
    
                         
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/plausibleUnitConceptIds.Rmd
       
diff --git a/docs/articles/checks/plausibleValueHigh.html b/docs/articles/checks/plausibleValueHigh.html
index fdcdc415..7879501f 100644
--- a/docs/articles/checks/plausibleValueHigh.html
+++ b/docs/articles/checks/plausibleValueHigh.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    plausibleValueHigh
    
                         
    Dymytry
 Dymshyts
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/plausibleValueHigh.Rmd
       
diff --git a/docs/articles/checks/plausibleValueLow.html b/docs/articles/checks/plausibleValueLow.html
index 86b7bf72..40d10b8a 100644
--- a/docs/articles/checks/plausibleValueLow.html
+++ b/docs/articles/checks/plausibleValueLow.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    plausibleValueLow
    
                         
    Dymytry
 Dymshyts
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/plausibleValueLow.Rmd
       
diff --git a/docs/articles/checks/sourceConceptRecordCompleteness.html b/docs/articles/checks/sourceConceptRecordCompleteness.html
index 7dba571d..2d4effb4 100644
--- a/docs/articles/checks/sourceConceptRecordCompleteness.html
+++ b/docs/articles/checks/sourceConceptRecordCompleteness.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    sourceConceptRecordCompleteness
    
                         
    Katy
 Sadowski
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/sourceConceptRecordCompleteness.Rmd
       
diff --git a/docs/articles/checks/sourceValueCompleteness.html b/docs/articles/checks/sourceValueCompleteness.html
index f52bd772..e02e771f 100644
--- a/docs/articles/checks/sourceValueCompleteness.html
+++ b/docs/articles/checks/sourceValueCompleteness.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    sourceValueCompleteness
    
                         
    Jared
 Houghtaling, Clair Blacketer
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/sourceValueCompleteness.Rmd
       
diff --git a/docs/articles/checks/standardConceptRecordCompleteness.html b/docs/articles/checks/standardConceptRecordCompleteness.html
index 4b1e3cbf..ee2794ce 100644
--- a/docs/articles/checks/standardConceptRecordCompleteness.html
+++ b/docs/articles/checks/standardConceptRecordCompleteness.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    standardConceptRecordCompleteness
    
                         
    Katy
 Sadowski
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/standardConceptRecordCompleteness.Rmd
       
diff --git a/docs/articles/checks/withinVisitDates.html b/docs/articles/checks/withinVisitDates.html
index 3089b657..1ca9c9b6 100644
--- a/docs/articles/checks/withinVisitDates.html
+++ b/docs/articles/checks/withinVisitDates.html
@@ -148,9 +148,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
 
  • 
@@ -184,7 +181,7 @@ 
    withinVisitDates
    
                         
    Clair
 Blacketer
    
             
-            
    2024-07-12
    
+            
    2024-07-11
    
       
       Source: vignettes/checks/withinVisitDates.Rmd
       
diff --git a/docs/articles/index.html b/docs/articles/index.html
index 1eadbf9d..a9c50ea9 100644
--- a/docs/articles/index.html
+++ b/docs/articles/index.html
@@ -128,9 +128,6 @@
     
  • 
       withinVisitDates
     
    • 
-    
  • 
-      measureConditionEraCompleteness
-    
    • 
   
 
  • 
   Changelog

From b26c91eaba5ed30c6b3a2092711aab84cf57649c Mon Sep 17 00:00:00 2001
From: Katy Sadowski 
Date: Fri, 12 Jul 2024 22:35:21 -0400
Subject: [PATCH 43/43] release prep

---
 DESCRIPTION                                   |   4 +-
 NEWS.md                                       |  32 +++
 R/calculateNotApplicableStatus.R              |  12 +-
 R/executeDqChecks.R                           |   2 +-
 R/sqlOnly.R                                   |   4 +-
 docs/404.html                                 |   4 +-
 docs/LICENSE-text.html                        |   4 +-
 docs/articles/AddNewCheck.html                |   9 +-
 docs/articles/CheckStatusDefinitions.html     |   9 +-
 docs/articles/CheckTypeDescriptions.html      |   9 +-
 docs/articles/DataQualityDashboard.html       |   9 +-
 docs/articles/DqdForCohorts.html              |   9 +-
 docs/articles/SqlOnly.html                    |   9 +-
 docs/articles/Thresholds.html                 |   9 +-
 docs/articles/checkIndex.html                 |  11 +-
 docs/articles/checks/cdmDatatype.html         |   9 +-
 docs/articles/checks/cdmField.html            |   9 +-
 docs/articles/checks/cdmTable.html            |   9 +-
 docs/articles/checks/fkClass.html             |   9 +-
 docs/articles/checks/fkDomain.html            |   9 +-
 docs/articles/checks/isForeignKey.html        |   9 +-
 docs/articles/checks/isPrimaryKey.html        |   9 +-
 docs/articles/checks/isRequired.html          |   9 +-
 .../checks/isStandardValidConcept.html        |  11 +-
 .../measureConditionEraCompleteness.html      |   4 +-
 .../checks/measurePersonCompleteness.html     |   9 +-
 .../checks/measureValueCompleteness.html      |   9 +-
 docs/articles/checks/plausibleAfterBirth.html |   9 +-
 .../articles/checks/plausibleBeforeDeath.html |   9 +-
 .../checks/plausibleGenderUseDescendants.html |   9 +-
 .../checks/plausibleStartBeforeEnd.html       |  11 +-
 .../checks/plausibleTemporalAfter.html        |   9 +-
 .../checks/plausibleUnitConceptIds.html       |   9 +-
 docs/articles/checks/plausibleValueHigh.html  |   9 +-
 docs/articles/checks/plausibleValueLow.html   |   9 +-
 .../sourceConceptRecordCompleteness.html      |   9 +-
 .../checks/sourceValueCompleteness.html       |   9 +-
 .../standardConceptRecordCompleteness.html    |   9 +-
 docs/articles/checks/withinVisitDates.html    |   9 +-
 docs/articles/index.html                      |   7 +-
 docs/authors.html                             |   4 +-
 docs/index.html                               |   4 +-
 docs/news/index.html                          |  41 +++-
 docs/pkgdown.yml                              |   2 +-
 docs/pull_request_template.html               |   2 +-
 .../reference/convertJsonResultsFileCase.html |   4 +-
 docs/reference/dot-applyNotApplicable.html    | 199 ++++++++++++++++++
 .../dot-calculateNotApplicableStatus.html     | 199 ++++++++++++++++++
 docs/reference/dot-containsNAchecks.html      | 199 ++++++++++++++++++
 docs/reference/dot-evaluateThresholds.html    |   4 +-
 docs/reference/dot-getCheckId.html            |   4 +-
 docs/reference/dot-hasNAchecks.html           | 199 ++++++++++++++++++
 docs/reference/dot-processCheck.html          |   4 +-
 docs/reference/dot-recordResult.html          |   4 +-
 docs/reference/dot-runCheck.html              |   4 +-
 docs/reference/dot-summarizeResults.html      |   4 +-
 docs/reference/dot-writeResultsToCsv.html     |   4 +-
 docs/reference/dot-writeResultsToJson.html    |   4 +-
 docs/reference/dot-writeResultsToTable.html   |   4 +-
 docs/reference/executeDqChecks.html           |   4 +-
 docs/reference/index.html                     |   4 +-
 docs/reference/listDqChecks.html              |   4 +-
 docs/reference/reEvaluateThresholds.html      |   4 +-
 docs/reference/viewDqDashboard.html           |   4 +-
 docs/reference/writeDBResultsToJson.html      |   4 +-
 docs/reference/writeJsonResultsToCsv.html     |   4 +-
 docs/reference/writeJsonResultsToTable.html   |   4 +-
 docs/sitemap.xml                              |  18 +-
 extras/DataQualityDashboard.pdf               | Bin 93577 -> 95023 bytes
 inst/doc/AddNewCheck.pdf                      | Bin 232110 -> 232602 bytes
 inst/doc/CheckStatusDefinitions.pdf           | Bin 159239 -> 159566 bytes
 inst/doc/CheckTypeDescriptions.pdf            | Bin 249669 -> 251396 bytes
 inst/doc/DataQualityDashboard.pdf             | Bin 240492 -> 240526 bytes
 inst/doc/DqdForCohorts.pdf                    | Bin 191607 -> 191816 bytes
 inst/doc/SqlOnly.pdf                          | Bin 231754 -> 232074 bytes
 inst/doc/Thresholds.pdf                       | Bin 274453 -> 274987 bytes
 man/dot-applyNotApplicable.Rd                 |  15 ++
 man/dot-calculateNotApplicableStatus.Rd       |  15 ++
 man/dot-containsNAchecks.Rd                   |  15 ++
 man/dot-hasNAchecks.Rd                        |  15 ++
 tests/testthat/setup.R                        |   2 +-
 .../test-calculateNotApplicableStatus.R       |   4 +-
 tests/testthat/test-convertResultsCase.R      |   8 +-
 tests/testthat/test-executeDqChecks.R         |   2 +-
 vignettes/checks/isStandardValidConcept.Rmd   |   2 +-
 vignettes/checks/plausibleStartBeforeEnd.Rmd  |   2 +-
 vignettes/checks/plausibleValueHigh.Rmd       |   2 +-
 vignettes/checks/sourceValueCompleteness.Rmd  |   2 +-
 88 files changed, 1206 insertions(+), 175 deletions(-)
 create mode 100644 docs/reference/dot-applyNotApplicable.html
 create mode 100644 docs/reference/dot-calculateNotApplicableStatus.html
 create mode 100644 docs/reference/dot-containsNAchecks.html
 create mode 100644 docs/reference/dot-hasNAchecks.html
 create mode 100644 man/dot-applyNotApplicable.Rd
 create mode 100644 man/dot-calculateNotApplicableStatus.Rd
 create mode 100644 man/dot-containsNAchecks.Rd
 create mode 100644 man/dot-hasNAchecks.Rd

diff --git a/DESCRIPTION b/DESCRIPTION
index c2e22641..4994be96 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -1,8 +1,8 @@
 Package: DataQualityDashboard
 Type: Package
 Title: Execute and View Data Quality Checks on OMOP CDM Database
-Version: 2.6.0
-Date: 2024-02-21
+Version: 2.6.1
+Date: 2024-07-12
 Authors@R: c(
   person("Katy", "Sadowski", email = "sadowski@ohdsi.org", role = c("aut", "cre")),
   person("Clair", "Blacketer", role = c("aut")),
diff --git a/NEWS.md b/NEWS.md
index 0c91c12c..b06f3e99 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,3 +1,35 @@
+DataQualityDashboard 2.6.1
+==========================
+This release includes: 
+
+### Bugfixes
+
+- Checks
+  - `plausibleStartBeforeEnd` was failing if SOURCE_RELEASE_DATE was before CDM_RELEASE_DATE in the CDM_SOURCE table. This is the opposite of the correct logic!  The check is now updated to fail if the CDM_RELEASE_DATE is before the SOURCE_RELEASE_DATE
+  - `plausibleTemporalAfter` was throwing a syntax error in BigQuery due to the format of a hardcoded date in the SQL query.  This query has now been updated to be compliant with SqlRender and the issue has been resolved
+- A dependency issue was causing `viewDqDashboard` to error out in newer versions of R.  This has now been resolved
+- `SqlOnly` mode was failing due to the format of the new check `plausibleGenderUseDescendants`, which takes multiple concepts as an input.  This has now been fixed
+
+### New Results Field
+
+- A new field has been added to the DQD results output - `executionTimeSeconds`.  This field stores the execution time in seconds of each check in numeric format.  (The existing `executionTime` field stores execution time as a string, making it difficult to use in analysis.)
+
+### Check Threshold Updates
+
+The default thresholds for 2 checks were discovered to be inconsistently populated and occasionally set to illogical levels.  These have now been fixed as detailed below.
+
+- The default thresholds for `sourceValueCompleteness` have been updated as follows:
+  - 10% for `_source_value` columns in condition_occurrence, measurement, procedure_occurrence, drug_exposure, and visit_occurrence tables
+  - 100% for all other `_source_value` columns
+- The default thresholds for `sourceConceptRecordCompleteness` have been updated as follows:
+  - 10% for `_source_concept_id` columns in condition_occurrence, drug_exposure, measurement, procedure_occurrence, device_exposure, and observation tables
+  - 100% for all other `_source_concept_id` columns
+
+### New Documentation
+We have continued (and nearly completed) our initiative to add more comprehensive user documentation at the data quality check level.  A dedicated documentation page is being created for each check type.  Each check's page includes detailed information about how its result is generated and what to do if it fails.  Guidance is provided for both ETL developers and data users.
+
+Check out the newly added pages [here](https://ohdsi.github.io/DataQualityDashboard/articles/checkIndex.html) and please reach out with feedback as we continue improving our documentation!
+
 DataQualityDashboard 2.6.0
 ==========================
 This release includes: 
diff --git a/R/calculateNotApplicableStatus.R b/R/calculateNotApplicableStatus.R
index eeafb203..c00b486e 100644
--- a/R/calculateNotApplicableStatus.R
+++ b/R/calculateNotApplicableStatus.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of DataQualityDashboard
 #
@@ -151,8 +151,12 @@
   checkResults$notApplicable <- NA
   checkResults$notApplicableReason <- NA
 
-  conditionOccurrenceIsMissing <- missingTables %>% dplyr::filter(.data$cdmTableName == "CONDITION_OCCURRENCE") %>% dplyr::pull(tableIsMissing)
-  conditionOccurrenceIsEmpty <- emptyTables %>% dplyr::filter(.data$cdmTableName == "CONDITION_OCCURRENCE") %>% dplyr::pull(tableIsEmpty)
+  conditionOccurrenceIsMissing <- missingTables %>%
+    dplyr::filter(.data$cdmTableName == "CONDITION_OCCURRENCE") %>%
+    dplyr::pull(.data$tableIsMissing)
+  conditionOccurrenceIsEmpty <- emptyTables %>%
+    dplyr::filter(.data$cdmTableName == "CONDITION_OCCURRENCE") %>%
+    dplyr::pull(.data$tableIsEmpty)
   for (i in seq_len(nrow(checkResults))) {
     # Special rule for measureConditionEraCompleteness, which should be notApplicable if CONDITION_OCCURRENCE is empty
     if (checkResults[i, "checkName"] == "measureConditionEraCompleteness") {
@@ -178,7 +182,7 @@
           .data$tableIsEmpty ~ sprintf("Table %s is empty.", .data$cdmTableName),
           .data$fieldIsEmpty ~ sprintf("Field %s.%s is not populated.", .data$cdmTableName, .data$cdmFieldName),
           .data$conceptIsMissing ~ sprintf("%s=%s is missing from the %s table.", .data$cdmFieldName, .data$conceptId, .data$cdmTableName),
-          .data$conceptAndUnitAreMissing ~ sprintf("Combination of %s=%s, unitConceptId=%s and VALUE_AS_NUMBER IS NOT NULL is missing from the %s table.", .data$cdmFieldName, .data$conceptId, .data$unitConceptId, .data$cdmTableName) #nolint
+          .data$conceptAndUnitAreMissing ~ sprintf("Combination of %s=%s, unitConceptId=%s and VALUE_AS_NUMBER IS NOT NULL is missing from the %s table.", .data$cdmFieldName, .data$conceptId, .data$unitConceptId, .data$cdmTableName) # nolint
         ),
         NA
       ),
diff --git a/R/executeDqChecks.R b/R/executeDqChecks.R
index e153b3c9..b5a1ea67 100644
--- a/R/executeDqChecks.R
+++ b/R/executeDqChecks.R
@@ -327,7 +327,7 @@ executeDqChecks <- function(connectionDetails,
       startTimestamp = startTime,
       endTimestamp = endTime,
       executionTime = sprintf("%.0f %s", delta, attr(delta, "units")),
-      #new variable executionTimeSeconds added to store execution time in seconds
+      # new variable executionTimeSeconds added to store execution time in seconds
       executionTimeSeconds = as.numeric(delta),
       CheckResults = checkResults,
       Metadata = metadata,
diff --git a/R/sqlOnly.R b/R/sqlOnly.R
index cb794868..f1222e12 100644
--- a/R/sqlOnly.R
+++ b/R/sqlOnly.R
@@ -232,7 +232,7 @@
       )
     } else if (checkLevel == "CONCEPT") {
       if (is.na(unitConceptId) &&
-          grepl(",", conceptId)) {
+        grepl(",", conceptId)) {
         thresholdFilter <- sprintf(
           "conceptChecks$%s[conceptChecks$cdmTableName == '%s' &
                                   conceptChecks$cdmFieldName == '%s' &
@@ -243,7 +243,7 @@
           conceptId
         )
       } else if (is.na(unitConceptId) &&
-                 !grepl(",", conceptId)) {
+        !grepl(",", conceptId)) {
         thresholdFilter <- sprintf(
           "conceptChecks$%s[conceptChecks$cdmTableName == '%s' &
                                   conceptChecks$cdmFieldName == '%s' &
diff --git a/docs/404.html b/docs/404.html
index f3724215..a4317ac4 100644
--- a/docs/404.html
+++ b/docs/404.html
@@ -32,7 +32,7 @@
       
       
         DataQualityDashboard
-        2.6.0
+        2.6.1
    • @@ -158,7 +158,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • @@ -156,7 +159,7 @@
  • Changelog
    • - hadesLogo +
    • @@ -157,6 +157,43 @@

      Changelog

      Source:       NEWS.md +
      +

      DataQualityDashboard 2.6.1

      +

      This release includes:

      +
      +

      Bugfixes

      +
      • Checks +
        • +plausibleStartBeforeEnd was failing if SOURCE_RELEASE_DATE was before CDM_RELEASE_DATE in the CDM_SOURCE table. This is the opposite of the correct logic! The check is now updated to fail if the CDM_RELEASE_DATE is before the SOURCE_RELEASE_DATE
          • +
        • +plausibleTemporalAfter was throwing a syntax error in BigQuery due to the format of a hardcoded date in the SQL query. This query has now been updated to be compliant with SqlRender and the issue has been resolved
          • +
        • +
      • A dependency issue was causing viewDqDashboard to error out in newer versions of R. This has now been resolved
        • +
      • +SqlOnly mode was failing due to the format of the new check plausibleGenderUseDescendants, which takes multiple concepts as an input. This has now been fixed
        • +
      +
      +

      New Results Field

      +
      • A new field has been added to the DQD results output - executionTimeSeconds. This field stores the execution time in seconds of each check in numeric format. (The existing executionTime field stores execution time as a string, making it difficult to use in analysis.)
        • +
      +
      +

      Check Threshold Updates

      +

      The default thresholds for 2 checks were discovered to be inconsistently populated and occasionally set to illogical levels. These have now been fixed as detailed below.

      +
      • The default thresholds for sourceValueCompleteness have been updated as follows: +
        • 10% for _source_value columns in condition_occurrence, measurement, procedure_occurrence, drug_exposure, and visit_occurrence tables
          • +
        • 100% for all other _source_value columns
          • +
        • +
      • The default thresholds for sourceConceptRecordCompleteness have been updated as follows: +
        • 10% for _source_concept_id columns in condition_occurrence, drug_exposure, measurement, procedure_occurrence, device_exposure, and observation tables
          • +
        • 100% for all other _source_concept_id columns
          • +
        • +
      +
      +

      New Documentation

      +

      We have continued (and nearly completed) our initiative to add more comprehensive user documentation at the data quality check level. A dedicated documentation page is being created for each check type. Each check’s page includes detailed information about how its result is generated and what to do if it fails. Guidance is provided for both ETL developers and data users.

      +

      Check out the newly added pages here and please reach out with feedback as we continue improving our documentation!

      +
      +

      DataQualityDashboard 2.6.0

      This release includes:

      diff --git a/docs/pkgdown.yml b/docs/pkgdown.yml index deb660db..db534a4c 100644 --- a/docs/pkgdown.yml +++ b/docs/pkgdown.yml @@ -34,7 +34,7 @@ articles: standardConceptRecordCompleteness: checks/standardConceptRecordCompleteness.html Thresholds: Thresholds.html withinVisitDates: checks/withinVisitDates.html -last_built: 2024-07-13T01:28Z +last_built: 2024-07-13T02:25Z urls: reference: https://ohdsi.github.io/DataQualityDashboard/reference article: https://ohdsi.github.io/DataQualityDashboard/articles diff --git a/docs/pull_request_template.html b/docs/pull_request_template.html index c8bcda13..f2d83fcb 100644 --- a/docs/pull_request_template.html +++ b/docs/pull_request_template.html @@ -100,7 +100,7 @@ Changelog