Merge pull request #530 from OHDSI/develop

Develop
OHDSI · Feb 22, 2024 · ac1baba · ac1baba
2 parents 81230bd + 8ba1c4a
commit ac1baba
Show file tree

Hide file tree

Showing 189 changed files with 13,794 additions and 15,360 deletions.
diff --git a/.Rbuildignore b/.Rbuildignore
@@ -9,4 +9,5 @@ compare_versions
 .github
 docs
 ^LICENSE$
-^inst/doc/.*\.pdf$
+^inst/doc/.*\.pdf$
+.lintr
diff --git a/.gitignore b/.gitignore
@@ -28,14 +28,17 @@ inst/results.json
 # Other
 .DS_Store
 
+# Visual Studio Code
+.vscode/
+.lintr
+
 errorReport.txt
 output/
 results/
 thresholds/
 hs_err_pid*.log
 docs/Gemfile
 docs/Gemfile.lock
-sql
 vignettes/*.log
 /doc/
 /Meta/
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,8 +1,8 @@
 Package: DataQualityDashboard
 Type: Package
 Title: Execute and View Data Quality Checks on OMOP CDM Database
-Version: 2.5.0
-Date: 2023-11-04
+Version: 2.6.0
+Date: 2024-02-21
 Authors@R: c(
   person("Katy", "Sadowski", email = "sadowski@ohdsi.org", role = c("aut", "cre")),
   person("Clair", "Blacketer", role = c("aut")),

diff --git a/NEWS.md b/NEWS.md
@@ -1,3 +1,24 @@
+DataQualityDashboard 2.6.0
+==========================
+This release includes: 
+
+### New Checks
+4 new data quality check types have been added in this release:
+
+- `plausibleStartBeforeEnd`: The number and percent of records with a value in the **cdmFieldName** field of the **cdmTableName** that occurs after the date in the **plausibleStartBeforeEndFieldName**.
+- `plausibleAfterBirth`: The number and percent of records with a date value in the **cdmFieldName** field of the **cdmTableName** table that occurs prior to birth.
+- `plausibleBeforeDeath`: The number and percent of records with a date value in the **cdmFieldName** field of the **cdmTableName** table that occurs after death.
+- `plausibleGenderUseDescendants`: For descendants of CONCEPT_ID **conceptId** (**conceptName**), the number and percent of records associated with patients with an implausible gender (correct gender = **plausibleGenderUseDescendants**).
+
+The 3 temporal plausibilty checks are intended to **replace** `plausibleTemporalAfter` and `plausibleDuringLife`, for a more comprehensive and clear approach to various temporality scenarios.  `plausibleGenderUseDescendants` is intended to **replace** `plausibleGender`, to enhance readability of the DQD results and improve performance.  The replaced checks are still available and enabled by default in DQD; however, in a future major release, these checks will be deprecated.  Please plan accordingly.
+
+For more information on the new checks, please check the [Check Type Definitions](https://ohdsi.github.io/DataQualityDashboard/articles/CheckTypeDescriptions.html) documentation page.  If you'd like to disable the deprecated checks, please see the suggested check exclusion workflow in our Getting Started code [here](https://ohdsi.github.io/DataQualityDashboard/articles/DataQualityDashboard.html).
+
+### New Documentation
+We have begun an initiative to add more comprehensive user documentation at the data quality check level.  A dedicated documentation page is being created for each check type.  Each check's page will include detailed information about how its result is generated and what to do if it fails.  Guidance is provided for both ETL developers and data users.
+
+9 pages have been added so far, and the rest will come in a future release.  Check them out [here](https://ohdsi.github.io/DataQualityDashboard/articles/checkIndex.html) and please reach out with feedback as we continue improving our documentation!
+
 DataQualityDashboard 2.5.0
 ==========================
 This release includes: 

diff --git a/R/constants.R b/R/constants.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of DataQualityDashboard
 #

diff --git a/R/convertResultsCase.R b/R/convertResultsCase.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of DataQualityDashboard
 #

diff --git a/R/evaluateThresholds.R b/R/evaluateThresholds.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of DataQualityDashboard
 #
@@ -78,7 +78,28 @@
           checkResults[i, ]$cdmFieldName
         )
       } else if (checkResults[i, ]$checkLevel == "CONCEPT") {
-        if (is.na(checkResults[i, ]$unitConceptId)) {
+        if (is.na(checkResults[i, ]$unitConceptId) &&
+          grepl(",", checkResults[i, ]$conceptId)) {
+          thresholdFilter <- sprintf(
+            "conceptChecks$%s[conceptChecks$cdmTableName == '%s' &
+                                  conceptChecks$cdmFieldName == '%s' &
+                                  conceptChecks$conceptId == '%s']",
+            thresholdField,
+            checkResults[i, ]$cdmTableName,
+            checkResults[i, ]$cdmFieldName,
+            checkResults[i, ]$conceptId
+          )
+          notesFilter <- sprintf(
+            "conceptChecks$%s[conceptChecks$cdmTableName == '%s' &
+                                  conceptChecks$cdmFieldName == '%s' &
+                                  conceptChecks$conceptId == '%s']",
+            notesField,
+            checkResults[i, ]$cdmTableName,
+            checkResults[i, ]$cdmFieldName,
+            checkResults[i, ]$conceptId
+          )
+        } else if (is.na(checkResults[i, ]$unitConceptId) &&
+          !grepl(",", checkResults[i, ]$conceptId)) {
           thresholdFilter <- sprintf(
             "conceptChecks$%s[conceptChecks$cdmTableName == '%s' &
                                   conceptChecks$cdmFieldName == '%s' &

diff --git a/R/executeDqChecks.R b/R/executeDqChecks.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of DataQualityDashboard
 #
@@ -123,6 +123,7 @@ executeDqChecks <- function(connectionDetails,
     }
   }
 
+
   # temporary patch to work around vroom 1.6.4 bug
   readr::local_edition(1)
 
@@ -258,6 +259,18 @@ executeDqChecks <- function(connectionDetails,
     stop("No checks are available based on excluded tables. Please review tablesToExclude.")
   }
 
+  if ("plausibleDuringLife" %in% checkDescriptionsDf$checkName) {
+    warning("DEPRECATION WARNING - The plausibleDuringLife check has been reimplemented with the plausibleBeforeDeath check.")
+  }
+
+  if ("plausibleTemporalAfter" %in% checkDescriptionsDf$checkName) {
+    warning("DEPRECATION WARNING - The plausibleTemporalAfter check has been reimplemented with the plausibleAfterBirth and plausibleStartBeforeEnd checks.")
+  }
+
+  if ("plausibleGender" %in% checkDescriptionsDf$checkName) {
+    warning("DEPRECATION WARNING - The plausibleGender check has been reimplemented with the plausibleGenderUseDescendants check.")
+  }
+
   checkDescriptions <- split(checkDescriptionsDf, seq_len(nrow(checkDescriptionsDf)))
 
   connection <- NULL

diff --git a/R/getCheckId.R b/R/getCheckId.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of DataQualityDashboard
 #

diff --git a/R/listChecks.R b/R/listChecks.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of DataQualityDashboard
 #

diff --git a/R/processCheck.R b/R/processCheck.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of DataQualityDashboard
 #

diff --git a/R/reEvaluateThresholds.R b/R/reEvaluateThresholds.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of DataQualityDashboard
 #

diff --git a/R/readThresholdFile.R b/R/readThresholdFile.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of DataQualityDashboard
 #

diff --git a/R/recordResult.R b/R/recordResult.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of DataQualityDashboard
 #

diff --git a/R/runCheck.R b/R/runCheck.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of DataQualityDashboard
 #

diff --git a/R/sqlOnly.R b/R/sqlOnly.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of DataQualityDashboard
 #

diff --git a/R/summarizeResults.R b/R/summarizeResults.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of DataQualityDashboard
 #

diff --git a/R/view.R b/R/view.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of DataQualityDashboard
 #

diff --git a/R/writeDBResultsTo.R b/R/writeDBResultsTo.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of DataQualityDashboard
 #

diff --git a/R/writeJsonResultsTo.R b/R/writeJsonResultsTo.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of DataQualityDashboard
 #

diff --git a/R/writeResultsTo.R b/R/writeResultsTo.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2024 Observational Health Data Sciences and Informatics
 #
 # This file is part of DataQualityDashboard
 #

diff --git a/_pkgdown.yml b/_pkgdown.yml
@@ -16,6 +16,7 @@ navbar:
     - intro
     - reference
     - articles
+    - checks
     - news
     right: [hades, github]
   components:
@@ -31,6 +32,44 @@ navbar:
     news:
       text: Changelog
       href: news/index.html
+    articles:
+      text: Articles
+      menu:
+      - text: Check Type Definitions
+        href: articles/CheckTypeDescriptions.html
+      - text: DQ Check Failure Thresholds
+        href: articles/Thresholds.html
+      - text: DQ Check Statuses
+        href: articles/CheckStatusDefinitions.html
+      - text: Adding a New Data Quality Check
+        href: articles/AddNewCheck.html
+      - text: DQD for Cohorts
+        href: articles/DqdForCohorts.html
+      - text: SQL-only Mode
+        href: articles/SqlOnly.html
+    checks:
+      text: NEW! Data Quality Check Types
+      menu:
+      - text: Index
+        href: articles/checkIndex.html
+      - text: cdmTable
+        href: articles/checks/cdmTable.html
+      - text: cdmField
+        href: articles/checks/cdmField.html
+      - text: cdmDatatype
+        href: articles/checks/cdmDatatype.html
+      - text: isPrimaryKey
+        href: articles/checks/isPrimaryKey.html
+      - text: isForeignKey
+        href: articles/checks/isForeignKey.html
+      - text: isRequired
+        href: articles/checks/isRequired.html
+      - text: fkDomain
+        href: articles/checks/fkDomain.html
+      - text: fkClass
+        href: articles/checks/fkClass.html
+      - text: plausibleAfterBirth
+        href: articles/checks/plausibleAfterBirth.html
     hades:
       text: hadesLogo
       href: https://ohdsi.github.io/Hades
@@ -67,3 +106,5 @@ reference:
     desc: > 
       Function to write DQD results from a database table into a JSON file
     contents: writeDBResultsToJson
+
+url: https://ohdsi.github.io/DataQualityDashboard/