v1.4 release candidate (#465)

* downloadEunomiaData support 5.4 (#414) * require omopgenerics 0.1.2 or higher (#433) * Compatibility with new release of omopgenerics (#439) * add dplyr db specific tests (#436) - A start to adding general dplyr translation tests that downstream analytic packages will rely on working - palmerpenguins added to suggests as it is used in these tests (so that we have a known source of truth) * benchmarkCDMConnector functionality (#441) * benchmarkCDMConnector functionality - add benchmarkCDMConnector function - add test of benchmarkCDMConnector function - add to website * update test name * fix workflow files for github actions * fix actions * update workflow * update workflows: * update workflows * debug actions * debug github actions * debug actions * debug actions --------- Co-authored-by: Adam Black <ablack3@users.noreply.github.com> * debug actions * bind + cohort_table (#447) * CirceR from carn (#453) * update license statements (#457) * additional omopgenerics reexports (#456) * additional omopgenerics reexports - deprecate version, reexport omopgenerics cdmVersion - use omopgenerics::cdmName(cdm) * fix reexport * deprecate_warn computeQuery (#455) * Empty concepts (#454) * return empty cohorts if missing concepts * fix test * remove temporary argument * also handle edge case with empty cs * keep limit in settings * deprecate_soft CQ (#458) * version 1.3.2 for cran (#460) - bump version to 1.3.2 - rebuild all docs - update cran comments - fix some examples that gave errors when run manually - add an example to benchmarkCDMConnector * reorganize test files a bit * move test file * r check vs dbms (#462) r check ran locally - passing on postgres * arrange for redshift tests (#464) * arrange for redshift tests * more ordering * generateConceptCohortSet now gives warning if no concepts ids are in the cdm * switch version to cdmVersion * tiny formatting changes * a few small changes to get tests working. * new omopgenerics methods (#418) * update docs * fix workflow files * update docs * turn off parallel tests * update cran comments * fix cdmTableFromSource.db_cdm * update insertTable to match with omopgenerics * update site --------- Co-authored-by: ginberg <ginberg@gmail.com> Co-authored-by: edward-burn <9583964+edward-burn@users.noreply.github.com> Co-authored-by: catalamarti <marticatalasabate@gmail.com>
darwin-eu · May 3, 2024 · 5a9d9c1 · 5a9d9c1
1 parent e08fe8e
commit 5a9d9c1
Show file tree

Hide file tree

Showing 283 changed files with 2,905 additions and 1,007 deletions.
diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml
@@ -58,6 +58,8 @@ jobs:
             install.packages("arrow")
             install.packages("tibble")
             install.packages("visR")
+            install.packages("tictoc")
+            install.packages("palmerpenguins")
         shell: Rscript {0}
 
       - name: Install CirceR

diff --git a/.github/workflows/duckdb-test.yaml b/.github/workflows/duckdb-test.yaml
@@ -11,7 +11,7 @@ jobs:
     # if: github.repository == 'darwin-eu/CDMConnector'
     runs-on: ubuntu-latest
     container:
-      image: executionengine.azurecr.io/darwin-base:v0.2
+      image: executionengine.azurecr.io/darwin-base:v0.3
       env:
         GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
         TESTTHAT_CPUS : 1
@@ -24,7 +24,17 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
+      - name: Install packages
+        run: |
+          print(getwd())
+          print(list.files())
+          install.packages("devtools", repos = "https://cloud.r-project.org/")
+          install.packages("omopgenerics", repos = "https://cloud.r-project.org/")
+          install.packages("palmerpenguins", repos = "https://cloud.r-project.org/")
+          install.packages("tictoc", repos = "https://cloud.r-project.org/")
+        shell: Rscript {0}
+
       - name: Run tests on duckdb
-        run: R -e 'withr::with_envvar(new = c("CI_TEST_DB" = "duckdb", "TESTTHAT_CPUS" = 1), testthat::test_local())'
+        run: R -e 'devtools::install(); withr::with_envvar(new = c("CI_TEST_DB" = "postgres"), testthat::test_local())'
 
 
diff --git a/.github/workflows/postgres-test.yaml b/.github/workflows/postgres-test.yaml
@@ -11,7 +11,7 @@ jobs:
     # if: github.repository == 'darwin-eu/CDMConnector'
     runs-on: ubuntu-latest
     container:
-      image: executionengine.azurecr.io/darwin-base:v0.2
+      image: executionengine.azurecr.io/darwin-base:v0.3
       env:
         CDM5_POSTGRESQL_USER:           ${{ secrets.CDM5_POSTGRESQL_USER }}
         CDM5_POSTGRESQL_PORT:           ${{ secrets.CDM5_POSTGRESQL_PORT }}
@@ -38,13 +38,17 @@ jobs:
       - name: Run one R command
         run: R -e "getwd()"
 
-      - name: Run several R commands
+      - name: Install Packages
         run: |
           print(getwd())
           print(list.files())
+          install.packages("devtools", repos = "https://cloud.r-project.org/")
+          install.packages("omopgenerics", repos = "https://cloud.r-project.org/")
+          install.packages("palmerpenguins", repos = "https://cloud.r-project.org/")
+          install.packages("tictoc", repos = "https://cloud.r-project.org/")
         shell: Rscript {0}
 
       - name: Run tests on postgres with RPostgres driver
-        run: R -e 'withr::with_envvar(new = c("CI_TEST_DB" = "postgres"), testthat::test_local())'
+        run: R -e 'devtools::install(); withr::with_envvar(new = c("CI_TEST_DB" = "postgres"), testthat::test_local())'
 
 
diff --git a/.github/workflows/redshift-test.yaml b/.github/workflows/redshift-test.yaml
@@ -11,7 +11,7 @@ jobs:
     # if: github.repository == 'darwin-eu/CDMConnector'
     runs-on: ubuntu-latest
     container:
-      image: executionengine.azurecr.io/darwin-base:v0.2
+      image: executionengine.azurecr.io/darwin-base:v0.3
       env:
         CDM5_REDSHIFT_USER:           ${{ secrets.CDM5_REDSHIFT_USER }}
         CDM5_REDSHIFT_PORT:           ${{ secrets.CDM5_REDSHIFT_PORT }}
@@ -33,7 +33,17 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
+      - name: Install R packages
+        run: |
+          print(getwd())
+          print(list.files())
+          install.packages("devtools", repos = "https://cloud.r-project.org/")
+          install.packages("omopgenerics", repos = "https://cloud.r-project.org/")
+          install.packages("palmerpenguins", repos = "https://cloud.r-project.org/")
+          install.packages("tictoc", repos = "https://cloud.r-project.org/")
+        shell: Rscript {0}
+
       - name: Run tests on Redshift with RPostgres driver
-        run: R -e 'withr::with_envvar(new = c("CI_TEST_DB" = "redshift"), testthat::test_local())'
+        run: R -e 'devtools::install(); withr::with_envvar(new = c("CI_TEST_DB" = "postgres"), testthat::test_local())'
 
 
diff --git a/.github/workflows/snowflake-odbc-test.yaml b/.github/workflows/snowflake-odbc-test.yaml
@@ -11,7 +11,7 @@ jobs:
     # if: github.repository == 'darwin-eu/CDMConnector'
     runs-on: ubuntu-latest
     container:
-      image: executionengine.azurecr.io/darwin-base:v0.2
+      image: executionengine.azurecr.io/darwin-base:v0.3
       env:
         SNOWFLAKE_CONNECTION_STRING: ${{ secrets.SNOWFLAKE_CONNECTION_STRING }}
         SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
@@ -33,7 +33,17 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
+      - name: Install R packages
+        run: |
+          print(getwd())
+          print(list.files())
+          install.packages("devtools", repos = "https://cloud.r-project.org/")
+          install.packages("omopgenerics", repos = "https://cloud.r-project.org/")
+          install.packages("palmerpenguins", repos = "https://cloud.r-project.org/")
+          install.packages("tictoc", repos = "https://cloud.r-project.org/")
+        shell: Rscript {0}
+
       - name: Run tests on Snowflake with odbc driver
-        run: R -e 'withr::with_envvar(new = c("CI_TEST_DB" = "snowflake"), testthat::test_local())'
+        run: R -e 'devtools::install(); withr::with_envvar(new = c("CI_TEST_DB" = "postgres"), testthat::test_local())'
 
 
diff --git a/.github/workflows/sqlserver-odbc-test.yaml b/.github/workflows/sqlserver-odbc-test.yaml
@@ -11,7 +11,7 @@ jobs:
     # if: github.repository == 'darwin-eu/CDMConnector'
     runs-on: ubuntu-latest
     container:
-      image: executionengine.azurecr.io/darwin-base:v0.2
+      image: executionengine.azurecr.io/darwin-base:v0.3
       env:
         CDM5_SQL_SERVER_USER:           ${{ secrets.CDM5_SQL_SERVER_USER }}
         CDM5_SQL_SERVER_PORT:           ${{ secrets.CDM5_SQL_SERVER_PORT }}
@@ -34,7 +34,17 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
+      - name: Install R packages
+        run: |
+          print(getwd())
+          print(list.files())
+          install.packages("devtools", repos = "https://cloud.r-project.org/")
+          install.packages("omopgenerics", repos = "https://cloud.r-project.org/")
+          install.packages("palmerpenguins", repos = "https://cloud.r-project.org/")
+          install.packages("tictoc", repos = "https://cloud.r-project.org/")
+        shell: Rscript {0}
+
       - name: Run tests on SQL Server with odbc driver
-        run: R -e 'withr::with_envvar(new = c("CI_TEST_DB" = "sqlserver"), testthat::test_local())'
+        run: R -e 'devtools::install(); withr::with_envvar(new = c("CI_TEST_DB" = "postgres"), testthat::test_local())'
 
 
diff --git a/CRAN-SUBMISSION b/CRAN-SUBMISSION
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: CDMConnector
 Title: Connect to an OMOP Common Data Model
-Version: 1.3.2
+Version: 1.4.0
 Authors@R: c(
     person("Adam", "Black", , "black@ohdsi.org", role = c("aut", "cre"),
            comment = c(ORCID = "0000-0001-5576-8701")),
@@ -40,9 +40,10 @@ Imports:
     fs,
     generics,
     tidyr,
-    omopgenerics (>= 0.0.2)
+    omopgenerics (>= 0.1.2)
 Suggests: 
     SqlRender,
+    CirceR,
     rJava,
     covr,
     knitr,
@@ -59,10 +60,11 @@ Suggests:
     testthat (>= 3.0.0),
     pool,
     snakecase,
-    visR
+    visR,
+    palmerpenguins,
+    tictoc
 Enhances:
     arrow,
-    CirceR,
     Capr
 Config/testthat/edition: 3
 Config/testthat/parallel: false
@@ -71,6 +73,7 @@ Additional_repositories: https://OHDSI.github.io/drat
 Collate: 
     'CDMConnector-package.R'
     'Eunomia.R'
+    'benchmarkCDMConnector.R'
     'cdm.R'
     'cdmSubset.R'
     'cdm_from_environment.R'

diff --git a/NAMESPACE b/NAMESPACE
@@ -1,10 +1,14 @@
 # Generated by roxygen2: do not edit by hand
 
+S3method(cdmTableFromSource,db_cdm)
 S3method(compute,db_cdm)
 S3method(dbplyr_edition,BigQueryConnection)
+S3method(dropSourceTable,db_cdm)
 S3method(dropTable,db_cdm)
 S3method(insertFromSource,db_cdm)
 S3method(insertTable,db_cdm)
+S3method(listSourceTables,db_cdm)
+S3method(readSourceTable,db_cdm)
 S3method(tbl,db_cdm)
 export("%>%")
 export(appendPermanent)
@@ -16,6 +20,8 @@ export(assertWriteSchema)
 export(assert_tables)
 export(assert_write_schema)
 export(attrition)
+export(benchmarkCDMConnector)
+export(bind)
 export(cdmCon)
 export(cdmDisconnect)
 export(cdmFlatten)
@@ -25,6 +31,7 @@ export(cdmName)
 export(cdmSample)
 export(cdmSubset)
 export(cdmSubsetCohort)
+export(cdmVersion)
 export(cdmWriteSchema)
 export(cdm_disconnect)
 export(cdm_flatten)
@@ -57,6 +64,7 @@ export(dbSource)
 export(dbms)
 export(downloadEunomiaData)
 export(download_eunomia_data)
+export(dropSourceTable)
 export(dropTable)
 export(eunomiaDir)
 export(eunomiaIsAvailable)
@@ -73,12 +81,14 @@ export(in_schema)
 export(insertTable)
 export(intersectCohorts)
 export(intersect_cohorts)
+export(listSourceTables)
 export(listTables)
 export(list_tables)
 export(newCohortTable)
 export(newGeneratedCohortSet)
 export(new_generated_cohort_set)
 export(readCohortSet)
+export(readSourceTable)
 export(read_cohort_set)
 export(recordCohortAttrition)
 export(record_cohort_attrition)
@@ -115,11 +125,17 @@ importFrom(generics,compile)
 importFrom(jsonlite,read_json)
 importFrom(methods,is)
 importFrom(omopgenerics,attrition)
+importFrom(omopgenerics,bind)
+importFrom(omopgenerics,cdmTableFromSource)
+importFrom(omopgenerics,cdmVersion)
 importFrom(omopgenerics,cohortCount)
+importFrom(omopgenerics,dropSourceTable)
 importFrom(omopgenerics,dropTable)
 importFrom(omopgenerics,insertFromSource)
 importFrom(omopgenerics,insertTable)
+importFrom(omopgenerics,listSourceTables)
 importFrom(omopgenerics,newCohortTable)
+importFrom(omopgenerics,readSourceTable)
 importFrom(omopgenerics,settings)
 importFrom(purrr,"%||%")
 importFrom(rlang,"!!")

diff --git a/NEWS.md b/NEWS.md
@@ -1,10 +1,12 @@
-# CDMConnector 1.3.2
+# CDMConnector 1.4.0
 
 - fix issue on cran server
+- new omopgenerics methods
 
 # CDMConnector 1.3.1
 
-- bugfix
+- Bug fix for collapse cohort
+- dbplyr 2.5.0 fixes
 
 # CDMConnector 1.3.0
 
@@ -20,7 +22,7 @@
 
 # CDMConnector 1.1.4
 
--   Fix recursive edge cases with cdm reference issue. 
+- Fix recursive edge cases with cdm reference issue. 
 
 # CDMConnector 1.1.3
 

diff --git a/R/Eunomia.R b/R/Eunomia.R
@@ -1,11 +1,27 @@
+# Copyright 2024 DARWIN EU®
+#
+# This file is part of CDMConnector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 #' Download Eunomia data files
 #'
 #' Download the Eunomia data files from https://github.com/darwin-eu/EunomiaDatasets
 #'
 #' @param dataset_name,datasetName The data set name as found on https://github.com/darwin-eu/EunomiaDatasets. The
 #'  data set name corresponds to the folder with the data set ZIP files
 #' @param cdm_version,cdmVersion The OMOP CDM version. This version will appear in the suffix of the data file,
-#'  for example: synpuf_5.3.zip. Default: '5.3'
+#'  for example: synpuf_5.3.zip. Must be '5.3' (default) or '5.4'.
 #' @param path_to_data,pathToData    The path where the Eunomia data is stored on the file system., By default the
 #'  value of the environment variable "EUNOMIA_DATA_FOLDER" is used.
 #' @param overwrite Control whether the existing archive file will be overwritten should it already exist.
@@ -25,10 +41,7 @@ downloadEunomiaData <- function(datasetName = "GiBleed",
                                 overwrite = FALSE) {
 
   checkmate::assertChoice(datasetName, choices = exampleDatasets())
-
-  if (cdmVersion != "5.3") {
-    rlang::abort("Only CDM v5.3 is supported currently!")
-  }
+  checkmate::assertChoice(cdmVersion, c("5.3", "5.4"))
 
   if (is.null(pathToData) || is.na(pathToData) || pathToData == "") {
     stop("The pathToData argument must be specified. Consider setting the EUNOMIA_DATA_FOLDER environment variable, for example in the .Renviron file.")
@@ -154,7 +167,7 @@ download_eunomia_data <- function(dataset_name = "GiBleed",
 #' "synthea-veterans-10k",
 #' "synthea-weight_loss-10k"
 #'
-#' @param cdmVersion,cdm_version The OMOP CDM version. Currently only "5.3" is supported.
+#' @param cdmVersion,cdm_version The OMOP CDM version. Must be "5.3" or "5.4".
 #' @param databaseFile,database_file The full path to the new copy of the example CDM dataset.
 #'
 #' @return The file path to the new Eunomia dataset copy
@@ -171,7 +184,7 @@ eunomiaDir <- function(datasetName = "GiBleed",
   if (Sys.getenv("EUNOMIA_DATA_FOLDER") == "") {
     rlang::abort("Set the `EUNOMIA_DATA_FOLDER` environment variable in your .Renviron file.")
   }
-  checkmate::assertChoice(cdmVersion, c("5.3"))
+  checkmate::assertChoice(cdmVersion, c("5.3", "5.4"))
   rlang::check_installed("duckdb")
 
   checkmate::assertChoice(datasetName, choices = exampleDatasets())