tar_map over patterns and targets

[CorradoLanera]

Help

• I understand and agree to https://books.ropensci.org/targets/help.html.

Description

Doing analyses based both on multiple DBs to import, and considering multiple (identically analyzed) scenarios, I have found my list of targets extremely redundant, copy-pasted error-prone, and not that easy to quickly find steps (all because of numerous targets created).

I started investigating tar_map to extract common patterns in targets' generation. After some research in the documentation (and here, e.g., #162, #150) and some experiments, I have found a working solution.

The aim of this discussion is, from one side to report my solution to others, from the other side ask if my approach is the correct/suggested/best one.

Thank you!
Corrado.

The idea

• import a few targets using map (think of importing multiple files in data frames (e.g.: a, b) from a few distinct folders (e.g.: db_1, db_2, db_3))
• use one of those targets to find some required insight (e.g., pids, to retain in the others)
• use that insights to manipulate all the other targets (e.g.: to filter each of the imported data frame)
• merge everything together (e.g.: in a final db)

next we would apply the same pattern to produce the various analyses, but the technique and approach should remain the same.

Standard and tar_map approaches

In both cases the network remain the same:

library(targets)


# Standard approach -----------------------------------------------

tar_dir({ # tar_dir() runs code from a temp dir for CRAN.
  tar_script({
    library(tarchetypes)

    db_1 <- list(
      a = tibble::tibble(pid = 1:2, a = 1:2, b = 3:4),
      b = tibble::tibble(pid = 3:4, a = 5:6, b = 7:8)
    )
    db_2 <- list(
      a = tibble::tibble(
        pid = c(1, 3), c = c("a", "b"), d = c("c", "d")
      ),
      b = tibble::tibble(
        pid = c(4, 7), c = c("e", "f"), d = c("g", "h")
      )
    )
    db_3 <- list(
      a = tibble::tibble(
        pid = c(2, 4), e = c("i", "j"), f = c("k", "l")
      ),
      b = tibble::tibble(
        pid = c(3, 6), e = c("m", "n"), f = c("o", "p")
      )
    )

    filter_db <- function(db, pids) {
      cat(str(pids))
      dplyr::filter(db, pid %in% unique(pids))
    }

    list(
      tar_target(id, letters[1:2]),

      tar_target(
        raw_db_1,
        db_1[[id]],
        pattern = map(id)
      ),
      tar_target(
        raw_db_2,
        db_2[[id]],
        pattern = map(id)
      ),
      tar_target(
        raw_db_3,
        db_3[[id]],
        pattern = map(id)
      ),

      tar_target(
        desired_pids,
        raw_db_1[["pid"]],
        map(raw_db_1)
      ),

      tar_target(
        filtered_raw_db_2,
        filter_db(raw_db_2, desired_pids)
      ),
      tar_target(
        filtered_raw_db_3,
        filter_db(raw_db_3, desired_pids)
      ),

      tar_target(
        db,
        raw_db_1 |>
          dplyr::left_join(filtered_raw_db_2) |>
          dplyr::left_join(filtered_raw_db_3)
      )
    )

  }, ask = FALSE)
  print(tar_manifest())
  print(tar_visnetwork(targets_only = TRUE, outdated = FALSE))
  tar_make()
  print(tar_read(raw_db_1))
  print(tar_read(filtered_raw_db_2))
  print(tar_read(db))
})
#> # A tibble: 8 × 3
#>   name              command                                              pattern
#>   <chr>             <chr>                                                <chr>  
#> 1 id                "letters[1:2]"                                       <NA>   
#> 2 raw_db_3          "db_3[[id]]"                                         map(id)
#> 3 raw_db_2          "db_2[[id]]"                                         map(id)
#> 4 raw_db_1          "db_1[[id]]"                                         map(id)
#> 5 desired_pids      "raw_db_1[[\"pid\"]]"                                map(ra…
#> 6 filtered_raw_db_2 "filter_db(raw_db_2, desired_pids)"                  <NA>   
#> 7 filtered_raw_db_3 "filter_db(raw_db_3, desired_pids)"                  <NA>   
#> 8 db                "dplyr::left_join(dplyr::left_join(raw_db_1, filter… <NA>   
#> ▶ start target id
#> ● built target id [0 seconds]
#> ▶ start branch raw_db_3_7bf64d1d
#> ● built branch raw_db_3_7bf64d1d [0 seconds]
#> ▶ start branch raw_db_3_069939b9
#> ● built branch raw_db_3_069939b9 [0 seconds]
#> ● built pattern raw_db_3
#> ▶ start branch raw_db_2_7bf64d1d
#> ● built branch raw_db_2_7bf64d1d [0 seconds]
#> ▶ start branch raw_db_2_069939b9
#> ● built branch raw_db_2_069939b9 [0 seconds]
#> ● built pattern raw_db_2
#> ▶ start branch raw_db_1_7bf64d1d
#> ● built branch raw_db_1_7bf64d1d [0 seconds]
#> ▶ start branch raw_db_1_069939b9
#> ● built branch raw_db_1_069939b9 [0 seconds]
#> ● built pattern raw_db_1
#> ▶ start branch desired_pids_871ca8f4
#> ● built branch desired_pids_871ca8f4 [0 seconds]
#> ▶ start branch desired_pids_91d808d0
#> ● built branch desired_pids_91d808d0 [0 seconds]
#> ● built pattern desired_pids
#>  Named int [1:4] 1 2 3 4
#>  - attr(*, "names")= chr [1:4] "desired_pids_871ca8f4_1" "desired_pids_871ca8f4_2" "desired_pids_91d808d0_1" "desired_pids_91d808d0_2"
#> ▶ start target filtered_raw_db_2
#> ● built target filtered_raw_db_2 [0.02 seconds]
#>  Named int [1:4] 1 2 3 4
#>  - attr(*, "names")= chr [1:4] "desired_pids_871ca8f4_1" "desired_pids_871ca8f4_2" "desired_pids_91d808d0_1" "desired_pids_91d808d0_2"
#> ▶ start target filtered_raw_db_3
#> ● built target filtered_raw_db_3 [0.01 seconds]
#> ▶ start target db
#> Joining with `by = join_by(pid)`
#> Joining with `by = join_by(pid)`
#> ● built target db [0.12 seconds]
#> ▶ end pipeline [0.53 seconds]
#> # A tibble: 4 × 3
#>     pid     a     b
#>   <int> <int> <int>
#> 1     1     1     3
#> 2     2     2     4
#> 3     3     5     7
#> 4     4     6     8
#> # A tibble: 3 × 3
#>     pid c     d    
#>   <dbl> <chr> <chr>
#> 1     1 a     c    
#> 2     3 b     d    
#> 3     4 e     g    
#> # A tibble: 4 × 7
#>     pid     a     b c     d     e     f    
#>   <dbl> <int> <int> <chr> <chr> <chr> <chr>
#> 1     1     1     3 a     c     <NA>  <NA> 
#> 2     2     2     4 <NA>  <NA>  i     k    
#> 3     3     5     7 b     d     m     o    
#> 4     4     6     8 e     g     j     l




# tar_map approach ------------------------------------------------

tar_dir({ # tar_dir() runs code from a temp dir for CRAN.
  tar_script({
    library(tarchetypes)

    db_1 <- list(
      a = tibble::tibble(pid = 1:2, a = 1:2, b = 3:4),
      b = tibble::tibble(pid = 3:4, a = 5:6, b = 7:8)
    )
    db_2 <- list(
      a = tibble::tibble(
        pid = c(1, 3), c = c("a", "b"), d = c("c", "d")
      ),
      b = tibble::tibble(
        pid = c(4, 7), c = c("e", "f"), d = c("g", "h")
      )
    )
    db_3 <- list(
      a = tibble::tibble(
        pid = c(2, 4), e = c("i", "j"), f = c("k", "l")
      ),
      b = tibble::tibble(
        pid = c(3, 6), e = c("m", "n"), f = c("o", "p")
      )
    )

    filter_db <- function(db, pids) {
      cat(str(pids))
      dplyr::filter(db, pid %in% unique(pids))
    }

    list(
      tar_target(id, letters[1:2]),

      tar_map(
        values = list(
          dbs = rlang::syms(c("db_1", "db_2", "db_3"))
        ),
        tar_target(raw, dbs[[id]], pattern = map(id))
      ),
      tar_target(
        desired_pids,
        raw_db_1[["pid"]],
        map(raw_db_1)
      ),

      tar_map(
        values = list(
          raw_dbs = rlang::syms(paste0("raw_db_", 2:3))
        ),
        tar_target(filtered, filter_db(raw_dbs, desired_pids))
      ),

      tar_target(
        db,
        raw_db_1 |>
          dplyr::left_join(filtered_raw_db_2) |>
          dplyr::left_join(filtered_raw_db_3)
      )
    )

  }, ask = FALSE)
  print(tar_manifest())
  print(tar_visnetwork(targets_only = TRUE, outdated = FALSE))
  tar_make()
  print(tar_read(raw_db_1))
  print(tar_read(filtered_raw_db_2))
  print(tar_read(db))
})
#> # A tibble: 8 × 3
#>   name              command                                              pattern
#>   <chr>             <chr>                                                <chr>  
#> 1 id                "letters[1:2]"                                       <NA>   
#> 2 raw_db_3          "db_3[[id]]"                                         map(id)
#> 3 raw_db_2          "db_2[[id]]"                                         map(id)
#> 4 raw_db_1          "db_1[[id]]"                                         map(id)
#> 5 desired_pids      "raw_db_1[[\"pid\"]]"                                map(ra…
#> 6 filtered_raw_db_2 "filter_db(raw_db_2, desired_pids)"                  <NA>   
#> 7 filtered_raw_db_3 "filter_db(raw_db_3, desired_pids)"                  <NA>   
#> 8 db                "dplyr::left_join(dplyr::left_join(raw_db_1, filter… <NA>   
#> ▶ start target id
#> ● built target id [0 seconds]
#> ▶ start branch raw_db_3_7bf64d1d
#> ● built branch raw_db_3_7bf64d1d [0 seconds]
#> ▶ start branch raw_db_3_069939b9
#> ● built branch raw_db_3_069939b9 [0 seconds]
#> ● built pattern raw_db_3
#> ▶ start branch raw_db_2_7bf64d1d
#> ● built branch raw_db_2_7bf64d1d [0 seconds]
#> ▶ start branch raw_db_2_069939b9
#> ● built branch raw_db_2_069939b9 [0 seconds]
#> ● built pattern raw_db_2
#> ▶ start branch raw_db_1_7bf64d1d
#> ● built branch raw_db_1_7bf64d1d [0 seconds]
#> ▶ start branch raw_db_1_069939b9
#> ● built branch raw_db_1_069939b9 [0 seconds]
#> ● built pattern raw_db_1
#> ▶ start branch desired_pids_871ca8f4
#> ● built branch desired_pids_871ca8f4 [0 seconds]
#> ▶ start branch desired_pids_91d808d0
#> ● built branch desired_pids_91d808d0 [0 seconds]
#>  Named int [1:4] 1 2 3 4
#>  - attr(*, "names")= chr [1:4] "desired_pids_871ca8f4_1" "desired_pids_871ca8f4_2" "desired_pids_91d808d0_1" "desired_pids_91d808d0_2"
#> ● built pattern desired_pids
#> ▶ start target filtered_raw_db_2
#> ● built target filtered_raw_db_2 [0.02 seconds]
#>  Named int [1:4] 1 2 3 4
#>  - attr(*, "names")= chr [1:4] "desired_pids_871ca8f4_1" "desired_pids_871ca8f4_2" "desired_pids_91d808d0_1" "desired_pids_91d808d0_2"
#> ▶ start target filtered_raw_db_3
#> ● built target filtered_raw_db_3 [0.02 seconds]
#> ▶ start target db
#> Joining with `by = join_by(pid)`
#> Joining with `by = join_by(pid)`
#> ● built target db [0.13 seconds]
#> ▶ end pipeline [0.56 seconds]
#> # A tibble: 4 × 3
#>     pid     a     b
#>   <int> <int> <int>
#> 1     1     1     3
#> 2     2     2     4
#> 3     3     5     7
#> 4     4     6     8
#> # A tibble: 3 × 3
#>     pid c     d    
#>   <dbl> <chr> <chr>
#> 1     1 a     c    
#> 2     3 b     d    
#> 3     4 e     g    
#> # A tibble: 4 × 7
#>     pid     a     b c     d     e     f    
#>   <dbl> <int> <int> <chr> <chr> <chr> <chr>
#> 1     1     1     3 a     c     <NA>  <NA> 
#> 2     2     2     4 <NA>  <NA>  i     k    
#> 3     3     5     7 b     d     m     o    
#> 4     4     6     8 e     g     j     l

^{Created on 2023-12-14 with reprex v2.0.2}

[wlandau]

At a glance, this looks like a nice way to structure your pipeline. This kind of dynamic-within-static branching actually comes up so much that targetypes has a special target factory for it: https://docs.ropensci.org/tarchetypes/reference/tar_map_rep.html. tar_map_rep() and tar_rep() are the branching-related target factories that I use the most often in my own work.

[CorradoLanera]

Wow, thank you! Didn't know about those yet! Just read the documentation, looking forward to apply and explore them 😊👍