Merge pull request #576 from carterbourette/with-uk-csl

feat: Add environment configuration to disable UK CSL download

README.md

@@ -220,6 +220,7 @@ PONG
 | `DPL_DOWNLOAD_TEMPLATE` | HTTP address for downloading the DPL. | `https://www.bis.doc.gov/dpl/%s` |
 | `EU_CSL_DOWNLOAD_URL` | Use an alternate URL for downloading EU Consolidated Screening List | Subresource of `webgate.ec.europa.eu` |
 | `WITH_EU_SCREENING_LIST` | Download and parse the EU Consolidated Screening List | Default: `true` |
+| `WITH_UK_CSL_SANCTIONS_LIST` | Download and parse the UK CSL Sanctions List on startup. | Default: `true` |
 | `UK_CSL_DOWNLOAD_URL` | Use an alternate URL for downloading UK Consolidated Screening List | Subresource of `www.gov.uk` |
 | `UK_SANCTIONS_LIST_URL` | Use an alternate URL for downloading UK Sanctions List | Subresource of `www.gov.uk` |
 | `WITH_UK_SANCTIONS_LIST` | Download and parse the UK Sanctions List on startup. | Default: `false` |

cmd/server/download.go

@@ -286,25 +286,28 @@ func (s *searcher) refreshData(initialDir string) (*DownloadStats, error) {
 	}
 	dps := precomputeDPs(deniedPersons, s.pipe)
 
-	var euConsolidatedList []*csl.EUCSLRecord
+	var euCSLs []*Result[csl.EUCSLRecord]
 	withEUScreeningList := cmp.Or(os.Getenv("WITH_EU_SCREENING_LIST"), "true")
 	if strx.Yes(withEUScreeningList) {
-		euConsolidatedList, err = euCSLRecords(s.logger, initialDir)
+		euConsolidatedList, err := euCSLRecords(s.logger, initialDir)
 		if err != nil {
 			lastDataRefreshFailure.WithLabelValues("EUCSL").Set(float64(time.Now().Unix()))
 			stats.Errors = append(stats.Errors, fmt.Errorf("EUCSL: %v", err))
 		}
+		euCSLs = precomputeCSLEntities[csl.EUCSLRecord](euConsolidatedList, s.pipe)
 	}
-	euCSLs := precomputeCSLEntities[csl.EUCSLRecord](euConsolidatedList, s.pipe)
 
-	ukConsolidatedList, err := ukCSLRecords(s.logger, initialDir)
-	if err != nil {
-		lastDataRefreshFailure.WithLabelValues("UKCSL").Set(float64(time.Now().Unix()))
-		stats.Errors = append(stats.Errors, fmt.Errorf("UKCSL: %v", err))
+	var ukCSLs []*Result[csl.UKCSLRecord]
+	withUKCSLSanctionsList := cmp.Or(os.Getenv("WITH_UK_CSL_SANCTIONS_LIST"), "true")
+	if strx.Yes(withUKCSLSanctionsList) {
+		ukConsolidatedList, err := ukCSLRecords(s.logger, initialDir)
+		if err != nil {
+			lastDataRefreshFailure.WithLabelValues("UKCSL").Set(float64(time.Now().Unix()))
+			stats.Errors = append(stats.Errors, fmt.Errorf("UKCSL: %v", err))
+		}
+		ukCSLs = precomputeCSLEntities[csl.UKCSLRecord](ukConsolidatedList, s.pipe)
 	}
 
-	ukCSLs := precomputeCSLEntities[csl.UKCSLRecord](ukConsolidatedList, s.pipe)
-
 	var ukSLs []*Result[csl.UKSanctionsListRecord]
 	withUKSanctionsList := os.Getenv("WITH_UK_SANCTIONS_LIST")
 	if strings.ToLower(withUKSanctionsList) == "true" {