PROBLEM-3017 add skip fields (#461)

* PROBLEM-3017 add skip fields * PROBLEM-3017 add skip fields * PROBLEM-3017 add skip fields * Fix Go formatting * PROBLEM-3017 add skip fields * PROBLEM-3017 add skip fields * Fix Go formatting * PROBLEM-3017 add skip fields --------- Co-authored-by: MickStanciu <1661195+MickStanciu@users.noreply.github.com>
SafetyCulture · Mar 25, 2024 · 2dfaafe · 2dfaafe
1 parent 680650b
commit 2dfaafe
Show file tree

Hide file tree

Showing 8 changed files with 347 additions and 3 deletions.
diff --git a/pkg/api/configuration_manager.go b/pkg/api/configuration_manager.go
@@ -54,6 +54,9 @@ type ExporterConfiguration struct {
 			SkipIds               []string `yaml:"skip_ids"`
 			WebReportLink         string   `yaml:"web_report_link"`
 		} `yaml:"inspection"`
+		InspectionItems struct {
+			SkipFields []string `yaml:"skip_fields"`
+		} `yaml:"inspection_items"`
 		Issue struct {
 			Limit int `yaml:"limit"`
 		} `yaml:"issue"`
@@ -283,6 +286,7 @@ func BuildConfigurationWithDefaults() *ExporterConfiguration {
 	cfg.Export.Inspection.Limit = 100
 	cfg.Export.Inspection.SkipIds = []string{}
 	cfg.Export.Inspection.WebReportLink = "private"
+	cfg.Export.InspectionItems.SkipFields = []string{}
 	cfg.Export.Issue.Limit = 100
 	cfg.Export.Path = exportLocation
 	cfg.Export.MediaPath = mediaPathLocation
@@ -353,6 +357,7 @@ func (ec *ExporterConfiguration) ToExporterConfig() *feed.ExporterFeedCfg {
 		ExportInspectionCompleted:             ec.Export.Inspection.Completed,
 		ExportInspectionIncludedInactiveItems: ec.Export.Inspection.IncludedInactiveItems,
 		ExportInspectionWebReportLink:         ec.Export.Inspection.WebReportLink,
+		ExportInspectionItemsSkipFields:       ec.Export.InspectionItems.SkipFields,
 		ExportScheduleResumeDownload:          ec.Export.Schedule.ResumeDownload,
 		ExportIncremental:                     ec.Export.Incremental,
 		ExportInspectionLimit:                 ec.Export.Inspection.Limit,

diff --git a/pkg/api/configuration_manager_test.go b/pkg/api/configuration_manager_test.go
@@ -180,6 +180,7 @@ func TestConfigurationManager_SaveConfiguration(t *testing.T) {
 	assert.EqualValues(t, []string{"PDF"}, newCm.Configuration.Report.Format)
 	assert.EqualValues(t, 15, newCm.Configuration.Report.RetryTimeout)
 	assert.EqualValues(t, false, newCm.Configuration.Export.Schedule.ResumeDownload)
+	assert.Empty(t, cm.Configuration.Export.InspectionItems.SkipFields)
 
 	_ = os.Remove("fake_file.yaml")
 }
@@ -207,6 +208,9 @@ func TestMapViperConfigToConfigurationOptions_CustomValues(t *testing.T) {
 	require.Nil(t, err)
 	require.NotNil(t, cm)
 	assert.EqualValues(t, "COMPLETION_STATUS_ALL", cm.Configuration.Export.Course.Progress.CompletionStatus)
+
+	expected := []string{"field_1", "field_2"}
+	assert.EqualValues(t, expected, cm.Configuration.Export.InspectionItems.SkipFields)
 }
 
 func TestNewConfigurationManagerFromFile_WhenZeroLengthFile(t *testing.T) {
@@ -235,6 +239,7 @@ func TestNewConfigurationManagerFromFile_WhenZeroLengthFile(t *testing.T) {
 	assert.EqualValues(t, "export", cm.Configuration.Export.Path)
 	assert.EqualValues(t, "export/media", cm.Configuration.Export.MediaPath)
 	assert.EqualValues(t, "0001-01-01", cm.Configuration.Export.ModifiedAfter.UTC().Format(util.TimeISO8601))
+	assert.Empty(t, cm.Configuration.Export.InspectionItems.SkipFields)
 }
 
 func TestNewConfigurationManagerFromFile_WhenFileIsCorrupt(t *testing.T) {

diff --git a/pkg/api/feed_inspection_item_test.go b/pkg/api/feed_inspection_item_test.go
@@ -8,6 +8,7 @@ import (
 
 	"github.com/SafetyCulture/safetyculture-exporter/pkg/internal/feed"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"gopkg.in/h2non/gock.v1"
 )
 
@@ -16,6 +17,9 @@ func TestInspectionItemFeedExport_should_export_rows_to_sql_db(t *testing.T) {
 	assert.NoError(t, err)
 
 	apiClient := GetTestClient()
+	defer resetMocks(apiClient.HTTPClient())
+	gock.InterceptClient(apiClient.HTTPClient())
+
 	initMockFeedsSet1(apiClient.HTTPClient())
 
 	inspectionItemFeed := feed.InspectionItemFeed{
@@ -37,6 +41,42 @@ func TestInspectionItemFeedExport_should_export_rows_to_sql_db(t *testing.T) {
 	assert.Equal(t, 5, len(rows))
 }
 
+func TestInspectionItemFeedExport_should_ignore_media_hyper(t *testing.T) {
+	exporter, err := getInmemorySQLExporter("")
+	assert.NoError(t, err)
+
+	apiClient := GetTestClient()
+	defer resetMocks(apiClient.HTTPClient())
+	gock.InterceptClient(apiClient.HTTPClient())
+
+	initMockInspectionItemsFeed(apiClient.HTTPClient())
+
+	inspectionItemFeed := feed.InspectionItemFeed{
+		SkipIDs:       []string{},
+		ModifiedAfter: time.Now(),
+		TemplateIDs:   []string{},
+		Archived:      "both",
+		Completed:     "both",
+		Incremental:   true,
+		SkipFields:    []string{"some_unknown_field", "media_hypertext_reference", "some_unknown_field"},
+	}
+
+	err = inspectionItemFeed.Export(context.Background(), apiClient, exporter, "")
+	assert.NoError(t, err)
+
+	var rows []feed.InspectionItem
+	resp := exporter.DB.Table("inspection_items").Scan(&rows)
+	assert.Nil(t, resp.Error)
+
+	require.Equal(t, 7, len(rows))
+	for _, row := range rows {
+		if row.ID == "audit_493215ad4986455688bfebb8de93197e_5aa3170b-b1a0-4a99-8425-5417cea967b7" {
+			assert.Empty(t, row.MediaHypertextReference)
+		}
+	}
+
+}
+
 func TestInspectionItemFeedExportWithMedia(t *testing.T) {
 	dir, err := os.MkdirTemp("", "export")
 	assert.NoError(t, err)
@@ -52,7 +92,9 @@ func TestInspectionItemFeedExportWithMedia(t *testing.T) {
 	req.SetHeader("Content-Type", "image/test-content")
 
 	apiClient := GetTestClient()
+	defer resetMocks(apiClient.HTTPClient())
 	gock.InterceptClient(apiClient.HTTPClient())
+
 	initMockFeedsSet1(apiClient.HTTPClient())
 
 	inspectionItemFeed := feed.InspectionItemFeed{

diff --git a/pkg/api/fixtures/test_custom_values.yaml b/pkg/api/fixtures/test_custom_values.yaml
@@ -28,6 +28,8 @@ export:
     limit: 100
     skip_ids: ["ID1", "ID2"]
     web_report_link: private
+  inspection_items:
+    skip_fields: ["field_1", "field_2"]
   media: false
   media_path: ./export/media/
   modified_after: ""

diff --git a/pkg/api/mock_feeds_test.go b/pkg/api/mock_feeds_test.go
@@ -389,6 +389,15 @@ func initMockIssuesFeed(httpClient *http.Client) {
 		File("mocks/set_1/feed_issues_2.json")
 }
 
+func initMockInspectionItemsFeed(httpClient *http.Client) {
+	gock.InterceptClient(httpClient)
+
+	gock.New("http://localhost:9999").
+		Get("/feed/inspection_items").
+		Reply(200).
+		File("mocks/items_ignore_fields/feed_inspection_items.json")
+}
+
 func resetMocks(httpClient *http.Client) {
 	gock.Off()
 	gock.Clean()