Test sources require an API key (#852)

* test providers require key

* remove redundant tests

* ignore failing(often) test

* added crtsh to ignore list

---------

Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>

.github/workflows/build-test.yml

@@ -28,19 +28,39 @@ jobs:
         working-directory: v2/
 
       - name: Run tests
+        env:
+          BEVIGIL_API_KEY: ${{secrets.BEVIGIL_API_KEY}}
+          BINARYEDGE_API_KEY: ${{secrets.BINARYEDGE_API_KEY}}
+          BUFFEROVER_API_KEY: ${{secrets.BUFFEROVER_API_KEY}}
+          C99_API_KEY: ${{secrets.C99_API_KEY}}
+          CENSYS_API_KEY: ${{secrets.CENSYS_API_KEY}}
+          CERTSPOTTER_API_KEY: ${{secrets.CERTSPOTTER_API_KEY}}
+          CHAOS_API_KEY: ${{secrets.CHAOS_API_KEY}}
+          CHINAZ_API_KEY: ${{secrets.CHINAZ_API_KEY}}
+          DNSDB_API_KEY: ${{secrets.DNSDB_API_KEY}}
+          DNSREPO_API_KEY: ${{secrets.DNSREPO_API_KEY}}
+          FOFA_API_KEY: ${{secrets.FOFA_API_KEY}}
+          FULLHUNT_API_KEY: ${{secrets.FULLHUNT_API_KEY}}
+          GITHUB_API_KEY: ${{secrets.GITHUB_API_KEY}}
+          HUNTER_API_KEY: ${{secrets.HUNTER_API_KEY}}
+          INTELX_API_KEY: ${{secrets.INTELX_API_KEY}}
+          LEAKIX_API_KEY: ${{secrets.LEAKIX_API_KEY}}
+          PASSIVETOTAL_API_KEY: ${{secrets.PASSIVETOTAL_API_KEY}}
+          QUAKE_API_KEY: ${{secrets.QUAKE_API_KEY}}
+          ROBTEX_API_KEY: ${{secrets.ROBTEX_API_KEY}}
+          SECURITYTRAILS_API_KEY: ${{secrets.SECURITYTRAILS_API_KEY}}
+          SHODAN_API_KEY: ${{secrets.SHODAN_API_KEY}}
+          THREATBOOK_API_KEY: ${{secrets.THREATBOOK_API_KEY}}
+          VIRUSTOTAL_API_KEY: ${{secrets.VIRUSTOTAL_API_KEY}}
+          WHOISXMLAPI_API_KEY: ${{secrets.WHOISXMLAPI_API_KEY}}
+          ZOOMEYE_API_KEY: ${{secrets.ZOOMEYE_API_KEY}}
+          ZOOMEYEAPI_API_KEY: ${{secrets.ZOOMEYEAPI_API_KEY}}
         uses: nick-invision/retry@v2
         with:
           timeout_seconds: 360
           max_attempts: 3
           command: cd v2; go test ./... -v
 
-      - name: Integration Tests
-        env:
-          GH_ACTION: true
-          DNSREPO_API_KEY: ${{secrets.DNSREPO_API}}
-        run: bash run.sh
-        working-directory: v2/cmd/integration-test/
-
       - name: Race Condition Tests
         run: go build -race ./...
         working-directory: v2/

v2/pkg/passive/sources_w_auth_test.go

@@ -0,0 +1,61 @@
+package passive
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/gologger/levels"
+	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
+)
+
+func TestSourcesWithKeys(t *testing.T) {
+	domain := "hackerone.com"
+	timeout := 60
+
+	gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug)
+
+	ctx := context.Background()
+	session, err := subscraping.NewSession(domain, "", 0, timeout)
+	assert.Nil(t, err)
+
+	var expected = subscraping.Result{Type: subscraping.Subdomain, Value: domain, Error: nil}
+
+	for _, source := range AllSources {
+		if !source.NeedsKey() {
+			continue
+		}
+
+		apiKey := os.Getenv(fmt.Sprintf("%s_API_KEY", strings.ToUpper(source.Name())))
+		if apiKey == "" {
+			fmt.Printf("Skipping %s as no API key is provided\n", source.Name())
+			continue
+		}
+		source.AddApiKeys([]string{apiKey})
+
+		t.Run(source.Name(), func(t *testing.T) {
+			var results []subscraping.Result
+
+			for result := range source.Run(ctx, domain, session) {
+				results = append(results, result)
+
+				assert.Equal(t, source.Name(), result.Source, "wrong source name")
+
+				if result.Type != subscraping.Error {
+					assert.True(t, strings.HasSuffix(strings.ToLower(result.Value), strings.ToLower(expected.Value)),
+						fmt.Sprintf("result(%s) is not subdomain of %s", strings.ToLower(result.Value), expected.Value))
+				} else {
+					assert.Equal(t, reflect.TypeOf(expected.Error), reflect.TypeOf(result.Error), fmt.Sprintf("%s: %s", result.Source, result.Error))
+				}
+			}
+
+			assert.GreaterOrEqual(t, len(results), 1, fmt.Sprintf("No result found for %s", source.Name()))
+		})
+	}
+}

v2/pkg/passive/sources_wo_auth_test.go

@@ -8,13 +8,20 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"golang.org/x/exp/slices"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/gologger/levels"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
 )
 
 func TestSourcesWithoutKeys(t *testing.T) {
+	ignoredSources := []string{
+		"commoncrawl", // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ
+		"riddler",     // Fails with 403: There might be too much traffic or a configuration error
+		"crtsh",     // Fails in GH Action (possibly IP-based ban) causing a timeout.
+	}
+
 	domain := "hackerone.com"
 	timeout := 60
 
@@ -27,14 +34,10 @@ func TestSourcesWithoutKeys(t *testing.T) {
 	var expected = subscraping.Result{Type: subscraping.Subdomain, Value: domain, Error: nil}
 
 	for _, source := range AllSources {
-		if source.NeedsKey() {
+		if source.NeedsKey() || slices.Contains(ignoredSources, source.Name()) {
 			continue
 		}
 
-		if source.Name() == "commoncrawl" {
-			continue // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ
-		}
-
 		t.Run(source.Name(), func(t *testing.T) {
 			var results []subscraping.Result