From 9c305b2e001ce378d12d58b6e2495a593ba1ca53 Mon Sep 17 00:00:00 2001 From: Gby56 Date: Tue, 24 Sep 2024 18:33:54 +0200 Subject: [PATCH 1/4] fix: shodan source filter subdomains properly --- v2/pkg/subscraping/sources/shodan/shodan.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/v2/pkg/subscraping/sources/shodan/shodan.go b/v2/pkg/subscraping/sources/shodan/shodan.go index 688f07bee..316f563ac 100644 --- a/v2/pkg/subscraping/sources/shodan/shodan.go +++ b/v2/pkg/subscraping/sources/shodan/shodan.go @@ -4,6 +4,7 @@ package shodan import ( "context" "fmt" + "strings" "time" jsoniter "github.com/json-iterator/go" @@ -75,10 +76,13 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, data := range response.Subdomains { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", data, domain), + value := fmt.Sprintf("%s.%s", data, response.Domain) + if strings.HasSuffix(value, domain) { + results <- subscraping.Result{ + Source: s.Name(), Type: subscraping.Subdomain, Value: value, + } + s.results++ } - s.results++ } if !response.More { From 556df0266223933b41662d112c609391e0800f3c Mon Sep 17 00:00:00 2001 From: Gabriel Marquet Date: Thu, 26 Sep 2024 17:48:06 +0200 Subject: [PATCH 2/4] Update v2/pkg/subscraping/sources/shodan/shodan.go Co-authored-by: Dwi Siswanto --- v2/pkg/subscraping/sources/shodan/shodan.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/pkg/subscraping/sources/shodan/shodan.go b/v2/pkg/subscraping/sources/shodan/shodan.go index 316f563ac..f58119971 100644 --- a/v2/pkg/subscraping/sources/shodan/shodan.go +++ b/v2/pkg/subscraping/sources/shodan/shodan.go @@ -77,7 +77,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se for _, data := range response.Subdomains { value := fmt.Sprintf("%s.%s", data, response.Domain) - if strings.HasSuffix(value, domain) { + suffix := fmt.Sprintf(".%s", strings.TrimPrefix(domain, ".")) + if strings.HasSuffix(value, suffix) { results <- subscraping.Result{ Source: s.Name(), Type: subscraping.Subdomain, Value: value, } From 89a47f193eb9cf5584b991345fdda67860ac9e51 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Fri, 27 Sep 2024 20:47:12 +0700 Subject: [PATCH 3/4] Revert "Update v2/pkg/subscraping/sources/shodan/shodan.go" This reverts commit 556df0266223933b41662d112c609391e0800f3c. --- v2/pkg/subscraping/sources/shodan/shodan.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/v2/pkg/subscraping/sources/shodan/shodan.go b/v2/pkg/subscraping/sources/shodan/shodan.go index f58119971..316f563ac 100644 --- a/v2/pkg/subscraping/sources/shodan/shodan.go +++ b/v2/pkg/subscraping/sources/shodan/shodan.go @@ -77,8 +77,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se for _, data := range response.Subdomains { value := fmt.Sprintf("%s.%s", data, response.Domain) - suffix := fmt.Sprintf(".%s", strings.TrimPrefix(domain, ".")) - if strings.HasSuffix(value, suffix) { + if strings.HasSuffix(value, domain) { results <- subscraping.Result{ Source: s.Name(), Type: subscraping.Subdomain, Value: value, } From c3228fdc9fdce1cd1821d7539edaa167570f4c09 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Fri, 27 Sep 2024 20:49:12 +0700 Subject: [PATCH 4/4] feat(shodan): rm `strings.HasSuffix` filter Signed-off-by: Dwi Siswanto --- v2/pkg/subscraping/sources/shodan/shodan.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/v2/pkg/subscraping/sources/shodan/shodan.go b/v2/pkg/subscraping/sources/shodan/shodan.go index 316f563ac..8d4925d79 100644 --- a/v2/pkg/subscraping/sources/shodan/shodan.go +++ b/v2/pkg/subscraping/sources/shodan/shodan.go @@ -4,7 +4,6 @@ package shodan import ( "context" "fmt" - "strings" "time" jsoniter "github.com/json-iterator/go" @@ -77,12 +76,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se for _, data := range response.Subdomains { value := fmt.Sprintf("%s.%s", data, response.Domain) - if strings.HasSuffix(value, domain) { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: value, - } - s.results++ + results <- subscraping.Result{ + Source: s.Name(), Type: subscraping.Subdomain, Value: value, } + s.results++ } if !response.More {