internal/frontend: move urlinfo to its own package

These are also being moved so that the code in fetch.go can be moved to
a new package without depending on internal/frontend.

A couple of functions that were only used by details.go are moved to
that file.

For #61399

Change-Id: Ic299069ea0b3aaeb80dbbf7f1ed4fedf7d1787df
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/518816
Reviewed-by: Robert Findley <rfindley@google.com>
kokoro-CI: kokoro <noreply+kokoro@google.com>
Run-TryBot: Michael Matloob <matloob@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>

internal/frontend/404.go

@@ -23,6 +23,7 @@ import (
 	"golang.org/x/pkgsite/internal/experiment"
 	"golang.org/x/pkgsite/internal/frontend/page"
 	"golang.org/x/pkgsite/internal/frontend/serrors"
+	"golang.org/x/pkgsite/internal/frontend/urlinfo"
 	"golang.org/x/pkgsite/internal/log"
 	"golang.org/x/pkgsite/internal/stdlib"
 	"golang.org/x/pkgsite/internal/version"
@@ -187,7 +188,7 @@ func githubPathRedirect(fullPath string) string {
 // pathNotFoundError returns a page with an option on how to
 // add a package or module to the site.
 func pathNotFoundError(ctx context.Context, fullPath, requestedVersion string) error {
-	if !isSupportedVersion(fullPath, requestedVersion) {
+	if !urlinfo.IsSupportedVersion(fullPath, requestedVersion) {
 		return invalidVersionError(fullPath, requestedVersion)
 	}
 	if stdlib.Contains(fullPath) {

internal/frontend/details.go

@@ -12,6 +12,7 @@ import (
 
 	"golang.org/x/pkgsite/internal/frontend/page"
 	"golang.org/x/pkgsite/internal/frontend/serrors"
+	"golang.org/x/pkgsite/internal/frontend/urlinfo"
 	mstats "golang.org/x/pkgsite/internal/middleware/stats"
 
 	"github.com/google/safehtml/template"
@@ -51,26 +52,26 @@ func (s *Server) serveDetails(w http.ResponseWriter, r *http.Request, ds interna
 		ctx = setExperimentsFromQueryParam(ctx, r)
 	}
 
-	urlInfo, err := extractURLPathInfo(r.URL.Path)
+	urlInfo, err := urlinfo.ExtractURLPathInfo(r.URL.Path)
 	if err != nil {
 		var epage *page.ErrorPage
-		if uerr := new(userError); errors.As(err, &uerr) {
-			epage = &page.ErrorPage{MessageData: uerr.userMessage}
+		if uerr := new(urlinfo.UserError); errors.As(err, &uerr) {
+			epage = &page.ErrorPage{MessageData: uerr.UserMessage}
 		}
 		return &serrors.ServerError{
 			Status: http.StatusBadRequest,
 			Err:    err,
 			Epage:  epage,
 		}
 	}
-	if !isSupportedVersion(urlInfo.fullPath, urlInfo.requestedVersion) {
-		return invalidVersionError(urlInfo.fullPath, urlInfo.requestedVersion)
+	if !urlinfo.IsSupportedVersion(urlInfo.FullPath, urlInfo.RequestedVersion) {
+		return invalidVersionError(urlInfo.FullPath, urlInfo.RequestedVersion)
 	}
-	if urlPath := stdlibRedirectURL(urlInfo.fullPath); urlPath != "" {
+	if urlPath := stdlibRedirectURL(urlInfo.FullPath); urlPath != "" {
 		http.Redirect(w, r, urlPath, http.StatusMovedPermanently)
 		return
 	}
-	if err := checkExcluded(ctx, ds, urlInfo.fullPath); err != nil {
+	if err := checkExcluded(ctx, ds, urlInfo.FullPath); err != nil {
 		return err
 	}
 	return s.serveUnitPage(ctx, w, r, ds, urlInfo)
@@ -140,3 +141,19 @@ func recordVersionTypeMetric(ctx context.Context, requestedVersion string) {
 		tag.Upsert(keyVersionType, v),
 	}, versionTypeResults.M(1))
 }
+
+func checkExcluded(ctx context.Context, ds internal.DataSource, fullPath string) error {
+	db, ok := ds.(internal.PostgresDB)
+	if !ok {
+		return nil
+	}
+	excluded, err := db.IsExcluded(ctx, fullPath)
+	if err != nil {
+		return err
+	}
+	if excluded {
+		// Return NotFound; don't let the user know that the package was excluded.
+		return &serrors.ServerError{Status: http.StatusNotFound}
+	}
+	return nil
+}

internal/frontend/experiments.go

@@ -0,0 +1,54 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package frontend
+
+import (
+	"context"
+	"net/http"
+	"strings"
+
+	"golang.org/x/pkgsite/internal/experiment"
+	"golang.org/x/pkgsite/internal/log"
+)
+
+func setExperimentsFromQueryParam(ctx context.Context, r *http.Request) context.Context {
+	if err := r.ParseForm(); err != nil {
+		log.Errorf(ctx, "ParseForm: %v", err)
+		return ctx
+	}
+	return newContextFromExps(ctx, r.Form["exp"])
+}
+
+// newContextFromExps adds and removes experiments from the context's experiment
+// set, creates a new set with the changes, and returns a context with the new
+// set. Each string in expMods can be either an experiment name, which means
+// that the experiment should be added, or "!" followed by an experiment name,
+// meaning that it should be removed.
+func newContextFromExps(ctx context.Context, expMods []string) context.Context {
+	var (
+		exps   []string
+		remove = map[string]bool{}
+	)
+	set := experiment.FromContext(ctx)
+	for _, exp := range expMods {
+		if strings.HasPrefix(exp, "!") {
+			exp = exp[1:]
+			if set.IsActive(exp) {
+				remove[exp] = true
+			}
+		} else if !set.IsActive(exp) {
+			exps = append(exps, exp)
+		}
+	}
+	if len(exps) == 0 && len(remove) == 0 {
+		return ctx
+	}
+	for _, a := range set.Active() {
+		if !remove[a] {
+			exps = append(exps, a)
+		}
+	}
+	return experiment.NewContext(ctx, exps...)
+}

internal/frontend/experments_test.go

@@ -0,0 +1,42 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package frontend
+
+import (
+	"context"
+	"sort"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"golang.org/x/pkgsite/internal/experiment"
+)
+
+func TestNewContextFromExps(t *testing.T) {
+	for _, test := range []struct {
+		mods []string
+		want []string
+	}{
+		{
+			mods: []string{"c", "a", "b"},
+			want: []string{"a", "b", "c"},
+		},
+		{
+			mods: []string{"d", "a"},
+			want: []string{"a", "b", "c", "d"},
+		},
+		{
+			mods: []string{"d", "!b", "!a", "c"},
+			want: []string{"c", "d"},
+		},
+	} {
+		ctx := experiment.NewContext(context.Background(), "a", "b", "c")
+		ctx = newContextFromExps(ctx, test.mods)
+		got := experiment.FromContext(ctx).Active()
+		sort.Strings(got)
+		if !cmp.Equal(got, test.want) {
+			t.Errorf("mods=%v:\ngot  %v\nwant %v", test.mods, got, test.want)
+		}
+	}
+}

internal/frontend/fetch.go

@@ -25,6 +25,7 @@ import (
 	"golang.org/x/pkgsite/internal/experiment"
 	"golang.org/x/pkgsite/internal/fetch"
 	"golang.org/x/pkgsite/internal/frontend/serrors"
+	"golang.org/x/pkgsite/internal/frontend/urlinfo"
 	"golang.org/x/pkgsite/internal/log"
 	"golang.org/x/pkgsite/internal/proxy"
 	"golang.org/x/pkgsite/internal/queue"
@@ -39,7 +40,7 @@ var (
 	// this module version in version_map.
 	errModuleDoesNotExist = errors.New("module does not exist")
 	// errPathDoesNotExistInModule indicates that a module for the path prefix
-	// exists, but within that module version, this fullPath could not be found.
+	// exists, but within that module version, this FullPath could not be found.
 	errPathDoesNotExistInModule = errors.New("path does not exist in module")
 	fetchTimeout                = 30 * time.Second
 	pollEvery                   = 1 * time.Second
@@ -103,11 +104,11 @@ func (s *Server) serveFetch(w http.ResponseWriter, r *http.Request, ds internal.
 		return &serrors.ServerError{Status: http.StatusNotFound}
 	}
 
-	urlInfo, err := extractURLPathInfo(strings.TrimPrefix(r.URL.Path, "/fetch"))
+	urlInfo, err := urlinfo.ExtractURLPathInfo(strings.TrimPrefix(r.URL.Path, "/fetch"))
 	if err != nil {
 		return &serrors.ServerError{Status: http.StatusBadRequest}
 	}
-	status, responseText := s.fetchAndPoll(r.Context(), ds, urlInfo.modulePath, urlInfo.fullPath, urlInfo.requestedVersion)
+	status, responseText := s.fetchAndPoll(r.Context(), ds, urlInfo.ModulePath, urlInfo.FullPath, urlInfo.RequestedVersion)
 	if status != http.StatusOK {
 		return &serrors.ServerError{Status: status, ResponseText: responseText}
 	}
@@ -134,14 +135,14 @@ func (s *Server) fetchAndPoll(ctx context.Context, ds internal.DataSource, modul
 		recordFrontendFetchMetric(ctx, status, time.Since(start))
 	}()
 
-	if !isSupportedVersion(fullPath, requestedVersion) {
+	if !urlinfo.IsSupportedVersion(fullPath, requestedVersion) {
 		return http.StatusBadRequest, http.StatusText(http.StatusBadRequest)
 	}
 	if !experiment.IsActive(ctx, internal.ExperimentEnableStdFrontendFetch) && stdlib.Contains(fullPath) {
 		return http.StatusBadRequest, http.StatusText(http.StatusBadRequest)
 	}
 
-	// Generate all possible module paths for the fullPath.
+	// Generate all possible module paths for the FullPath.
 	db := ds.(internal.PostgresDB)
 	modulePaths, err := modulePathsToFetch(ctx, db, fullPath, modulePath)
 	if err != nil {
@@ -165,7 +166,7 @@ func (s *Server) fetchAndPoll(ctx context.Context, ds internal.DataSource, modul
 }
 
 // checkPossibleModulePaths checks all modulePaths at the requestedVersion, to see
-// if the fullPath exists. For each module path, it first checks version_map to
+// if the FullPath exists. For each module path, it first checks version_map to
 // see if we already attempted to fetch the module. If not, and shouldQueue is
 // true, it will enqueue the module to the frontend task queue to be fetched.
 // checkPossibleModulePaths will then poll the database for each module path,
@@ -391,7 +392,7 @@ func checkForPath(ctx context.Context, db internal.PostgresDB,
 	vm, err := db.GetVersionMap(ctx, modulePath, requestedVersion)
 	if err != nil {
 		// If an error is returned, there are two possibilities:
-		// (1) A row for this modulePath and version does not exist.
+		// (1) A row for this ModulePath and version does not exist.
 		// This means that the fetch request is not done yet, so return
 		// statusNotFoundInVersionMap so the fetchHandler will call checkForPath
 		// again in a few seconds.
@@ -520,10 +521,10 @@ func candidateModulePaths(fullPath string) (_ []string, err error) {
 	if fullPath == stdlib.ModulePath {
 		return []string{stdlib.ModulePath}, nil
 	}
-	if !isValidPath(fullPath) {
+	if !urlinfo.IsValidPath(fullPath) {
 		return nil, &serrors.ServerError{
 			Status: http.StatusBadRequest,
-			Err:    fmt.Errorf("isValidPath(%q): false", fullPath),
+			Err:    fmt.Errorf("urlinfo.IsValidPath(%q): false", fullPath),
 		}
 	}
 	paths := internal.CandidateModulePaths(fullPath)

internal/frontend/server.go

@@ -28,6 +28,7 @@ import (
 	"golang.org/x/pkgsite/internal/experiment"
 	pagepkg "golang.org/x/pkgsite/internal/frontend/page"
 	"golang.org/x/pkgsite/internal/frontend/serrors"
+	"golang.org/x/pkgsite/internal/frontend/urlinfo"
 	"golang.org/x/pkgsite/internal/godoc/dochtml"
 	"golang.org/x/pkgsite/internal/licenses"
 	"golang.org/x/pkgsite/internal/log"
@@ -329,12 +330,12 @@ func detailsTTLForPath(ctx context.Context, urlPath, tab string) time.Duration {
 	if urlPath == "/" {
 		return defaultTTL
 	}
-	info, err := parseDetailsURLPath(urlPath)
+	info, err := urlinfo.ParseDetailsURLPath(urlPath)
 	if err != nil {
 		log.Errorf(ctx, "falling back to default TTL: %v", err)
 		return defaultTTL
 	}
-	if info.requestedVersion == version.Latest {
+	if info.RequestedVersion == version.Latest {
 		return shortTTL
 	}
 	if tab == "importedby" || tab == "versions" {

internal/frontend/unit.go

@@ -19,6 +19,7 @@ import (
 	"golang.org/x/pkgsite/internal/derrors"
 	"golang.org/x/pkgsite/internal/frontend/page"
 	"golang.org/x/pkgsite/internal/frontend/serrors"
+	"golang.org/x/pkgsite/internal/frontend/urlinfo"
 	"golang.org/x/pkgsite/internal/log"
 	"golang.org/x/pkgsite/internal/middleware/stats"
 	"golang.org/x/pkgsite/internal/stdlib"
@@ -106,7 +107,7 @@ type UnitPage struct {
 
 // serveUnitPage serves a unit page for a path.
 func (s *Server) serveUnitPage(ctx context.Context, w http.ResponseWriter, r *http.Request,
-	ds internal.DataSource, info *urlPathInfo) (err error) {
+	ds internal.DataSource, info *urlinfo.URLPathInfo) (err error) {
 	defer derrors.Wrap(&err, "serveUnitPage(ctx, w, r, ds, %v)", info)
 	defer stats.Elapsed(ctx, "serveUnitPage")()
 
@@ -121,12 +122,12 @@ func (s *Server) serveUnitPage(ctx context.Context, w http.ResponseWriter, r *ht
 		return nil
 	}
 
-	um, err := ds.GetUnitMeta(ctx, info.fullPath, info.modulePath, info.requestedVersion)
+	um, err := ds.GetUnitMeta(ctx, info.FullPath, info.ModulePath, info.RequestedVersion)
 	if err != nil {
 		if !errors.Is(err, derrors.NotFound) {
 			return err
 		}
-		return s.servePathNotFoundPage(w, r, ds, info.fullPath, info.modulePath, info.requestedVersion)
+		return s.servePathNotFoundPage(w, r, ds, info.FullPath, info.ModulePath, info.RequestedVersion)
 	}
 
 	makeDepsDevURL := depsDevURLGenerator(ctx, um)
@@ -137,16 +138,16 @@ func (s *Server) serveUnitPage(ctx context.Context, w http.ResponseWriter, r *ht
 	// It's also okay to provide just one (e.g. GOOS=windows), which will select
 	// the first doc with that value, ignoring the other one.
 	bc := internal.BuildContext{GOOS: r.FormValue("GOOS"), GOARCH: r.FormValue("GOARCH")}
-	d, err := fetchDetailsForUnit(ctx, r, tab, ds, um, info.requestedVersion, bc, s.vulnClient)
+	d, err := fetchDetailsForUnit(ctx, r, tab, ds, um, info.RequestedVersion, bc, s.vulnClient)
 	if err != nil {
 		return err
 	}
 	if s.shouldServeJSON(r) {
 		return s.serveJSONPage(w, r, d)
 	}
 
-	recordVersionTypeMetric(ctx, info.requestedVersion)
-	if _, ok := internal.DefaultBranches[info.requestedVersion]; ok {
+	recordVersionTypeMetric(ctx, info.RequestedVersion)
+	if _, ok := internal.DefaultBranches[info.RequestedVersion]; ok {
 		// Since path@master is a moving target, we don't want it to be stale.
 		// As a result, we enqueue every request of path@master to the frontend
 		// task queue, which will initiate a fetch request depending on the
@@ -160,17 +161,17 @@ func (s *Server) serveUnitPage(ctx context.Context, w http.ResponseWriter, r *ht
 				Err:    err,
 				Epage: &page.ErrorPage{
 					MessageData: fmt.Sprintf(`Default branches like "@%s" are not supported. Omit to get the current version.`,
-						info.requestedVersion),
+						info.RequestedVersion),
 				},
 			}
 		}
 		go func() {
 			ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
 			defer cancel()
-			log.Infof(ctx, "serveUnitPage: Scheduling %q@%q to be fetched", um.ModulePath, info.requestedVersion)
-			if _, err := s.queue.ScheduleFetch(ctx, um.ModulePath, info.requestedVersion, nil); err != nil {
+			log.Infof(ctx, "serveUnitPage: Scheduling %q@%q to be fetched", um.ModulePath, info.RequestedVersion)
+			if _, err := s.queue.ScheduleFetch(ctx, um.ModulePath, info.RequestedVersion, nil); err != nil {
 				log.Errorf(ctx, "serveUnitPage(%q): scheduling fetch for %q@%q: %v",
-					r.URL.Path, um.ModulePath, info.requestedVersion, err)
+					r.URL.Path, um.ModulePath, info.RequestedVersion, err)
 			}
 		}()
 	}
@@ -185,7 +186,7 @@ func (s *Server) serveUnitPage(ctx context.Context, w http.ResponseWriter, r *ht
 	// If we've already called GetUnitMeta for an unknown module path and the latest version, pass
 	// it to GetLatestInfo to avoid a redundant call.
 	var latestUnitMeta *internal.UnitMeta
-	if info.modulePath == internal.UnknownModulePath && info.requestedVersion == version.Latest {
+	if info.ModulePath == internal.UnknownModulePath && info.RequestedVersion == version.Latest {
 		latestUnitMeta = um
 	}
 	latestInfo := s.GetLatestInfo(ctx, um.Path, um.ModulePath, latestUnitMeta)
@@ -202,16 +203,16 @@ func (s *Server) serveUnitPage(ctx context.Context, w http.ResponseWriter, r *ht
 	if tabSettings.Name == "" {
 		basePage.UseResponsiveLayout = true
 	}
-	lv := linkVersion(um.ModulePath, info.requestedVersion, um.Version)
+	lv := linkVersion(um.ModulePath, info.RequestedVersion, um.Version)
 	page := UnitPage{
 		BasePage:              basePage,
 		Unit:                  um,
-		Breadcrumb:            displayBreadcrumb(um, info.requestedVersion),
+		Breadcrumb:            displayBreadcrumb(um, info.RequestedVersion),
 		Title:                 title,
 		SelectedTab:           tabSettings,
-		URLPath:               constructUnitURL(um.Path, um.ModulePath, info.requestedVersion),
-		CanonicalURLPath:      canonicalURLPath(um.Path, um.ModulePath, info.requestedVersion, um.Version),
-		DisplayVersion:        displayVersion(um.ModulePath, info.requestedVersion, um.Version),
+		URLPath:               constructUnitURL(um.Path, um.ModulePath, info.RequestedVersion),
+		CanonicalURLPath:      canonicalURLPath(um.Path, um.ModulePath, info.RequestedVersion, um.Version),
+		DisplayVersion:        displayVersion(um.ModulePath, info.RequestedVersion, um.Version),
 		LinkVersion:           lv,
 		LatestURL:             constructUnitURL(um.Path, um.ModulePath, version.Latest),
 		LatestMinorClass:      latestMinorClass(lv, latestInfo),