fix: when using --urls, allow skipping root path

Fixes #163
harlan-zw · Mar 3, 2024 · dc2563f · dc2563f
1 parent 574bf87
commit dc2563f
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 12 deletions.
diff --git a/packages/cli/src/util.ts b/packages/cli/src/util.ts
@@ -7,28 +7,29 @@ import { handleError } from './errors'
 import type { CiOptions, CliOptions } from './types'
 
 export async function validateHost(resolvedConfig: ResolvedUserConfig) {
+  const site = resolvedConfig.site
   const logger = useLogger()
   // site will not be set from integrations yet
-  if (resolvedConfig.site) {
+  if (site) {
     // test HTTP response from site
-    logger.debug(`Testing Site \`${resolvedConfig.site}\` is valid.`)
-    const { valid, response, error, redirected, redirectUrl } = await fetchUrlRaw(resolvedConfig.site, resolvedConfig)
+    logger.debug(`Testing Site \`${site}\` is valid.`)
+    const { valid, response, error, redirected, redirectUrl } = await fetchUrlRaw(site, resolvedConfig)
     if (!valid) {
       // something is wrong with the site, bail
       if (response?.status)
-        logger.warn(`Request to site \`${resolvedConfig.site}\` returned an invalid http status code \`${response.status}\`. Please check the URL is valid.`)
+        logger.warn(`Request to site \`${site}\` returned an invalid http status code \`${response.status}\`. lease check the URL is valid and not blocking crawlers.`)
       else
-        logger.warn(`Request to site \`${resolvedConfig.site}\` threw an unhandled exception. Please check the URL is valid.`, error)
+        logger.warn(`Request to site \`${site}\` threw an unhandled exception. Please check the URL is valid and not blocking crawlers.`, error)
       logger.error('Site check failed. will attempt to proceed but may fail.')
     }
     else if (response) {
       // change the URL to the redirect one, make sure it's not to a file (i.e /index.php)
       if (redirected && redirectUrl && !redirectUrl.includes('.')) {
-        logger.success(`Request to site \`${resolvedConfig.site}\` redirected to \`${redirectUrl}\`, using that as the site.`)
+        logger.success(`Request to site \`${site}\` redirected to \`${redirectUrl}\`, using that as the site.`)
         resolvedConfig.site = normaliseHost(redirectUrl)
       }
       else {
-        logger.success(`Successfully connected to \`${resolvedConfig.site}\`. (Status: \`${response.status}\`).`)
+        logger.success(`Successfully connected to \`${site}\`. (Status: \`${response.status}\`).`)
       }
     }
   }
@@ -45,10 +46,12 @@ export function isValidUrl(s: string) {
 }
 
 export function validateOptions(resolvedOptions: UserConfig) {
-  if (!resolvedOptions.site)
+  if (!resolvedOptions.site && resolvedOptions.urls?.length)
+    resolvedOptions.site = resolvedOptions.urls[0]
+  if (!resolvedOptions.configFile && !resolvedOptions.site)
     return handleError('Please provide a site to scan with --site <url>.')
 
-  if (!isValidUrl(resolvedOptions.site))
+  if (resolvedOptions.site && !isValidUrl(resolvedOptions.site))
     return handleError('Please provide a valid site URL.')
 
   if (resolvedOptions?.ci?.reporter === 'lighthouseServer') {

diff --git a/packages/core/src/discovery/routes.ts b/packages/core/src/discovery/routes.ts
@@ -21,7 +21,7 @@ export const resolveReportableRoutes: () => Promise<NormalisedRoute[]> = async (
   const logger = useLogger()
   const { resolvedConfig, hooks, worker, routeDefinitions } = useUnlighthouse()
 
-  const urls = new Set<string>([resolvedConfig.site])
+  const urls = new Set<string>([])
   // the urls function may be null
   if (resolvedConfig.urls) {
     let urlsToAdd
@@ -33,11 +33,15 @@ export const resolveReportableRoutes: () => Promise<NormalisedRoute[]> = async (
     urlsToAdd.forEach(url => urls.add(url))
     if (urlsToAdd.length) {
       resolvedConfig.scanner.sitemap = false
+      resolvedConfig.scanner.robotsTxt = false
       resolvedConfig.scanner.crawler = false
       resolvedConfig.scanner.dynamicSampling = false
-      logger.info(`The \`url\` config has been provided with ${urlsToAdd.length} paths for scanning. Disabling sitemap, sampling and crawler.`)
+      logger.info(`The \`url\` config has been provided with ${urlsToAdd.length} paths for scanning. Disabling sitemap, robots, sampling and crawler.`)
     }
   }
+  else {
+    urls.add(resolvedConfig.site)
+  }
 
   if (resolvedConfig.scanner.robotsTxt) {
     const robotsTxt = await fetchRobotsTxt(resolvedConfig.site)

diff --git a/packages/core/src/resolveConfig.ts b/packages/core/src/resolveConfig.ts
@@ -35,6 +35,9 @@ export const resolveUserConfig: (userConfig: UserConfig) => Promise<ResolvedUser
   })
   const config = merger(userConfig, defaultConfig)
 
+  if (!config.site && config.urls?.[0])
+    config.site = config.urls[0]
+
   // it's possible we don't know the site at runtime
   if (config.site) {
     // normalise site

diff --git a/packages/core/src/util.ts b/packages/core/src/util.ts
@@ -72,7 +72,10 @@ export function hashPathName(path: string) {
 export function normaliseHost(host: string) {
   if (!host.startsWith('http'))
     host = `http${host.startsWith('localhost') ? '' : 's'}://${host}`
-  return host.includes('.') ? host : withTrailingSlash(host)
+  host = host.includes('.') ? host : withTrailingSlash(host)
+  // strip pathname from host
+  const url = new URL(host)
+  return `${url.protocol}//${url.host}/`
 }
 
 /**