chore: fix robots tests

harlan-zw · Sep 29, 2024 · c34d06c · c34d06c
1 parent 722522d
commit c34d06c
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/packages/core/src/discovery/robotsTxt.ts b/packages/core/src/discovery/robotsTxt.ts
@@ -105,7 +105,7 @@ export function matchPathToRule(path: string, _rules: RobotsTxtRule[]): RobotsTx
 export function mergeRobotsTxtConfig(config: ResolvedUserConfig, { groups, sitemaps }: RobotsTxtParsed): void {
   config.scanner._robotsTxtRules = groups.filter((group) => {
     return group.userAgent.includes('*') || group.userAgent.includes(String(config.lighthouseOptions?.emulatedUserAgent))
-  }).map(group => group._rules)
+  }).flatMap(group => group._rules)
   if (config.scanner.sitemap !== false && sitemaps.length) {
     // allow overriding the robots.txt sitemaps with your own
     if (!Array.isArray(config.scanner.sitemap) || !config.scanner.sitemap.length)

diff --git a/packages/core/src/puppeteer/worker.ts b/packages/core/src/puppeteer/worker.ts
@@ -96,7 +96,7 @@ export async function createUnlighthouseWorker(tasks: Record<UnlighthouseTask, T
       return
 
     // do robots.txt test
-    if (resolvedConfig.scanner.robotsTxt) {
+    if (resolvedConfig.scanner.robotsTxt && resolvedConfig.scanner._robotsTxtRules?.length) {
       const rule = matchPathToRule(path, resolvedConfig.scanner._robotsTxtRules)
       if (rule && !rule.allow) {
         logger.info(`Skipping route based on robots.txt rule \`${rule.pattern}\``, { path })