Skip to content

Commit

Permalink
fix: --sitemaps option will override robots.txt sitemaps
Browse files Browse the repository at this point in the history
Fixes #169
  • Loading branch information
harlan-zw committed Mar 3, 2024
1 parent 3ec2531 commit 8f01db0
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 3 deletions.
2 changes: 1 addition & 1 deletion packages/cli/src/createCli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ export default function createCli() {

cli.option('--site <site>', 'Host URL to scan.')
cli.option('--router-prefix <site>', 'The URL path prefix for the client and API to run from.')
cli.option('--sitemaps <sitemaps>', 'Comma separated list of sitemaps to use for scanning.')
cli.option('--sitemaps <sitemaps>', 'Comma separated list of sitemaps to use for scanning. Providing these will override any in robots.txt.')
cli.option('--samples <samples>', 'Specify the amount of samples to run.')
cli.option('--throttle', 'Enable the throttling')
cli.option('--enable-javascript', 'When inspecting the HTML wait for the javascript to execute. Useful for SPAs.')
Expand Down
7 changes: 5 additions & 2 deletions packages/core/src/discovery/robotsTxt.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ export function mergeRobotsTxtConfig(config: ResolvedUserConfig, { groups, sitem
...normalisedGroups.flatMap(group => group.allow),
])].filter(isValidRegex)
}
if (config.scanner.sitemap !== false && sitemaps.length)
config.scanner.sitemap = [...new Set([...(Array.isArray(config.scanner.sitemap) ? config.scanner.sitemap : []), ...sitemaps])]
if (config.scanner.sitemap !== false && sitemaps.length) {
// allow overriding the robots.txt sitemaps with your own
if (!Array.isArray(config.scanner.sitemap) || !config.scanner.sitemap.length)
config.scanner.sitemap = [...new Set([...(Array.isArray(config.scanner.sitemap) ? config.scanner.sitemap : []), ...sitemaps])]
}
}

0 comments on commit 8f01db0

Please sign in to comment.