From cc900662d5a8552ded8b10c31110eeb73ae35272 Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Tue, 3 Dec 2024 12:00:26 +0100 Subject: [PATCH] draft --- TODO.md | 101 +++++++++ launcher.sh | 32 +++ packages/kbn-manifest/index.ts | 8 +- packages/kbn-manifest/manifest.ts | 71 ++++++- packages/kbn-manifest/relocate.ts | 339 ++++++++++++++++++++++++++++++ 5 files changed, 547 insertions(+), 4 deletions(-) create mode 100644 TODO.md create mode 100644 launcher.sh create mode 100644 packages/kbn-manifest/relocate.ts diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000000000..9068cee74c9dd --- /dev/null +++ b/TODO.md @@ -0,0 +1,101 @@ + +## PHASE 1 - ESLint constraints + +- Manifests (plugin export function + setup + start contracts) +- Import statements +- group: 'platform' | 'observability' | 'security' | 'search' +- visibility: 'private' | 'shared' # platform only, solutions are 'private' + +## PHASE 2 - Categorising modules (packages and plugins) + +- Not moving, only tagging +- Reveal inter-group dependencies + + +## PHASE 3 - Actually moving stuff +``` + 'src/platform/(packages|plugins)/(private|shared)', + 'x-pack/platform/(packages|plugins)/(private|shared)', + + 'x-pack/solutions/observability/plugins', + 'x-pack/solutions/observability/packages', + 'x-pack/solutions/security/plugins', + 'x-pack/solutions/security/packages', + 'x-pack/solutions/search/plugins', + 'x-pack/solutions/search/packages', +``` + +- Command-line tool to relocate modules, by owner +- 38 teams => 38 big PRs + + + +--- + + + + + + + + + + + + + + + + + + + +node scripts/manifest --relocate @elastic/appex-ai-infra + +Manually search for [\/\\] + +const TEAMS = [ + '@elastic/kibana-core', https://github.com/elastic/kibana/pull/201653 + '@elastic/appex-ai-infra', https://github.com/elastic/kibana/pull/202410 + '@elastic/appex-sharedux', ai-infra + '@elastic/docs', https://github.com/elastic/kibana/pull/202416 + '@elastic/fleet', shallow-2 https://github.com/elastic/kibana/pull/202422 + * x-pack/platform/plugins/shared/fleet/cypress/tasks/login.ts + * x-pack/platform/plugins/shared/fleet/cypress/reporter_config.json + '@elastic/kibana-cloud-security-posture', + '@elastic/kibana-data-discovery', + '@elastic/kibana-esql', + '@elastic/kibana-localization', + '@elastic/kibana-management', + '@elastic/kibana-operations', + '@elastic/kibana-presentation', + '@elastic/kibana-reporting-services', + '@elastic/kibana-security', + '@elastic/kibana-visualizations', + '@elastic/logstash', + '@elastic/ml-ui', + '@elastic/obs-ai-assistant', + '@elastic/obs-entities', + '@elastic/obs-knowledge-team', + '@elastic/obs-ux-infra_services-team', + '@elastic/obs-ux-logs-team', + '@elastic/obs-ux-management-team', + '@elastic/obs-ux-onboarding-team', + '@elastic/observability-ui', + '@elastic/response-ops', + '@elastic/search-kibana', + '@elastic/security-asset-management', + '@elastic/security-defend-workflows', + '@elastic/security-detection-engine', + '@elastic/security-detection-rule-management', + '@elastic/security-detections-response', + '@elastic/security-generative-ai', + '@elastic/security-scalability', + '@elastic/security-solution', + '@elastic/security-threat-hunting-explore', + '@elastic/security-threat-hunting-investigations', + '@elastic/security-threat-hunting', + '@elastic/stack-monitoring', + '@simianhacker @flash1293 @dgieselaar', + '@vigneshshanmugam', +]; diff --git a/launcher.sh b/launcher.sh new file mode 100644 index 0000000000000..5e3d735d39825 --- /dev/null +++ b/launcher.sh @@ -0,0 +1,32 @@ + + + + +cd /Users/gsoldevila/Work/kibana-presentation && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/kibana-presentation > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-presentation && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-reporting-services && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/kibana-reporting-services > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-reporting-services && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-security && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/kibana-security > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-security && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-visualizations && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/kibana-visualizations > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-visualizations && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-logstash && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/logstash > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-logstash && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-ml-ui && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/ml-ui > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-ml-ui && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-obs-ai-assistant && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/obs-ai-assistant > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-obs-ai-assistant && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-obs-entities && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/obs-entities > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-obs-entities && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-obs-knowledge-team && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/obs-knowledge-team > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-obs-knowledge-team && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-obs-ux-infra_services-team && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/obs-ux-infra_services-team > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-obs-ux-infra_services-team && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-obs-ux-logs-team && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/obs-ux-logs-team > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-obs-ux-logs-team && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-obs-ux-management-team && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/obs-ux-management-team > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-obs-ux-management-team && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-obs-ux-onboarding-team && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/obs-ux-onboarding-team > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-obs-ux-onboarding-team && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-observability-ui && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/observability-ui > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-observability-ui && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-response-ops && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/response-ops > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-response-ops && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-search-kibana && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/search-kibana > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-search-kibana && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-security-asset-management && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/security-asset-management > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-security-asset-management && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-security-defend-workflows && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/security-defend-workflows > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-security-defend-workflows && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-security-detection-engine && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/security-detection-engine > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-security-detection-engine && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-security-detection-rule-management && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/security-detection-rule-management > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-security-detection-rule-management && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-security-detections-response && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/security-detections-response > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-security-detections-response && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-security-generative-ai && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/security-generative-ai > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-security-generative-ai && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-security-scalability && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/security-scalability > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-security-scalability && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-security-solution && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/security-solution > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-security-solution && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-security-threat-hunting-explore && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/security-threat-hunting-explore > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-security-threat-hunting-explore && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-security-threat-hunting-investigations && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/security-threat-hunting-investigations > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-security-threat-hunting-investigations && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-security-threat-hunting && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/security-threat-hunting > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-security-threat-hunting && git add . && git restore --staged ./packages/kbn-manifest/* +cd /Users/gsoldevila/Work/kibana-stack-monitoring && git pull && cp ../kibana/packages/kbn-manifest/* ./packages/kbn-manifest/ && rm -fR node_modules && yarn kbn bootstrap && node scripts/manifest --max-old-space-size=16384 --relocate @elastic/stack-monitoring > relocate.out 2>relocate.err && git branch kbn-team-1309-move-kibana-stack-monitoring && git add . && git restore --staged ./packages/kbn-manifest/* diff --git a/packages/kbn-manifest/index.ts b/packages/kbn-manifest/index.ts index ce890742ea61f..e6e841d4a8f98 100644 --- a/packages/kbn-manifest/index.ts +++ b/packages/kbn-manifest/index.ts @@ -9,6 +9,7 @@ import { run } from '@kbn/dev-cli-runner'; import { listManifestFiles, printManifest, updateManifest } from './manifest'; +import { relocateModules } from './relocate'; /** * A CLI to manipulate Kibana package manifest files @@ -17,7 +18,9 @@ export const runKbnManifestCli = () => { run( async ({ log, flags }) => { if (flags.list === 'all') { - listManifestFiles(flags, log); + await listManifestFiles(flags, log); + } else if (typeof flags.relocate === 'string' && flags.relocate!.length > 0) { + await relocateModules(flags.relocate, log); } else { if (!flags.package && !flags.plugin) { throw new Error('You must specify the identifer of the --package or --plugin to update.'); @@ -31,10 +34,11 @@ export const runKbnManifestCli = () => { defaultLevel: 'info', }, flags: { - string: ['list', 'package', 'plugin', 'set', 'unset'], + string: ['list', 'relocate', 'package', 'plugin', 'set', 'unset'], help: ` Usage: node scripts/manifest --package --set group=platform --set visibility=private --list all List all the manifests + --relocate Relocate all modules (packages and plugins) belonging to the specified owner --package [packageId] Select a package to update. --plugin [pluginId] Select a plugin to update. --set [property]=[value] Set the desired "[property]": "[value]" diff --git a/packages/kbn-manifest/manifest.ts b/packages/kbn-manifest/manifest.ts index a839dba7b4077..01f93233d532f 100644 --- a/packages/kbn-manifest/manifest.ts +++ b/packages/kbn-manifest/manifest.ts @@ -8,13 +8,14 @@ */ import { join } from 'path'; -import { writeFile } from 'fs/promises'; +import { readFile, writeFile } from 'fs/promises'; import { flatMap, unset } from 'lodash'; import { set } from '@kbn/safer-lodash-set'; import type { ToolingLog } from '@kbn/tooling-log'; import type { Flags } from '@kbn/dev-cli-runner'; -import { type Package, getPackages } from '@kbn/repo-packages'; +import { type Package, getPackages, Jsonc } from '@kbn/repo-packages'; import { REPO_ROOT } from '@kbn/repo-info'; +import { existsSync, readFileSync } from 'fs'; const MANIFEST_FILE = 'kibana.jsonc'; @@ -42,6 +43,72 @@ const getKibanaJsonc = (flags: Flags, log: ToolingLog): Package[] => { ); }; +interface TsConfigObject { + kbn_references: string[]; +} + +export const readTsConfig = async (directory: string): Promise => { + const tsConfigPath = join(directory, 'tsconfig.json'); + if (existsSync(tsConfigPath)) { + const tsConfigString = await readFile(tsConfigPath); + const tsConfig = Jsonc.parse(tsConfigString.toString()) as TsConfigObject; + return tsConfig; + } +}; + +export const writeTsConfig = async (directory: string, data: object) => { + const tsConfigPath = join(directory, 'tsconfig.json'); + return await writeFile(tsConfigPath, JSON.stringify(data, null, 2)); +}; + +export const dependsOn = (potentialDependency: Package, module: Package): boolean => { + const tsConfigPath = join(potentialDependency.directory, 'tsconfig.json'); + if (existsSync(tsConfigPath)) { + const tsConfigString = readFileSync(tsConfigPath); + const tsConfig = Jsonc.parse(tsConfigString.toString()) as TsConfigObject; + return tsConfig?.kbn_references?.includes(module.id); + } else { + return false; + } +}; + +export const noDependants = (current: Package): boolean => { + const modules = getPackages(REPO_ROOT); + const dependants = modules.filter((module) => dependsOn(module, current)); + return dependants.length === 0; +}; + +export const checkDependants = (current: Package): boolean => { + const modules = getPackages(REPO_ROOT); + const dependants = modules.filter((module) => dependsOn(module, current)); + const solutionDependants = dependants + .filter( + (module) => + module.group === 'observability' || module.group === 'security' || module.group === 'search' + ) + .map((module) => module.group); + + if (solutionDependants.length) { + console.log( + 'Module', + current.id, + 'has the following dependant(s)', + dependants.map((module) => `${module.id} - ${module.group}/${module.visibility}`) + ); + } + return true; + // dependants.length > 0 && + // dependants.every( + // (module) => + // module.group !== 'observability' && module.group !== 'security' && module.group !== 'search' + // ) && + // dependants.some( + // (module) => + // module.group === 'observability' || module.group === 'security' || module.group === 'search' + // ) + // dependants.some((module) => module.group === 'platform') +}; + export const listManifestFiles = (flags: Flags, log: ToolingLog) => { const modules = getPackages(REPO_ROOT); modules diff --git a/packages/kbn-manifest/relocate.ts b/packages/kbn-manifest/relocate.ts new file mode 100644 index 0000000000000..99c8b82289a6d --- /dev/null +++ b/packages/kbn-manifest/relocate.ts @@ -0,0 +1,339 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import util from 'util'; +import dedent from 'dedent'; +import { join } from 'path'; +import { exec } from 'child_process'; +import { existsSync } from 'fs'; +import { rename, mkdir, rm } from 'fs/promises'; +import { orderBy } from 'lodash'; +import type { ToolingLog } from '@kbn/tooling-log'; +import { type Package, getPackages } from '@kbn/repo-packages'; +import { REPO_ROOT } from '@kbn/repo-info'; + +const execAsync = util.promisify(exec); +const safeExec = async (command: string, log: ToolingLog) => { + try { + const result = await execAsync(command); + if (result.stderr) { + log.error(result.stderr); + } + return result; + } catch (err) { + const message = `Error executing ${command}: ${err}`; + log.error(message); + return { stdout: '', stderr: message }; + } +}; + +const BASE_FOLDER = process.cwd() + '/'; +const KIBANA_FOLDER = process.cwd().split('/').pop()!; +const BASE_FOLDER_DEPTH = 5; +const EXCLUDED_MODULES = ['@kbn/core']; +const TARGET_FOLDERS = [ + 'src/platform/plugins/', + 'src/platform/packages/', + 'x-pack/platform/plugins/', + 'x-pack/platform/packages/', + 'x-pack/solutions/', +]; +const EXTENSIONS = [ + 'eslintignore', + 'gitignore', + 'js', + 'txt', + 'json', + 'lock', + 'bazel', + 'md', + 'mdz', + 'asciidoc', + 'ts', + 'jsonc', + 'yaml', + 'yml', +]; + +const EXCLUDED_FOLDERS = [ + './.chromium', + './.devcontainer', + './.es', + './.git', + './.github', + './.native_modules', + './.node_binaries', + './.vscode', + './.yarn-local-mirror', + './build', + './core_http.codeql', + './data', + './node_modules', + './target', + './test.codeql', + './test2.codeql', + './trash', + './trash', +]; + +const NO_GREP = EXCLUDED_FOLDERS.map((f) => `--exclude-dir "${f}"`).join(' '); + +const UPDATED_REFERENCES = new Set(); +const UPDATED_RELATIVE_PATHS = new Set(); + +const calculateModuleTargetFolder = (module: Package): string => { + const group = module.manifest.group!; + const isPlugin = module.manifest.type === 'plugin'; + const fullPath = join(BASE_FOLDER, module.directory); + let moduleDelimiter = isPlugin ? '/plugins/' : '/packages/'; + if (TARGET_FOLDERS.some((folder) => module.directory.includes(folder)) && group === 'platform') { + // if a module has already been relocated, strip the /private/ or /shared/ part too + moduleDelimiter += `${module.visibility}/`; + } + const moduleFolder = fullPath.split(moduleDelimiter).pop()!; + + if (group === 'platform') { + const isXpack = fullPath.includes(`/${KIBANA_FOLDER}/x-pack/`); + const visibility = module.manifest.visibility!; + + return join( + BASE_FOLDER, + isXpack ? 'x-pack' : 'src', + group, + isPlugin ? 'plugins' : 'packages', + visibility, + moduleFolder + ); + } else { + return join( + BASE_FOLDER, + 'x-pack', // all solution modules are 'x-pack' + 'solutions', + group, + isPlugin ? 'plugins' : 'packages', + moduleFolder + ); + } +}; + +const belongsTo = (module: Package, owner: string): boolean => { + return Array.from(module.manifest.owner)[0] === owner; +}; + +const stripFirstChunk = (path: string): string => { + const chunks = path.split('/'); + chunks.shift(); + return chunks.join('/'); +}; + +const replaceReferences = async (module: Package, destination: string, log: ToolingLog) => { + const source = module.directory.startsWith('/Users') + ? module.directory + : join(BASE_FOLDER, module.directory); + const relativeSource = source.replace(BASE_FOLDER, ''); + const relativeDestination = destination.replace(BASE_FOLDER, ''); + + await replaceReferencesInternal(relativeSource, relativeDestination, log); + if ( + (relativeSource.startsWith('src') && relativeDestination.startsWith('src')) || + (relativeSource.startsWith('x-pack') && relativeDestination.startsWith('x-pack')) + ) { + await replaceReferencesInternal( + stripFirstChunk(relativeSource), + stripFirstChunk(relativeDestination), + log + ); + } +}; + +const replaceReferencesInternal = async ( + relativeSource: string, + relativeDestination: string, + log: ToolingLog +) => { + log.info(`Finding and replacing "${relativeSource}" by "${relativeDestination}"`); + + const src = relativeSource.replaceAll('/', '\\/'); + const dst = relativeDestination.replaceAll('/', '\\/'); + + const result = await safeExec( + `grep -I -s -R -l ${EXTENSIONS.map((ext) => `--include="*.${ext}"`).join(' ')} \ + ${NO_GREP} --exclude-dir "./node_modules" "${relativeSource}"`, + log + ); + + const matchingFiles = result.stdout.split('\n').filter(Boolean); + + for (let i = 0; i < matchingFiles.length; ++i) { + const file = matchingFiles[i]; + + const md5Before = (await safeExec(`md5 ${file} --quiet`, log)).stdout.trim(); + // if we are updating packages/cloud references, we must pay attention to not update packages/cloud_defend too + await safeExec(`sed -i '' -E "/${src}[\-_a-zA-Z0-9]/! s/${src}/${dst}/g" ${file}`, log); + const md5After = (await safeExec(`md5 ${file} --quiet`, log)).stdout.trim(); + + if (md5Before !== md5After) { + UPDATED_REFERENCES.add(file); + } + } +}; + +const getRelativeDepth = (directory: string): number => { + const fullPath = directory.startsWith(BASE_FOLDER) ? directory : join(BASE_FOLDER, directory); + return fullPath.split('/').length - BASE_FOLDER_DEPTH; +}; + +const replaceRelativePaths = async (module: Package, destination: string, log: ToolingLog) => { + log.info('Updating relative paths at fault'); + + const relativeDepthBefore = getRelativeDepth(module.directory); + const relativeDepthAfter = getRelativeDepth(destination); + const relativeDepthDiff = relativeDepthAfter - relativeDepthBefore; + + const result = await safeExec( + `grep -I -s -R -n -o ${NO_GREP} -E "\\.\\.(/\\.\\.)+/?" ${destination}`, + log + ); + const matches = result.stdout.split('\n').filter(Boolean); + + const brokenReferences = orderBy( + matches + .map((line) => line.split(':')) + .map(([path, line, match]) => { + if (match.endsWith('/')) { + match = match.substring(0, match.length - 1); + } + let moduleRelativePath = path.replace(destination, ''); + if (moduleRelativePath.startsWith('/')) { + moduleRelativePath = moduleRelativePath.substring(1); + } + const moduleRelativeDepth = moduleRelativePath.split('/').length - 1; // do not count filename + const matchDepth = match.split('/').length; + + return { path, line, moduleRelativeDepth, match, matchDepth }; + }) + .filter(({ matchDepth, moduleRelativeDepth }) => matchDepth > moduleRelativeDepth), + 'matchDepth', + 'desc' + ); + + for (let i = 0; i < brokenReferences.length; ++i) { + const { path, line, match, matchDepth } = brokenReferences[i]; + const pathLine = `${path}:${line}`; + + if (UPDATED_RELATIVE_PATHS.has(pathLine)) { + log.error( + `Cannot replace multiple occurrences of "${match}" in the same line, please fix manually:\t${pathLine}` + ); + } else { + const escapedMatch = match.replaceAll('/', '\\/').replaceAll('.', '\\.'); // escape '.' too (regexp any char) + const escapedReplacement = new Array(matchDepth + relativeDepthDiff).fill('..').join('\\/'); + + await safeExec(`sed -i '' "${line}s/${escapedMatch}/${escapedReplacement}/" ${path}`, log); + UPDATED_RELATIVE_PATHS.add(pathLine); + } + } +}; + +const relocateModule = async (module: Package, log: ToolingLog) => { + const destination = calculateModuleTargetFolder(module); + log.info(`Moving ${module.directory} to ${destination}`); + const chunks = destination.split('/'); + chunks.pop(); // discard module folder + if (existsSync(destination)) { + await rm(destination, { recursive: true }); + } + await mkdir(join('/', ...chunks), { recursive: true }); + await rename(module.directory, destination); + await replaceReferences(module, destination, log); + await replaceRelativePaths(module, destination, log); +}; + +export const playground = (modules: Package[], log: ToolingLog) => { + // modules.forEach((module) => log.info(`${module.id}: ${module.directory}`)); + const teams = new Set( + modules.flatMap((module) => + Array.isArray(module.manifest?.owner) ? module.manifest?.owner : [module.manifest?.owner] + ) + ); + log.info('TEAMS', teams); +}; + +export const showRelocatePlan = (modules: Package[], log: ToolingLog) => { + const plugins = modules.filter((module) => module.manifest.type === 'plugin'); + const packages = modules.filter((module) => module.manifest.type !== 'plugin'); + + const target = (module: Package) => calculateModuleTargetFolder(module).replace(BASE_FOLDER, ''); + + log.info(dedent`${plugins.length} plugin(s) are going to be relocated:\n + | Id | Target folder | + | -- | ------------- | + ${plugins.map((plg) => `| \`${plg.id}\` | \`${target(plg)}\` |`).join('\n')} + `); + + log.info(dedent`${packages.length} package(s) are going to be relocated:\n + | Id | Target folder | + | -- | ------------- | + ${packages.map((pkg) => `| \`${pkg.id}\` | \`${target(pkg)}\` |`).join('\n')} + `); +}; + +export const showRelocateSummary = (log: ToolingLog) => { + if (UPDATED_REFERENCES.size > 0) { + log.info('\n\nThe following files have been updated to replace references to modules:'); + UPDATED_REFERENCES.forEach((ref) => log.info(ref)); + } + + if (UPDATED_RELATIVE_PATHS.size > 0) { + log.info('\n\nThe following files contain relative paths that have been updated:'); + UPDATED_RELATIVE_PATHS.forEach((ref) => log.info(ref)); + } +}; + +export const relocateModules = async (owner: string, log: ToolingLog) => { + const modules = getPackages(REPO_ROOT); + + const toMove = modules.filter( + (module) => + belongsTo(module, owner) && + module.manifest.group && + !module.manifest.devOnly && + !EXCLUDED_MODULES.includes(module.id) && + !module.id.startsWith('@kbn/core-') && + !module.directory.includes(`/${KIBANA_FOLDER}/test/`) && + !module.directory.includes(`/${KIBANA_FOLDER}/x-pack/test/`) + ); + + // return playground(toMove, log); + showRelocatePlan(toMove, log); + + for (let i = 0; i < toMove.length; ++i) { + const module = toMove[i]; + + if (TARGET_FOLDERS.some((folder) => module.directory.includes(folder))) { + // skip modules that are already moved + continue; + } + log.info('\n--------------------------------------------------------------------------------'); + log.info(`\t${module.id} (${i + 1} of ${toMove.length})`); + log.info('--------------------------------------------------------------------------------'); + await relocateModule(module, log); + } + + showRelocateSummary(log); + + // after move operations + await safeExec('yarn kbn bootstrap', log); + await safeExec('node scripts/build_plugin_list_docs', log); + await safeExec('node scripts/generate codeowners', log); + await safeExec('node scripts/lint_packages --fix', log); + await safeExec('node scripts/telemetry_check --fix', log); + await safeExec('node scripts/eslint --fix', log); + await safeExec('node scripts/precommit_hook --fix', log); +};