From 5083d8e47308b3e4a31309890ee2922ae1679c5a Mon Sep 17 00:00:00 2001 From: Dominic Sudy Date: Thu, 3 Aug 2023 23:05:51 +0200 Subject: [PATCH] feat: Component filter for packages --- docs/PROJECT-CONFIG.md | 46 ++++++++- src/commands/exec/index.ts | 11 +- src/commands/init/index.ts | 2 +- src/commands/package/index.ts | 1 - src/commands/sync/index.ts | 23 +++-- src/modules/component.ts | 172 +++++++++++++++++--------------- src/modules/exec.ts | 10 +- src/modules/package.ts | 18 +--- src/modules/project-config.ts | 15 ++- src/modules/setup.ts | 8 +- src/modules/variables.ts | 1 - test/commands/sync/sync.test.ts | 6 +- test/unit/variables.test.ts | 7 +- test/utils/mockConfig.ts | 7 -- 14 files changed, 183 insertions(+), 144 deletions(-) diff --git a/docs/PROJECT-CONFIG.md b/docs/PROJECT-CONFIG.md index 9f7a4a40..3f8e7332 100644 --- a/docs/PROJECT-CONFIG.md +++ b/docs/PROJECT-CONFIG.md @@ -1,17 +1,55 @@ # Project configuration +The project configuration describes which packages your project is using and in which version. The versions of the referenced packages can be upgraded using the `upgrade` command. If you only want to see which new versions are available use `upgrade --dry-run`. Each package may expose variables which need to be set from the project configuration. If multiple different packages all expose the same named variable `foo`, setting this variable once in the project configuration will pass the value to all packages. + +Read more about variables [here](./features/VARIABLES.md). + +```json +{ + "packages": [ + { + "repo": "package-A", + "version": "v1.0.0" + }, + { + "repo": "package-B", + "version": "v2.3.1-dev" + } + ], + "variables": { + "repoUrl": "https://github.com/eclipse-velocitas/cli", + "copyrightYear": 2023, + "autoGenerateVehicleModel": true + } +} +``` + +## Components + +A package always exposes 1 to n *components* each of which should provide distinct functionality, i.e. to set up a devContainer or to integrate with Github. + +By default, all components of a package will be used, but if desired the used components can be filtered. By providing a list of component configurations in the project configuration: + ```json { "packages": [ { - "name": "package-A", + "repo": "package-A", "version": "v1.0.0" }, { - "name": "package-B", + "repo": "package-B", "version": "v2.3.1-dev" } ], + "components": [ + { + "id": "component-exposed-by-pkg-a" + }, + { + "id": "component-exposed-by-pkg-b" + }, + ], "variables": { "repoUrl": "https://github.com/eclipse-velocitas/cli", "copyrightYear": 2023, @@ -20,6 +58,10 @@ } ``` +The project above will only used the components `component-exposed-by-pkg-a` and `component-exposed-by-pkg-b`, ignoring any other components exposed by the packages. + +## File Structure + ### `packages` - Array[[`PackageConfig`](#packageconfig)] Array of packages used in the project. diff --git a/src/commands/exec/index.ts b/src/commands/exec/index.ts index 7a94e465..eea5c6ac 100644 --- a/src/commands/exec/index.ts +++ b/src/commands/exec/index.ts @@ -85,11 +85,16 @@ Executing script... const appManifestData = readAppManifest(); - const [packageConfig, componentConfig, component] = findComponentByName(projectConfig, args.component); + const componentContext = findComponentByName(projectConfig, args.component); - const variables = VariableCollection.build(projectConfig, packageConfig, componentConfig, component); + const variables = VariableCollection.build( + projectConfig, + componentContext.packageReference, + componentContext.config, + componentContext.manifest, + ); - const envVars = createEnvVars(packageConfig.getPackageDirectoryWithVersion(), variables, appManifestData); + const envVars = createEnvVars(componentContext.packageReference.getPackageDirectoryWithVersion(), variables, appManifestData); try { await runExecSpec(execSpec, args.component, projectConfig, envVars, { verbose: flags.verbose }); diff --git a/src/commands/init/index.ts b/src/commands/init/index.ts index f1b905fe..d04b5d74 100644 --- a/src/commands/init/index.ts +++ b/src/commands/init/index.ts @@ -33,7 +33,7 @@ async function runPostInitHook( console.log(`... > Running post init hook for '${component.id}'`); - const maybeComponentConfig = packageConfig.components?.find((c) => c.id === component.id); + const maybeComponentConfig = projectConfig.components?.find((c) => c.id === component.id); const componentConfig = maybeComponentConfig ? maybeComponentConfig : new ComponentConfig(); const variables = VariableCollection.build(projectConfig, packageConfig, componentConfig, component); diff --git a/src/commands/package/index.ts b/src/commands/package/index.ts index bf91b76b..e8b4b873 100644 --- a/src/commands/package/index.ts +++ b/src/commands/package/index.ts @@ -79,7 +79,6 @@ $ velocitas component --get-path devenv-runtime-local this.log(`${' '.repeat(4)}components:`); for (const component of packageManifest.components) { this.log(`${' '.repeat(5)} - id: ${component.id}`); - this.log(`${' '.repeat(8)}type: ${component.type}`); if (component.variables && component.variables.length > 0) { this.log(`${' '.repeat(8)}variables:`); for (const exposedVariable of component.variables) { diff --git a/src/commands/sync/index.ts b/src/commands/sync/index.ts index f8e521df..fbc66455 100644 --- a/src/commands/sync/index.ts +++ b/src/commands/sync/index.ts @@ -13,7 +13,7 @@ // SPDX-License-Identifier: Apache-2.0 import { Command } from '@oclif/core'; -import { ComponentType, findComponentsByType, getComponentConfig, SetupComponent } from '../../modules/component'; +import { getComponentConfig, getComponents } from '../../modules/component'; import { ProjectConfig } from '../../modules/project-config'; import { installComponent } from '../../modules/setup'; import { VariableCollection } from '../../modules/variables'; @@ -22,22 +22,25 @@ export default class Sync extends Command { static description = 'Syncs Velocitas components into your repo.'; static examples = [ - `$ velocitas update MyAwesomeApp --lang cpp + `$ velocitas sync Syncing Velocitas components! -... syncing 'devenv-github-workflows' -... syncing 'devenv-github-templates'`, +... syncing 'github-workflows' +... syncing 'github-templates'`, ]; async run(): Promise { this.log(`Syncing Velocitas components!`); const projectConfig = ProjectConfig.read(); - const setupComponents = findComponentsByType(projectConfig, ComponentType.setup); - for (const setupComponent of setupComponents) { - this.log(`... syncing '${setupComponent[0].getPackageName()}'`); - const componentConfig = getComponentConfig(setupComponent[0], setupComponent[2].id); - const variables = VariableCollection.build(projectConfig, setupComponent[0], componentConfig, setupComponent[2]); - installComponent(setupComponent[0], setupComponent[2] as SetupComponent, variables); + for (const component of getComponents(projectConfig)) { + if (!component.manifest.files || component.manifest.files.length === 0) { + continue; + } + + this.log(`... syncing '${component.manifest.id}'`); + const componentConfig = getComponentConfig(projectConfig, component.manifest.id); + const variables = VariableCollection.build(projectConfig, component.packageReference, componentConfig, component.manifest); + installComponent(component.packageReference, component.manifest, variables); } } } diff --git a/src/modules/component.ts b/src/modules/component.ts index b1ad55d4..7c2e94f8 100644 --- a/src/modules/component.ts +++ b/src/modules/component.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022 Robert Bosch GmbH +// Copyright (c) 2022-2023 Robert Bosch GmbH // // This program and the accompanying materials are made available under the // terms of the Apache License, Version 2.0 which is available at @@ -12,44 +12,70 @@ // // SPDX-License-Identifier: Apache-2.0 -import { getComponentByType, PackageConfig, PackageManifest } from './package'; +import { PackageConfig } from './package'; import { ComponentConfig, ProjectConfig } from './project-config'; import { VariableDefinition } from './variables'; -type IComponent = new () => { readonly type: ComponentType }; - -const subcomponentTypes: Record = {}; - -function serializable(constructor: T) { - subcomponentTypes[new constructor().type] = constructor; - return constructor; -} - +/** + * Specification of a program that is exported by a component to be used via `velocitas exec`. + */ export interface ProgramSpec { + // Unique ID of the program. Needs to be unique within one component. id: string; + + // Short description of the program and that it is doing. description?: string; + + // Path to the executable (relative to the package root) of the exposed program. executable: string; + + // Default arguments passed to the invoked program upon execution. args?: Array; } +/** + * Execution specification which invokes an exposed program and is able to model + * execution dependencies as well as successful startup. + */ export interface ExecSpec { + // Reference to the id of the exposed program. ref: string; + + // Additional arguments to be passed to the exposed program. args?: Array; + + // Regular expression which identifies a successful startup of the program. startupLine?: string; + + // A reference to another exposed program that this one depends on. dependsOn?: string; } -export enum ComponentType { - runtime = 'runtime', - deployment = 'deployment', - setup = 'setup', +/** + * File copy specification. Describes a file or set of files to be copied from a + * source to a destination. + */ +export interface FileSpec { + // The source file path or directory path (relative to the package root). + src: string; + + // The destination file path or directory path (relative to the workspace root). + dst: string; + + // The condition which has to be fulfilled for the copy to execute. Is evaluated as JS condition. + condition: string; } -// Interface definition for implementing components +/** + * Interface definition for implementing components + */ export interface Component { // Unique ID of the component. Needs to be unique over all installed components. id: string; + // A list of files that need to be copied from source to target when running `velocitas sync`. + files?: Array; + // A list of all variable definitions exposed by this component. variables?: Array; @@ -58,70 +84,29 @@ export interface Component { // Hook which is called after the component has been initialized. onPostInit?: Array; - - // The type of the component. - readonly type: ComponentType; -} - -@serializable -export class RuntimeComponent implements Component { - id = ''; - alias = ''; - readonly type = ComponentType.runtime; - programs? = new Array(); - onPostInit? = new Array(); - variables? = new Array(); -} - -export interface FileSpec { - src: string; - dst: string; - condition: string; } -@serializable -export class SetupComponent implements Component { - id = ''; - readonly type = ComponentType.setup; - files? = new Array(); - programs? = new Array(); - onPostInit? = new Array(); - variables? = new Array(); -} +/** The context in which a component is used. It holds all necessary information to operate on a component. */ +export class ComponentContext { + public packageReference: PackageConfig; + public manifest: Component; + public config: ComponentConfig; -@serializable -export class DeployComponent implements Component { - id = ''; - alias = ''; - readonly type = ComponentType.deployment; - programs? = new Array(); - onPostInit? = new Array(); - variables? = new Array(); -} - -export function findComponentsByType( - projectConfig: ProjectConfig, - type: ComponentType, -): Array<[PackageConfig, PackageManifest, TComponentType]> { - const result = new Array<[PackageConfig, PackageManifest, TComponentType]>(); - for (const packageConfig of projectConfig.packages) { - const componentManifest = packageConfig.readPackageManifest(); - try { - result.push([packageConfig, componentManifest, getComponentByType(componentManifest, type) as TComponentType]); - } catch (e) {} + constructor(packageReference: PackageConfig, manifest: Component, config: ComponentConfig) { + this.packageReference = packageReference; + this.manifest = manifest; + this.config = config; } - - return result; } -export function findComponentByName(projectConfig: ProjectConfig, componentId: string): [PackageConfig, ComponentConfig, Component] { - let result: [PackageConfig, ComponentConfig, Component] | undefined; +export function findComponentByName(projectConfig: ProjectConfig, componentId: string): ComponentContext { + let result: ComponentContext | undefined; for (const packageConfig of projectConfig.packages) { const packageManifest = packageConfig.readPackageManifest(); const matchingComponent = packageManifest.components.find((c) => c.id === componentId); - const matchingComponentConfig = getComponentConfig(packageConfig, componentId); + const matchingComponentConfig = getComponentConfig(projectConfig, componentId); if (matchingComponent) { - result = [packageConfig, matchingComponentConfig, matchingComponent]; + result = new ComponentContext(packageConfig, matchingComponent, matchingComponentConfig); break; } } @@ -133,22 +118,47 @@ export function findComponentByName(projectConfig: ProjectConfig, componentId: s return result; } -export function getComponentConfig(packageConfig: PackageConfig, componentId: string): ComponentConfig { +/** + * Return the configuration of a component. + * + * @param projectConfig The project configuration. + * @param componentId The ID of the component. + * @returns The configuration of the component. + */ +export function getComponentConfig(projectConfig: ProjectConfig, componentId: string): ComponentConfig { var maybeComponentConfig: ComponentConfig | undefined; - if (packageConfig.components) { - maybeComponentConfig = packageConfig.components.find((c) => c.id === componentId); + if (projectConfig.components) { + maybeComponentConfig = projectConfig.components.find((c) => c.id === componentId); } return maybeComponentConfig ? maybeComponentConfig : new ComponentConfig(); } -const reviver = (_: string, v: any) => { - if (typeof v === 'object' && 'type' in v && v.type in subcomponentTypes) { - return Object.assign(new subcomponentTypes[v.type](), v); - } - return v; -}; - // use this to deserialize JSON instead of plain JSON.parse export function deserializeComponentJSON(json: string) { - return JSON.parse(json, reviver); + return JSON.parse(json); +} + +/** + * Return all components used by the project. If the project specifies no components explicitly, + * all components are used by default. + * + * @param projectConfig The project configuration to use as an input. + * @returns A list of all components used by the project. + */ +export function getComponents(projectConfig: ProjectConfig): Array { + var componentTuples = new Array(); + + const usedComponents = projectConfig.components; + + for (const packageConfig of projectConfig.packages) { + const packageManifest = packageConfig.readPackageManifest(); + + for (const component of packageManifest.components) { + if (usedComponents.length === 0 || usedComponents.find((c) => c.id === component.id)) { + componentTuples.push(new ComponentContext(packageConfig, component, getComponentConfig(projectConfig, component.id))); + } + } + } + + return componentTuples; } diff --git a/src/modules/exec.ts b/src/modules/exec.ts index 00f50ce0..d324a81c 100644 --- a/src/modules/exec.ts +++ b/src/modules/exec.ts @@ -107,18 +107,18 @@ export async function runExecSpec( console.info(`Starting ${componentId}/${execSpec.ref}`); } - const [packageConfig, , component] = findComponentByName(projectConfig, componentId); + const componentContext = findComponentByName(projectConfig, componentId); - if (!component.programs) { + if (!componentContext.manifest.programs) { throw new Error(`Component '${componentId}' has no exposed programs!`); } - const programSpec = component.programs.find((prog) => prog.id === execSpec.ref); + const programSpec = componentContext.manifest.programs.find((prog) => prog.id === execSpec.ref); if (!programSpec) { - throw new Error(`No program found for item '${execSpec.ref}' referenced in program list of '${component.id}'`); + throw new Error(`No program found for item '${execSpec.ref}' referenced in program list of '${componentId}'`); } - const cwd = join(packageConfig.getPackageDirectory(), packageConfig.version); + const cwd = join(componentContext.packageReference.getPackageDirectory(), componentContext.packageReference.version); let programArgs = programSpec.args ? programSpec.args : []; if (execSpec.args && execSpec.args.length > 0) { diff --git a/src/modules/package.ts b/src/modules/package.ts index 9b1813b8..e31c4e45 100644 --- a/src/modules/package.ts +++ b/src/modules/package.ts @@ -15,9 +15,8 @@ import { existsSync, readFileSync } from 'fs-extra'; import { homedir } from 'os'; import { join } from 'path'; -import { Component, ComponentType, deserializeComponentJSON } from './component'; +import { Component, deserializeComponentJSON } from './component'; import { DEFAULT_BUFFER_ENCODING } from './constants'; -import { ComponentConfig } from './project-config'; import { packageDownloader } from './package-downloader'; export const MANIFEST_FILE_NAME = 'manifest.json'; @@ -28,7 +27,6 @@ export interface PackageManifest { export class PackageConfig { // name of the package to the package repository - // @deprecated use repo instead repo: string = ''; // version of the package to use @@ -37,15 +35,11 @@ export class PackageConfig { // package-wide variable configuration variables?: Map; - // per-component configuration - components?: ComponentConfig[]; - constructor(config?: any) { - const { name, version, variables, components } = config; + const { name, version, variables } = config; this.repo = name; this.version = version; this.variables = variables; - this.components = components; } private _isCustomPackage(repository: string): boolean { @@ -132,11 +126,3 @@ export function getVelocitasRoot(): string { function getPackageFolderPath(): string { return join(getVelocitasRoot(), 'packages'); } - -export function getComponentByType(packageManifest: PackageManifest, type: ComponentType): Component { - const component = packageManifest.components.find((component: Component) => component.type === type); - if (component === undefined) { - throw new TypeError(`No Subcomponent with type "${type}" found!`); - } - return component; -} diff --git a/src/modules/project-config.ts b/src/modules/project-config.ts index c0b1c2a9..d654bf66 100644 --- a/src/modules/project-config.ts +++ b/src/modules/project-config.ts @@ -23,6 +23,7 @@ export const DEFAULT_CONFIG_FILE_PATH = resolve(cwd(), './.velocitas.json'); interface ProjectConfigOptions { packages: PackageConfig[]; + components?: ComponentConfig[]; variables: Map; } @@ -30,6 +31,9 @@ export class ProjectConfig implements ProjectConfigOptions { // packages used in the project packages: Array = new Array(); + // components used in the project + components: Array = new Array(); + // project-wide variable configuration variables: Map = new Map(); @@ -43,6 +47,7 @@ export class ProjectConfig implements ProjectConfigOptions { constructor(config?: ProjectConfigOptions) { this.packages = config?.packages ? ProjectConfig._parsePackageConfig(config.packages) : this.packages; + this.components = config?.components ? config.components : this.components; this.variables = config?.variables ? config.variables : this.variables; } @@ -63,12 +68,12 @@ export class ProjectConfig implements ProjectConfigOptions { if (packageConfig.variables) { packageConfig.variables = new Map(Object.entries(packageConfig.variables)); } + } - if (packageConfig.components) { - for (let componentConfig of packageConfig.components) { - if (componentConfig.variables) { - componentConfig.variables = new Map(Object.entries(componentConfig.variables)); - } + if (config.components) { + for (let componentConfig of config.components) { + if (componentConfig.variables) { + componentConfig.variables = new Map(Object.entries(componentConfig.variables)); } } } diff --git a/src/modules/setup.ts b/src/modules/setup.ts index 089abcc6..b50a8cac 100644 --- a/src/modules/setup.ts +++ b/src/modules/setup.ts @@ -18,9 +18,9 @@ import path, { join } from 'path'; import { cwd } from 'process'; import copy from 'recursive-copy'; import { TransformCallback, TransformOptions } from 'stream'; -import { SetupComponent } from './component'; import { PackageConfig } from './package'; import { VariableCollection } from './variables'; +import { Component } from './component'; const SUPPORTED_TEXT_FILES_ARRAY = ['.md', '.yaml', '.yml', '.txt', '.json', '.sh', '.html', '.htm', '.xml', '.tpl']; @@ -80,9 +80,9 @@ class ReplaceVariablesStream extends Transform { } } -export function installComponent(packageConfig: PackageConfig, setupComponent: SetupComponent, variables: VariableCollection) { - if (setupComponent.files) { - for (const spec of setupComponent.files) { +export function installComponent(packageConfig: PackageConfig, component: Component, variables: VariableCollection) { + if (component.files) { + for (const spec of component.files) { const src = variables.substitute(spec.src); const dst = variables.substitute(spec.dst); let ifCondition = spec.condition ? variables.substitute(spec.condition) : 'true'; diff --git a/src/modules/variables.ts b/src/modules/variables.ts index f38aedc6..7dde8735 100644 --- a/src/modules/variables.ts +++ b/src/modules/variables.ts @@ -77,7 +77,6 @@ export class VariableCollection { map.set('builtin.package.github.repo', packageConfig.getPackageName()); map.set('builtin.package.github.ref', packageConfig.version); map.set('builtin.component.id', component.id); - map.set('builtin.component.type', component.type); return new VariableCollection(map); } diff --git a/test/commands/sync/sync.test.ts b/test/commands/sync/sync.test.ts index f95af5be..2a6ec213 100644 --- a/test/commands/sync/sync.test.ts +++ b/test/commands/sync/sync.test.ts @@ -13,7 +13,7 @@ // SPDX-License-Identifier: Apache-2.0 import { expect, test } from '@oclif/test'; -import { velocitasConfigMock } from '../../utils/mockConfig'; +import { runtimeComponentManifestMock, setupComponentManifestMock, velocitasConfigMock } from '../../utils/mockConfig'; import { mockFolders, mockRestore } from '../../utils/mockfs'; describe('sync', () => { @@ -27,7 +27,7 @@ describe('sync', () => { .command(['sync']) .it('syncing components into project directory', (ctx) => { expect(ctx.stdout).to.contain('Syncing Velocitas components!'); - expect(ctx.stdout).to.contain(`... syncing '${velocitasConfigMock.packages[1].name}'`); - expect(ctx.stdout).to.not.contain(`... syncing '${velocitasConfigMock.packages[0].name}'`); + expect(ctx.stdout).to.contain(`... syncing '${setupComponentManifestMock.components[0].id}'`); + expect(ctx.stdout).to.not.contain(`... syncing '${runtimeComponentManifestMock.components[0].id}'`); }); }); diff --git a/test/unit/variables.test.ts b/test/unit/variables.test.ts index b4dd8b1e..c736a612 100644 --- a/test/unit/variables.test.ts +++ b/test/unit/variables.test.ts @@ -14,15 +14,15 @@ import { expect } from 'chai'; import 'mocha'; -import { ComponentType, SetupComponent } from '../../src/modules/component'; import { PackageConfig } from '../../src/modules/package'; import { ComponentConfig, ProjectConfig } from '../../src/modules/project-config'; import { VariableCollection } from '../../src/modules/variables'; +import { Component } from '../../src/modules/component'; let projectConfig: ProjectConfig; let packageConfig: PackageConfig; let componentConfig: ComponentConfig; -let componentManifest: SetupComponent; +let componentManifest: Component; let variablesObject: { [key: string]: any }; let variablesMap: Map; @@ -61,7 +61,6 @@ describe('variables - module', () => { default: false, }, ], - type: ComponentType.setup, }; }); describe('VariableCollection', () => { @@ -132,7 +131,6 @@ describe('variables - module', () => { expect(vars.substitute('${{ builtin.package.github.repo }}')).to.equal('test-package'); expect(vars.substitute('${{ builtin.package.github.ref }}')).to.equal('v1.1.1'); expect(vars.substitute('${{ builtin.component.id }}')).to.equal('test-component'); - expect(vars.substitute('${{ builtin.component.type }}')).to.equal('setup'); }); it('should transform variable names into allowed environment variable names', () => { const vars = VariableCollection.build(projectConfig, packageConfig, componentConfig, componentManifest); @@ -143,7 +141,6 @@ describe('variables - module', () => { expect(envVars['builtin_package_github_repo']).to.equal('test-package'); expect(envVars['builtin_package_github_ref']).to.equal('v1.1.1'); expect(envVars['builtin_component_id']).to.equal('test-component'); - expect(envVars['builtin_component_type']).to.equal('setup'); }); }); }); diff --git a/test/utils/mockConfig.ts b/test/utils/mockConfig.ts index e04c8be2..377ee608 100644 --- a/test/utils/mockConfig.ts +++ b/test/utils/mockConfig.ts @@ -12,8 +12,6 @@ // // SPDX-License-Identifier: Apache-2.0 -import { ComponentType } from '../../src/modules/component'; - export const velocitasConfigMock = { packages: [ { @@ -46,7 +44,6 @@ export const setupComponentManifestMock = { components: [ { id: 'github-workflows', - type: 'setup', files: [ { src: 'src/common', @@ -91,8 +88,6 @@ export const runtimeComponentManifestMock = { components: [ { id: 'test-runtime-local', - alias: 'local', - type: ComponentType.runtime, programs: [ { id: 'test-script-1', @@ -112,8 +107,6 @@ export const runtimeComponentManifestMock = { }, { id: 'test-runtime-deploy-local', - alias: 'local', - type: ComponentType.deployment, programs: [ { id: 'test-script-1',