diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cfea225eb..cf5fad6bae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Add `buf registry plugin label {archive,info,list,unarchive}` to manage BSR plugin commits. - Move `buf registry module update` to `buf registry module settings update`. Command `buf registry module update` is now deprecated. +- Support remote check plugins in `buf lint` and `buf breaking` commands. ## [v1.47.2] - 2024-11-14 diff --git a/private/buf/bufctl/controller.go b/private/buf/bufctl/controller.go index 1cb04b8e30..a287a7c88d 100644 --- a/private/buf/bufctl/controller.go +++ b/private/buf/bufctl/controller.go @@ -29,6 +29,7 @@ import ( "github.com/bufbuild/buf/private/buf/bufwkt/bufwktstore" "github.com/bufbuild/buf/private/buf/bufworkspace" "github.com/bufbuild/buf/private/bufpkg/bufanalysis" + "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimageutil" @@ -48,6 +49,7 @@ import ( "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/storage/storageos" "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/bufbuild/buf/private/pkg/wasm" "github.com/bufbuild/protovalidate-go" "google.golang.org/protobuf/proto" ) @@ -89,11 +91,19 @@ type Controller interface { workspace bufworkspace.Workspace, options ...FunctionOption, ) (bufimage.Image, error) - GetTargetImageWithConfigs( + // GetTargetImageWithConfigsAndCheckClient gets the target ImageWithConfigs + // with a configured bufcheck Client. + // + // ImageWithConfig scopes the configuration per image for use with breaking + // and lint checks. The check Client is bound to the input to ensure that the + // correct remote plugin dependencies are used. A wasmRuntime is provided + // to evaluate Wasm plugins. + GetTargetImageWithConfigsAndCheckClient( ctx context.Context, input string, + wasmRuntime wasm.Runtime, options ...FunctionOption, - ) ([]ImageWithConfig, error) + ) ([]ImageWithConfig, bufcheck.Client, error) // GetImportableImageFileInfos gets the importable .proto FileInfos for the given input. // // This includes all files that can be possible imported. For example, if a Module @@ -128,6 +138,16 @@ type Controller interface { defaultMessageEncoding buffetch.MessageEncoding, options ...FunctionOption, ) error + // GetCheckClientForWorkspace returns a new bufcheck Client for the given Workspace. + // + // Clients are bound to a specific Workspace to ensure that the correct + // plugin dependencies are used. A wasmRuntime is provided to evaluate + // Wasm plugins. + GetCheckClientForWorkspace( + ctx context.Context, + workspace bufworkspace.Workspace, + wasmRuntime wasm.Runtime, + ) (bufcheck.Client, error) } func NewController( @@ -243,6 +263,7 @@ func newController( graphProvider, moduleDataProvider, commitProvider, + pluginKeyProvider, ) controller.workspaceDepManagerProvider = bufworkspace.NewWorkspaceDepManagerProvider( logger, @@ -333,11 +354,12 @@ func (c *controller) GetImageForWorkspace( return c.getImageForWorkspace(ctx, workspace, functionOptions) } -func (c *controller) GetTargetImageWithConfigs( +func (c *controller) GetTargetImageWithConfigsAndCheckClient( ctx context.Context, input string, + wasmRuntime wasm.Runtime, options ...FunctionOption, -) (_ []ImageWithConfig, retErr error) { +) (_ []ImageWithConfig, _ bufcheck.Client, retErr error) { defer c.handleFileAnnotationSetRetError(&retErr) functionOptions := newFunctionOptions(c) for _, option := range options { @@ -345,42 +367,41 @@ func (c *controller) GetTargetImageWithConfigs( } ref, err := c.buffetchRefParser.GetRef(ctx, input) if err != nil { - return nil, err + return nil, nil, err } + var workspace bufworkspace.Workspace switch t := ref.(type) { case buffetch.ProtoFileRef: - workspace, err := c.getWorkspaceForProtoFileRef(ctx, t, functionOptions) + workspace, err = c.getWorkspaceForProtoFileRef(ctx, t, functionOptions) if err != nil { - return nil, err + return nil, nil, err } - return c.buildTargetImageWithConfigs(ctx, workspace, functionOptions) case buffetch.SourceRef: - workspace, err := c.getWorkspaceForSourceRef(ctx, t, functionOptions) + workspace, err = c.getWorkspaceForSourceRef(ctx, t, functionOptions) if err != nil { - return nil, err + return nil, nil, err } - return c.buildTargetImageWithConfigs(ctx, workspace, functionOptions) case buffetch.ModuleRef: - workspace, err := c.getWorkspaceForModuleRef(ctx, t, functionOptions) + workspace, err = c.getWorkspaceForModuleRef(ctx, t, functionOptions) if err != nil { - return nil, err + return nil, nil, err } - return c.buildTargetImageWithConfigs(ctx, workspace, functionOptions) case buffetch.MessageRef: image, err := c.getImageForMessageRef(ctx, t, functionOptions) if err != nil { - return nil, err + return nil, nil, err } bucket, err := c.storageosProvider.NewReadWriteBucket( ".", storageos.ReadWriteBucketWithSymlinksIfSupported(), ) if err != nil { - return nil, err + return nil, nil, err } lintConfig := bufconfig.DefaultLintConfigV1 breakingConfig := bufconfig.DefaultBreakingConfigV1 var pluginConfigs []bufconfig.PluginConfig + pluginKeyProvider := bufplugin.NopPluginKeyProvider bufYAMLFile, err := bufconfig.GetBufYAMLFileForPrefixOrOverride( ctx, bucket, @@ -389,16 +410,15 @@ func (c *controller) GetTargetImageWithConfigs( ) if err != nil { if !errors.Is(err, fs.ErrNotExist) { - return nil, err + return nil, nil, err } // We did not find a buf.yaml in our current directory, and there was no config override. // Use the defaults. } else { - pluginConfigs = bufYAMLFile.PluginConfigs() if topLevelLintConfig := bufYAMLFile.TopLevelLintConfig(); topLevelLintConfig == nil { // Ensure that this is a v2 config if fileVersion := bufYAMLFile.FileVersion(); fileVersion != bufconfig.FileVersionV2 { - return nil, syserror.Newf("non-v2 version with no top-level lint config: %s", fileVersion) + return nil, nil, syserror.Newf("non-v2 version with no top-level lint config: %s", fileVersion) } // v2 config without a top-level lint config, use v2 default lintConfig = bufconfig.DefaultLintConfigV2 @@ -407,26 +427,87 @@ func (c *controller) GetTargetImageWithConfigs( } if topLevelBreakingConfig := bufYAMLFile.TopLevelBreakingConfig(); topLevelBreakingConfig == nil { if fileVersion := bufYAMLFile.FileVersion(); fileVersion != bufconfig.FileVersionV2 { - return nil, syserror.Newf("non-v2 version with no top-level breaking config: %s", fileVersion) + return nil, nil, syserror.Newf("non-v2 version with no top-level breaking config: %s", fileVersion) } // v2 config without a top-level breaking config, use v2 default breakingConfig = bufconfig.DefaultBreakingConfigV2 } else { breakingConfig = topLevelBreakingConfig } + // The directory path is resolved to a buf.yaml file and a buf.lock file. If the + // buf.yaml file is found, the PluginConfigs from the buf.yaml file and the PluginKeys + // from the buf.lock file are resolved to create the PluginKeyProvider. + pluginConfigs = bufYAMLFile.PluginConfigs() + // If a config override is provided, the PluginConfig remote Refs use the BSR + // to resolve the PluginKeys. No buf.lock is required. + // If the buf.yaml file is not found, the bufplugin.NopPluginKeyProvider is returned. + // If the buf.lock file is not found, the bufplugin.NopPluginKeyProvider is returned. + if functionOptions.configOverride != "" { + // To support remote plugins in the override, we need to resolve the remote + // Refs to PluginKeys. A buf.lock file is not required for this operation. + // We use the BSR to resolve any remote plugin Refs. + pluginKeyProvider = c.pluginKeyProvider + } else if bufYAMLFile.FileVersion() == bufconfig.FileVersionV2 { + var pluginKeys []bufplugin.PluginKey + if bufLockFile, err := bufconfig.GetBufLockFileForPrefix( + ctx, + bucket, + // buf.lock files live next to the buf.yaml + ".", + ); err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return nil, nil, err + } + // We did not find a buf.lock in our current directory. + // Remote plugins are not available. + pluginKeys = nil + } else { + pluginKeys = bufLockFile.RemotePluginKeys() + } + pluginKeyProvider, err = newStaticPluginKeyProviderForPluginConfigs( + pluginConfigs, + pluginKeys, + ) + if err != nil { + return nil, nil, err + } + } } - return []ImageWithConfig{ + imageWithConfigs := []ImageWithConfig{ newImageWithConfig( image, lintConfig, breakingConfig, pluginConfigs, ), - }, nil + } + pluginRunnerProvider := bufcheck.NewLocalRunnerProvider( + wasmRuntime, + pluginKeyProvider, + c.pluginDataProvider, + ) + checkClient, err := bufcheck.NewClient( + c.logger, + pluginRunnerProvider, + bufcheck.ClientWithStderr(c.container.Stderr()), + ) + if err != nil { + return nil, nil, err + } + return imageWithConfigs, checkClient, nil default: // This is a system error. - return nil, syserror.Newf("invalid Ref: %T", ref) + return nil, nil, syserror.Newf("invalid Ref: %T", ref) + } + targetImageWithConfigs, err := c.buildTargetImageWithConfigs(ctx, workspace, functionOptions) + if err != nil { + return nil, nil, err } + checkClient, err := c.GetCheckClientForWorkspace(ctx, workspace, wasmRuntime) + if err != nil { + return nil, nil, err + } + return targetImageWithConfigs, checkClient, err } func (c *controller) GetImportableImageFileInfos( @@ -706,6 +787,30 @@ func (c *controller) PutMessage( return errors.Join(err, writeCloser.Close()) } +func (c *controller) GetCheckClientForWorkspace( + ctx context.Context, + workspace bufworkspace.Workspace, + wasmRuntime wasm.Runtime, +) (_ bufcheck.Client, retErr error) { + pluginKeyProvider, err := newStaticPluginKeyProviderForPluginConfigs( + workspace.PluginConfigs(), + workspace.RemotePluginKeys(), + ) + if err != nil { + return nil, err + } + pluginRunnerProvider := bufcheck.NewLocalRunnerProvider( + wasmRuntime, + pluginKeyProvider, + c.pluginDataProvider, + ) + return bufcheck.NewClient( + c.logger, + pluginRunnerProvider, + bufcheck.ClientWithStderr(c.container.Stderr()), + ) +} + func (c *controller) getImage( ctx context.Context, input string, @@ -1342,3 +1447,36 @@ func validateFileAnnotationErrorFormat(fileAnnotationErrorFormat string) error { fileAnnotationErrorFormatFlagName := "error-format" return appcmd.NewInvalidArgumentErrorf("--%s: invalid format: %q", fileAnnotationErrorFormatFlagName, fileAnnotationErrorFormat) } + +// newStaticPluginKeyProvider creates a new PluginKeyProvider for the set of PluginKeys. +// +// The PluginKeys come from the buf.lock file. The PluginKeyProvider is static +// and does not change. PluginConfigs are validated to ensure that all remote +// PluginConfigs are pinned in the buf.lock file. +func newStaticPluginKeyProviderForPluginConfigs( + pluginConfigs []bufconfig.PluginConfig, + pluginKeys []bufplugin.PluginKey, +) (_ bufplugin.PluginKeyProvider, retErr error) { + // Validate that all remote PluginConfigs are present in the buf.lock file. + pluginKeysByFullName, err := slicesext.ToUniqueValuesMap(pluginKeys, func(pluginKey bufplugin.PluginKey) string { + return pluginKey.FullName().String() + }) + if err != nil { + return nil, fmt.Errorf("failed to validate remote PluginKeys: %w", err) + } + // Remote PluginConfig Refs are any PluginConfigs that have a Ref. + remotePluginRefs := slicesext.Filter( + slicesext.Map(pluginConfigs, func(pluginConfig bufconfig.PluginConfig) bufparse.Ref { + return pluginConfig.Ref() + }), + func(pluginRef bufparse.Ref) bool { + return pluginRef != nil + }, + ) + for _, remotePluginRef := range remotePluginRefs { + if _, ok := pluginKeysByFullName[remotePluginRef.FullName().String()]; !ok { + return nil, fmt.Errorf(`remote plugin %q is not in the buf.lock file, use "buf plugin update" to pin remote refs`, remotePluginRef) + } + } + return bufplugin.NewStaticPluginKeyProvider(pluginKeys) +} diff --git a/private/buf/buflsp/buflsp.go b/private/buf/buflsp/buflsp.go index 536cf71d43..2d440f64f7 100644 --- a/private/buf/buflsp/buflsp.go +++ b/private/buf/buflsp/buflsp.go @@ -25,11 +25,11 @@ import ( "sync/atomic" "github.com/bufbuild/buf/private/buf/bufctl" - "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/pkg/app/appext" "github.com/bufbuild/buf/private/pkg/slogext" "github.com/bufbuild/buf/private/pkg/storage" "github.com/bufbuild/buf/private/pkg/storage/storageos" + "github.com/bufbuild/buf/private/pkg/wasm" "go.lsp.dev/jsonrpc2" "go.lsp.dev/protocol" "go.uber.org/zap" @@ -43,7 +43,7 @@ func Serve( wktBucket storage.ReadBucket, container appext.Container, controller bufctl.Controller, - checkClient bufcheck.Client, + wasmRuntime wasm.Runtime, stream jsonrpc2.Stream, ) (jsonrpc2.Conn, error) { // The LSP protocol deals with absolute filesystem paths. This requires us to @@ -68,7 +68,7 @@ func Serve( container: container, logger: container.Logger(), controller: controller, - checkClient: checkClient, + wasmRuntime: wasmRuntime, rootBucket: bucket, wktBucket: wktBucket, } @@ -96,7 +96,7 @@ type lsp struct { logger *slog.Logger controller bufctl.Controller - checkClient bufcheck.Client + wasmRuntime wasm.Runtime rootBucket storage.ReadBucket fileManager *fileManager diff --git a/private/buf/buflsp/file.go b/private/buf/buflsp/file.go index 437d11bb2d..d5e9288adb 100644 --- a/private/buf/buflsp/file.go +++ b/private/buf/buflsp/file.go @@ -63,8 +63,9 @@ type file struct { version int32 hasText bool // Whether this file has ever had text read into it. - workspace bufworkspace.Workspace - module bufmodule.Module + workspace bufworkspace.Workspace + module bufmodule.Module + checkClient bufcheck.Client againstStrategy againstStrategy againstGitRef string @@ -394,6 +395,13 @@ func (f *file) FindModule(ctx context.Context) { return } + // Get the check client for this workspace. + checkClient, err := f.lsp.controller.GetCheckClientForWorkspace(ctx, workspace, f.lsp.wasmRuntime) + if err != nil { + f.lsp.logger.Warn("could not get check client", slogext.ErrorAttr(err)) + return + } + // Figure out which module this file belongs to. var module bufmodule.Module for _, mod := range workspace.Modules() { @@ -421,6 +429,7 @@ func (f *file) FindModule(ctx context.Context) { f.workspace = workspace f.module = module + f.checkClient = checkClient } // IndexImports finds URIs for all of the files imported by this file. @@ -664,9 +673,13 @@ func (f *file) RunLints(ctx context.Context) bool { f.lsp.logger.Warn(fmt.Sprintf("could not find image for %q", f.uri)) return false } + if f.checkClient == nil { + f.lsp.logger.Warn(fmt.Sprintf("could not find check client for %q", f.uri)) + return false + } f.lsp.logger.Debug(fmt.Sprintf("running lint for %q in %v", f.uri, f.module.FullName())) - return f.appendLintErrors("buf lint", f.lsp.checkClient.Lint( + return f.appendLintErrors("buf lint", f.checkClient.Lint( ctx, f.workspace.GetLintConfigForOpaqueID(f.module.OpaqueID()), f.image, @@ -687,9 +700,13 @@ func (f *file) RunBreaking(ctx context.Context) bool { f.lsp.logger.Warn(fmt.Sprintf("could not find --against image for %q", f.uri)) return false } + if f.checkClient == nil { + f.lsp.logger.Warn(fmt.Sprintf("could not find check client for %q", f.uri)) + return false + } f.lsp.logger.Debug(fmt.Sprintf("running breaking for %q in %v", f.uri, f.module.FullName())) - return f.appendLintErrors("buf breaking", f.lsp.checkClient.Breaking( + return f.appendLintErrors("buf breaking", f.checkClient.Breaking( ctx, f.workspace.GetBreakingConfigForOpaqueID(f.module.OpaqueID()), f.image, diff --git a/private/buf/bufworkspace/workspace.go b/private/buf/bufworkspace/workspace.go index 68ea77ac06..a1b76b7367 100644 --- a/private/buf/bufworkspace/workspace.go +++ b/private/buf/bufworkspace/workspace.go @@ -18,6 +18,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/bufpkg/bufparse" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/slicesext" ) @@ -72,8 +73,14 @@ type Workspace interface { // detector ignoring these configs anyways. GetBreakingConfigForOpaqueID(opaqueID string) bufconfig.BreakingConfig // PluginConfigs gets the configured PluginConfigs of the Workspace. + // + // These come from the buf.lock file. Only v2 supports plugins. PluginConfigs() []bufconfig.PluginConfig - // ConfiguredDepModuleRefs returns the configured dependencies of the Workspace as ModuleRefs. + // RemotePluginKeys gets the remote PluginKeys of the Workspace. + // + // These come from the buf.lock file. Only v2 supports plugins. + RemotePluginKeys() []bufplugin.PluginKey + // ConfiguredDepModuleRefs returns the configured dependencies of the Workspace as Refs. // // These come from buf.yaml files. // @@ -105,6 +112,7 @@ type workspace struct { opaqueIDToLintConfig map[string]bufconfig.LintConfig opaqueIDToBreakingConfig map[string]bufconfig.BreakingConfig pluginConfigs []bufconfig.PluginConfig + remotePluginKeys []bufplugin.PluginKey configuredDepModuleRefs []bufparse.Ref // If true, the workspace was created from v2 buf.yamls. @@ -117,6 +125,7 @@ func newWorkspace( opaqueIDToLintConfig map[string]bufconfig.LintConfig, opaqueIDToBreakingConfig map[string]bufconfig.BreakingConfig, pluginConfigs []bufconfig.PluginConfig, + remotePluginKeys []bufplugin.PluginKey, configuredDepModuleRefs []bufparse.Ref, isV2 bool, ) *workspace { @@ -125,6 +134,7 @@ func newWorkspace( opaqueIDToLintConfig: opaqueIDToLintConfig, opaqueIDToBreakingConfig: opaqueIDToBreakingConfig, pluginConfigs: pluginConfigs, + remotePluginKeys: remotePluginKeys, configuredDepModuleRefs: configuredDepModuleRefs, isV2: isV2, } @@ -142,6 +152,10 @@ func (w *workspace) PluginConfigs() []bufconfig.PluginConfig { return slicesext.Copy(w.pluginConfigs) } +func (w *workspace) RemotePluginKeys() []bufplugin.PluginKey { + return slicesext.Copy(w.remotePluginKeys) +} + func (w *workspace) ConfiguredDepModuleRefs() []bufparse.Ref { return slicesext.Copy(w.configuredDepModuleRefs) } diff --git a/private/buf/bufworkspace/workspace_provider.go b/private/buf/bufworkspace/workspace_provider.go index a9811072cf..e2a191eb62 100644 --- a/private/buf/bufworkspace/workspace_provider.go +++ b/private/buf/bufworkspace/workspace_provider.go @@ -25,6 +25,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/bufpkg/bufparse" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/normalpath" "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/slogext" @@ -77,12 +78,14 @@ func NewWorkspaceProvider( graphProvider bufmodule.GraphProvider, moduleDataProvider bufmodule.ModuleDataProvider, commitProvider bufmodule.CommitProvider, + pluginKeyProvider bufplugin.PluginKeyProvider, ) WorkspaceProvider { return newWorkspaceProvider( logger, graphProvider, moduleDataProvider, commitProvider, + pluginKeyProvider, ) } @@ -93,6 +96,10 @@ type workspaceProvider struct { graphProvider bufmodule.GraphProvider moduleDataProvider bufmodule.ModuleDataProvider commitProvider bufmodule.CommitProvider + + // pluginKeyProvider is only used for getting remote plugin keys for a single module + // when an override is specified. + pluginKeyProvider bufplugin.PluginKeyProvider } func newWorkspaceProvider( @@ -100,12 +107,14 @@ func newWorkspaceProvider( graphProvider bufmodule.GraphProvider, moduleDataProvider bufmodule.ModuleDataProvider, commitProvider bufmodule.CommitProvider, + pluginKeyProvider bufplugin.PluginKeyProvider, ) *workspaceProvider { return &workspaceProvider{ logger: logger, graphProvider: graphProvider, moduleDataProvider: moduleDataProvider, commitProvider: commitProvider, + pluginKeyProvider: pluginKeyProvider, } } @@ -136,8 +145,11 @@ func (w *workspaceProvider) GetWorkspaceForModuleKey( targetModuleConfig := bufconfig.DefaultModuleConfigV1 // By default, there will be no plugin configs, however, similar to the lint and breaking // configs, there may be an override, in which case, we need to populate the plugin configs - // from the override. - var pluginConfigs []bufconfig.PluginConfig + // from the override. Any remote plugin refs will be resolved by the pluginKeyProvider. + var ( + pluginConfigs []bufconfig.PluginConfig + remotePluginKeys []bufplugin.PluginKey + ) if config.configOverride != "" { bufYAMLFile, err := bufconfig.GetBufYAMLFileForOverride(config.configOverride) if err != nil { @@ -150,7 +162,7 @@ func (w *workspaceProvider) GetWorkspaceForModuleKey( case 1: // If we have a single ModuleConfig, we assume that regardless of whether or not // This ModuleConfig has a name, that this is what the user intends to associate - // with the tqrget module. This also handles the v1 case - v1 buf.yamls will always + // with the target module. This also handles the v1 case - v1 buf.yamls will always // only have a single ModuleConfig, and it was expected pre-refactor that regardless // of if the ModuleConfig had a name associated with it or not, the lint and breaking // config that came from it would be associated. @@ -172,6 +184,27 @@ func (w *workspaceProvider) GetWorkspaceForModuleKey( } if bufYAMLFile.FileVersion() == bufconfig.FileVersionV2 { pluginConfigs = bufYAMLFile.PluginConfigs() + // To support remote plugins when using a config override, we need to resolve the remote + // Refs to PluginKeys. We use the pluginKeyProvider to resolve any remote plugin Refs. + remotePluginRefs := slicesext.Filter( + slicesext.Map(pluginConfigs, func(pluginConfig bufconfig.PluginConfig) bufparse.Ref { + return pluginConfig.Ref() + }), + func(ref bufparse.Ref) bool { + return ref != nil + }, + ) + if len(remotePluginRefs) > 0 { + var err error + remotePluginKeys, err = w.pluginKeyProvider.GetPluginKeysForPluginRefs( + ctx, + remotePluginRefs, + bufplugin.DigestTypeP1, + ) + if err != nil { + return nil, err + } + } } } @@ -209,18 +242,18 @@ func (w *workspaceProvider) GetWorkspaceForModuleKey( opaqueIDToLintConfig, opaqueIDToBreakingConfig, pluginConfigs, + remotePluginKeys, nil, false, ), nil } -func (w *workspaceProvider) GetWorkspaceForBucket( +func (w *workspaceProvider) getWorkspaceTargetingForBucket( ctx context.Context, bucket storage.ReadBucket, bucketTargeting buftarget.BucketTargeting, options ...WorkspaceBucketOption, -) (Workspace, error) { - defer slogext.DebugProfile(w.logger)() +) (*workspaceTargeting, error) { config, err := newWorkspaceBucketConfig(options) if err != nil { return nil, err @@ -232,7 +265,7 @@ func (w *workspaceProvider) GetWorkspaceForBucket( return nil, err } } - workspaceTargeting, err := newWorkspaceTargeting( + return newWorkspaceTargeting( ctx, w.logger, config, @@ -241,6 +274,21 @@ func (w *workspaceProvider) GetWorkspaceForBucket( overrideBufYAMLFile, config.ignoreAndDisallowV1BufWorkYAMLs, ) +} + +func (w *workspaceProvider) GetWorkspaceForBucket( + ctx context.Context, + bucket storage.ReadBucket, + bucketTargeting buftarget.BucketTargeting, + options ...WorkspaceBucketOption, +) (Workspace, error) { + defer slogext.DebugProfile(w.logger)() + workspaceTargeting, err := w.getWorkspaceTargetingForBucket( + ctx, + bucket, + bucketTargeting, + options..., + ) if err != nil { return nil, err } @@ -358,7 +406,8 @@ func (w *workspaceProvider) getWorkspaceForBucketAndModuleDirPathsV1Beta1OrV1( return w.getWorkspaceForBucketModuleSet( moduleSet, v1WorkspaceTargeting.bucketIDToModuleConfig, - nil, + nil, // No PluginConfigs for v1 + nil, // No remote PluginKeys for v1 v1WorkspaceTargeting.allConfiguredDepModuleRefs, false, ) @@ -370,6 +419,7 @@ func (w *workspaceProvider) getWorkspaceForBucketBufYAMLV2( v2Targeting *v2Targeting, ) (*workspace, error) { moduleSetBuilder := bufmodule.NewModuleSetBuilder(ctx, w.logger, w.moduleDataProvider, w.commitProvider) + var remotePluginKeys []bufplugin.PluginKey bufLockFile, err := bufconfig.GetBufLockFileForPrefix( ctx, bucket, @@ -398,6 +448,7 @@ func (w *workspaceProvider) getWorkspaceForBucketBufYAMLV2( false, ) } + remotePluginKeys = bufLockFile.RemotePluginKeys() } // Only check for duplicate module description in v2, which would be an user error, i.e. // This is not a system error: @@ -455,6 +506,7 @@ func (w *workspaceProvider) getWorkspaceForBucketBufYAMLV2( moduleSet, v2Targeting.bucketIDToModuleConfig, v2Targeting.bufYAMLFile.PluginConfigs(), + remotePluginKeys, v2Targeting.bufYAMLFile.ConfiguredDepModuleRefs(), true, ) @@ -465,6 +517,7 @@ func (w *workspaceProvider) getWorkspaceForBucketModuleSet( moduleSet bufmodule.ModuleSet, bucketIDToModuleConfig map[string]bufconfig.ModuleConfig, pluginConfigs []bufconfig.PluginConfig, + remotePluginKeys []bufplugin.PluginKey, // Expected to already be unique by FullName. configuredDepModuleRefs []bufparse.Ref, isV2 bool, @@ -490,6 +543,7 @@ func (w *workspaceProvider) getWorkspaceForBucketModuleSet( opaqueIDToLintConfig, opaqueIDToBreakingConfig, pluginConfigs, + remotePluginKeys, configuredDepModuleRefs, isV2, ), nil diff --git a/private/buf/bufworkspace/workspace_test.go b/private/buf/bufworkspace/workspace_test.go index 0006c48217..bc2f6b96e7 100644 --- a/private/buf/bufworkspace/workspace_test.go +++ b/private/buf/bufworkspace/workspace_test.go @@ -24,6 +24,7 @@ import ( "github.com/bufbuild/buf/private/buf/buftarget" "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduletesting" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/dag/dagtest" "github.com/bufbuild/buf/private/pkg/ioext" "github.com/bufbuild/buf/private/pkg/normalpath" @@ -321,6 +322,8 @@ func testNewWorkspaceProvider(t *testing.T, testModuleDatas ...bufmoduletesting. bsrProvider, bsrProvider, bsrProvider, + // TODO: add support for plugins to bufmoduletesting.NewOmniProvider. + bufplugin.NopPluginKeyProvider, ) } diff --git a/private/buf/cmd/buf/command/beta/lsp/lsp.go b/private/buf/cmd/buf/command/beta/lsp/lsp.go index bdf7a21229..e7b7b2fc4e 100644 --- a/private/buf/cmd/buf/command/beta/lsp/lsp.go +++ b/private/buf/cmd/buf/command/beta/lsp/lsp.go @@ -26,8 +26,6 @@ import ( "github.com/bufbuild/buf/private/buf/bufcli" "github.com/bufbuild/buf/private/buf/buflsp" - "github.com/bufbuild/buf/private/bufpkg/bufcheck" - "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appext" "github.com/bufbuild/buf/private/pkg/ioext" @@ -114,20 +112,8 @@ func run( defer func() { retErr = errors.Join(retErr, wasmRuntime.Close(ctx)) }() - checkClient, err := bufcheck.NewClient( - container.Logger(), - bufcheck.NewLocalRunnerProvider( - wasmRuntime, - bufplugin.NopPluginKeyProvider, - bufplugin.NopPluginDataProvider, - ), - bufcheck.ClientWithStderr(container.Stderr()), - ) - if err != nil { - return err - } - conn, err := buflsp.Serve(ctx, wktBucket, container, controller, checkClient, jsonrpc2.NewStream(transport)) + conn, err := buflsp.Serve(ctx, wktBucket, container, controller, wasmRuntime, jsonrpc2.NewStream(transport)) if err != nil { return err } diff --git a/private/buf/cmd/buf/command/breaking/breaking.go b/private/buf/cmd/buf/command/breaking/breaking.go index 0a5b6990e5..1ddbf6359e 100644 --- a/private/buf/cmd/buf/command/breaking/breaking.go +++ b/private/buf/cmd/buf/command/breaking/breaking.go @@ -25,7 +25,6 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufanalysis" "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/bufpkg/bufimage" - "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appext" "github.com/bufbuild/buf/private/pkg/slicesext" @@ -162,11 +161,23 @@ func run( if err != nil { return err } + wasmRuntimeCacheDir, err := bufcli.CreateWasmRuntimeCacheDir(container) + if err != nil { + return err + } + wasmRuntime, err := wasm.NewRuntime(ctx, wasm.WithLocalCacheDir(wasmRuntimeCacheDir)) + if err != nil { + return err + } + defer func() { + retErr = errors.Join(retErr, wasmRuntime.Close(ctx)) + }() // Do not exclude imports here. bufcheck's Client requires all imports. // Use bufcheck's BreakingWithExcludeImports. - imageWithConfigs, err := controller.GetTargetImageWithConfigs( + imageWithConfigs, checkClient, err := controller.GetTargetImageWithConfigsAndCheckClient( ctx, input, + wasmRuntime, bufctl.WithTargetPaths(flags.Paths, flags.ExcludePaths), bufctl.WithConfigOverride(flags.Config), ) @@ -184,9 +195,10 @@ func run( } // Do not exclude imports here. bufcheck's Client requires all imports. // Use bufcheck's BreakingWithExcludeImports. - againstImageWithConfigs, err := controller.GetTargetImageWithConfigs( + againstImageWithConfigs, _, err := controller.GetTargetImageWithConfigsAndCheckClient( ctx, flags.Against, + wasm.UnimplementedRuntime, bufctl.WithTargetPaths(externalPaths, flags.ExcludePaths), bufctl.WithConfigOverride(flags.AgainstConfig), ) @@ -206,38 +218,15 @@ func run( len(againstImageWithConfigs), ) } - wasmRuntimeCacheDir, err := bufcli.CreateWasmRuntimeCacheDir(container) - if err != nil { - return err - } - wasmRuntime, err := wasm.NewRuntime(ctx, wasm.WithLocalCacheDir(wasmRuntimeCacheDir)) - if err != nil { - return err - } - defer func() { - retErr = errors.Join(retErr, wasmRuntime.Close(ctx)) - }() var allFileAnnotations []bufanalysis.FileAnnotation for i, imageWithConfig := range imageWithConfigs { - client, err := bufcheck.NewClient( - container.Logger(), - bufcheck.NewLocalRunnerProvider( - wasmRuntime, - bufplugin.NopPluginKeyProvider, - bufplugin.NopPluginDataProvider, - ), - bufcheck.ClientWithStderr(container.Stderr()), - ) - if err != nil { - return err - } breakingOptions := []bufcheck.BreakingOption{ bufcheck.WithPluginConfigs(imageWithConfig.PluginConfigs()...), } if flags.ExcludeImports { breakingOptions = append(breakingOptions, bufcheck.BreakingWithExcludeImports()) } - if err := client.Breaking( + if err := checkClient.Breaking( ctx, imageWithConfig.BreakingConfig(), imageWithConfig, diff --git a/private/buf/cmd/buf/command/config/internal/internal.go b/private/buf/cmd/buf/command/config/internal/internal.go index 83566a4a95..66fb68c483 100644 --- a/private/buf/cmd/buf/command/config/internal/internal.go +++ b/private/buf/cmd/buf/command/config/internal/internal.go @@ -22,9 +22,9 @@ import ( "buf.build/go/bufplugin/check" "github.com/bufbuild/buf/private/buf/bufcli" + "github.com/bufbuild/buf/private/buf/bufctl" "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/bufpkg/bufconfig" - "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appext" "github.com/bufbuild/buf/private/pkg/normalpath" @@ -162,7 +162,6 @@ func lsRun( return appcmd.NewInvalidArgumentErrorf("--%s must be set if --%s is specified", configuredOnlyFlagName, modulePathFlagName) } } - configOverride := flags.Config if flags.Version != "" { configOverride = fmt.Sprintf(`{"version":"%s"}`, flags.Version) @@ -195,19 +194,26 @@ func lsRun( defer func() { retErr = errors.Join(retErr, wasmRuntime.Close(ctx)) }() - client, err := bufcheck.NewClient( - container.Logger(), - bufcheck.NewLocalRunnerProvider( - wasmRuntime, - bufplugin.NopPluginKeyProvider, - bufplugin.NopPluginDataProvider, - ), - bufcheck.ClientWithStderr(container.Stderr()), + controller, err := bufcli.NewController(container) + if err != nil { + return err + } + workspace, err := controller.GetWorkspace( + ctx, + ".", + bufctl.WithConfigOverride(configOverride), + ) + if err != nil { + return err + } + checkClient, err := controller.GetCheckClientForWorkspace( + ctx, + workspace, + wasmRuntime, ) if err != nil { return err } - var rules []bufcheck.Rule if flags.ConfiguredOnly { moduleConfigs := bufYAMLFile.ModuleConfigs() @@ -248,7 +254,7 @@ func lsRun( configuredRuleOptions := []bufcheck.ConfiguredRulesOption{ bufcheck.WithPluginConfigs(bufYAMLFile.PluginConfigs()...), } - rules, err = client.ConfiguredRules( + rules, err = checkClient.ConfiguredRules( ctx, ruleType, checkConfig, @@ -261,7 +267,7 @@ func lsRun( allRulesOptions := []bufcheck.AllRulesOption{ bufcheck.WithPluginConfigs(bufYAMLFile.PluginConfigs()...), } - rules, err = client.AllRules( + rules, err = checkClient.AllRules( ctx, ruleType, bufYAMLFile.FileVersion(), diff --git a/private/buf/cmd/buf/command/lint/lint.go b/private/buf/cmd/buf/command/lint/lint.go index 9a40b720c0..98b8c6debf 100644 --- a/private/buf/cmd/buf/command/lint/lint.go +++ b/private/buf/cmd/buf/command/lint/lint.go @@ -23,7 +23,6 @@ import ( "github.com/bufbuild/buf/private/buf/bufctl" "github.com/bufbuild/buf/private/bufpkg/bufanalysis" "github.com/bufbuild/buf/private/bufpkg/bufcheck" - "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appext" "github.com/bufbuild/buf/private/pkg/stringutil" @@ -122,15 +121,6 @@ func run( if err != nil { return err } - imageWithConfigs, err := controller.GetTargetImageWithConfigs( - ctx, - input, - bufctl.WithTargetPaths(flags.Paths, flags.ExcludePaths), - bufctl.WithConfigOverride(flags.Config), - ) - if err != nil { - return err - } wasmRuntimeCacheDir, err := bufcli.CreateWasmRuntimeCacheDir(container) if err != nil { return err @@ -142,24 +132,22 @@ func run( defer func() { retErr = errors.Join(retErr, wasmRuntime.Close(ctx)) }() + imageWithConfigs, checkClient, err := controller.GetTargetImageWithConfigsAndCheckClient( + ctx, + input, + wasmRuntime, + bufctl.WithTargetPaths(flags.Paths, flags.ExcludePaths), + bufctl.WithConfigOverride(flags.Config), + ) + if err != nil { + return err + } var allFileAnnotations []bufanalysis.FileAnnotation for _, imageWithConfig := range imageWithConfigs { - client, err := bufcheck.NewClient( - container.Logger(), - bufcheck.NewLocalRunnerProvider( - wasmRuntime, - bufplugin.NopPluginKeyProvider, - bufplugin.NopPluginDataProvider, - ), - bufcheck.ClientWithStderr(container.Stderr()), - ) - if err != nil { - return err - } lintOptions := []bufcheck.LintOption{ bufcheck.WithPluginConfigs(imageWithConfig.PluginConfigs()...), } - if err := client.Lint( + if err := checkClient.Lint( ctx, imageWithConfig.LintConfig(), imageWithConfig, diff --git a/private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go b/private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go index 1df99b4e0f..dcfac584e5 100644 --- a/private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go +++ b/private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go @@ -148,6 +148,7 @@ func upload( var plugin bufplugin.Plugin switch { case flags.Binary != "": + // We create a local plugin reference to the Wasm binary. var err error plugin, err = bufplugin.NewLocalWasmPlugin( pluginFullName, diff --git a/private/bufpkg/bufcheck/breaking_test.go b/private/bufpkg/bufcheck/breaking_test.go index a5c2d1d6b5..32b4716baa 100644 --- a/private/bufpkg/bufcheck/breaking_test.go +++ b/private/bufpkg/bufcheck/breaking_test.go @@ -1313,6 +1313,7 @@ func testBreaking( bufmodule.NopGraphProvider, bufmodule.NopModuleDataProvider, bufmodule.NopCommitProvider, + bufplugin.NopPluginKeyProvider, ) previousWorkspace, err := workspaceProvider.GetWorkspaceForBucket( ctx, diff --git a/private/bufpkg/bufcheck/bufcheck.go b/private/bufpkg/bufcheck/bufcheck.go index 4ced80116d..d5cee8cfc9 100644 --- a/private/bufpkg/bufcheck/bufcheck.go +++ b/private/bufpkg/bufcheck/bufcheck.go @@ -170,7 +170,8 @@ func (r RunnerProviderFunc) NewRunner(pluginConfig bufconfig.PluginConfig) (plug return r(pluginConfig) } -// NewLocalRunnerProvider returns a new RunnerProvider for the wasm.Runtime. +// NewLocalRunnerProvider returns a new RunnerProvider for the wasm.Runtime and +// the given plugin providers. // // This implementation should only be used for local applications. It is safe to // use concurrently. @@ -182,6 +183,10 @@ func (r RunnerProviderFunc) NewRunner(pluginConfig bufconfig.PluginConfig) (plug // - bufconfig.PluginConfigTypeRemoteWasm // // If the PluginConfigType is not supported, an error is returned. +// To disable support for Wasm plugins, set wasmRuntime to wasm.UnimplementedRuntime. +// To disable support for bufconfig.PluginConfigTypeRemoteWasm Plugins, set +// pluginKeyProvider and pluginDataProvider to bufplugin.NopPluginKeyProvider +// and bufplugin.NopPluginDataProvider. func NewLocalRunnerProvider( wasmRuntime wasm.Runtime, pluginKeyProvider bufplugin.PluginKeyProvider, diff --git a/private/bufpkg/bufcheck/lint_test.go b/private/bufpkg/bufcheck/lint_test.go index 29c99f6b78..cbc8ce64cd 100644 --- a/private/bufpkg/bufcheck/lint_test.go +++ b/private/bufpkg/bufcheck/lint_test.go @@ -1319,6 +1319,7 @@ func testLintWithOptions( bufmodule.NopGraphProvider, bufmodule.NopModuleDataProvider, bufmodule.NopCommitProvider, + bufplugin.NopPluginKeyProvider, ).GetWorkspaceForBucket( ctx, readWriteBucket, diff --git a/private/bufpkg/bufcheck/runner_provider.go b/private/bufpkg/bufcheck/runner_provider.go index e3541f355d..5d7da852ab 100644 --- a/private/bufpkg/bufcheck/runner_provider.go +++ b/private/bufpkg/bufcheck/runner_provider.go @@ -48,7 +48,7 @@ func newRunnerProvider( func (r *runnerProvider) NewRunner(pluginConfig bufconfig.PluginConfig) (pluginrpc.Runner, error) { switch pluginConfig.Type() { case bufconfig.PluginConfigTypeLocal: - return pluginrpcutil.NewRunner( + return pluginrpcutil.NewLocalRunner( pluginConfig.Name(), pluginConfig.Args()..., ), nil diff --git a/private/bufpkg/bufconfig/plugin_config.go b/private/bufpkg/bufconfig/plugin_config.go index ea53cbaa47..0b402b28fc 100644 --- a/private/bufpkg/bufconfig/plugin_config.go +++ b/private/bufpkg/bufconfig/plugin_config.go @@ -75,18 +75,35 @@ func NewLocalPluginConfig( // NewLocalWasmPluginConfig returns a new PluginConfig for a local Wasm plugin. // -// The first path argument is the path to the Wasm plugin and must end with .wasm. -// The remaining path arguments are the arguments to the Wasm plugin. These are passed -// to the Wasm plugin as command line arguments. +// The name is the path to the Wasm plugin and must end with .wasm. +// The args are the arguments to the Wasm plugin. These are passed to the Wasm plugin +// as command line arguments. func NewLocalWasmPluginConfig( name string, options map[string]any, - path []string, + args []string, ) (PluginConfig, error) { return newLocalWasmPluginConfig( name, options, - path, + args, + ) +} + +// NewRemoteWasmPluginConfig returns a new PluginConfig for a remote Wasm plugin. +// +// The pluginRef is the remote reference to the plugin. +// The args are the arguments to the remote plugin. These are passed to the remote plugin +// as command line arguments. +func NewRemoteWasmPluginConfig( + pluginRef bufparse.Ref, + options map[string]any, + args []string, +) (PluginConfig, error) { + return newRemotePluginConfig( + pluginRef, + options, + args, ) } diff --git a/private/bufpkg/bufplugin/plugin_key_provider.go b/private/bufpkg/bufplugin/plugin_key_provider.go index fcde880280..71ff358e7a 100644 --- a/private/bufpkg/bufplugin/plugin_key_provider.go +++ b/private/bufpkg/bufplugin/plugin_key_provider.go @@ -16,9 +16,11 @@ package bufplugin import ( "context" + "fmt" "io/fs" "github.com/bufbuild/buf/private/bufpkg/bufparse" + "github.com/bufbuild/buf/private/pkg/slicesext" ) var ( @@ -41,6 +43,17 @@ type PluginKeyProvider interface { GetPluginKeysForPluginRefs(context.Context, []bufparse.Ref, DigestType) ([]PluginKey, error) } +// NewStaticPluginKeyProvider returns a new PluginKeyProvider for a static set of PluginKeys. +// +// The set of PluginKeys must be unique by FullName. If there are duplicates, +// an error will be returned. +// +// When resolving Refs, the Ref will be matched to the PluginKey by FullName. +// If the Ref is not found in the set of provided keys, an fs.ErrNotExist will be returned. +func NewStaticPluginKeyProvider(pluginKeys []PluginKey) (PluginKeyProvider, error) { + return newStaticPluginKeyProvider(pluginKeys) +} + // *** PRIVATE *** type nopPluginKeyProvider struct{} @@ -52,3 +65,49 @@ func (nopPluginKeyProvider) GetPluginKeysForPluginRefs( ) ([]PluginKey, error) { return nil, fs.ErrNotExist } + +type staticPluginKeyProvider struct { + pluginKeysByFullName map[string]PluginKey +} + +func newStaticPluginKeyProvider(pluginKeys []PluginKey) (*staticPluginKeyProvider, error) { + var pluginKeysByFullName map[string]PluginKey + if len(pluginKeys) > 0 { + var err error + pluginKeysByFullName, err = slicesext.ToUniqueValuesMap(pluginKeys, func(pluginKey PluginKey) string { + return pluginKey.FullName().String() + }) + if err != nil { + return nil, err + } + } + return &staticPluginKeyProvider{ + pluginKeysByFullName: pluginKeysByFullName, + }, nil +} + +func (s staticPluginKeyProvider) GetPluginKeysForPluginRefs( + _ context.Context, + refs []bufparse.Ref, + digestType DigestType, +) ([]PluginKey, error) { + pluginKeys := make([]PluginKey, len(refs)) + for i, ref := range refs { + // Only the FullName is used to match the PluginKey. The Ref is not + // validated to match the PluginKey as there is not enough information + // to do so. + pluginKey, ok := s.pluginKeysByFullName[ref.FullName().String()] + if !ok { + return nil, fs.ErrNotExist + } + digest, err := pluginKey.Digest() + if err != nil { + return nil, err + } + if digest.Type() != digestType { + return nil, fmt.Errorf("expected DigestType %v, got %v", digestType, digest.Type()) + } + pluginKeys[i] = pluginKey + } + return pluginKeys, nil +} diff --git a/private/pkg/pluginrpcutil/pluginrpcutil.go b/private/pkg/pluginrpcutil/pluginrpcutil.go index a585a47946..45c56fe68d 100644 --- a/private/pkg/pluginrpcutil/pluginrpcutil.go +++ b/private/pkg/pluginrpcutil/pluginrpcutil.go @@ -22,8 +22,11 @@ import ( "pluginrpc.com/pluginrpc" ) -// NewRunner returns a new pluginrpc.Runner for the program name. -func NewRunner(programName string, programArgs ...string) pluginrpc.Runner { +// NewLocalRunner returns a new pluginrpc.Runner for the local program. +// +// The programName is the path or name of the program. Any program args are passed to +// the program when it is run. The programArgs may be nil. +func NewLocalRunner(programName string, programArgs ...string) pluginrpc.Runner { return newRunner(programName, programArgs...) }