Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Workflow bundling for runtime #4

Merged
merged 14 commits into from
Sep 25, 2024
Merged

Workflow bundling for runtime #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
5735492
simaple workflow bundler
evgenyk Sep 24, 2024
3f564b4
defined interface for kinde project
evgenyk Sep 24, 2024
50a4181
wip tests
evgenyk Sep 24, 2024
9cdfb8c
added workflow introspection to bundler
evgenyk Sep 24, 2024
04fbe28
introspection improvements
evgenyk Sep 25, 2024
c1c9572
checking other settings
evgenyk Sep 25, 2024
83a0118
bundle hash for redundant deployments support
evgenyk Sep 25, 2024
b86fc17
fixed a typo
evgenyk Sep 25, 2024
900fc48
kinde project discovers the root configuration
evgenyk Sep 25, 2024
abc4b39
configuration
evgenyk Sep 25, 2024
4a5ae01
locating enviornment workflows
evgenyk Sep 25, 2024
aeff3ce
package bundling
evgenyk Sep 25, 2024
f3e0344
added API binding
evgenyk Sep 25, 2024
c544d0f
test: add test-coveage reporting
DanielRivers Sep 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,11 @@ jobs:
uses: golang/govulncheck-action@v1
with:
go-package: ./...

- name: Run tests
run: go test -coverprofile=coverage.txt
Comment on lines +29 to +30
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

Fix YAML syntax for the "Run tests" step.

The current syntax for the "Run tests" step is incorrect. The run key should be at the same indentation level as the name key.

Please apply the following change to fix the YAML syntax:

    - name: Run tests
-        run: go test -coverprofile=coverage.txt
+      run: go test -coverprofile=coverage.txt
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
- name: Run tests
run: go test -coverprofile=coverage.txt
- name: Run tests
run: go test -coverprofile=coverage.txt
🧰 Tools
actionlint

30-30: could not parse as YAML: yaml: line 30: mapping values are not allowed in this context

(syntax-check)


- name: Upload results to Codecov
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
5 changes: 5 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,19 @@ require github.com/dop251/goja v0.0.0-20240919115326-6c7d1df7ff05

require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
golang.org/x/sys v0.25.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

require (
github.com/dlclark/regexp2 v1.11.4 // indirect
github.com/evanw/esbuild v0.24.0
github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
github.com/google/pprof v0.0.0-20240910150728-a0b0bb1d4134 // indirect
github.com/rs/zerolog v1.33.0
github.com/stretchr/testify v1.9.0
golang.org/x/net v0.29.0
golang.org/x/text v0.18.0 // indirect
Expand Down
19 changes: 19 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,21 +1,40 @@
github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dop251/goja v0.0.0-20240919115326-6c7d1df7ff05 h1:oK4+QcKsczZjHYTHD0JAdkvq5w74JEkG95J0XNBx/BI=
github.com/dop251/goja v0.0.0-20240919115326-6c7d1df7ff05/go.mod h1:MxLav0peU43GgvwVgNbLAj1s/bSGboKkhuULvq/7hx4=
github.com/evanw/esbuild v0.24.0 h1:GZ78naTLp7FKr+K7eNuM/SLs5maeiHYRPsTg6kmdsSE=
github.com/evanw/esbuild v0.24.0/go.mod h1:D2vIQZqV/vIf/VRHtViaUtViZmG7o+kKmlBfVQuRi48=
github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q=
github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/google/pprof v0.0.0-20240910150728-a0b0bb1d4134 h1:c5FlPPgxOn7kJz3VoPLkQYQXGBS3EklQ4Zfi57uOuqQ=
github.com/google/pprof v0.0.0-20240910150728-a0b0bb1d4134/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
Expand Down
160 changes: 114 additions & 46 deletions gojaRuntime/gojaRuntime.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,44 @@ type (
Context map[string]interface{} `json:"context"`
ExitResult interface{} `json:"exit_result"`
}
introspectedExport struct {
value interface{}
}

introspectionResult struct {
exports map[string]introspectedExport
}
)

// HasExport implements runtime_registry.IntrospectedExport.
func (i introspectedExport) HasExport() bool {
return i.value != nil
}

// Value implements runtime_registry.IntrospectedExport.
func (i introspectedExport) Value() interface{} {
return i.value
}

// ValueAsMap implements runtime_registry.IntrospectedExport.
func (i introspectedExport) ValueAsMap() map[string]interface{} {
if i.value == nil {
return map[string]interface{}{}
}
return i.value.(map[string]interface{})
}
Comment on lines +45 to +51
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

Consider safer type assertion in ValueAsMap method

The ValueAsMap method uses a type assertion that could potentially panic if the value is not of type map[string]interface{}.

Consider using a type assertion with the ok idiom to safely handle cases where the type assertion may fail:

func (i introspectedExport) ValueAsMap() map[string]interface{} {
	if i.value == nil {
		return map[string]interface{}{}
	}
-	return i.value.(map[string]interface{})
+	if m, ok := i.value.(map[string]interface{}); ok {
+		return m
+	}
+	return map[string]interface{}{}
}
Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
// ValueAsMap implements runtime_registry.IntrospectedExport.
func (i introspectedExport) ValueAsMap() map[string]interface{} {
if i.value == nil {
return map[string]interface{}{}
}
return i.value.(map[string]interface{})
}
// ValueAsMap implements runtime_registry.IntrospectedExport.
func (i introspectedExport) ValueAsMap() map[string]interface{} {
if i.value == nil {
return map[string]interface{}{}
}
if m, ok := i.value.(map[string]interface{}); ok {
return m
}
return map[string]interface{}{}
}


// GetExport implements runtime_registry.IntrospectionResult.
func (i introspectionResult) GetExport(name string) runtimesRegistry.IntrospectedExport {
return i.exports[name]
}

func (i introspectionResult) recordExport(name string, value interface{}) {
i.exports[name] = introspectedExport{
value: value,
}
}

// GetConsoleError implements runtime_registry.Result.
func (a *actionResult) GetConsoleError() []interface{} {
return a.ConsoleError
Expand Down Expand Up @@ -82,69 +118,46 @@ func newGojaRunner() runtimesRegistry.Runner {
return &runner
}

func (e *gojaRunnerV1) Execute(ctx context.Context, workflow runtimesRegistry.WorkflowDescriptor, startOptions runtimesRegistry.StartOptions) (runtimesRegistry.Result, error) {
// Introspect implements runtime_registry.Runner.
func (e *gojaRunnerV1) Introspect(ctx context.Context, workflow runtimesRegistry.WorkflowDescriptor, options runtimesRegistry.IntrospectionOptions) (runtimesRegistry.IntrospectionResult, error) {
vm := goja.New()
_, returnErr := setupVM(ctx, vm, e, workflow)

registry.Enable(vm)

e.maxExecutionTimeout(ctx, vm, workflow.Limits.MaxExecutionDuration)
vm.SetTimeSource(func() time.Time { return time.Now() }) //static time source

executionResult := &actionResult{
ConsoleLog: []interface{}{},
ConsoleError: []interface{}{},
Context: map[string]interface{}{},
if returnErr != nil {
return nil, returnErr
}
module := vm.Get("module").ToObject(vm)
exports := module.Get("exports").ToObject(vm)

for name, binding := range workflow.Bindings.GlobalModules {
if module, ok := availableModules[name]; ok {
module(e, vm, vm.NewObject(), executionResult, binding)
}
introspectionResult := introspectionResult{
exports: map[string]introspectedExport{},
}

vm.Set("kinde", vm.NewObject())
for name, binding := range workflow.Bindings.KindeAPIs {
kindeMountPoint := vm.Get("kinde").(*goja.Object)
if apiFunc, ok := kindeAPIs[name]; ok {
kindeMountPoint.Set(name, e.callRegisteredAPI(binding, apiFunc))
for _, exportToIntrospect := range options.Exports {
exportIntrospect := exports.Get(exportToIntrospect)
if exportIntrospect != nil {
mapped := exportIntrospect.Export()
introspectionResult.recordExport(exportToIntrospect, mapped)
} else {
introspectionResult.recordExport(exportToIntrospect, nil)
}
}

workflowHash := workflow.GetHash()
program, err := e.Cache.cacheProgram(workflowHash, func() (*goja.Program, error) {
ast, err := goja.Parse("main", string(workflow.ProcessedSource.Source))

if err != nil {
return nil, fmt.Errorf("error parsing %w", err)
}

program, err := goja.CompileAST(ast, false)

if err != nil {
return nil, fmt.Errorf("error compiling %w", err)
}
return introspectionResult, nil
}

return program, nil
func (e *gojaRunnerV1) Execute(ctx context.Context, workflow runtimesRegistry.WorkflowDescriptor, startOptions runtimesRegistry.StartOptions) (runtimesRegistry.ExecutionResult, error) {

})
vm := goja.New()
executionResult, returnErr := setupVM(ctx, vm, e, workflow)

if err != nil {
return nil, err
}

_, err = vm.RunProgram(program)
if err != nil {
return nil, fmt.Errorf("%v", err.Error())
if returnErr != nil {
return executionResult, returnErr
Comment on lines +154 to +155
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

Ensure consistent error handling between Introspect and Execute methods

In the Execute method, when setupVM returns an error, you return executionResult and returnErr (line 126). However, in the Introspect method, you return nil and returnErr (line 100) when an error occurs. This inconsistency might lead to confusion and potential nil pointer dereferences if the caller expects a nil result when an error occurs. Consider returning nil as the result in the Execute method when an error occurs to maintain consistency.

Apply this diff to standardize error handling:

if returnErr != nil {
-    return executionResult, returnErr
+    return nil, returnErr
}
Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
if returnErr != nil {
return executionResult, returnErr
if returnErr != nil {
return nil, returnErr

}

module := vm.Get("module").ToObject(vm)
exports := module.Get("exports").ToObject(vm)

settingsExport := exports.Get("workflowSettings")
if settingsExport != nil {
executionResult.Context["workflowSettings"] = settingsExport.Export()
}

defaultExport := exports.Get("default")
if defaultExport == nil {
return nil, fmt.Errorf("no default export")
Expand Down Expand Up @@ -196,6 +209,61 @@ func (e *gojaRunnerV1) Execute(ctx context.Context, workflow runtimesRegistry.Wo
return executionResult, nil
}

func setupVM(ctx context.Context, vm *goja.Runtime, runner *gojaRunnerV1, workflow runtimesRegistry.WorkflowDescriptor) (*actionResult, error) {
registry.Enable(vm)

runner.maxExecutionTimeout(ctx, vm, workflow.Limits.MaxExecutionDuration)
vm.SetTimeSource(func() time.Time { return time.Now() })

executionResult := &actionResult{
ConsoleLog: []interface{}{},
ConsoleError: []interface{}{},
Context: map[string]interface{}{},
}

for name, binding := range workflow.Bindings.GlobalModules {
if module, ok := availableModules[name]; ok {
module(runner, vm, vm.NewObject(), executionResult, binding)
}
}

vm.Set("kinde", vm.NewObject())
for name, binding := range workflow.Bindings.KindeAPIs {
kindeMountPoint := vm.Get("kinde").(*goja.Object)
if apiFunc, ok := kindeAPIs[name]; ok {
kindeMountPoint.Set(name, runner.callRegisteredAPI(binding, apiFunc))
}
}

workflowHash := workflow.GetHash()
program, err := runner.Cache.cacheProgram(workflowHash, func() (*goja.Program, error) {
ast, err := goja.Parse("main", string(workflow.ProcessedSource.Source))

if err != nil {
return nil, fmt.Errorf("error parsing %w", err)
}

program, err := goja.CompileAST(ast, false)

if err != nil {
return nil, fmt.Errorf("error compiling %w", err)
}

return program, nil

})

if err != nil {
return nil, err
}

_, err = vm.RunProgram(program)
if err != nil {
return nil, fmt.Errorf("%v", err.Error())
}
return executionResult, nil
}

func (*gojaRunnerV1) maxExecutionTimeout(ctx context.Context, vm *goja.Runtime, maxExecutionDuration time.Duration) {
go func() {
timer := time.NewTimer(maxExecutionDuration)
Expand Down
Loading