Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(convergence): Serialize access to observed state #12150

Open
wants to merge 1 commit into
base: main
Choose a base branch
from

Conversation

anantadwi13
Copy link

What I did
fixing race condition while accessing observedState map

Related issue
Got an error while reloading container using docker compose up -d after updating the docker-compose.yaml file

Log
fatal error: concurrent map read and map write

goroutine 69 [running]:
github.com/docker/compose/v2/pkg/compose.(*convergence).stopDependentContainers(_, {_, _}, _, {{0xc000615468, 0x7}, {0x0, 0x0, 0x0}, 0x0, ...})
        github.com/docker/compose/v2/pkg/compose/convergence.go:226 +0x113
github.com/docker/compose/v2/pkg/compose.(*convergence).ensureService(_, {_, _}, _, {{0xc000615468, 0x7}, {0x0, 0x0, 0x0}, 0x0, ...}, ...)
        github.com/docker/compose/v2/pkg/compose/convergence.go:162 +0x694
github.com/docker/compose/v2/pkg/compose.(*composeService).create.(*convergence).apply.func1.1({0x2747e68, 0xc0006db5c0})
        github.com/docker/compose/v2/pkg/compose/convergence.go:103 +0x10b
github.com/docker/compose/v2/internal/tracing.SpanWrapFunc.func1({0x2747ea0, 0xc0007da0a0})
        github.com/docker/compose/v2/internal/tracing/wrap.go:43 +0x1c4
github.com/docker/compose/v2/pkg/compose.(*composeService).create.(*convergence).apply.func1({0x2747ea0, 0xc0007da0a0}, {0xc000615468, 0x7})
        github.com/docker/compose/v2/pkg/compose/convergence.go:104 +0x2e3
github.com/docker/compose/v2/pkg/compose.(*graphTraversal).run.func1()
        github.com/docker/compose/v2/pkg/compose/dependencies.go:184 +0x99
golang.org/x/sync/errgroup.(*Group).Go.func1()
        golang.org/x/sync@v0.8.0/errgroup/errgroup.go:78 +0x56
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 12
        golang.org/x/sync@v0.8.0/errgroup/errgroup.go:75 +0x96

goroutine 1 [semacquire]:
sync.runtime_Semacquire(0xc00059bad0?)
        runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0x21ecf60?)
        sync/waitgroup.go:116 +0x48
golang.org/x/sync/errgroup.(*Group).Wait(0xc0007f0dc0)
        golang.org/x/sync@v0.8.0/errgroup/errgroup.go:56 +0x25
github.com/docker/compose/v2/pkg/progress.RunWithStatus({0x2747e68, 0xc000791cb0}, 0xc00032a980, 0xc0003537a0, {0x23a0d45, 0x7})
        github.com/docker/compose/v2/pkg/progress/writer.go:97 +0x1ef
github.com/docker/compose/v2/pkg/progress.Run({0x2747e68, 0xc000791cb0}, 0xc0007f0d80, 0xc0003537a0)
        github.com/docker/compose/v2/pkg/progress/writer.go:61 +0x85
github.com/docker/compose/v2/pkg/compose.(*composeService).Up(0xc000437640, {0x2747e68, 0xc000791cb0}, _, {{0xc0001cb300, {0xc000645e50, 0x0, 0x1}, 0x0, 0x0, ...}, ...})
        github.com/docker/compose/v2/pkg/compose/up.go:40 +0x233
github.com/docker/compose/v2/cmd/compose.runUp({_, _}, {_, _}, {_, _}, {0x0, 0x0, {0x239f110, 0x6}, ...}, ...)
        github.com/docker/compose/v2/cmd/compose/up.go:295 +0xc14
github.com/docker/compose/v2/cmd/compose.upCommand.func2({0x2747e68, 0xc000791cb0}, 0xc0005faab0, {0xc000645e50, 0x0, 0x1})
        github.com/docker/compose/v2/cmd/compose/up.go:136 +0x2d8
github.com/docker/compose/v2/cmd/compose.upCommand.(*ProjectOptions).WithServices.func4({0x2747ea0, 0xc0000cc550}, {0xc000645e50, 0x0, 0x1})
        github.com/docker/compose/v2/cmd/compose/compose.go:170 +0x1a3
github.com/docker/compose/v2/cmd/compose.upCommand.(*ProjectOptions).WithServices.Adapt.func6({0x2747ea0?, 0xc0000cc550?}, 0x2?, {0xc000645e50?, 0x272d670?, 0xb5c344?})
        github.com/docker/compose/v2/cmd/compose/compose.go:125 +0x30
github.com/docker/compose/v2/cmd/compose.upCommand.(*ProjectOptions).WithServices.Adapt.AdaptCmd.func7(0xc000421508, {0xc000645e50, 0x0, 0x1})
        github.com/docker/compose/v2/cmd/compose/compose.go:101 +0x154
github.com/docker/cli/cli-plugins/plugin.RunPlugin.func1.1.2(0xc000421508, {0xc000645e50, 0x0, 0x1})
        github.com/docker/cli@v27.3.0-rc.2+incompatible/cli-plugins/plugin/plugin.go:64 +0x6c
github.com/docker/compose/v2/cmd/cmdtrace.Setup.wrapRunE.func2(0xc000421508?, {0xc000645e50?, 0x0?, 0x1?})
        github.com/docker/compose/v2/cmd/cmdtrace/cmd_span.go:85 +0x63
github.com/spf13/cobra.(*Command).execute(0xc000421508, {0xc0003791f0, 0x1, 0x1})
        github.com/spf13/cobra@v1.8.1/command.go:985 +0xaca
github.com/spf13/cobra.(*Command).ExecuteC(0xc0003d6f08)
        github.com/spf13/cobra@v1.8.1/command.go:1117 +0x3ff
github.com/spf13/cobra.(*Command).Execute(...)
        github.com/spf13/cobra@v1.8.1/command.go:1041
github.com/docker/cli/cli-plugins/plugin.RunPlugin(0xc0003b88c0, 0xc000421208, {{0x239dbc7, 0x5}, {0x23a7af5, 0xb}, {0x271ce78, 0x7}, {0x0, 0x0}, ...})
        github.com/docker/cli@v27.3.0-rc.2+incompatible/cli-plugins/plugin/plugin.go:79 +0x145
github.com/docker/cli/cli-plugins/plugin.Run(0x24ec6a8, {{0x239dbc7, 0x5}, {0x23a7af5, 0xb}, {0x271ce78, 0x7}, {0x0, 0x0}, {0x0, ...}})
        github.com/docker/cli@v27.3.0-rc.2+incompatible/cli-plugins/plugin/plugin.go:94 +0x165
main.pluginMain()
        github.com/docker/compose/v2/cmd/main.go:38 +0xa5
main.main()
        github.com/docker/compose/v2/cmd/main.go:98 +0x19c

goroutine 36 [IO wait]:
internal/poll.runtime_pollWait(0x7a4fd6c35700, 0x72)
        runtime/netpoll.go:345 +0x85
internal/poll.(*pollDesc).wait(0xc000427700?, 0xc000093fbf?, 0x0)
        internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
        internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000427700, {0xc000093fbf, 0x1, 0x1})
        internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc000427700, {0xc000093fbf?, 0x0?, 0x0?})
        net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc00063c040, {0xc000093fbf?, 0x0?, 0x0?})
        net/net.go:185 +0x45
github.com/docker/cli/cli-plugins/socket.ConnectAndWait.func1()
        github.com/docker/cli@v27.3.0-rc.2+incompatible/cli-plugins/socket/socket.go:162 +0x45
created by github.com/docker/cli/cli-plugins/socket.ConnectAndWait in goroutine 1
        github.com/docker/cli@v27.3.0-rc.2+incompatible/cli-plugins/socket/socket.go:159 +0x118

goroutine 11 [select]:
github.com/docker/compose/v2/pkg/progress.(*ttyWriter).Start(0xc0003e0a80, {0x2747b28, 0x39ed700})
        github.com/docker/compose/v2/pkg/progress/tty.go:54 +0x105
github.com/docker/compose/v2/pkg/progress.RunWithStatus.func1()
        github.com/docker/compose/v2/pkg/progress/writer.go:83 +0x2a
golang.org/x/sync/errgroup.(*Group).Go.func1()
        golang.org/x/sync@v0.8.0/errgroup/errgroup.go:78 +0x56
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
        golang.org/x/sync@v0.8.0/errgroup/errgroup.go:75 +0x96

goroutine 24 [select]:
go.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue(0xc0005940a0)
        go.opentelemetry.io/otel/sdk@v1.21.0/trace/batch_span_processor.go:312 +0x11d
go.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func1()
        go.opentelemetry.io/otel/sdk@v1.21.0/trace/batch_span_processor.go:128 +0x54
created by go.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor in goroutine 1
        go.opentelemetry.io/otel/sdk@v1.21.0/trace/batch_span_processor.go:126 +0x2e5

goroutine 50 [syscall]:
os/signal.signal_recv()
        runtime/sigqueue.go:152 +0x29
os/signal.loop()
        os/signal/signal_unix.go:23 +0x13
created by os/signal.Notify.func1.1 in goroutine 1
        os/signal/signal.go:151 +0x1f

goroutine 51 [chan receive]:
github.com/docker/compose/v2/cmd/compose.upCommand.AdaptCmd.func3.1()
        github.com/docker/compose/v2/cmd/compose/compose.go:95 +0x27
created by github.com/docker/compose/v2/cmd/compose.upCommand.AdaptCmd.func3 in goroutine 1
        github.com/docker/compose/v2/cmd/compose/compose.go:94 +0x11c

goroutine 52 [chan receive]:
github.com/docker/compose/v2/cmd/compose.upCommand.(*ProjectOptions).WithServices.Adapt.AdaptCmd.func7.1()
        github.com/docker/compose/v2/cmd/compose/compose.go:95 +0x27
created by github.com/docker/compose/v2/cmd/compose.upCommand.(*ProjectOptions).WithServices.Adapt.AdaptCmd.func7 in goroutine 1
        github.com/docker/compose/v2/cmd/compose/compose.go:94 +0x11c

goroutine 12 [semacquire]:
sync.runtime_Semacquire(0x1d21f88?)
        runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0x5?)
        sync/waitgroup.go:116 +0x48
golang.org/x/sync/errgroup.(*Group).Wait(0xc00012ca80)
        golang.org/x/sync@v0.8.0/errgroup/errgroup.go:56 +0x25
github.com/docker/compose/v2/pkg/compose.(*graphTraversal).visit(0xc0007da050, {0x2747e68, 0xc0005565d0}, 0xc0007f63a0)
        github.com/docker/compose/v2/pkg/compose/dependencies.go:163 +0x226
github.com/docker/compose/v2/pkg/compose.InDependencyOrder({0x2747e68, 0xc0005565d0}, 0xc0003e6f40?, 0xc00013a380, {0x0, 0x0, 0xc000437640?})
        github.com/docker/compose/v2/pkg/compose/dependencies.go:88 +0xf1
github.com/docker/compose/v2/pkg/compose.(*convergence).apply(...)
        github.com/docker/compose/v2/pkg/compose/convergence.go:92
github.com/docker/compose/v2/pkg/compose.(*composeService).create(0xc000437640, {0x2747e68, 0xc0005565d0}, 0xc0005faab0, {0xc0001cb300, {0xc0002c8700, 0xa, 0x10}, 0x0, 0x0, ...})
        github.com/docker/compose/v2/pkg/compose/create.go:118 +0x5c5
github.com/docker/compose/v2/pkg/compose.(*composeService).Up.func1({0x2747e68, 0xc0005565d0})
        github.com/docker/compose/v2/pkg/compose/up.go:41 +0x85
github.com/docker/compose/v2/pkg/compose.(*composeService).Up.SpanWrapFunc.func5({0x2747e68, 0xc0005565a0})
        github.com/docker/compose/v2/internal/tracing/wrap.go:43 +0x152
github.com/docker/compose/v2/pkg/progress.Run.func1({0x2747e68?, 0xc0005565a0?})
        github.com/docker/compose/v2/pkg/progress/writer.go:62 +0x22
github.com/docker/compose/v2/pkg/progress.RunWithStatus.func2()
        github.com/docker/compose/v2/pkg/progress/writer.go:90 +0x7f
golang.org/x/sync/errgroup.(*Group).Go.func1()
        golang.org/x/sync@v0.8.0/errgroup/errgroup.go:78 +0x56
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
        golang.org/x/sync@v0.8.0/errgroup/errgroup.go:75 +0x96

goroutine 68 [select]:
github.com/docker/compose/v2/pkg/compose.(*graphTraversal).visit.func1()
        github.com/docker/compose/v2/pkg/compose/dependencies.go:147 +0x10c
golang.org/x/sync/errgroup.(*Group).Go.func1()
        golang.org/x/sync@v0.8.0/errgroup/errgroup.go:78 +0x56
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 12
        golang.org/x/sync@v0.8.0/errgroup/errgroup.go:75 +0x96

goroutine 71 [runnable]:
fmt.(*pp).fmtBytes(0xc00083c0d0, {0xc00089e000, 0x20, 0x20?}, 0x78, {0x239f29c?, 0x6?})
        fmt/print.go:509 +0x6cc
fmt.(*pp).printArg(0xc00083c0d0, {0x1e9b300, 0xc000012780}, 0x78)
        fmt/print.go:743 +0x11a
fmt.(*pp).doPrintf(0xc00083c0d0, {0x239c5d2, 0x2}, {0xc00006f7c8, 0x1, 0x1})
        fmt/print.go:1075 +0x37e
fmt.Sprintf({0x239c5d2, 0x2}, {0xc00006f7c8, 0x1, 0x1})
        fmt/print.go:239 +0x53
github.com/opencontainers/go-digest.Algorithm.Encode(...)
        github.com/opencontainers/go-digest@v1.0.0/algorithm.go:143
github.com/opencontainers/go-digest.NewDigestFromBytes({0x239f158, 0x6}, {0xc00089e000?, 0x0?, 0x240?})
        github.com/opencontainers/go-digest@v1.0.0/digest.go:49 +0x6a
github.com/opencontainers/go-digest.NewDigest({0x239f158, 0x6}, {0x274b020?, 0xc0005a0000?})
        github.com/opencontainers/go-digest@v1.0.0/digest.go:41 +0x4a
github.com/opencontainers/go-digest.(*digester).Digest(0xc0005a0000?)
        github.com/opencontainers/go-digest@v1.0.0/digester.go:39 +0x25
github.com/opencontainers/go-digest.Algorithm.FromBytes({0x239f158, 0x6}, {0xc0001d0a00, 0x262, 0x280})
        github.com/opencontainers/go-digest@v1.0.0/algorithm.go:170 +0x9b
github.com/docker/compose/v2/pkg/compose.ServiceHash({{0xc00052ad80, 0x9}, {0x0, 0x0, 0x0}, 0x0, 0x0, 0x0, 0x0, 0x0, ...})
        github.com/docker/compose/v2/pkg/compose/hash.go:41 +0xad
github.com/docker/compose/v2/pkg/compose.mustRecreate({{0xc00052ad80, 0x9}, {0x0, 0x0, 0x0}, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...)
        github.com/docker/compose/v2/pkg/compose/convergence.go:322 +0x8c
github.com/docker/compose/v2/pkg/compose.(*convergence).ensureService(_, {_, _}, _, {{0xc00052ad80, 0x9}, {0x0, 0x0, 0x0}, 0x0, ...}, ...)
        github.com/docker/compose/v2/pkg/compose/convergence.go:157 +0x60e
github.com/docker/compose/v2/pkg/compose.(*composeService).create.(*convergence).apply.func1.1({0x2747e68, 0xc0005e8030})
        github.com/docker/compose/v2/pkg/compose/convergence.go:103 +0x10b
github.com/docker/compose/v2/internal/tracing.SpanWrapFunc.func1({0x2747ea0, 0xc0007da0a0})
        github.com/docker/compose/v2/internal/tracing/wrap.go:43 +0x1c4
github.com/docker/compose/v2/pkg/compose.(*composeService).create.(*convergence).apply.func1({0x2747ea0, 0xc0007da0a0}, {0xc00052ad80, 0x9})
        github.com/docker/compose/v2/pkg/compose/convergence.go:104 +0x2e3
github.com/docker/compose/v2/pkg/compose.(*graphTraversal).run.func1()
        github.com/docker/compose/v2/pkg/compose/dependencies.go:184 +0x99
golang.org/x/sync/errgroup.(*Group).Go.func1()
        golang.org/x/sync@v0.8.0/errgroup/errgroup.go:78 +0x56
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 12
        golang.org/x/sync@v0.8.0/errgroup/errgroup.go:75 +0x96

goroutine 98 [IO wait]:
internal/poll.runtime_pollWait(0x7a4fd6c35418, 0x72)
        runtime/netpoll.go:345 +0x85
internal/poll.(*pollDesc).wait(0xc00071e080?, 0xc000986000?, 0x0)
        internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
        internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00071e080, {0xc000986000, 0x1000, 0x1000})
        internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc00071e080, {0xc000986000?, 0xc0002ca0c0?, 0xc00021d180?})
        net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc000984000, {0xc000986000?, 0x0?, 0xc000501180?})
        net/net.go:185 +0x45
net/http.(*persistConn).Read(0xc0004a4900, {0xc000986000?, 0xc0009880c0?, 0xc000889d38?})
        net/http/transport.go:1977 +0x4a
bufio.(*Reader).fill(0xc000564000)
        bufio/bufio.go:110 +0x103
bufio.(*Reader).Peek(0xc000564000, 0x1)
        bufio/bufio.go:148 +0x53
net/http.(*persistConn).readLoop(0xc0004a4900)
        net/http/transport.go:2141 +0x1b9
created by net/http.(*Transport).dialConn in goroutine 53
        net/http/transport.go:1799 +0x152f

goroutine 99 [select]:
net/http.(*persistConn).writeLoop(0xc0004a4900)
        net/http/transport.go:2458 +0xf0
created by net/http.(*Transport).dialConn in goroutine 53
        net/http/transport.go:1800 +0x1585

goroutine 146 [IO wait]:
internal/poll.runtime_pollWait(0x7a4fd6c34d50, 0x72)
        runtime/netpoll.go:345 +0x85
internal/poll.(*pollDesc).wait(0xc00079e080?, 0xc000874000?, 0x0)
        internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
        internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00079e080, {0xc000874000, 0x1000, 0x1000})
        internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc00079e080, {0xc000874000?, 0x0?, 0xc000876000?})
        net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc0008160c0, {0xc000874000?, 0x0?, 0xc000583a40?})
        net/net.go:185 +0x45
net/http.(*persistConn).Read(0xc0004d2120, {0xc000874000?, 0xc00083e2a0?, 0xc0007a4d38?})
        net/http/transport.go:1977 +0x4a
bufio.(*Reader).fill(0xc0008482a0)
        bufio/bufio.go:110 +0x103
bufio.(*Reader).Peek(0xc0008482a0, 0x1)
        bufio/bufio.go:148 +0x53
net/http.(*persistConn).readLoop(0xc0004d2120)
        net/http/transport.go:2141 +0x1b9
created by net/http.(*Transport).dialConn in goroutine 38
        net/http/transport.go:1799 +0x152f

goroutine 147 [select]:
net/http.(*persistConn).writeLoop(0xc0004d2120)
        net/http/transport.go:2458 +0xf0
created by net/http.(*Transport).dialConn in goroutine 38
        net/http/transport.go:1800 +0x1585

goroutine 73 [runnable]:
github.com/docker/compose/v2/internal/tracing.timeAttr({0x23ba4fd, 0x14}, {0x0?, 0xede2f2fe2?, 0x0?})
        github.com/docker/compose/v2/internal/tracing/attributes.go:167 +0x189
github.com/docker/compose/v2/internal/tracing.unixTimeAttr({0x23ba4fd?, 0x0?}, 0x0?)
        github.com/docker/compose/v2/internal/tracing/attributes.go:172 +0x7e
github.com/docker/compose/v2/internal/tracing.ContainerOptions({{0xc00005cc80, 0x40}, {0xc0003e6b30, 0x1, 0x1}, {0xc000322fc0, 0x24}, {0xc0006d43c0, 0x47}, {0xc0001794ea, ...}, ...})
        github.com/docker/compose/v2/internal/tracing/attributes.go:147 +0x219
github.com/docker/compose/v2/pkg/compose.(*convergence).ensureService(_, {_, _}, _, {{0xc0000151d0, 0x9}, {0x0, 0x0, 0x0}, 0x0, ...}, ...)
        github.com/docker/compose/v2/pkg/compose/convergence.go:168 +0x73f
github.com/docker/compose/v2/pkg/compose.(*composeService).create.(*convergence).apply.func1.1({0x2747e68, 0xc0007ee000})
        github.com/docker/compose/v2/pkg/compose/convergence.go:103 +0x10b
github.com/docker/compose/v2/internal/tracing.SpanWrapFunc.func1({0x2747ea0, 0xc0007da0a0})
        github.com/docker/compose/v2/internal/tracing/wrap.go:43 +0x1c4
github.com/docker/compose/v2/pkg/compose.(*composeService).create.(*convergence).apply.func1({0x2747ea0, 0xc0007da0a0}, {0xc0000151d0, 0x9})
        github.com/docker/compose/v2/pkg/compose/convergence.go:104 +0x2e3
github.com/docker/compose/v2/pkg/compose.(*graphTraversal).run.func1()
        github.com/docker/compose/v2/pkg/compose/dependencies.go:184 +0x99
golang.org/x/sync/errgroup.(*Group).Go.func1()
        golang.org/x/sync@v0.8.0/errgroup/errgroup.go:78 +0x56
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 12
        golang.org/x/sync@v0.8.0/errgroup/errgroup.go:75 +0x96

Signed-off-by: Ananta Dwi Prasetya Purna Yuda <hi@anantadwi13.com>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
None yet
Development

Successfully merging this pull request may close these issues.

1 participant