From 0ef4968c03308945c0f2c76cd13f7b752c34f903 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Wed, 21 Aug 2024 16:28:44 +0200 Subject: [PATCH] Add Python development compatibility flag for tests (#2560) This also moves toward deriving the bundle name from the compat date. --- src/workerd/io/compatibility-date.c++ | 8 +++++++ src/workerd/io/compatibility-date.capnp | 10 ++++++++- src/workerd/io/compatibility-date.h | 1 + src/workerd/server/tests/python/BUILD.bazel | 6 ++--- .../server/tests/python/env-param/env.wd-test | 2 +- .../server/tests/python/hello/hello.wd-test | 2 +- .../server/tests/python/import_tests.bzl | 2 +- .../server/tests/python/py_wd_test.bzl | 4 ++-- .../server/tests/python/random/random.wd-test | 2 +- .../python/subdirectory/subdirectory.wd-test | 2 +- src/workerd/server/workerd-api.c++ | 22 ++++++------------- 11 files changed, 35 insertions(+), 26 deletions(-) diff --git a/src/workerd/io/compatibility-date.c++ b/src/workerd/io/compatibility-date.c++ index 793bbcbf7c6..094c663aaee 100644 --- a/src/workerd/io/compatibility-date.c++ +++ b/src/workerd/io/compatibility-date.c++ @@ -355,4 +355,12 @@ kj::Maybe getPythonSnapshotRelease( return result; } +kj::String getPythonBundleName(PythonSnapshotRelease::Reader pyodideRelease) { + if (pyodideRelease.getPyodide() == "dev") { + return kj::str("dev"); + } + return kj::str(pyodideRelease.getPyodide(), "_", pyodideRelease.getPyodideRevision(), "_", + pyodideRelease.getBackport()); +} + } // namespace workerd diff --git a/src/workerd/io/compatibility-date.capnp b/src/workerd/io/compatibility-date.capnp index 190e91366a7..5f323ab863f 100644 --- a/src/workerd/io/compatibility-date.capnp +++ b/src/workerd/io/compatibility-date.capnp @@ -424,7 +424,8 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef { pythonWorkers @43 :Bool $compatEnableFlag("python_workers") $pythonSnapshotRelease(pyodide = "0.26.0a2", pyodideRevision = "2024-03-01", - packages = "2024-03-01", backport = 0); + packages = "2024-03-01", backport = 0) + $impliedByAfterDate(name = "pythonWorkersDevPyodide", date = "2000-01-01"); # Enables Python Workers. Access to this flag is not restricted, instead bundles containing # Python modules are restricted in EWC. # @@ -572,4 +573,11 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef { # would not be cleared until the next time the queue was processed. This behavior leads # to a situtation where the stream can hang if the consumer stops consuming. When set, # this flag changes the behavior to clear the queue immediately upon abort. + + pythonWorkersDevPyodide @58 :Bool + $compatEnableFlag("python_workers_development") + $pythonSnapshotRelease(pyodide = "dev", pyodideRevision = "dev", + packages = "2024-03-01", backport = 0) + $experimental; + # Enables Python Workers and uses the bundle from the Pyodide source directory directly. For testing only. } diff --git a/src/workerd/io/compatibility-date.h b/src/workerd/io/compatibility-date.h index 8633b49fb2b..2a35ce56e37 100644 --- a/src/workerd/io/compatibility-date.h +++ b/src/workerd/io/compatibility-date.h @@ -53,6 +53,7 @@ kj::String currentDateStr(); kj::Maybe getPythonSnapshotRelease( CompatibilityFlags::Reader featureFlags); +kj::String getPythonBundleName(PythonSnapshotRelease::Reader pyodideRelease); // These values come from src/workerd/io/compatibility-date.capnp static constexpr uint64_t COMPAT_ENABLE_FLAG_ANNOTATION_ID = 0xb6dabbc87cd1b03eull; diff --git a/src/workerd/server/tests/python/BUILD.bazel b/src/workerd/server/tests/python/BUILD.bazel index 2f9d85c9966..94bdaa39f57 100644 --- a/src/workerd/server/tests/python/BUILD.bazel +++ b/src/workerd/server/tests/python/BUILD.bazel @@ -4,14 +4,14 @@ load("//src/workerd/server/tests/python:py_wd_test.bzl", "py_wd_test") load("@bazel_skylib//rules:copy_file.bzl", "copy_file") -# pyodide-dev.capnp.bin represents a custom pyodide version "dev" that is generated +# pyodide_dev.capnp.bin represents a custom pyodide version "dev" that is generated # at build time using the latest contents of the src/pyodide directory. # This is used to run tests to ensure that they are always run against the latest build of # the Pyodide bundle. copy_file( - name = "pyodide-dev.capnp.bin@rule", + name = "pyodide_dev.capnp.bin@rule", src = "//src/pyodide:pyodide.capnp.bin", - out = "pyodide-bundle-cache/pyodide-dev.capnp.bin" + out = "pyodide-bundle-cache/pyodide_dev.capnp.bin" ) py_wd_test( diff --git a/src/workerd/server/tests/python/env-param/env.wd-test b/src/workerd/server/tests/python/env-param/env.wd-test index b2c27541a52..0c311291e6e 100644 --- a/src/workerd/server/tests/python/env-param/env.wd-test +++ b/src/workerd/server/tests/python/env-param/env.wd-test @@ -14,7 +14,7 @@ const unitTests :Workerd.Config = ( ), ], compatibilityDate = "2024-01-15", - compatibilityFlags = ["python_workers"], + compatibilityFlags = ["python_workers_development"], ) ), ], diff --git a/src/workerd/server/tests/python/hello/hello.wd-test b/src/workerd/server/tests/python/hello/hello.wd-test index 7b10cd9986f..368df13e1e2 100644 --- a/src/workerd/server/tests/python/hello/hello.wd-test +++ b/src/workerd/server/tests/python/hello/hello.wd-test @@ -8,7 +8,7 @@ const unitTests :Workerd.Config = ( (name = "worker.py", pythonModule = embed "worker.py") ], compatibilityDate = "2024-01-15", - compatibilityFlags = ["python_workers"], + compatibilityFlags = ["python_workers_development"], ) ), ], diff --git a/src/workerd/server/tests/python/import_tests.bzl b/src/workerd/server/tests/python/import_tests.bzl index 850dc807be8..02b13da96dd 100644 --- a/src/workerd/server/tests/python/import_tests.bzl +++ b/src/workerd/server/tests/python/import_tests.bzl @@ -23,7 +23,7 @@ const unitTests :Workerd.Config = ( (name = "{}", pythonRequirement = ""), ], compatibilityDate = "2024-05-02", - compatibilityFlags = ["python_workers"], + compatibilityFlags = ["python_workers_development"], ) ), ], diff --git a/src/workerd/server/tests/python/py_wd_test.bzl b/src/workerd/server/tests/python/py_wd_test.bzl index 3c768cadee9..a6fdd0b7545 100644 --- a/src/workerd/server/tests/python/py_wd_test.bzl +++ b/src/workerd/server/tests/python/py_wd_test.bzl @@ -7,8 +7,8 @@ def py_wd_test( args = [], **kwargs ): - data += ["pyodide-dev.capnp.bin@rule"] - args += ["--pyodide-bundle-disk-cache-dir", "$(location pyodide-dev.capnp.bin@rule)/.."] + data += ["pyodide_dev.capnp.bin@rule"] + args += ["--pyodide-bundle-disk-cache-dir", "$(location pyodide_dev.capnp.bin@rule)/.."] wd_test( src = src, diff --git a/src/workerd/server/tests/python/random/random.wd-test b/src/workerd/server/tests/python/random/random.wd-test index 7b10cd9986f..368df13e1e2 100644 --- a/src/workerd/server/tests/python/random/random.wd-test +++ b/src/workerd/server/tests/python/random/random.wd-test @@ -8,7 +8,7 @@ const unitTests :Workerd.Config = ( (name = "worker.py", pythonModule = embed "worker.py") ], compatibilityDate = "2024-01-15", - compatibilityFlags = ["python_workers"], + compatibilityFlags = ["python_workers_development"], ) ), ], diff --git a/src/workerd/server/tests/python/subdirectory/subdirectory.wd-test b/src/workerd/server/tests/python/subdirectory/subdirectory.wd-test index 92f00b095d3..3078b4e96bb 100644 --- a/src/workerd/server/tests/python/subdirectory/subdirectory.wd-test +++ b/src/workerd/server/tests/python/subdirectory/subdirectory.wd-test @@ -10,7 +10,7 @@ const unitTests :Workerd.Config = ( (name = "subdir/a.py", pythonModule = embed "./subdir/a.py"), ], compatibilityDate = "2023-12-18", - compatibilityFlags = ["python_workers"], + compatibilityFlags = ["python_workers_development"], ) ), ], diff --git a/src/workerd/server/workerd-api.c++ b/src/workerd/server/workerd-api.c++ index 72194da4738..202777c8441 100644 --- a/src/workerd/server/workerd-api.c++ +++ b/src/workerd/server/workerd-api.c++ @@ -411,7 +411,7 @@ kj::Maybe WorkerdApi::tryCompileModule(jsg::Loc namespace { kj::Path getPyodideBundleFileName(kj::StringPtr version) { - return kj::Path(kj::str("pyodide-", version, ".capnp.bin")); + return kj::Path(kj::str("pyodide_", version, ".capnp.bin")); } kj::Maybe> getPyodideBundleFile( @@ -476,7 +476,7 @@ kj::Maybe fetchPyodideBundle( kj::HttpHeaders headers(table); kj::String url = - kj::str("https://pyodide.runtime-playground.workers.dev/pyodide-capnp-bin/pyodide-", + kj::str("https://pyodide.runtime-playground.workers.dev/pyodide-capnp-bin/pyodide_", version, ".capnp.bin"); auto req = client->request(kj::HttpMethod::GET, kj::StringPtr(url), headers); @@ -511,19 +511,11 @@ void WorkerdApi::compileModules(jsg::Lock& lockParam, "The python_workers compatibility flag is required to use Python."); // Inject Pyodide bundle if (util::Autogate::isEnabled(util::AutogateKey::PYODIDE_LOAD_EXTERNAL)) { - KJ_IF_SOME(bundle, fetchPyodideBundle(impl->pythonConfig, "dev"_kj)) { - modules->addBuiltinBundle(bundle, kj::none); - } else { - auto pythonRelease = KJ_ASSERT_NONNULL(getPythonSnapshotRelease(featureFlags)); - auto pyodide = pythonRelease.getPyodide(); - auto pyodideRevision = pythonRelease.getPyodideRevision(); - auto backport = pythonRelease.getBackport(); - - auto version = kj::str(pyodide, "_", pyodideRevision, "_", backport); - auto bundle = KJ_ASSERT_NONNULL( - fetchPyodideBundle(impl->pythonConfig, version), "Failed to get Pyodide bundle"); - modules->addBuiltinBundle(bundle, kj::none); - } + auto pythonRelease = KJ_ASSERT_NONNULL(getPythonSnapshotRelease(featureFlags)); + auto version = getPythonBundleName(pythonRelease); + auto bundle = KJ_ASSERT_NONNULL( + fetchPyodideBundle(impl->pythonConfig, version), "Failed to get Pyodide bundle"); + modules->addBuiltinBundle(bundle, kj::none); } // Inject pyodide bootstrap module (TODO: load this from the capnproto bundle?) {