Add Python development compatibility flag for tests (#2560)

This also moves toward deriving the bundle name from the compat date.

src/workerd/io/compatibility-date.c++

@@ -355,4 +355,12 @@ kj::Maybe<PythonSnapshotRelease::Reader> getPythonSnapshotRelease(
   return result;
 }
 
+kj::String getPythonBundleName(PythonSnapshotRelease::Reader pyodideRelease) {
+  if (pyodideRelease.getPyodide() == "dev") {
+    return kj::str("dev");
+  }
+  return kj::str(pyodideRelease.getPyodide(), "_", pyodideRelease.getPyodideRevision(), "_",
+      pyodideRelease.getBackport());
+}
+
 }  // namespace workerd

src/workerd/io/compatibility-date.capnp

@@ -424,7 +424,8 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef {
   pythonWorkers @43 :Bool
       $compatEnableFlag("python_workers")
       $pythonSnapshotRelease(pyodide = "0.26.0a2", pyodideRevision = "2024-03-01",
-          packages = "2024-03-01", backport = 0);
+          packages = "2024-03-01", backport = 0)
+      $impliedByAfterDate(name = "pythonWorkersDevPyodide", date = "2000-01-01");
   # Enables Python Workers. Access to this flag is not restricted, instead bundles containing
   # Python modules are restricted in EWC.
   #
@@ -572,4 +573,11 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef {
   # would not be cleared until the next time the queue was processed. This behavior leads
   # to a situtation where the stream can hang if the consumer stops consuming. When set,
   # this flag changes the behavior to clear the queue immediately upon abort.
+
+  pythonWorkersDevPyodide @58 :Bool
+    $compatEnableFlag("python_workers_development")
+    $pythonSnapshotRelease(pyodide = "dev", pyodideRevision = "dev",
+          packages = "2024-03-01", backport = 0)
+    $experimental;
+  # Enables Python Workers and uses the bundle from the Pyodide source directory directly. For testing only.
 }

src/workerd/io/compatibility-date.h

@@ -53,6 +53,7 @@ kj::String currentDateStr();
 
 kj::Maybe<PythonSnapshotRelease::Reader> getPythonSnapshotRelease(
     CompatibilityFlags::Reader featureFlags);
+kj::String getPythonBundleName(PythonSnapshotRelease::Reader pyodideRelease);
 
 // These values come from src/workerd/io/compatibility-date.capnp
 static constexpr uint64_t COMPAT_ENABLE_FLAG_ANNOTATION_ID = 0xb6dabbc87cd1b03eull;

src/workerd/server/tests/python/BUILD.bazel

@@ -4,14 +4,14 @@ load("//src/workerd/server/tests/python:py_wd_test.bzl", "py_wd_test")
 
 load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
 
-# pyodide-dev.capnp.bin represents a custom pyodide version "dev" that is generated
+# pyodide_dev.capnp.bin represents a custom pyodide version "dev" that is generated
 # at build time using the latest contents of the src/pyodide directory.
 # This is used to run tests to ensure that they are always run against the latest build of
 # the Pyodide bundle.
 copy_file(
-    name = "pyodide-dev.capnp.bin@rule",
+    name = "pyodide_dev.capnp.bin@rule",
     src = "//src/pyodide:pyodide.capnp.bin",
-    out = "pyodide-bundle-cache/pyodide-dev.capnp.bin"
+    out = "pyodide-bundle-cache/pyodide_dev.capnp.bin"
 )
 
 py_wd_test(

src/workerd/server/tests/python/env-param/env.wd-test

@@ -14,7 +14,7 @@ const unitTests :Workerd.Config = (
           ),
         ],
         compatibilityDate = "2024-01-15",
-        compatibilityFlags = ["python_workers"],
+        compatibilityFlags = ["python_workers_development"],
       )
     ),
   ],

src/workerd/server/tests/python/hello/hello.wd-test

@@ -8,7 +8,7 @@ const unitTests :Workerd.Config = (
           (name = "worker.py", pythonModule = embed "worker.py")
         ],
         compatibilityDate = "2024-01-15",
-        compatibilityFlags = ["python_workers"],
+        compatibilityFlags = ["python_workers_development"],
       )
     ),
   ],

src/workerd/server/tests/python/import_tests.bzl

@@ -23,7 +23,7 @@ const unitTests :Workerd.Config = (
           (name = "{}", pythonRequirement = ""),
         ],
         compatibilityDate = "2024-05-02",
-        compatibilityFlags = ["python_workers"],
+        compatibilityFlags = ["python_workers_development"],
       )
     ),
   ],

src/workerd/server/tests/python/py_wd_test.bzl

@@ -7,8 +7,8 @@ def py_wd_test(
   args = [],
   **kwargs
 ):
-    data += ["pyodide-dev.capnp.bin@rule"]
-    args += ["--pyodide-bundle-disk-cache-dir", "$(location pyodide-dev.capnp.bin@rule)/.."]
+    data += ["pyodide_dev.capnp.bin@rule"]
+    args += ["--pyodide-bundle-disk-cache-dir", "$(location pyodide_dev.capnp.bin@rule)/.."]
 
     wd_test(
         src = src,

src/workerd/server/tests/python/random/random.wd-test

@@ -8,7 +8,7 @@ const unitTests :Workerd.Config = (
           (name = "worker.py", pythonModule = embed "worker.py")
         ],
         compatibilityDate = "2024-01-15",
-        compatibilityFlags = ["python_workers"],
+        compatibilityFlags = ["python_workers_development"],
       )
     ),
   ],

src/workerd/server/tests/python/subdirectory/subdirectory.wd-test

@@ -10,7 +10,7 @@ const unitTests :Workerd.Config = (
           (name = "subdir/a.py", pythonModule = embed "./subdir/a.py"),
         ],
         compatibilityDate = "2023-12-18",
-        compatibilityFlags = ["python_workers"],
+        compatibilityFlags = ["python_workers_development"],
       )
     ),
   ],

src/workerd/server/workerd-api.c++

@@ -411,7 +411,7 @@ kj::Maybe<jsg::ModuleRegistry::ModuleInfo> WorkerdApi::tryCompileModule(jsg::Loc
 
 namespace {
 kj::Path getPyodideBundleFileName(kj::StringPtr version) {
-  return kj::Path(kj::str("pyodide-", version, ".capnp.bin"));
+  return kj::Path(kj::str("pyodide_", version, ".capnp.bin"));
 }
 
 kj::Maybe<kj::Own<const kj::ReadableFile>> getPyodideBundleFile(
@@ -476,7 +476,7 @@ kj::Maybe<jsg::Bundle::Reader> fetchPyodideBundle(
       kj::HttpHeaders headers(table);
 
       kj::String url =
-          kj::str("https://pyodide.runtime-playground.workers.dev/pyodide-capnp-bin/pyodide-",
+          kj::str("https://pyodide.runtime-playground.workers.dev/pyodide-capnp-bin/pyodide_",
               version, ".capnp.bin");
 
       auto req = client->request(kj::HttpMethod::GET, kj::StringPtr(url), headers);
@@ -511,19 +511,11 @@ void WorkerdApi::compileModules(jsg::Lock& lockParam,
           "The python_workers compatibility flag is required to use Python.");
       // Inject Pyodide bundle
       if (util::Autogate::isEnabled(util::AutogateKey::PYODIDE_LOAD_EXTERNAL)) {
-        KJ_IF_SOME(bundle, fetchPyodideBundle(impl->pythonConfig, "dev"_kj)) {
-          modules->addBuiltinBundle(bundle, kj::none);
-        } else {
-          auto pythonRelease = KJ_ASSERT_NONNULL(getPythonSnapshotRelease(featureFlags));
-          auto pyodide = pythonRelease.getPyodide();
-          auto pyodideRevision = pythonRelease.getPyodideRevision();
-          auto backport = pythonRelease.getBackport();
-
-          auto version = kj::str(pyodide, "_", pyodideRevision, "_", backport);
-          auto bundle = KJ_ASSERT_NONNULL(
-              fetchPyodideBundle(impl->pythonConfig, version), "Failed to get Pyodide bundle");
-          modules->addBuiltinBundle(bundle, kj::none);
-        }
+        auto pythonRelease = KJ_ASSERT_NONNULL(getPythonSnapshotRelease(featureFlags));
+        auto version = getPythonBundleName(pythonRelease);
+        auto bundle = KJ_ASSERT_NONNULL(
+            fetchPyodideBundle(impl->pythonConfig, version), "Failed to get Pyodide bundle");
+        modules->addBuiltinBundle(bundle, kj::none);
       }
       // Inject pyodide bootstrap module (TODO: load this from the capnproto bundle?)
       {