Merge pull request #834 from chrispyles/beta-739

Image build improvements

.github/workflows/run-docker-tests.yml

@@ -42,7 +42,6 @@ jobs:
       - uses: r-lib/actions/setup-pandoc@v2
 
       - uses: liskin/gh-pipx@v1
-        if: ${{ steps.check-branch.outputs.run == 'true' }}
         with:
           packages: >-
             poetry
@@ -56,7 +55,6 @@ jobs:
           cache-environment-key: requirements-${{ hashFiles('pyproject.toml') }}
 
       - name: Install dependencies
-        if: ${{ steps.check-branch.outputs.run == 'true' }}
         run: |
           micromamba activate otter-grader
           poetry install --with test --all-extras

CHANGELOG.md

@@ -2,11 +2,16 @@
 
 **v6.0.0 (unreleased):**
 
+* Switched to [poetry](https://python-poetry.org/) for packaging
 * Removed compatibility patches for nbconvert < 6 per [#777](https://github.com/ucbds-infra/otter-grader/issues/777)
 * Updated Otter Export to throw an error if nbconvert<6.0.0 is found
 * Converted Otter Export's PDF via HTML exporter to use nbconvert's WebPDF exporter per [#781](https://github.com/ucbds-infra/otter-grader/issues/781)
 * Removed pdfkit from dependencies
 * Added ability to export PDFs via HTML in grading containers per [#782](https://github.com/ucbds-infra/otter-grader/issues/782)
+* Set default Python version for grading images to 3.12
+* Remove support for Python versions < 3.9 per [#668](https://github.com/ucbds-infra/otter-grader/issues/668)
+* Removed `setuptools` and `pkg_resources` dependencies
+* Remove dependencies not strictly required by Otter in the grading environment per [#739](https://github.com/ucbds-infra/otter-grader/issues/739)
 
 **v5.6.0:**
 

docs/_static/grading-environment-r.yml

@@ -4,7 +4,7 @@ channels:
   - conda-forge
   - r
 dependencies:
-  - python=3.9
+  - python=3.12
   - pip
   - nb_conda_kernels
   - r-base>=4.0.0
@@ -35,5 +35,5 @@ dependencies:
       - dill
       - numpy
       - gspread
-      - otter-grader==5.6.0
+      - otter-grader[grading,plugins]==5.6.0
       - rpy2

docs/_static/grading-environment.yml

@@ -3,7 +3,7 @@ channels:
   - defaults
   - conda-forge
 dependencies:
-  - python=3.9
+  - python=3.12
   - pip
   - nb_conda_kernels
   - pip:
@@ -23,4 +23,4 @@ dependencies:
       - numpy
       - gspread
       - pypdf
-      - otter-grader==5.6.0
+      - otter-grader[grading,plugins]==5.6.0

docs/index.rst

@@ -68,7 +68,7 @@ each with a command-line interface:
 Installation
 ------------
 
-Otter is a Python package that is compatible with Python 3.8+. The PDF export internals require 
+Otter is a Python package that is compatible with Python 3.9+. The PDF export internals require 
 either LaTeX and Pandoc or Playwright and Chromium to be installed. Docker is also required to grade
 assignments locally with containerization. Otter's Python package can be installed using pipx_ or pip.
 

environment.yml

@@ -3,7 +3,7 @@ channels:
     - conda-forge
     - defaults
 dependencies:
-    - python>=3.8
+    - python>=3.9
     - r-base>=4.0.0
     - r-essentials
     - r-devtools

examples/data-88e-proj03/proj03.ipynb

@@ -74,7 +74,6 @@
     "\n",
     "warnings.simplefilter(action='ignore')\n",
     "%matplotlib inline\n",
-    "plt.style.use('seaborn-muted')\n",
     "plt.rcParams[\"figure.figsize\"] = [10,7]"
    ]
   },
@@ -2286,7 +2285,7 @@
     "new_features = [\"bill_aug05\", \"bill_jul05\", \"paid_aug05\", \"paid_jul05\", \"sex_male\", ...]\n",
     "\"\"\" # END PROMPT\n",
     "\n",
-    "new_X = defaults[new_features]                   # SOLUTION\n",
+    "new_X = defaults[new_features].astype(float)     # SOLUTION\n",
     "\n",
     "new_model = sm.OLS(Y, sm.add_constant(new_X))    # SOLUTION\n",
     "new_result = new_model.fit()                     # SOLUTION\n",
@@ -2418,7 +2417,7 @@
     {
      "data": {
       "text/plain": [
-       "True"
+       "np.True_"
       ]
      },
      "execution_count": 188,

examples/data-88e-proj03/requirements.txt

@@ -1 +1,3 @@
 statsmodels
+matplotlib
+numpy

otter/api.py

@@ -4,12 +4,7 @@
 
 import os
 
-from contextlib import redirect_stdout
-
-try:
-    from contextlib import nullcontext
-except ImportError:
-    from .utils import nullcontext  # nullcontext is new in Python 3.7
+from contextlib import nullcontext, redirect_stdout
 
 from .export import export_notebook
 from .run import main as run_grader

otter/assign/assignment.py

@@ -219,7 +219,7 @@ class TestsValue(fica.Config):
     )
 
     python_version: Optional[Union[str, int, float]] = fica.Key(
-        description = "the version of Python to use in the grading image (must be 3.6+)",
+        description = "the version of Python to use in the grading image (must be 3.9+)",
         default = None,
     )
 

otter/export/exporters/base_exporter.py

@@ -1,16 +1,17 @@
 """ABC for Otter Export exporters"""
 
+import importlib.resources
 import nbformat
-import pkg_resources
 
 from abc import ABC, abstractmethod
 
+from . import __name__ as pkg_name
 from .utils import has_begin, has_end, sub_end_for_new_page
 
 from ...utils import NBFORMAT_VERSION
 
 
-TEMPLATE_DIR = pkg_resources.resource_filename(__name__, "templates")
+TEMPLATE_DIR = importlib.resources.files(pkg_name) / "templates"
 
 
 class ExportFailedException(Exception):

otter/export/exporters/via_html.py

@@ -27,13 +27,24 @@ def convert_notebook(cls, nb_path, dest, **kwargs):
 
         nb = cls.load_notebook(nb_path, filtering=options["filtering"], pagebreaks=options["pagebreaks"])
 
-        nbconvert.TemplateExporter.extra_template_basedirs = [TEMPLATE_DIR]
+        nbconvert.TemplateExporter.extra_template_basedirs = [str(TEMPLATE_DIR)]
         orig_template_name = nbconvert.TemplateExporter.template_name
         nbconvert.TemplateExporter.template_name = options["template"]
 
         exporter = nbconvert.WebPDFExporter()
 
-        pdf, _ = nbconvert.export(exporter, nb)
+        try:
+            pdf, _ = nbconvert.export(exporter, nb)
+        except RuntimeError as e:
+            # Replace nbconvert's error about installing chromium since their flag can't be passed
+            # to Otter.
+            if "--allow-chromium-download" in str(e):
+                raise RuntimeError(
+                    "No suitable version of chromium was found; please install chromium by running "
+                    "'playwright install chromium'"
+                )
+            raise e
+
         pdf_path = os.path.splitext(dest)[0] + ".pdf"
         with open(pdf_path, "wb+") as f:
             f.write(pdf)

otter/export/exporters/via_latex.py

@@ -36,7 +36,7 @@ def convert_notebook(cls, nb_path, dest, xecjk=False, **kwargs):
 
         nb = cls.load_notebook(nb_path, filtering=options["filtering"], pagebreaks=options["pagebreaks"])
 
-        nbconvert.TemplateExporter.extra_template_basedirs = [TEMPLATE_DIR]
+        nbconvert.TemplateExporter.extra_template_basedirs = [str(TEMPLATE_DIR)]
         orig_template_name = nbconvert.TemplateExporter.template_name
         nbconvert.TemplateExporter.template_name = options["template"]
 

otter/generate/__init__.py

@@ -1,9 +1,9 @@
 """Autograder configuration generator for Otter-Grader"""
 
+import importlib.resources
 import json
 import os
 import pathlib
-import pkg_resources
 import re
 import yaml
 import zipfile
@@ -22,11 +22,10 @@
 from ..version import __version__
 
 
-DEFAULT_PYTHON_VERSION = "3.9"
+DEFAULT_PYTHON_VERSION = "3.12"
 OTTER_ENV_NAME = "otter-env"
 OTTR_VERSION = "1.5.0"
-TEMPLATE_DIR = pkg_resources.resource_filename(__name__, "templates")
-GENERAL_TEMPLATE_DIR = os.path.join(TEMPLATE_DIR, "general")
+TEMPLATE_DIR = importlib.resources.files(__name__) / "templates"
 
 
 @dataclass
@@ -63,47 +62,31 @@ def to_dict(self):
             environment["dependencies"].extend([
                 "gcc_linux-64",
                 "gxx_linux-64",
-                "r-base>=4.0.0",
-                "r-essentials",
-                "r-devtools",
                 "libgit2",
                 "libgomp",
+                "r-base>=4.0.0",
+                "r-devtools",
+                "r-essentials",
                 "r-gert",
-                "r-usethis",
-                "r-testthat",
-                "r-startup",
                 "r-rmarkdown",
+                "r-startup",
                 "r-stringi",
+                "r-testthat",
+                "r-usethis",
                 f"r-ottr=={OTTR_VERSION}",
             ])
 
+        r_extra = ""
+        if self.is_r:
+            r_extra = ",r"
+
         pip_deps = self.requirements if self.overwrite_requirements else [
-            "datascience",
-            "jupyter_client",
-            "ipykernel",
-            "matplotlib",
-            "pandas",
-            "ipywidgets",
-            "scipy",
-            "seaborn",
-            "scikit-learn",
-            "jinja2",
-            "nbconvert",
-            "nbconvert[webpdf]",
-            "nbformat",
-            "dill",
-            "numpy",
-            "gspread",
-            "pypdf",
-            f"otter-grader=={__version__}",
+            f"otter-grader[grading,plugins{r_extra}]=={__version__}",
             *self.requirements,
         ]
 
         environment["dependencies"].append({"pip": pip_deps})
 
-        if self.is_r:
-            environment["dependencies"][-1]["pip"].append("rpy2")
-
         if self.user_environment:
             environment = merge_conda_environments(
                 self.user_environment, environment, OTTER_ENV_NAME)
@@ -115,20 +98,20 @@ def to_str(self):
 
 
 COMMON_TEMPLATES = [
-    os.path.join(TEMPLATE_DIR, "common", "run_autograder"),
-    os.path.join(TEMPLATE_DIR, "common", "run_otter.py"),
+    TEMPLATE_DIR / "common" / "run_autograder",
+    TEMPLATE_DIR / "common" / "run_otter.py",
 ]
 
 LANGUAGE_BASED_CONFIGURATIONS = {
     "python": {
         "test_file_pattern": "*.py",
         "requirements_filename": "requirements.txt",
-        "templates": [*COMMON_TEMPLATES, os.path.join(TEMPLATE_DIR, "python", "setup.sh")]
+        "templates": [*COMMON_TEMPLATES, TEMPLATE_DIR / "python" / "setup.sh"]
     },
     "r": {
         "test_file_pattern": "*.[Rr]",
         "requirements_filename": "requirements.R",
-        "templates": [*COMMON_TEMPLATES, os.path.join(TEMPLATE_DIR, "r", "setup.sh")]
+        "templates": [*COMMON_TEMPLATES, TEMPLATE_DIR / "r" / "setup.sh"]
     },
 }
 
@@ -234,10 +217,9 @@ def main(
     lang_config = LANGUAGE_BASED_CONFIGURATIONS[ag_config.lang]
 
     templates = {}
-    for template_path in lang_config["templates"]:
-        fn = os.path.basename(template_path)
-        with open(template_path) as f:
-            templates[fn] = Template(f.read())
+    for template in lang_config["templates"]:
+        fn = os.path.basename(template)
+        templates[fn] = Template(template.read_text())
 
     if python_version is not None:
         match = re.match(r"(\d+)\.(\d+)(\.\d+)?", python_version)

otter/grade/containers.py

@@ -1,9 +1,9 @@
 """Docker container management for Otter Grade"""
 
+import importlib.resources
 import json
 import os
 import pathlib
-import pkg_resources
 import shutil
 import tempfile
 import zipfile
@@ -13,6 +13,7 @@
 from textwrap import indent
 from typing import List, Optional
 
+from . import __name__ as pkg_name
 from .utils import OTTER_DOCKER_IMAGE_NAME, merge_scores_to_df, TimeoutException
 
 from ..run.run_autograder.autograder_config import AutograderConfig
@@ -38,7 +39,7 @@ def build_image(ag_zip_path: str, base_image: str, tag: str, config: AutograderC
         ``str``: the tag of the newly-build Docker image
     """
     image = OTTER_DOCKER_IMAGE_NAME + ":" + tag
-    dockerfile_path = pkg_resources.resource_filename(__name__, "Dockerfile")
+    dockerfile_path = str(importlib.resources.files(pkg_name) / "Dockerfile")
 
     LOGGER.info(f"Building image using {base_image} as base image")
 

otter/run/run_autograder/__init__.py

@@ -5,13 +5,14 @@
 import pandas as pd
 import zipfile
 
+from contextlib import nullcontext
 from glob import glob
 
 from .runners import create_runner
 from .utils import capture_run_output, OtterRuntimeError, print_output
 
 from ...version import LOGO_WITH_VERSION
-from ...utils import chdir, loggers, nullcontext, OTTER_CONFIG_FILENAME
+from ...utils import chdir, loggers, OTTER_CONFIG_FILENAME
 
 
 __all__ = ["capture_run_output", "main"]

otter/utils.py

@@ -146,15 +146,6 @@ def get_source(cell):
     raise TypeError(f"Unknown cell source type: {type(source)}")
 
 
-@contextmanager
-def nullcontext():
-    """
-    Yields an empty context. Added because ``contextlib.nullcontext`` was added in Python 3.7, so
-    earlier versions of Python require this patch.
-    """
-    yield
-
-
 @contextmanager
 def load_default_file(provided_fn, default_fn, default_disabled=False):
     """