Merge pull request #480 from pretendWhale/v2.4.0

V2.4.0

.github/workflows/test_ci.yml

@@ -13,6 +13,8 @@ jobs:
           - 3.8
           - 3.9
           - '3.10'
+          - 3.11
+          - 3.12
         test-dir:
           - client
           - server

.pre-commit-config.yaml

@@ -1,9 +1,9 @@
 repos:
   - repo: https://github.com/psf/black
-    rev: 22.12.0
+    rev: 23.12.1
     hooks:
     - id: black
   - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
     - id: flake8

Changelog.md

@@ -1,6 +1,13 @@
 # CHANGELOG
 All notable changes to this project will be documented here.
 
+## [v2.4.0]
+- Fix bug that prevented test results from being returned when a feedback file could not be found (#458)
+- Add support for Python 3.11 and 3.12 (#467)
+- Track test environment setup status and report errors when running tests if environment setup is in progress or raised an error (#468)
+- Update Haskell tester to use [Stack](https://docs.haskellstack.org/en/stable/) to install dependencies (#469)
+- Improve default error message when a test group times out (#470)
+
 ## [v2.3.3]
 - Updated python-ta to 2.6.2 (#454)
 

README.md

@@ -66,7 +66,7 @@ Both the autotester and the API are designed to be run on Ubuntu 20.04 (or suffi
 6. [Configure the autotester](#autotester-configuration-options)
 7. Optionally install additional python versions.
    
-   The `py` (python3) and `pyta` testers can be run using any version of python between versions 3.7 and 3.10. When
+   The `py` (python3) and `pyta` testers can be run using any version of python between versions 3.7 and 3.12. When
    these testers are installed the autotester will search the PATH for available python executables. If you want users
    to be able to run tests with a specific python version, ensure that it is visible in the PATH of both the user running
    the autotester and all users who run tests.

client/.dockerfiles/Dockerfile

@@ -2,11 +2,14 @@ ARG UBUNTU_VERSION
 
 FROM ubuntu:$UBUNTU_VERSION
 
-RUN apt-get update -y && apt-get install -y python3 python3-venv
+RUN apt-get update -y && \
+    apt-get -y install software-properties-common && \
+    add-apt-repository -y ppa:deadsnakes/ppa && \
+    apt-get install -y python3.11 python3.11-venv
 
 COPY ./requirements.txt /requirements.txt
 
-RUN python3 -m venv /markus_venv && \
+RUN python3.11 -m venv /markus_venv && \
     /markus_venv/bin/pip install wheel && \
     /markus_venv/bin/pip install -r /requirements.txt
 

client/autotest_client/__init__.py

@@ -213,6 +213,17 @@ def update_settings(settings_id, user):
 @app.route("/settings/<settings_id>/test", methods=["PUT"])
 @authorize
 def run_tests(settings_id, user):
+    test_settings = json.loads(REDIS_CONNECTION.hget("autotest:settings", key=settings_id))
+    env_status = test_settings.get("_env_status")
+    if env_status == "setup":
+        raise Exception("Setting up test environment. Please try again later.")
+    elif env_status == "error":
+        msg = "Settings Error"
+        settings_error = test_settings.get("_error", "")
+        if settings_error:
+            msg += f": {settings_error}"
+        raise Exception(msg)
+
     test_data = request.json["test_data"]
     categories = request.json["categories"]
     high_priority = request.json.get("request_high_priority")
@@ -221,7 +232,7 @@ def run_tests(settings_id, user):
 
     timeout = 0
 
-    for settings_ in settings(settings_id)["testers"]:
+    for settings_ in test_settings["testers"]:
         for data in settings_["test_data"]:
             timeout += data["timeout"]
 

client/requirements.txt

@@ -1,5 +1,10 @@
-flask==2.2.2
-python-dotenv==0.21.0
-rq==1.11.1
-redis==4.3.4
-jsonschema==4.16.0
+flask==2.2.5;python_version<"3.8"
+flask==2.3.3;python_version>="3.8"
+python-dotenv==0.21.1;python_version<"3.8"
+python-dotenv==1.0.0;python_version>="3.8"
+rq==1.15.1
+redis==5.0.1
+jsonschema==4.17.3;python_version<"3.8"
+jsonschema==4.20.0;python_version>="3.8"
+Werkzeug==2.2.3;python_version<"3.8"
+Werkzeug==2.3.8;python_version>="3.8"

docker-compose.yml

@@ -77,5 +77,5 @@ volumes:
 
 networks:
   markus_dev:
-    external:
-      name: markus_dev
+    name: markus_dev
+    external: true

server/.dockerfiles/Dockerfile

@@ -18,6 +18,10 @@ RUN apt-get update -y && \
                        python3.9-venv \
                        python3.10 \
                        python3.10-venv \
+                       python3.11 \
+                       python3.11-venv \
+                       python3.12 \
+                       python3.12-venv \
                        redis-server \
                        postgresql-client \
                        libpq-dev \
@@ -34,7 +38,7 @@ RUN useradd -ms /bin/bash $LOGIN_USER && \
 
 COPY . /app
 
-RUN python3.10 -m venv /markus_venv && \
+RUN python3.11 -m venv /markus_venv && \
     /markus_venv/bin/pip install wheel && \
     /markus_venv/bin/pip install -r /app/requirements.txt && \
     find /app/autotest_server/testers -name requirements.system -exec {} \; && \

server/autotest_server/__init__.py

@@ -1,3 +1,4 @@
+from __future__ import annotations
 import os
 import sys
 import shutil
@@ -134,9 +135,9 @@ def _get_env_vars(test_username: str) -> Dict[str, str]:
     return env_vars
 
 
-def _get_feedback(test_data, tests_path, test_id):
+def _get_feedback(test_data, tests_path, test_id) -> tuple[dict, str]:
     feedback_files = test_data.get("feedback_file_names", [])
-    feedback = []
+    feedback, feedback_errors = [], []
     for feedback_file in feedback_files:
         feedback_path = os.path.join(tests_path, feedback_file)
         if os.path.isfile(feedback_path):
@@ -155,8 +156,8 @@ def _get_feedback(test_data, tests_path, test_id):
                     }
                 )
         else:
-            raise Exception(f"Cannot find feedback file at '{feedback_path}'.")
-    return feedback
+            feedback_errors.append(feedback_file)
+    return feedback, feedback_errors
 
 
 def _update_env_vars(base_env: Dict, test_env: Dict) -> Dict:
@@ -203,7 +204,8 @@ def _run_test_specs(
                 timeout_expired = None
                 timeout = test_data.get("timeout")
                 try:
-                    env_vars = {**os.environ, **_get_env_vars(test_username), **settings["_env"]}
+                    env = settings.get("_env", {})
+                    env_vars = {**os.environ, **_get_env_vars(test_username), **env}
                     env_vars = _update_env_vars(env_vars, test_env_vars)
                     proc = subprocess.Popen(
                         args,
@@ -215,7 +217,7 @@ def _run_test_specs(
                         stdin=subprocess.PIPE,
                         preexec_fn=set_rlimits_before_test,
                         universal_newlines=True,
-                        env={**os.environ, **env_vars, **settings["_env"]},
+                        env={**os.environ, **env_vars, **env},
                     )
                     try:
                         settings_json = json.dumps({**settings, "test_data": test_data})
@@ -227,13 +229,22 @@ def _run_test_specs(
                         else:
                             _kill_user_processes(test_username)
                         out, err = proc.communicate()
+                        if err == "Killed\n":  # Default message from shell
+                            test_group_name = test_data.get("extra_info", {}).get("name", "").strip()
+                            if test_group_name:
+                                err = f"Tests for {test_group_name} did not complete within time limit ({timeout}s)\n"
+                            else:
+                                err = f"Tests did not complete within time limit ({timeout}s)\n"
                         timeout_expired = timeout
                 except Exception as e:
                     err += "\n\n{}".format(e)
                 finally:
                     duration = int(round(time.time() - start, 3) * 1000)
                     extra_info = test_data.get("extra_info", {})
-                    feedback = _get_feedback(test_data, tests_path, test_id)
+                    feedback, feedback_errors = _get_feedback(test_data, tests_path, test_id)
+                    if feedback_errors:
+                        msg = "Cannot find feedback file(s): " + ", ".join(feedback_errors)
+                        err = err + "\n\n" + msg if err else msg
                     results.append(_create_test_group_result(out, err, duration, extra_info, feedback, timeout_expired))
     return results
 
@@ -355,6 +366,11 @@ def ignore_missing_dir_error(
 
 
 def update_test_settings(user, settings_id, test_settings, file_url):
+    test_settings["_user"] = user
+    test_settings["_last_access"] = int(time.time())
+    test_settings["_env_status"] = "setup"
+    redis_connection().hset("autotest:settings", key=settings_id, value=json.dumps(test_settings))
+
     try:
         settings_dir = os.path.join(TEST_SCRIPT_DIR, str(settings_id))
 
@@ -387,8 +403,10 @@ def update_test_settings(user, settings_id, test_settings, file_url):
             test_settings["testers"][i] = tester_settings
         test_settings["_files"] = files_dir
         test_settings.pop("_error", None)
+        test_settings["_env_status"] = "ready"
     except Exception as e:
         test_settings["_error"] = str(e)
+        test_settings["_env_status"] = "error"
         raise
     finally:
         test_settings["_user"] = user

server/autotest_server/config.py

@@ -14,7 +14,6 @@
 
 
 class _Config:
-
     _replacement_pattern: ClassVar[Pattern] = re.compile(r".*?\${(\w+)}.*?")
     _not_found_key: ClassVar[str] = ""
 

server/autotest_server/testers/haskell/haskell_tester.py

@@ -7,6 +7,8 @@
 from ..tester import Tester, Test, TestError
 from ..specs import TestSpecs
 
+STACK_OPTIONS = ["--resolver=lts-14.27", "--system-ghc", "--allow-different-user"]
+
 
 class HaskellTest(Test):
     def __init__(
@@ -104,11 +106,23 @@ def run_haskell_tests(self) -> Dict[str, List[Dict[str, Union[int, str]]]]:
         haskell_lib = os.path.join(os.path.dirname(os.path.realpath(__file__)), "lib")
         for test_file in self.specs["test_data", "script_files"]:
             with tempfile.NamedTemporaryFile(dir=this_dir) as f:
-                cmd = ["tasty-discover", ".", "_", f.name] + self._test_run_flags(test_file)
+                cmd = [
+                    "stack",
+                    "exec",
+                    *STACK_OPTIONS,
+                    "--",
+                    "tasty-discover",
+                    ".",
+                    "_",
+                    f.name,
+                    *self._test_run_flags(test_file),
+                ]
                 subprocess.run(cmd, stdout=subprocess.DEVNULL, universal_newlines=True, check=True)
                 with tempfile.NamedTemporaryFile(mode="w+", dir=this_dir) as sf:
-                    cmd = ["runghc", "--", f"-i={haskell_lib}", f.name, f"--stats={sf.name}"]
-                    subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, universal_newlines=True)
+                    cmd = ["stack", "runghc", *STACK_OPTIONS, "--", f"-i={haskell_lib}", f.name, f"--stats={sf.name}"]
+                    subprocess.run(
+                        cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, universal_newlines=True, check=True
+                    )
                     results[test_file] = self._parse_test_results(csv.reader(sf))
         return results
 

server/autotest_server/testers/haskell/requirements.system

@@ -1,11 +1,8 @@
 #!/usr/bin/env bash
 
-if ! dpkg -l ghc cabal-install &> /dev/null; then
+if ! dpkg -l ghc cabal-install haskell-stack &> /dev/null; then
   apt-get -y update
-  DEBIAN_FRONTEND=noninteractive apt-get install -y -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' ghc cabal-install
+  DEBIAN_FRONTEND=noninteractive apt-get install -y -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' ghc cabal-install haskell-stack
 fi
 
-# TODO: install these without cabal so they can be properly isolated/uninstalled
-cabal update
-ghc-pkg describe tasty-discover &>/dev/null || cabal install tasty-discover --global
-ghc-pkg describe tasty-quickcheck &>/dev/null || cabal install tasty-quickcheck --global
+stack update

server/autotest_server/testers/haskell/setup.py

@@ -3,7 +3,14 @@
 import subprocess
 
 
+HASKELL_TEST_DEPS = ["tasty-discover", "tasty-quickcheck"]
+
+
 def create_environment(_settings, _env_dir, default_env_dir):
+    resolver = "lts-14.27"
+    cmd = ["stack", "build", "--resolver", resolver, "--system-ghc", *HASKELL_TEST_DEPS]
+    subprocess.run(cmd, check=True)
+
     return {"PYTHON": os.path.join(default_env_dir, "bin", "python3")}
 
 

server/autotest_server/testers/java/java_tester.py

@@ -30,7 +30,6 @@ def run(self):
 
 
 class JavaTester(Tester):
-
     JUNIT_TESTER_JAR = os.path.join(os.path.dirname(__file__), "lib", "junit-platform-console-standalone.jar")
     JUNIT_JUPITER_RESULT = "TEST-junit-jupiter.xml"
     JUNIT_VINTAGE_RESULT = "TEST-junit-vintage.xml"

server/autotest_server/testers/jupyter/setup.py

@@ -18,7 +18,7 @@ def create_environment(settings_, env_dir, _default_env_dir):
 def settings():
     with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "settings_schema.json")) as f:
         settings_ = json.load(f)
-    py_versions = [f"3.{x}" for x in range(7, 11) if shutil.which(f"python3.{x}")]
+    py_versions = [f"3.{x}" for x in range(7, 13) if shutil.which(f"python3.{x}")]
     python_versions = settings_["properties"]["env_data"]["properties"]["python_version"]
     python_versions["enum"] = py_versions
     python_versions["default"] = py_versions[-1]

server/autotest_server/testers/py/lib/sql_helper.py

@@ -144,7 +144,6 @@ def execute_psql_file(
 
 
 class PSQLTest:
-
     connection: ClassVar[Optional[ConnectionType]] = None
 
     SCHEMA_COPY_STR = """

server/autotest_server/testers/py/setup.py

@@ -18,7 +18,7 @@ def create_environment(settings_, env_dir, _default_env_dir):
 def settings():
     with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "settings_schema.json")) as f:
         settings_ = json.load(f)
-    py_versions = [f"3.{x}" for x in range(7, 11) if shutil.which(f"python3.{x}")]
+    py_versions = [f"3.{x}" for x in range(7, 13) if shutil.which(f"python3.{x}")]
     python_versions = settings_["properties"]["env_data"]["properties"]["python_version"]
     python_versions["enum"] = py_versions
     python_versions["default"] = py_versions[-1]

server/autotest_server/testers/pyta/pyta_tester.py

@@ -14,7 +14,6 @@ class PytaReporter(python_ta.reporters.json_reporter.JSONReporter, python_ta.rep
 
 
 class PytaTest(Test):
-
     ERROR_MSGS = {"reported": "{} error(s)"}
 
     def __init__(

server/autotest_server/testers/pyta/requirements.txt

@@ -1,3 +1,3 @@
 python-ta==1.4.2;python_version<"3.8"
-python-ta==2.6.2; python_version>="3.8"
+python-ta==2.7.0; python_version>="3.8"
 isort<5;python_version<"3.8"

server/autotest_server/testers/pyta/setup.py

@@ -18,7 +18,7 @@ def create_environment(settings_, env_dir, _default_env_dir):
 def settings():
     with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "settings_schema.json")) as f:
         settings_ = json.load(f)
-    py_versions = [f"3.{x}" for x in range(7, 11) if shutil.which(f"python3.{x}")]
+    py_versions = [f"3.{x}" for x in range(7, 13) if shutil.which(f"python3.{x}")]
     python_versions = settings_["properties"]["env_data"]["properties"]["python_version"]
     python_versions["enum"] = py_versions
     python_versions["default"] = py_versions[-1]

server/autotest_server/testers/racket/racket_tester.py

@@ -36,7 +36,6 @@ def run(self) -> str:
 
 
 class RacketTester(Tester):
-
     ERROR_MSGS = {"bad_json": "Unable to parse test results: {}"}
 
     def __init__(self, specs, test_class: Type[RacketTest] = RacketTest) -> None:

server/requirements.txt

@@ -1,8 +1,9 @@
-rq==1.11.1
-click==8.1.3
-redis==4.3.4
-pyyaml==6.0
-jsonschema==4.16.0
-requests==2.28.1
-psycopg2-binary==2.9.5
-supervisor==4.2.4
+rq==1.15.1
+click==8.1.7
+redis==5.0.1
+pyyaml==6.0.1
+jsonschema==4.17.3;python_version<"3.8"
+jsonschema==4.20.0;python_version>="3.8"
+requests==2.31.0
+psycopg2-binary==2.9.9
+supervisor==4.2.5