Add tool to update pyproject.toml in subprojects

- Also add CI check to ensure it stays in sync

.github/workflows/dist.yml

@@ -41,6 +41,20 @@ jobs:
     - uses: actions/checkout@v4
     - uses: psf/black@stable
 
+    - uses: actions/setup-python@v4
+      with:
+        python-version: "3.12"
+
+    - name: Install deps
+      shell: bash
+      run: |
+        python -m pip --disable-pip-version-check install -r rdev_requirements.txt
+
+    - name: Check pyproject / rdev synchronization
+      shell: bash
+      run: |
+        ./rdev.sh ci check-pyproject
+
 
   #
   # Build native wheels

devtools/__main__.py

@@ -2,6 +2,7 @@
 
 from .ctx import Context
 from . import ci
+from . import update_pyproject
 
 
 @click.group()
@@ -12,6 +13,7 @@ def main(ctx: click.Context):
 
 
 main.add_command(ci.ci)
+main.add_command(update_pyproject.update_pyproject)
 
 
 @main.command()

devtools/ci.py

@@ -2,20 +2,38 @@
 # CI commands
 #
 
-import inspect
-import pathlib
-import typing
+import sys
 
 import click
 
 from .ctx import Context
+from .update_pyproject import ProjectUpdater
 
 
 @click.group()
 def ci():
     """CI commands"""
 
 
+@ci.command()
+@click.pass_obj
+def check_pyproject(ctx: Context):
+    """
+    Ensures that all pyproject.toml files are in sync with rdev.toml
+    """
+    print("Checking for changes..")
+    updater = ProjectUpdater(ctx)
+    updater.update()
+    if updater.changed:
+        print(
+            "ERROR: please use ./rdev.sh update-pyproject to synchronize pyproject.toml and rdev.toml",
+            file=sys.stderr,
+        )
+        exit(1)
+    else:
+        print("OK")
+
+
 @ci.command()
 @click.option("--no-test", default=False, is_flag=True)
 @click.pass_obj

devtools/config.py

@@ -21,6 +21,10 @@ class Parameters(Model):
     wpilib_bin_version: str
     wpilib_bin_url: str
 
+    robotpy_build_req: str
+
+    exclude_artifacts: typing.Set[str]
+
 
 class UpdateConfig(Model):
     params: Parameters

devtools/ctx.py

@@ -1,4 +1,5 @@
 import pathlib
+import subprocess
 import sysconfig
 import typing
 
@@ -40,3 +41,23 @@ def __init__(self) -> None:
             subprojects[i].name: subprojects[i]
             for i in toposort.toposort_flatten(ti, sort=False)
         }
+
+    def git_commit(self, msg: str, *relpath: str):
+        subprocess.run(
+            ["git", "commit", "-F", "-", "--"] + list(relpath),
+            check=True,
+            cwd=self.root_path,
+            input=msg,
+            text=True,
+        )
+        subprocess.run(
+            ["git", "--no-pager", "log", "-1", "--stat"],
+            check=True,
+            cwd=self.root_path,
+        )
+
+    def git_is_file_dirty(self, relpath: str) -> bool:
+        output = subprocess.check_output(
+            ["git", "status", "--porcelain", relpath], cwd=self.root_path
+        ).decode("utf-8")
+        return output != ""

devtools/subproject.py

@@ -15,9 +15,11 @@ class Subproject:
     def __init__(self, cfg: SubprojectConfig, path: pathlib.Path) -> None:
         self.cfg = cfg
         self.path = path
+        self.pyproject_path = self.path / "pyproject.toml"
         self.name = path.name
 
-        with open(self.path / "pyproject.toml", "rb") as fp:
+        # Use tomli here because it's faster and we just need the data
+        with open(self.pyproject_path, "rb") as fp:
             self.pyproject_data = tomli.load(fp)
 
         self.requires = [

devtools/update_pyproject.py

@@ -0,0 +1,221 @@
+import dataclasses
+import pathlib
+import typing
+
+import click
+from packaging.requirements import Requirement
+from packaging.specifiers import SpecifierSet
+from packaging.version import Version
+import tomlkit
+
+from .ctx import Context
+
+
+@dataclasses.dataclass
+class ProjectInfo:
+    pyproject_toml: pathlib.Path
+    data: tomlkit.TOMLDocument
+    changed: bool = False
+
+
+class ProjectUpdater:
+    """
+    Updates pyproject.toml versions for robotpy-build projects
+    """
+
+    def __init__(self, ctx: Context) -> None:
+        self.ctx = ctx
+        self.cfg = ctx.cfg
+
+        self.commit_changes: typing.Set[str] = set()
+
+        # The required versions for everything
+        # - in theory projects could have different requirements, but in
+        #   practice this is simpler and we haven't had issues
+        self.version_specs: typing.Dict[str, SpecifierSet] = {}
+
+        # robotpy-build is special
+        self.version_specs["robotpy-build"] = SpecifierSet(
+            self.cfg.params.robotpy_build_req
+        )
+
+        # load all the pyproject.toml using tomlkit so we can make changes
+        # and retain all the comments
+        self.subprojects: typing.Dict[str, ProjectInfo] = {}
+        for name, project in self.ctx.subprojects.items():
+            with open(project.pyproject_path, "r") as fp:
+                data = tomlkit.load(fp)
+
+            self.subprojects[name] = ProjectInfo(
+                pyproject_toml=project.pyproject_path, data=data
+            )
+
+            self.version_specs[project.pyproject_name] = SpecifierSet(
+                f"~={project.cfg.min_version}"
+            )
+
+    @property
+    def changed(self) -> bool:
+        return len(self.commit_changes) > 0
+
+    @property
+    def wpilib_bin_version(self) -> str:
+        return self.cfg.params.wpilib_bin_version
+
+    @property
+    def wpilib_bin_url(self) -> str:
+        return self.cfg.params.wpilib_bin_url
+
+    def _update_requirements(
+        self,
+        info: ProjectInfo,
+        pypi_name: str,
+        what: str,
+        reqs: typing.List[str],
+    ) -> bool:
+        requires = list(reqs)
+        changes = []
+        for i, req_str in enumerate(requires):
+            req = Requirement(req_str)
+            # see if the requirement is in our list of managed dependencies; if so
+            # then change it if its different
+            new_spec = self.version_specs.get(req.name)
+            if new_spec is not None and new_spec != req.specifier:
+                old_spec = str(req.specifier)
+                req.specifier = new_spec
+                reqs[i] = str(req)
+                info.changed = True
+                self.commit_changes.add(f"{what}: {req}")
+                changes.append(f"{req.name}: '{old_spec}' => '{new_spec}'")
+
+        if changes:
+            print(f"* {pypi_name} {what}:")
+            for change in changes:
+                print("  -", change)
+            return True
+
+        return False
+
+    def update_requirements(self):
+        for info in self.subprojects.values():
+            data = info.data
+            pypi_name = data["tool"]["robotpy-build"]["metadata"]["name"]
+
+            # update build-system
+            self._update_requirements(
+                info,
+                pypi_name,
+                "build-system.requires",
+                data["build-system"]["requires"],
+            )
+
+            # update tool.robotpy-build.metadata: install_requires
+            self._update_requirements(
+                info,
+                pypi_name,
+                "metadata.install_requires",
+                data["tool"]["robotpy-build"]["metadata"]["install_requires"],
+            )
+
+    def _update_maven(self, info: ProjectInfo):
+        data = info.data
+        iter = list(data["tool"]["robotpy-build"]["wrappers"].items())
+        if "static_libs" in data["tool"]["robotpy-build"]:
+            iter += list(data["tool"]["robotpy-build"]["static_libs"].items())
+        for pkg, wrapper in iter:
+            if (
+                "maven_lib_download" not in wrapper
+                or wrapper["maven_lib_download"]["artifact_id"]
+                in self.cfg.params.exclude_artifacts
+            ):
+                continue
+
+            if wrapper["maven_lib_download"]["repo_url"] != self.wpilib_bin_url:
+                print(
+                    "* ",
+                    pkg,
+                    "repo url:",
+                    wrapper["maven_lib_download"]["repo_url"],
+                    "=>",
+                    self.wpilib_bin_url,
+                )
+                self.commit_changes.add(f"repo updated to {self.wpilib_bin_url}")
+                info.changed = True
+                wrapper["maven_lib_download"]["repo_url"] = self.wpilib_bin_url
+
+            if wrapper["maven_lib_download"]["version"] != self.wpilib_bin_version:
+                print(
+                    "* ",
+                    pkg,
+                    "so version:",
+                    wrapper["maven_lib_download"]["version"],
+                    "=>",
+                    self.wpilib_bin_version,
+                )
+                self.commit_changes.add(f"lib updated to {self.wpilib_bin_version}")
+                info.changed = True
+                wrapper["maven_lib_download"]["version"] = self.wpilib_bin_version
+
+    def update_maven(self):
+        for data in self.subprojects.values():
+            self._update_maven(data)
+
+    def update(self):
+        self.update_maven()
+        self.update_requirements()
+
+    def commit(self):
+        files = []
+
+        # check each file and fail if any are dirty
+        for info in self.subprojects.values():
+            if info.changed:
+                if self.ctx.git_is_file_dirty(str(info.pyproject_toml)):
+                    raise ValueError(f"{info.pyproject_toml} is dirty, aborting!")
+
+        for info in self.subprojects.values():
+            if info.changed:
+                s = tomlkit.dumps(info.data)
+                with open(info.pyproject_toml, "w") as fp:
+                    fp.write(s)
+
+                files.append(str(info.pyproject_toml))
+
+        # Make a single useful commit with our changes
+        msg = "Updated dependencies\n\n"
+        msg += "- " + "\n- ".join(sorted(self.commit_changes))
+
+        self.ctx.git_commit(msg, *files)
+
+
+@click.command()
+@click.option("--commit", default=False, is_flag=True)
+@click.pass_obj
+def update_pyproject(ctx: Context, commit: bool):
+    """
+    Updates pyproject.toml version requirements for all projects.
+
+    It is expected that first the wpilib_bin_url and wpilib_bin_version
+    are updated and pushed as a PR. If that succeeds, the PR is merged
+    to main and then the min_version is updated for each project if
+    needed. That commit should be pushed directly to main along with
+    an appropriate tag.
+    """
+
+    # TODO: Make a bot or something to run this process automatically?
+
+    if commit:
+        print("Making changes...")
+    else:
+        print("Checking for changes...")
+
+    updater = ProjectUpdater(ctx)
+    updater.update()
+
+    if not updater.changed:
+        print(".. no changes found")
+    elif commit:
+        updater.commit()
+        print("Changes committed")
+    else:
+        print("Use --commit to make changes")

rdev.toml

@@ -8,7 +8,12 @@ wpilib_bin_url = "https://frcmaven.wpi.edu/artifactory/release"
 wpilib_bin_version = "2024.1.1-beta-2"
 # wpilib_bin_url = "https://frcmaven.wpi.edu/artifactory/development"
 
-# robotpy_build = "2024.0.0b2"
+# Don't update these artifacts
+exclude_artifacts = [
+    "opencv-cpp"
+]
+
+robotpy_build_req = "<2025.0.0,~=2024.0.0b2"
 
 #
 # Subproject configuration