Skip to content

Commit

Permalink
Merge pull request #230 from robotpy/update-gen
Browse files Browse the repository at this point in the history
Update various generation related tooling
  • Loading branch information
virtuald authored Sep 4, 2024
2 parents 00080e1 + 8276c95 commit cf2c751
Show file tree
Hide file tree
Showing 12 changed files with 317 additions and 120 deletions.
27 changes: 26 additions & 1 deletion docs/tools.rst
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,20 @@ your python package.

This will scan all of your defined includes directories (including those of
downloaded artifacts) and output something you can paste into the ``generate``
key of ``pyproject.toml``.
key of ``pyproject.toml``. By default it will only show files that are not
present in ``pyproject.toml`` -- to show all files use the ``--all`` argument.

Often there are files that you don't want to wrap. You can add them to the
``pyproject.toml`` file and they will be ignored. The list accepts glob patterns
supported by the fnmatch module.

.. code-block:: toml
[tool.robotpy-build]
scan_headers_ignore = [
"ignored_header.h",
"ignore_dir/*",
]
.. _create_gen:

Expand All @@ -40,3 +53,15 @@ python package.
.. code-block:: sh
$ python -m robotpy_build create-imports rpydemo rpydemo._rpydemo
Use the ``--write`` argument to write the file.

To write a list of ``__init__.py`` files, you can specify them in the ``pyproject.toml``
file like so:

.. code-block:: toml
[tool.robotpy-build]
update_init = ["rpydemo rpydemo._rpydemo"]
To actually update the files, run ``python setup.py update_init``.
2 changes: 1 addition & 1 deletion robotpy_build/autowrap/generator_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ def get_function_data(
data = self._default_fn_data
report_data.deferred_signatures.append((fn, is_private))
elif not data.overloads:
report_data.deferred_signatures.append((fn, True))
report_data.deferred_signatures.append((fn, is_private))
else:
# When there is overload data present, we have to actually compute
# the signature of every function
Expand Down
79 changes: 79 additions & 0 deletions robotpy_build/command/_built_env.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
import importlib.util
import os
from os.path import abspath, exists, dirname, join

from setuptools import Command

from .util import get_install_root


class _BuiltEnv(Command):

user_options = [("build-lib=", "d", 'directory to "build" (copy) to')]

def initialize_options(self):
self.build_lib = None

def finalize_options(self):
self.set_undefined_options("build", ("build_lib", "build_lib"))

def setup_built_env(self):

# Gather information for n
data = {"mapping": {}}

# OSX-specific: need to set DYLD_LIBRARY_PATH otherwise modules don't
# work. Luckily, that information was computed when building the
# extensions...
env = os.environ.copy()
dyld_path = set()

# Requires information from build_ext to work
build_ext = self.get_finalized_command("build_ext")
if build_ext.inplace:
data["out"] = get_install_root(self)
else:
data["out"] = self.build_lib

# Ensure that the associated packages can always be found locally
for wrapper in build_ext.wrappers:
pkgdir = wrapper.package_name.split(".")
init_py = abspath(join(self.build_lib, *pkgdir, "__init__.py"))
if exists(init_py):
data["mapping"][wrapper.package_name] = init_py

# Ensure that the built extension can always be found
build_ext.resolve_libs()
for ext in build_ext.extensions:
fname = build_ext.get_ext_filename(ext.name)
data["mapping"][ext.name] = abspath(join(self.build_lib, fname))

rpybuild_libs = getattr(ext, "rpybuild_libs", None)
if rpybuild_libs:
for pth, _ in rpybuild_libs.values():
dyld_path.add(dirname(pth))

# OSX-specific
if dyld_path:
dyld_path = ":".join(dyld_path)
if "DYLD_LIBRARY_PATH" in env:
dyld_path += ":" + env["DYLD_LIBRARY_PATH"]
env["DYLD_LIBRARY_PATH"] = dyld_path

return data, env


class _PackageFinder:
"""
Custom loader to allow loading built modules from their location
in the build directory (as opposed to their install location)
"""

# Set this to mapping returned from _BuiltEnv.setup_built_env
mapping = {}

@classmethod
def find_spec(cls, fullname, path, target=None):
m = cls.mapping.get(fullname)
if m:
return importlib.util.spec_from_file_location(fullname, m)
21 changes: 21 additions & 0 deletions robotpy_build/command/build_ext.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,27 @@ def _spawn(cmd):
# Used in build_pyi
ext.rpybuild_libs = libs

def resolve_libs(self):
# used in _built_env
platform = get_platform()
if platform.os == "osx":
for wrapper in self.wrappers:
wrapper.finalize_extension()

from ..relink_libs import resolve_libs

install_root = get_install_root(self)

for ext in self.extensions:
libs = resolve_libs(
install_root,
ext.rpybuild_wrapper,
self.rpybuild_pkgcfg,
)

# Used in build_pyi
ext.rpybuild_libs = libs

def run(self):
# files need to be generated before building can occur
self.run_command("build_gen")
Expand Down
101 changes: 12 additions & 89 deletions robotpy_build/command/build_pyi.py
Original file line number Diff line number Diff line change
@@ -1,35 +1,29 @@
import importlib.util
import json
import os
from os.path import abspath, exists, dirname, join
from os.path import exists, dirname, join
import subprocess
import sys

import pybind11_stubgen
from setuptools import Command
from distutils.errors import DistutilsError

from .util import get_install_root
try:
from setuptools.errors import BaseError
except ImportError:
from distutils.errors import DistutilsError as BaseError

from ._built_env import _BuiltEnv, _PackageFinder

class GeneratePyiError(DistutilsError):

class GeneratePyiError(BaseError):
pass


class BuildPyi(Command):
class BuildPyi(_BuiltEnv):
base_package: str

command_name = "build_pyi"
description = "Generates pyi files from built extensions"

user_options = [("build-lib=", "d", 'directory to "build" (copy) to')]

def initialize_options(self):
self.build_lib = None

def finalize_options(self):
self.set_undefined_options("build", ("build_lib", "build_lib"))

def run(self):
# cannot build pyi files when cross-compiling
if (
Expand All @@ -40,50 +34,18 @@ def run(self):
return

# Gather information for needed stubs
data = {"mapping": {}, "stubs": []}

# OSX-specific: need to set DYLD_LIBRARY_PATH otherwise modules don't
# work. Luckily, that information was computed when building the
# extensions...
env = os.environ.copy()
dyld_path = set()

# Requires information from build_ext to work
build_ext = self.distribution.get_command_obj("build_ext")
if build_ext.inplace:
data["out"] = get_install_root(self)
else:
data["out"] = self.build_lib

# Ensure that the associated packages can always be found locally
for wrapper in build_ext.wrappers:
pkgdir = wrapper.package_name.split(".")
init_py = abspath(join(self.build_lib, *pkgdir, "__init__.py"))
if exists(init_py):
data["mapping"][wrapper.package_name] = init_py
data, env = self.setup_built_env()
data["stubs"] = []

# Ensure that the built extension can always be found
build_ext = self.get_finalized_command("build_ext")
for ext in build_ext.extensions:
fname = build_ext.get_ext_filename(ext.name)
data["mapping"][ext.name] = abspath(join(self.build_lib, fname))
data["stubs"].append(ext.name)

rpybuild_libs = getattr(ext, "rpybuild_libs", None)
if rpybuild_libs:
for pth, _ in rpybuild_libs.values():
dyld_path.add(dirname(pth))

# Don't do anything if nothing is needed
if not data["stubs"]:
return

# OSX-specific
if dyld_path:
dyld_path = ":".join(dyld_path)
if "DYLD_LIBRARY_PATH" in env:
dyld_path += ":" + env["DYLD_LIBRARY_PATH"]
env["DYLD_LIBRARY_PATH"] = dyld_path

data_json = json.dumps(data)

# Execute in a subprocess in case it crashes
Expand All @@ -101,45 +63,6 @@ def run(self):
pass


class _PackageFinder:
"""
Custom loader to allow loading built modules from their location
in the build directory (as opposed to their install location)
"""

mapping = {}

@classmethod
def find_spec(cls, fullname, path, target=None):
m = cls.mapping.get(fullname)
if m:
return importlib.util.spec_from_file_location(fullname, m)


def generate_pyi(module_name: str, pyi_filename: str):
print("generating", pyi_filename)

pybind11_stubgen.FunctionSignature.n_invalid_signatures = 0
module = pybind11_stubgen.ModuleStubsGenerator(module_name)
module.parse()
if pybind11_stubgen.FunctionSignature.n_invalid_signatures > 0:
print("FAILED to generate pyi for", module_name, file=sys.stderr)
return False

module.write_setup_py = False
with open(pyi_filename, "w") as fp:
fp.write("#\n# AUTOMATICALLY GENERATED FILE, DO NOT EDIT!\n#\n\n")
fp.write("\n".join(module.to_lines()))

typed = join(dirname(pyi_filename), "py.typed")
print("generating", typed)
if not exists(typed):
with open(typed, "w") as fp:
pass

return True


def main():
cfg = json.load(sys.stdin)

Expand Down
73 changes: 73 additions & 0 deletions robotpy_build/command/update_init.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
import json
import os
import subprocess
import sys
import typing

try:
from setuptools.errors import BaseError
except ImportError:
from distutils.errors import DistutilsError as BaseError

from ._built_env import _BuiltEnv, _PackageFinder


class UpdateInitError(BaseError):
pass


class UpdateInit(_BuiltEnv):
update_list: typing.List[str]

command_name = "update_init"
description = (
"Updates __init__.py files using settings from tool.robotpy-build.update_init"
)

def run(self):
# cannot use when cross-compiling
if (
"_PYTHON_HOST_PLATFORM" in os.environ
or "PYTHON_CROSSENV" in os.environ
or not self.update_list
):
return

data, env = self.setup_built_env()
data["update_list"] = self.update_list

data_json = json.dumps(data)

# Execute in a subprocess in case it crashes
args = [sys.executable, "-m", __name__]
try:
subprocess.run(args, input=data_json.encode("utf-8"), env=env, check=True)
except subprocess.CalledProcessError:
raise UpdateInitError(
"Failed to generate .pyi file (see above, or set RPYBUILD_SKIP_PYI=1 to ignore) via %s"
% (args,)
) from None


def main():
cfg = json.load(sys.stdin)

# Configure custom loader
_PackageFinder.mapping = cfg["mapping"]
sys.meta_path.insert(0, _PackageFinder)

from .. import tool

# Update init

for to_update in cfg["update_list"]:

sys.argv = ["<dummy>", "create-imports", "-w"] + to_update.split(" ", 1)

retval = tool.main()
if retval != 0:
break


if __name__ == "__main__":
main()
8 changes: 8 additions & 0 deletions robotpy_build/config/pyproject_toml.py
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,14 @@ class RobotpyBuildConfig(Model):
#: Python package to store version information and robotpy-build metadata in
base_package: str

#: List of headers for the scan-headers tool to ignore
scan_headers_ignore: List[str] = []

#: List of python packages with __init__.py to update when ``python setup.py update_init``
#: is called -- this is an argument to the ``robotpy-build create-imports`` command, and
#: may contain a space and the second argument to create-imports.
update_init: List[str] = []

#:
#: .. seealso:: :class:`.SupportedPlatform`
#:
Expand Down
Loading

0 comments on commit cf2c751

Please sign in to comment.