From 4bf4f5489018b168bed4525674c16d7638ebe471 Mon Sep 17 00:00:00 2001 From: Dustin Spicuzza Date: Sun, 24 Dec 2023 02:57:14 -0500 Subject: [PATCH 1/4] Update image match error message to match GradleRIO --- robotpy_installer/installer.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/robotpy_installer/installer.py b/robotpy_installer/installer.py index 305561d..fbd1279 100755 --- a/robotpy_installer/installer.py +++ b/robotpy_installer/installer.py @@ -153,7 +153,12 @@ def roborio_checks( if not ignore_image_version and version not in images: raise ClickException( - f"{name} image {images[-1]} is required! Use --ignore-image-version to install anyways" + f"{name} image {images[-1]} is required!\n" + "\n" + "See https://docs.wpilib.org/en/stable/docs/zero-to-robot/step-3/imaging-your-roborio.html\n" + "for information about upgrading the RoboRIO image.\n" + "\n" + "Use --ignore-image-version to install anyways" ) # From f61d44f31395619436dd1bc681da1dba3d1b8f11 Mon Sep 17 00:00:00 2001 From: Dustin Spicuzza Date: Sun, 24 Dec 2023 02:57:58 -0500 Subject: [PATCH 2/4] sshcontroller conn should be optional --- robotpy_installer/sshcontroller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/robotpy_installer/sshcontroller.py b/robotpy_installer/sshcontroller.py index 5532686..af8e76b 100644 --- a/robotpy_installer/sshcontroller.py +++ b/robotpy_installer/sshcontroller.py @@ -45,7 +45,7 @@ def __init__( hostname: str, username: str, password: str, - conn: typing.Optional[socket.socket], + conn: typing.Optional[socket.socket] = None, ): self.username = username self.password = password From 20008ccc587e7ca463c904398e36f45013ea1133 Mon Sep 17 00:00:00 2001 From: Dustin Spicuzza Date: Sun, 24 Dec 2023 03:00:46 -0500 Subject: [PATCH 3/4] Move pyfrc deploy utilities to robotpy-installer - Sets the stage for #70 --- robotpy_installer/deploy.py | 464 +++++++++++++++++++++++++++++++ robotpy_installer/deploy_info.py | 69 +++++ robotpy_installer/undeploy.py | 80 ++++++ robotpy_installer/utils.py | 14 + setup.cfg | 11 +- 5 files changed, 633 insertions(+), 5 deletions(-) create mode 100644 robotpy_installer/deploy.py create mode 100644 robotpy_installer/deploy_info.py create mode 100644 robotpy_installer/undeploy.py diff --git a/robotpy_installer/deploy.py b/robotpy_installer/deploy.py new file mode 100644 index 0000000..79f5c7d --- /dev/null +++ b/robotpy_installer/deploy.py @@ -0,0 +1,464 @@ +import argparse +import contextlib +import subprocess +import datetime +import socket +import inspect +import json +import os +import sys +import shutil +import tempfile +import threading +import getpass + +from os.path import abspath, basename, dirname, join, splitext +from pathlib import PurePosixPath + +from . import sshcontroller +from .utils import print_err, yesno + +import logging + +logger = logging.getLogger("deploy") + + +def relpath(path): + """Path helper, gives you a path relative to this file""" + return os.path.normpath( + os.path.join(os.path.abspath(os.path.dirname(__file__)), path) + ) + + +@contextlib.contextmanager +def wrap_ssh_error(msg: str): + try: + yield + except sshcontroller.SshExecError as e: + raise sshcontroller.SshExecError(f"{msg}: {str(e)}", e.retval) from e + + +class Deploy: + """ + Uploads your robot code to the robot and executes it immediately + """ + + def __init__(self, parser: argparse.ArgumentParser): + parser.add_argument( + "--builtin", + default=False, + action="store_true", + help="Use pyfrc's builtin tests if no tests are specified", + ) + + parser.add_argument( + "--skip-tests", + action="store_true", + default=False, + help="If specified, don't run tests before uploading code to robot (DANGEROUS)", + ) + + parser.add_argument( + "--debug", + action="store_true", + default=False, + help="If specified, runs the code in debug mode (which only currently enables verbose logging)", + ) + + parser.add_argument( + "--nonstandard", + action="store_true", + default=False, + help="When specified, allows you to deploy code in a file that isn't called robot.py", + ) + + parser.add_argument( + "--nc", + "--netconsole", + action="store_true", + default=False, + help="Attach netconsole listener and show robot stdout (requires DS to be connected)", + ) + + parser.add_argument( + "--nc-ds", + "--netconsole-ds", + action="store_true", + default=False, + help="Attach netconsole listener and show robot stdout (fakes a DS connection)", + ) + + parser.add_argument( + "-n", + "--no-version-check", + action="store_true", + default=False, + help="If specified, don't verify that your local wpilib install matches the version on the robot (not recommended)", + ) + + parser.add_argument( + "--large", + action="store_true", + default=False, + help="If specified, allow uploading large files (> 250k) to the RoboRIO", + ) + + robot_args = parser.add_mutually_exclusive_group() + + robot_args.add_argument( + "--robot", default=None, help="Set hostname or IP address of robot" + ) + + robot_args.add_argument( + "--team", default=None, type=int, help="Set team number to deploy robot for" + ) + + parser.add_argument( + "--no-resolve", + action="store_true", + default=False, + help="If specified, don't do a DNS lookup, allow ssh et al to do it instead", + ) + + def run(self, options, robot_class, **static_options): + # run the test suite before uploading + if not options.skip_tests: + test_args = [sys.executable, sys.modules["__main__"].__file__, "test"] + if options.builtin: + test_args.append("--builtin") + + logger.info("Running tests: %s", " ".join(test_args)) + proc = subprocess.run(test_args) + retval = proc.returncode + if retval != 0: + print_err("ERROR: Your robot tests failed, aborting upload.") + if not sys.stdin.isatty(): + print_err("- Use --skip-tests if you want to upload anyways") + return retval + + print() + if not yesno("- Upload anyways?"): + return retval + + if not yesno("- Are you sure? Your robot code may crash!"): + return retval + + print() + print("WARNING: Uploading code against my better judgement...") + + # upload all files in the robot.py source directory + robot_file = abspath(inspect.getfile(robot_class)) + robot_path = dirname(robot_file) + robot_filename = basename(robot_file) + cfg_filename = join(robot_path, ".deploy_cfg") + + if not options.nonstandard and robot_filename != "robot.py": + print_err( + f"ERROR: Your robot code must be in a file called robot.py (launched from {robot_filename})!" + ) + print_err() + print_err( + "If you really want to do this, then specify the --nonstandard argument" + ) + return 1 + + if not options.large and not self._check_large_files(robot_path): + return 1 + + hostname_or_team = options.robot + if not hostname_or_team and options.team: + hostname_or_team = options.team + + try: + with sshcontroller.ssh_from_cfg( + cfg_filename, + username="lvuser", + password="", + hostname=hostname_or_team, + no_resolve=options.no_resolve, + ) as ssh: + if not self._check_requirements(ssh, options.no_version_check): + return 1 + + if not self._do_deploy(ssh, options, robot_filename, robot_path): + return 1 + + except sshcontroller.SshExecError as e: + print_err("ERROR:", str(e)) + return 1 + + print("\nSUCCESS: Deploy was successful!") + return 0 + + def _generate_build_data(self, robot_path) -> dict: + """ + Generate a deploy.json + """ + + deploy_data = { + "deploy-host": socket.gethostname(), # os.uname doesn't work on systems that use non-unix os + "deploy-user": getpass.getuser(), + "deploy-date": datetime.datetime.now().replace(microsecond=0).isoformat(), + "code-path": robot_path, + } + + # Test if we're in a git repo or not + try: + revParseProcess = subprocess.run( + args=["git", "rev-parse", "--is-inside-work-tree"], + capture_output=True, + ) + in_git_repo = revParseProcess.stdout.decode().strip() == "true" + except FileNotFoundError: + in_git_repo = False + + # If we're in a git repo + if in_git_repo: + try: + hashProc = subprocess.run( + args=["git", "rev-parse", "HEAD"], capture_output=True + ) + + # Describe this repo + descProc = subprocess.run( + args=["git", "describe", "--dirty=-dirty", "--always"], + capture_output=True, + ) + + # Get the branch name + nameProc = subprocess.run( + args=["git", "rev-parse", "--abbrev-ref", "HEAD"], + capture_output=True, + ) + + # Insert this data into our deploy.json dict + deploy_data["git-hash"] = hashProc.stdout.decode().strip() + deploy_data["git-desc"] = descProc.stdout.decode().strip() + deploy_data["git-branch"] = nameProc.stdout.decode().strip() + except subprocess.CalledProcessError as e: + logging.exception(e) + else: + logging.info("Not including git hash in deploy.json: Not a git repo.") + + return deploy_data + + def _check_large_files(self, robot_path): + large_sz = 250000 + + large_files = [] + for fname in self._copy_to_tmpdir(None, robot_path, dry_run=True): + st = os.stat(fname) + if st.st_size > large_sz: + large_files.append((fname, st.st_size)) + + if large_files: + print_err(f"ERROR: large files found (larger than {large_sz} bytes)") + for fname, sz in sorted(large_files): + print_err(f"- {fname} ({sz} bytes)") + + if not yesno("Upload anyways?"): + return False + + return True + + def _check_requirements( + self, ssh: sshcontroller.SshController, no_wpilib_version_check: bool + ) -> bool: + # does python exist + with wrap_ssh_error("checking if python exists"): + if ssh.exec_cmd("[ -x /usr/local/bin/python3 ]").returncode != 0: + print_err( + "ERROR: python3 was not found on the roboRIO: have you installed robotpy?" + ) + print_err() + print_err( + f"See {sys.executable} -m robotpy-installer install-python --help" + ) + return False + + # does wpilib exist and does the version match + with wrap_ssh_error("checking for wpilib version"): + py = ";".join( + [ + "import os.path, site", + "version = 'unknown'", + "v = site.getsitepackages()[0] + '/wpilib/version.py'", + "exec(open(v).read(), globals()) if os.path.exists(v) else False", + "print(version)", + ] + ) + + result = ssh.exec_cmd( + f'/usr/local/bin/python3 -c "{py}"', check=True, get_output=True + ) + assert result.stdout is not None + + wpilib_version = result.stdout.strip() + if wpilib_version == "unknown": + print_err( + "WPILib was not found on the roboRIO: have you installed it on the RoboRIO?" + ) + return False + + print("RoboRIO has WPILib version", wpilib_version) + + try: + from wpilib import __version__ as local_wpilib_version # type: ignore + except ImportError: + local_wpilib_version = "unknown" + + if not no_wpilib_version_check and wpilib_version != local_wpilib_version: + print_err(f"ERROR: expected WPILib version {local_wpilib_version}") + print_err() + print_err("You should either:") + print_err( + "- If the robot version is older, upgrade the RobotPy on your robot" + ) + print_err("- Otherwise, upgrade pyfrc on your computer") + print_err() + print_err( + "Alternatively, you can specify --no-version-check to skip this check" + ) + return False + + return True + + def _do_deploy( + self, + ssh: sshcontroller.SshController, + options, + robot_filename: str, + robot_path: str, + ) -> bool: + # This probably should be configurable... oh well + + deploy_dir = PurePosixPath("/home/lvuser") + py_deploy_subdir = "py" + py_new_deploy_subdir = "py_new" + py_deploy_dir = deploy_dir / py_deploy_subdir + + # note below: deployed_cmd appears that it only can be a single line + + # In 2015, there were stdout/stderr issues. In 2016+, they seem to + # have been fixed, but need to use -u for it to really work properly + + if options.debug: + compileall_flags = "" + deployed_cmd = ( + "env LD_LIBRARY_PATH=/usr/local/frc/lib/ " + f"/usr/local/bin/python3 -u {py_deploy_dir}/{robot_filename} -v run" + ) + deployed_cmd_fname = "robotDebugCommand" + bash_cmd = "/bin/bash -cex" + else: + compileall_flags = "-O" + deployed_cmd = ( + "env LD_LIBRARY_PATH=/usr/local/frc/lib/ " + f"/usr/local/bin/python3 -u -O {py_deploy_dir}/{robot_filename} run" + ) + deployed_cmd_fname = "robotCommand" + bash_cmd = "/bin/bash -ce" + + py_new_deploy_dir = deploy_dir / py_new_deploy_subdir + replace_cmd = f"rm -rf {py_deploy_dir}; mv {py_new_deploy_dir} {py_deploy_dir}" + + with wrap_ssh_error("configuring command"): + ssh.exec_cmd( + f'echo "{deployed_cmd}" > {deploy_dir}/{deployed_cmd_fname}', check=True + ) + + if options.debug: + with wrap_ssh_error("touching frcDebug"): + ssh.exec_cmd("touch /tmp/frcdebug", check=True) + + with wrap_ssh_error("removing stale deploy directory"): + ssh.exec_cmd(f"rm -rf {py_new_deploy_dir}", check=True) + + # Copy the files over, copy to a temporary directory first + # -> this is inefficient, but it's easier in sftp + tmp_dir = tempfile.mkdtemp() + try: + py_tmp_dir = join(tmp_dir, py_new_deploy_subdir) + # Copy robot path contents to new deploy subdir + self._copy_to_tmpdir(py_tmp_dir, robot_path) + + # Copy 'build' artifacts to new deploy subdir + with open(join(py_tmp_dir, "deploy.json"), "w") as outf: + json.dump(self._generate_build_data(robot_path), outf) + + # sftp new deploy subdir to robot + ssh.sftp(py_tmp_dir, deploy_dir, mkdir=True) + finally: + shutil.rmtree(tmp_dir) + + # start the netconsole listener now if requested, *before* we + # actually start the robot code, so we can see all messages + nc_thread = None + if options.nc or options.nc_ds: + nc_thread = self._start_nc(ssh, options) + + # Restart the robot code and we're done! + sshcmd = ( + f"{bash_cmd} '" + f"{replace_cmd};" + f"/usr/local/bin/python3 {compileall_flags} -m compileall -q -r 5 /home/lvuser/py;" + ". /etc/profile.d/frc-path.sh; " + ". /etc/profile.d/natinst-path.sh; " + f"chown -R lvuser:ni {py_deploy_dir}; " + "sync; " + "/usr/local/frc/bin/frcKillRobot.sh -t -r || true" + "'" + ) + + logger.debug("SSH: %s", sshcmd) + + with wrap_ssh_error("starting robot code"): + ssh.exec_cmd(sshcmd, check=True, print_output=True) + + if nc_thread is not None: + nc_thread.join() + + return True + + def _start_nc(self, ssh, options): + from netconsole import run # type: ignore + + nc_event = threading.Event() + nc_thread = threading.Thread( + target=run, + args=(ssh.hostname,), + kwargs=dict(connect_event=nc_event, fakeds=options.nc_ds), + daemon=True, + ) + nc_thread.start() + nc_event.wait(5) + logger.info("Netconsole is listening...") + return nc_thread + + def _copy_to_tmpdir(self, tmp_dir, robot_path, dry_run=False): + upload_files = [] + ignore_exts = frozenset({".pyc", ".whl", ".ipk", ".zip", ".gz", ".wpilog"}) + + for root, dirs, files in os.walk(robot_path): + prefix = root[len(robot_path) + 1 :] + if not dry_run: + os.mkdir(join(tmp_dir, prefix)) + + # skip .svn, .git, .hg, etc directories + for d in dirs[:]: + if d.startswith(".") or d in ("__pycache__", "venv"): + dirs.remove(d) + + # skip .pyc files + for filename in files: + r, ext = splitext(filename) + if ext in ignore_exts or r.startswith("."): + continue + + fname = join(root, filename) + upload_files.append(fname) + + if not dry_run: + shutil.copy(fname, join(tmp_dir, prefix, filename)) + + return upload_files diff --git a/robotpy_installer/deploy_info.py b/robotpy_installer/deploy_info.py new file mode 100644 index 0000000..40d2a62 --- /dev/null +++ b/robotpy_installer/deploy_info.py @@ -0,0 +1,69 @@ +import argparse +import inspect + +import pathlib +import json + +from . import sshcontroller + +from .utils import print_err + + +class DeployInfo: + """ + Displays information about code deployed to robot. + """ + + def __init__(self, parser: argparse.ArgumentParser): + robot_args = parser.add_mutually_exclusive_group() + + robot_args.add_argument( + "--robot", default=None, help="Set hostname or IP address of robot" + ) + + robot_args.add_argument( + "--team", default=None, type=int, help="Set team number to deploy robot for" + ) + + parser.add_argument( + "--no-resolve", + action="store_true", + default=False, + help="If specified, don't do a DNS lookup, allow ssh et al to do it instead", + ) + + def run(self, options, robot_class, **static_options): + robot_file = pathlib.Path(inspect.getfile(robot_class)) + cfg_filename = robot_file.parent / ".deploy_cfg" + + hostname_or_team = options.robot + if not hostname_or_team and options.team: + hostname_or_team = options.team + + try: + with sshcontroller.ssh_from_cfg( + cfg_filename, + username="lvuser", + password="", + hostname=hostname_or_team, + no_resolve=options.no_resolve, + ) as ssh: + result = ssh.exec_cmd( + ( + "[ -f /home/lvuser/py/deploy.json ] && " + "cat /home/lvuser/py/deploy.json || " + "echo {}" + ), + get_output=True, + ) + if not result.stdout: + print("{}") + else: + data = json.loads(result.stdout) + print(json.dumps(data, indent=2, sort_keys=True)) + + except sshcontroller.SshExecError as e: + print_err("ERROR:", str(e)) + return 1 + + return 0 diff --git a/robotpy_installer/undeploy.py b/robotpy_installer/undeploy.py new file mode 100644 index 0000000..d30b6ca --- /dev/null +++ b/robotpy_installer/undeploy.py @@ -0,0 +1,80 @@ +import argparse +import inspect + +from os.path import abspath, dirname, join + +from . import sshcontroller +from .utils import print_err, yesno + + +class Undeploy: + """ + Removes current Python robot code from a RoboRIO + """ + + def __init__(self, parser: argparse.ArgumentParser): + robot_args = parser.add_mutually_exclusive_group() + + robot_args.add_argument( + "--robot", default=None, help="Set hostname or IP address of robot" + ) + + robot_args.add_argument( + "--team", default=None, type=int, help="Set team number to deploy robot for" + ) + + parser.add_argument( + "--no-resolve", + action="store_true", + default=False, + help="If specified, don't do a DNS lookup, allow ssh et al to do it instead", + ) + parser.add_argument( + "--yes", + "-y", + action="store_true", + default=False, + help="Do it without prompting", + ) + + def run(self, options, robot_class, **static_options): + robot_file = abspath(inspect.getfile(robot_class)) + robot_path = dirname(robot_file) + cfg_filename = join(robot_path, ".deploy_cfg") + + if not options.yes: + if not yesno( + "This will stop your robot code and delete it from the RoboRIO. Continue?" + ): + return 1 + + hostname_or_team = options.robot + if not hostname_or_team and options.team: + hostname_or_team = options.team + + try: + with sshcontroller.ssh_from_cfg( + cfg_filename, + username="lvuser", + password="", + hostname=hostname_or_team, + no_resolve=options.no_resolve, + ) as ssh: + # first, turn off the running program + ssh.exec_cmd("/usr/local/frc/bin/frcKillRobot.sh -t") + + # delete the code + ssh.exec_cmd("rm -rf /home/lvuser/py") + + # for good measure, delete the start command too + ssh.exec_cmd( + "rm -f /home/lvuser/robotDebugCommand /home/lvuser/robotCommand" + ) + + except sshcontroller.SshExecError as e: + print_err("ERROR:", str(e)) + return 1 + + print("SUCCESS: Files have been successfully wiped!") + + return 0 diff --git a/robotpy_installer/utils.py b/robotpy_installer/utils.py index 292ea66..6942b60 100644 --- a/robotpy_installer/utils.py +++ b/robotpy_installer/utils.py @@ -146,3 +146,17 @@ def _resolve_addr(hostname): return ip raise Error("Could not find robot at %s" % hostname) + + +def print_err(*args): + print(*args, file=sys.stderr) + + +def yesno(prompt: str) -> bool: + """Returns True if user answers 'y'""" + prompt += " [y/n]" + a = "" + while a not in ["y", "n"]: + a = input(prompt).lower() + + return a == "y" diff --git a/setup.cfg b/setup.cfg index b6d8f4b..5f89edd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,10 +16,6 @@ classifiers = License :: OSI Approved :: BSD License Operating System :: OS Independent Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 Topic :: Software Development Topic :: Scientific/Engineering @@ -30,10 +26,15 @@ packages = find: install_requires = click paramiko + pynetconsole~=2.0.2 setup_requires = setuptools_scm > 6 -python_requires = >=3.6 +python_requires = >=3.8 [options.entry_points] console_scripts = robotpy-installer = robotpy_installer.installer:main +robotpy = + deploy = robotpy_installer.deploy:Deploy + deploy-info = robotpy_installer.deploy_info:DeployInfo + undeploy = robotpy_installer.undeploy:Undeploy From f4b5c48eb53a08d3c18572674c0c65950b165248 Mon Sep 17 00:00:00 2001 From: Dustin Spicuzza Date: Sun, 24 Dec 2023 03:10:29 -0500 Subject: [PATCH 4/4] Update github actions --- .github/workflows/dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index 152e360..8e1f6d5 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -11,7 +11,7 @@ on: jobs: ci: - uses: robotpy/build-actions/.github/workflows/package-pure.yml@v2023 + uses: robotpy/build-actions/.github/workflows/package-pure.yml@v2024 with: enable_sphinx_check: false secrets: