diff --git a/robotpy_installer/cli_deploy.py b/robotpy_installer/cli_deploy.py index a07faf7..94eb69b 100644 --- a/robotpy_installer/cli_deploy.py +++ b/robotpy_installer/cli_deploy.py @@ -15,7 +15,7 @@ from os.path import join, splitext -from . import pyproject, sshcontroller +from . import pyproject, roborio_utils, sshcontroller from .installer import PipInstallError, PythonMissingError, RobotpyInstaller from .errors import Error from .utils import handle_cli_error, print_err, yesno @@ -327,6 +327,10 @@ def _ensure_requirements( python_exists = False requirements_installed = False + # does c++/java exist + with wrap_ssh_error("removing c++/java user programs"): + cpp_java_exists = roborio_utils.uninstall_cpp_java_lvuser(ssh) + # does python exist with wrap_ssh_error("checking if python exists"): python_exists = ( @@ -370,7 +374,7 @@ def _ensure_requirements( if force_install: requirements_installed = False - if not python_exists or not requirements_installed: + if cpp_java_exists or not python_exists or not requirements_installed: if no_install and not python_exists: raise Error( "python3 was not found on the roboRIO\n" @@ -378,6 +382,13 @@ def _ensure_requirements( "- Use 'python -m robotpy installer install-python' to install python separately" ) + # This also will give more memory + ssh.exec_bash( + ". /etc/profile.d/frc-path.sh", + ". /etc/profile.d/natinst-path.sh", + roborio_utils.kill_robot_cmd, + ) + installer = RobotpyInstaller() with installer.connect_to_robot( project_path=project_path, @@ -385,6 +396,9 @@ def _ensure_requirements( ignore_image_version=ignore_image_version, ssh=ssh, ): + if cpp_java_exists: + roborio_utils.uninstall_cpp_java_admin(installer.ssh) + if not python_exists: try: installer.install_python() @@ -395,6 +409,9 @@ def _ensure_requirements( ) from e if not requirements_installed: + # pip is greedy + installer.ensure_more_memory() + logger.info("Installing project requirements on RoboRIO:") assert project is not None packages = project.get_install_list() diff --git a/robotpy_installer/cli_installer.py b/robotpy_installer/cli_installer.py index a2573b7..aca3b2e 100644 --- a/robotpy_installer/cli_installer.py +++ b/robotpy_installer/cli_installer.py @@ -3,6 +3,7 @@ import shutil import typing +from . import roborio_utils from .utils import handle_cli_error from .installer import ( @@ -151,6 +152,33 @@ def run( installer.uninstall_python() +class InstallerUninstallJavaCpp: + """ + Uninstall FRC Java/C++ programs from a RoboRIO + """ + + def __init__(self, parser: argparse.ArgumentParser) -> None: + _add_ssh_options(parser) + + @handle_cli_error + def run( + self, + project_path: pathlib.Path, + main_file: pathlib.Path, + ignore_image_version: bool, + robot: typing.Optional[str], + ): + installer = RobotpyInstaller() + with installer.connect_to_robot( + project_path=project_path, + main_file=main_file, + robot_or_team=robot, + ignore_image_version=ignore_image_version, + ): + if not roborio_utils.uninstall_cpp_java_lvuser(installer.ssh): + roborio_utils.uninstall_cpp_java_admin(installer.ssh) + + # # Installer pip things # @@ -290,7 +318,7 @@ def run( main_file=main_file, robot_or_team=robot, ignore_image_version=ignore_image_version, - log_disk_usage=False, + log_usage=False, ): installer.pip_list() @@ -347,4 +375,5 @@ class Installer: ("list", InstallerList), ("uninstall", InstallerUninstall), ("uninstall-python", InstallerUninstallPython), + ("uninstall-frc-java-cpp", InstallerUninstallJavaCpp), ] diff --git a/robotpy_installer/cli_sync.py b/robotpy_installer/cli_sync.py index 67d0d97..549e26f 100644 --- a/robotpy_installer/cli_sync.py +++ b/robotpy_installer/cli_sync.py @@ -1,10 +1,15 @@ import argparse +import inspect import logging import os import pathlib +import subprocess import sys +import tempfile -from .utils import handle_cli_error +from packaging.version import Version + +from .utils import handle_cli_error, yesno from .installer import RobotpyInstaller @@ -83,6 +88,27 @@ def run( # parse pyproject.toml to determine the requirements project = pyproject.load(project_path, write_if_missing=True) + # Get the local version and don't accidentally downgrade them + try: + local_robotpy_version = Version(pyproject.robotpy_installed_version()) + if project.robotpy_version < local_robotpy_version: + logger.warning( + "pyproject.toml robotpy version is older than currently installed version" + ) + print() + msg = ( + f"Version currently installed: {local_robotpy_version}\n" + f"Version in `pyproject.toml`: {project.robotpy_version}\n" + "- Should we downgrade robotpy?" + ) + if not yesno(msg): + print( + "Please update your pyproject.toml with the desired version of robotpy" + ) + return False + except pyproject.NoRobotpyError: + pass + packages = project.get_install_list() logger.info("Robot project requirements:") @@ -123,4 +149,24 @@ def run( pip_args.append("--user") pip_args.extend(packages) - os.execv(sys.executable, pip_args) + # POSIX systems are easy, just execv and we're done + if sys.platform != "win32": + os.execv(sys.executable, pip_args) + + with tempfile.NamedTemporaryFile("w", delete=False, suffix=".py") as fp: + fp.write( + inspect.cleandoc( + f""" + import os, subprocess + subprocess.run({pip_args!r}) + print() + input("Install complete, press enter to continue") + os.unlink(__file__) + """ + ) + ) + + print("pip is launching in a new window to complete the installation") + subprocess.Popen( + [sys.executable, fp.name], creationflags=subprocess.CREATE_NEW_CONSOLE + ) diff --git a/robotpy_installer/installer.py b/robotpy_installer/installer.py index e7d6ae9..2dd7af8 100755 --- a/robotpy_installer/installer.py +++ b/robotpy_installer/installer.py @@ -73,6 +73,9 @@ def __init__(self, *, log_startup: bool = True): self._image_version_ok = False self._robot_pip_ok = False + self._webserver_stopped = False + self._webserver_needs_start = False + if log_startup: logger.info("RobotPy Installer %s", __version__) logger.info("-> caching files at %s", self.cache_root) @@ -85,7 +88,7 @@ def connect_to_robot( main_file: pathlib.Path, robot_or_team: typing.Union[None, str, int] = None, ignore_image_version: bool = False, - log_disk_usage: bool = True, + log_usage: bool = True, no_resolve: bool = False, ssh: typing.Optional[SshController] = None, ): @@ -106,13 +109,20 @@ def connect_to_robot( self.ensure_image_version(ignore_image_version) - if log_disk_usage: + if log_usage: self.show_disk_space() + self.show_mem_usage() yield - if log_disk_usage: + if self._webserver_needs_start: + self.ssh.exec_cmd("/etc/init.d/systemWebServer start") + self._webserver_needs_start = False + self._webserver_stopped = False + + if log_usage: self.show_disk_space() + self.show_mem_usage() self._ssh = None @@ -243,6 +253,48 @@ def show_disk_space( return size, used, pct + def show_mem_usage(self): + with catch_ssh_error("checking memory info"): + result = self.ssh.check_output("cat /proc/meminfo") + + total_kb = 0 + available_kb = 0 + found = 0 + + for line in result.strip().splitlines(): + if line.startswith("MemTotal:"): + total_kb = int(line.split()[1]) + found += 1 + elif line.startswith("MemAvailable"): + available_kb = int(line.split()[1]) + found += 1 + + if found == 2: + break + + used_kb = total_kb - available_kb + pct_free = (available_kb / float(total_kb)) * 100.0 + + logger.info( + "-> RoboRIO memory %.1fM/%.1fM (%.0f%% full)", + used_kb / 1000.0, + total_kb / 1000.0, + pct_free, + ) + + def ensure_more_memory(self): + if self._webserver_stopped: + return + + # This takes up a ton of memory and we need the memory... + with catch_ssh_error("Stopping NI webserver"): + result = self.ssh.exec_bash('[ -z "$(ps | grep NIWebServiceContainer)" ]') + if result.returncode != 0: + self.ssh.exec_cmd("/etc/init.d/systemWebServer stop") + self._webserver_needs_start = True + + self._webserver_stopped = True + def ensure_image_version(self, ignore_image_version: bool): if self._image_version_ok: return diff --git a/robotpy_installer/pyproject.py b/robotpy_installer/pyproject.py index 6a86153..9f9957a 100644 --- a/robotpy_installer/pyproject.py +++ b/robotpy_installer/pyproject.py @@ -48,9 +48,20 @@ class RobotPyProjectToml: """ + #: Version of robotpy that is depended on + robotpy_version: Version + + robotpy_extras: typing.List[str] = dataclasses.field(default_factory=list) + #: Requirement for the robotpy meta package -- all RobotPy projects must #: depend on it - robotpy_requires: Requirement + @property + def robotpy_requires(self) -> Requirement: + if self.robotpy_extras: + extras = f"[{','.join(self.robotpy_extras)}]" + else: + extras = "" + return Requirement(f"robotpy{extras}=={self.robotpy_version}") #: Requirements for requires: typing.List[Requirement] = dataclasses.field(default_factory=list) @@ -139,7 +150,7 @@ def load( if not pyproject_path.exists(): if default_if_missing: return RobotPyProjectToml( - robotpy_requires=Requirement(f"robotpy=={robotpy_installed_version()}") + robotpy_version=Version(robotpy_installed_version()) ) if write_if_missing: write_default_pyproject(project_path) @@ -175,13 +186,6 @@ def load( else: robotpy_extras = [str(robotpy_extras_any)] - # Construct the full requirement - robotpy_pkg = "robotpy" - if robotpy_extras: - extras_s = ",".join(robotpy_extras) - robotpy_pkg = f"robotpy[{extras_s}]" - robotpy_requires = Requirement(f"{robotpy_pkg}=={robotpy_version}") - requires_any = robotpy_data.get("requires") if isinstance(requires_any, list): requires = [] @@ -192,7 +196,11 @@ def load( else: requires = [] - return RobotPyProjectToml(robotpy_requires=robotpy_requires, requires=requires) + return RobotPyProjectToml( + robotpy_version=robotpy_version, + robotpy_extras=robotpy_extras, + requires=requires, + ) def are_requirements_met( diff --git a/robotpy_installer/roborio_utils.py b/robotpy_installer/roborio_utils.py new file mode 100644 index 0000000..94abef9 --- /dev/null +++ b/robotpy_installer/roborio_utils.py @@ -0,0 +1,69 @@ +import logging + +from .sshcontroller import SshController + +logger = logging.getLogger("robotpy.installer") + + +java_jars = "/home/lvuser/*.jar" +cpp_exe = "/home/lvuser/frcUserProgram" +robot_command = "/home/lvuser/robotCommand" +static_deploy = "/home/lvuser/deploy" + +third_party_libs = "/usr/local/frc/third-party/lib" + +kill_robot_cmd = "/usr/local/frc/bin/frcKillRobot.sh -t" + + +def uninstall_cpp_java_lvuser(ssh: SshController) -> bool: + """ + Frees up disk space by removing FRC C++/Java programs. This runs as lvuser or admin. + + :returns: True if success, False if uninstall_cpp_java_admin needs to be ran + """ + + logger.info("Clearing FRC C++/Java user programs if present") + + rm_paths = (java_jars, cpp_exe, robot_command) + + ssh.exec_bash( + ". /etc/profile.d/frc-path.sh", + ". /etc/profile.d/natinst-path.sh", + "set -x", + # Kill code only if java jar present + f"[ ! -f {java_jars} ] || {kill_robot_cmd}", + # Kill code only if cpp exe present + f"[ ! -f {cpp_exe} ] || {kill_robot_cmd}", + f"rm -rf {' '.join(rm_paths)}", + check=True, + print_output=True, + ) + + # Check if admin pieces need to run + result = ssh.exec_bash( + '[ -z "$(opkg list-installed frc*-openjdk-*)" ]' + f'[ ! -d {third_party_libs} ] || [ -z "$(ls /usr/local/frc/third-party/lib)" ]', + # This is copied with admin privs, can't delete as lvuser + f"[ ! -d {static_deploy} ]", + ) + return result.returncode == 0 + + +def uninstall_cpp_java_admin(ssh: SshController): + """ + Frees up disk space by removing FRC C++/Java programs. Fails if not ran as admin. + """ + + logger.info("Clearing FRC C++/Java program support") + + rm_paths = (third_party_libs,) + + ssh.exec_bash( + # Remove java ipk + 'opkg remove "frc*-openjdk*"', + # Remove third party libs not used by RobotPy + f"rm -rf {' '.join(rm_paths)}", + bash_opts="ex", + print_output=True, + check=True, + ) diff --git a/robotpy_installer/sshcontroller.py b/robotpy_installer/sshcontroller.py index 7f65be4..806f493 100644 --- a/robotpy_installer/sshcontroller.py +++ b/robotpy_installer/sshcontroller.py @@ -4,6 +4,7 @@ import os from os.path import exists, join, expanduser, split as splitpath from pathlib import Path, PurePath, PurePosixPath +import shlex import socket import sys import typing @@ -113,6 +114,30 @@ def exec_cmd( return SshExecResult(retval, output) + def exec_bash( + self, + /, + *commands: str, + bash_opts: str = "e", + check: bool = False, + get_output: bool = False, + print_output: bool = False, + ) -> SshExecResult: + """ + Executes a single giant shell command and returns the result + """ + + parts = ["/bin/bash"] + if bash_opts: + parts.append(f"-{bash_opts}") + parts.append("-c") + + parts.append(";".join(c for c in commands)) + cmd = shlex.join(parts) + return self.exec_cmd( + cmd, check=check, get_output=get_output, print_output=print_output + ) + def check_output(self, cmd: str, *, print_output: bool = False) -> str: result = self.exec_cmd( cmd,