From 49dbb9d8a5feabd8ef5b01c81837fa8fb20caac9 Mon Sep 17 00:00:00 2001 From: Tommaso Caiazzi Date: Fri, 31 Mar 2023 12:11:14 +0200 Subject: [PATCH 01/34] WiP on waiting startup commands while connecting to a device (#209) --- src/Kathara/manager/docker/DockerMachine.py | 49 ++++++++++++++++++--- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/src/Kathara/manager/docker/DockerMachine.py b/src/Kathara/manager/docker/DockerMachine.py index 983375e7..e780af66 100644 --- a/src/Kathara/manager/docker/DockerMachine.py +++ b/src/Kathara/manager/docker/DockerMachine.py @@ -1,6 +1,9 @@ import logging import re +import select import shlex +import sys +import time from itertools import islice from multiprocessing.dummy import Pool from typing import List, Dict, Generator, Optional, Set, Tuple, Union, Any @@ -77,7 +80,9 @@ "/hostlab/{machine_name}.startup &> /var/log/startup.log; fi", # Placeholder for user commands - "{machine_commands}" + "{machine_commands}", + + "touch /var/log/EOS" ] SHUTDOWN_COMMANDS = [ @@ -402,7 +407,9 @@ def start(self, machine: Machine) -> None: machine.meta['startup_commands'] = new_commands # Build the final startup commands string - startup_commands_string = "; ".join(STARTUP_COMMANDS).format( + startup_commands_string = "; ".join( + STARTUP_COMMANDS if machine.meta['startup_commands'] else STARTUP_COMMANDS[:-2] + STARTUP_COMMANDS[ + :-1]).format( machine_name=machine.name, machine_commands="; ".join(machine.meta['startup_commands']) ) @@ -490,7 +497,7 @@ def _undeploy_machine(self, machine_api_object: docker.models.containers.Contain EventDispatcher.get_instance().dispatch("machine_undeployed", item=machine_api_object) def connect(self, lab_hash: str, machine_name: str, user: str = None, shell: str = None, - logs: bool = False) -> None: + logs: bool = False, wait: bool = True) -> None: """Open a stream to the Docker container specified by machine_name using the specified shell. Args: @@ -499,6 +506,7 @@ def connect(self, lab_hash: str, machine_name: str, user: str = None, shell: str user (str): The name of a current user on the host. shell (str): The path to the desired shell. logs (bool): If True, print the logs of the startup command. + wait (bool): If True, wait the end of the startup commands before giving control to the user. Returns: None @@ -516,7 +524,37 @@ def connect(self, lab_hash: str, machine_name: str, user: str = None, shell: str else: shell = shlex.split(shell) - logging.debug("Connect to device `%s` with shell: %s" % (machine_name, shell)) + logging.debug(f"Connect to device `{machine_name}` with shell: {shell}") + + startup_waited = True + if wait: + logging.debug(f"Waiting startup commands execution for device {machine_name}") + exit_code = 1 + while exit_code != 0: + exec_result = self._exec_run(container, + cmd="cat /var/log/EOS", + stdout=True, + stderr=False, + privileged=False, + detach=False + ) + exit_code = exec_result['exit_code'] + + sys.stdout.write("\033[2J") + sys.stdout.write("\033[0;0H") + sys.stdout.write("Waiting startup commands execution. Press enter to take control of the device...") + sys.stdout.flush() + + to_break, _, _ = select.select([sys.stdin], [], [], 0.1) + if to_break: + startup_waited = False + break + + time.sleep(0.1) + + sys.stdout.write("\033[2J") + sys.stdout.write("\033[0;0H") + sys.stdout.flush() # Get the logs, if the command fails it means that the shell is not found. cat_logs_cmd = "cat /var/log/shared.log /var/log/startup.log" @@ -534,7 +572,8 @@ def connect(self, lab_hash: str, machine_name: str, user: str = None, shell: str if startup_output and logs and Setting.get_instance().print_startup_log: print("--- Startup Commands Log\n") print(startup_output) - print("--- End Startup Commands Log\n") + print("--- End Startup Commands Log\n" if startup_waited + else "--- Executing Other Commands in Background\n") resp = self.client.api.exec_create(container.id, shell, From 9839b245ed6355e2fc4e6e47ce44bd9c9c986016 Mon Sep 17 00:00:00 2001 From: Tommaso Caiazzi Date: Fri, 31 Mar 2023 15:30:39 +0200 Subject: [PATCH 02/34] Add multi-platform support (#209) --- src/Kathara/manager/docker/DockerMachine.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Kathara/manager/docker/DockerMachine.py b/src/Kathara/manager/docker/DockerMachine.py index e780af66..30937929 100644 --- a/src/Kathara/manager/docker/DockerMachine.py +++ b/src/Kathara/manager/docker/DockerMachine.py @@ -1,6 +1,5 @@ import logging import re -import select import shlex import sys import time @@ -526,6 +525,15 @@ def connect(self, lab_hash: str, machine_name: str, user: str = None, shell: str logging.debug(f"Connect to device `{machine_name}` with shell: {shell}") + def wait_user_input_linux(): + import select + to_break, _, _ = select.select([sys.stdin], [], [], 0.1) + return to_break + + def wait_user_input_windows(): + import msvcrt + return msvcrt.kbhit() + startup_waited = True if wait: logging.debug(f"Waiting startup commands execution for device {machine_name}") @@ -545,8 +553,7 @@ def connect(self, lab_hash: str, machine_name: str, user: str = None, shell: str sys.stdout.write("Waiting startup commands execution. Press enter to take control of the device...") sys.stdout.flush() - to_break, _, _ = select.select([sys.stdin], [], [], 0.1) - if to_break: + if utils.exec_by_platform(wait_user_input_linux, wait_user_input_windows, wait_user_input_linux): startup_waited = False break From a2e7bba8236e84a5195fbc7ca328f4bd95bb8025 Mon Sep 17 00:00:00 2001 From: Tommaso Caiazzi Date: Fri, 31 Mar 2023 16:04:34 +0200 Subject: [PATCH 03/34] Fix (#209) --- src/Kathara/manager/docker/DockerMachine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Kathara/manager/docker/DockerMachine.py b/src/Kathara/manager/docker/DockerMachine.py index 30937929..01a0fcce 100644 --- a/src/Kathara/manager/docker/DockerMachine.py +++ b/src/Kathara/manager/docker/DockerMachine.py @@ -407,8 +407,8 @@ def start(self, machine: Machine) -> None: # Build the final startup commands string startup_commands_string = "; ".join( - STARTUP_COMMANDS if machine.meta['startup_commands'] else STARTUP_COMMANDS[:-2] + STARTUP_COMMANDS[ - :-1]).format( + STARTUP_COMMANDS if machine.meta['startup_commands'] else STARTUP_COMMANDS[:-2] + STARTUP_COMMANDS[-1:] + ).format( machine_name=machine.name, machine_commands="; ".join(machine.meta['startup_commands']) ) From 64cb9a4aff73beb73f8062c869787d73854b5dcf Mon Sep 17 00:00:00 2001 From: Tommaso Caiazzi Date: Mon, 3 Apr 2023 10:36:54 +0200 Subject: [PATCH 04/34] Add comments (#209) --- src/Kathara/manager/docker/DockerMachine.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Kathara/manager/docker/DockerMachine.py b/src/Kathara/manager/docker/DockerMachine.py index 01a0fcce..c633c4b4 100644 --- a/src/Kathara/manager/docker/DockerMachine.py +++ b/src/Kathara/manager/docker/DockerMachine.py @@ -526,16 +526,19 @@ def connect(self, lab_hash: str, machine_name: str, user: str = None, shell: str logging.debug(f"Connect to device `{machine_name}` with shell: {shell}") def wait_user_input_linux(): + """Non-blocking input function for Linux and macOS.""" import select to_break, _, _ = select.select([sys.stdin], [], [], 0.1) return to_break def wait_user_input_windows(): + """Return True if a keypress is waiting to be read. Only for Windows.""" import msvcrt return msvcrt.kbhit() startup_waited = True if wait: + # Wait until the startup commands are executed or until the user requests the control over the device. logging.debug(f"Waiting startup commands execution for device {machine_name}") exit_code = 1 while exit_code != 0: @@ -548,17 +551,20 @@ def wait_user_input_windows(): ) exit_code = exec_result['exit_code'] + # To print the message on the same line at each loop sys.stdout.write("\033[2J") sys.stdout.write("\033[0;0H") sys.stdout.write("Waiting startup commands execution. Press enter to take control of the device...") sys.stdout.flush() + # If the user requests the control, break the while loop if utils.exec_by_platform(wait_user_input_linux, wait_user_input_windows, wait_user_input_linux): startup_waited = False break time.sleep(0.1) + # Clean the terminal output sys.stdout.write("\033[2J") sys.stdout.write("\033[0;0H") sys.stdout.flush() From a43865d7e60762dfe1f1cee43b8a14e709ade076 Mon Sep 17 00:00:00 2001 From: Tommaso Caiazzi Date: Mon, 3 Apr 2023 19:22:29 +0200 Subject: [PATCH 05/34] Manage the print of the startup logs on the container (#208) --- src/Kathara/manager/docker/DockerMachine.py | 28 ++++++++------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/src/Kathara/manager/docker/DockerMachine.py b/src/Kathara/manager/docker/DockerMachine.py index c633c4b4..c0950af6 100644 --- a/src/Kathara/manager/docker/DockerMachine.py +++ b/src/Kathara/manager/docker/DockerMachine.py @@ -570,26 +570,20 @@ def wait_user_input_windows(): sys.stdout.flush() # Get the logs, if the command fails it means that the shell is not found. - cat_logs_cmd = "cat /var/log/shared.log /var/log/startup.log" - startup_command = [item for item in shell] - startup_command.extend(['-c', cat_logs_cmd]) - exec_result = self._exec_run(container, - cmd=startup_command, - stdout=True, - stderr=False, - privileged=False, - detach=False - ) - startup_output = exec_result['output'].decode('utf-8') + cat_logs_cmd = "[ -f var/log/shared.log ] && " \ + "(echo '-- Shared Commands --'; cat /var/log/shared.log; echo '-- End Shared Commands --\n');" \ + "[ -f /var/log/startup.log ] && " \ + "(echo '-- Device Commands --'; cat /var/log/startup.log; echo '-- End Device Commands --')" + + command = f"{shell[0]} -c \"echo '--- Startup Commands Log\n';" \ + f"{cat_logs_cmd};" + command += "echo '\n--- End Startup Commands Log\n';" if startup_waited else \ + "echo '\n--- Executing other commands in background\n';" - if startup_output and logs and Setting.get_instance().print_startup_log: - print("--- Startup Commands Log\n") - print(startup_output) - print("--- End Startup Commands Log\n" if startup_waited - else "--- Executing Other Commands in Background\n") + command += f"{shell[0]}\"" resp = self.client.api.exec_create(container.id, - shell, + command, stdout=True, stderr=True, stdin=True, From 6d5e30fd3f279ac6762a3ef98edc8ea546df4a48 Mon Sep 17 00:00:00 2001 From: Tommaso Caiazzi Date: Mon, 3 Apr 2023 20:22:11 +0200 Subject: [PATCH 06/34] Fix logs print (#209) --- src/Kathara/manager/docker/DockerMachine.py | 28 ++++++++++++--------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/Kathara/manager/docker/DockerMachine.py b/src/Kathara/manager/docker/DockerMachine.py index c0950af6..746b76ed 100644 --- a/src/Kathara/manager/docker/DockerMachine.py +++ b/src/Kathara/manager/docker/DockerMachine.py @@ -569,18 +569,22 @@ def wait_user_input_windows(): sys.stdout.write("\033[0;0H") sys.stdout.flush() - # Get the logs, if the command fails it means that the shell is not found. - cat_logs_cmd = "[ -f var/log/shared.log ] && " \ - "(echo '-- Shared Commands --'; cat /var/log/shared.log; echo '-- End Shared Commands --\n');" \ - "[ -f /var/log/startup.log ] && " \ - "(echo '-- Device Commands --'; cat /var/log/startup.log; echo '-- End Device Commands --')" - - command = f"{shell[0]} -c \"echo '--- Startup Commands Log\n';" \ - f"{cat_logs_cmd};" - command += "echo '\n--- End Startup Commands Log\n';" if startup_waited else \ - "echo '\n--- Executing other commands in background\n';" - - command += f"{shell[0]}\"" + + command = "" + if logs: + # Print the startup logs inside the container and open the shell + cat_logs_cmd = "[ -f var/log/shared.log ] && " \ + "(echo '-- Shared Commands --'; cat /var/log/shared.log; " \ + "echo '-- End Shared Commands --\n');" \ + "[ -f /var/log/startup.log ] && " \ + "(echo '-- Device Commands --'; cat /var/log/startup.log; echo '-- End Device Commands --')" + command += f"{shell[0]} -c \"echo '--- Startup Commands Log\n';" \ + f"{cat_logs_cmd};" + command += "echo '\n--- End Startup Commands Log\n';" if startup_waited else \ + f"echo '\n--- Executing other commands in background\n';" + command += f"{shell[0]}\"" + else: + command += f"{shell[0]}" resp = self.client.api.exec_create(container.id, command, From ebc1486c7da5f89f63ee3af44511421313fa5483 Mon Sep 17 00:00:00 2001 From: Tommaso Caiazzi Date: Mon, 3 Apr 2023 20:25:18 +0200 Subject: [PATCH 07/34] Fix ExecCommand to accept non-single-word command strings --- src/Kathara/cli/command/ExecCommand.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Kathara/cli/command/ExecCommand.py b/src/Kathara/cli/command/ExecCommand.py index c0ed9c57..72d342a4 100644 --- a/src/Kathara/cli/command/ExecCommand.py +++ b/src/Kathara/cli/command/ExecCommand.py @@ -78,7 +78,11 @@ def run(self, current_path: str, argv: List[str]) -> None: except (Exception, IOError): lab = Lab(None, path=lab_path) - exec_output = Kathara.get_instance().exec(args['machine_name'], args['command'], lab_hash=lab.hash) + exec_output = Kathara.get_instance().exec( + args['machine_name'], + args['command'] if len(args['command']) > 1 else args['command'].pop(), + lab_hash=lab.hash + ) try: while True: From 99aa99b047661370e450910cf7e6682c780da2f6 Mon Sep 17 00:00:00 2001 From: Tommaso Caiazzi Date: Tue, 4 Apr 2023 14:05:52 +0200 Subject: [PATCH 08/34] Add --wait option to ExecCommand (#209) --- docs/kathara-exec.1.ronn | 3 + src/Kathara/cli/command/ExecCommand.py | 10 +- src/Kathara/foundation/manager/IManager.py | 3 +- src/Kathara/manager/Kathara.py | 5 +- src/Kathara/manager/docker/DockerMachine.py | 97 +++++++++++-------- src/Kathara/manager/docker/DockerManager.py | 5 +- .../manager/kubernetes/KubernetesManager.py | 6 +- src/Kathara/utils.py | 13 +++ 8 files changed, 92 insertions(+), 50 deletions(-) diff --git a/docs/kathara-exec.1.ronn b/docs/kathara-exec.1.ronn index de640907..9dfb168f 100644 --- a/docs/kathara-exec.1.ronn +++ b/docs/kathara-exec.1.ronn @@ -35,6 +35,9 @@ Execute a command in the Kathara device DEVICE_NAME. * `--no-stderr`: Disable stderr of the executed command. +* `--wait`: + Wait startup commands execution. + * `: Name of the device to execute the command into. diff --git a/src/Kathara/cli/command/ExecCommand.py b/src/Kathara/cli/command/ExecCommand.py index 72d342a4..36268d70 100644 --- a/src/Kathara/cli/command/ExecCommand.py +++ b/src/Kathara/cli/command/ExecCommand.py @@ -52,6 +52,13 @@ def __init__(self) -> None: action="store_true", help='Disable stderr of the executed command.', ) + self.parser.add_argument( + '--wait', + dest="wait", + action="store_true", + default=False, + help='Wait startup commands execution.', + ) self.parser.add_argument( 'machine_name', metavar='DEVICE_NAME', @@ -81,7 +88,8 @@ def run(self, current_path: str, argv: List[str]) -> None: exec_output = Kathara.get_instance().exec( args['machine_name'], args['command'] if len(args['command']) > 1 else args['command'].pop(), - lab_hash=lab.hash + lab_hash=lab.hash, + wait=args['wait'] ) try: diff --git a/src/Kathara/foundation/manager/IManager.py b/src/Kathara/foundation/manager/IManager.py index f38ad478..4e27a479 100644 --- a/src/Kathara/foundation/manager/IManager.py +++ b/src/Kathara/foundation/manager/IManager.py @@ -175,7 +175,7 @@ def connect_tty(self, machine_name: str, lab_hash: Optional[str] = None, lab_nam @abstractmethod def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = None, - lab_name: Optional[str] = None) -> Generator[Tuple[bytes, bytes], None, None]: + lab_name: Optional[str] = None, wait: bool = False) -> Generator[Tuple[bytes, bytes], None, None]: """Exec a command on a device in a running network scenario. Args: @@ -183,6 +183,7 @@ def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = command (List[str]): The command to exec on the device. lab_hash (Optional[str]): The hash of the network scenario where the device is deployed. lab_name (Optional[str]): The name of the network scenario where the device is deployed. + wait (bool): If True, wait the end of the startup before executing the command. Returns: Generator[Tuple[bytes, bytes]]: A generator of tuples containing the stdout and stderr in bytes. diff --git a/src/Kathara/manager/Kathara.py b/src/Kathara/manager/Kathara.py index 713da019..d27a1082 100644 --- a/src/Kathara/manager/Kathara.py +++ b/src/Kathara/manager/Kathara.py @@ -200,7 +200,7 @@ def connect_tty(self, machine_name: str, lab_hash: Optional[str] = None, lab_nam self.manager.connect_tty(machine_name, lab_hash, lab_name, shell, logs) def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = None, - lab_name: Optional[str] = None) -> Generator[Tuple[bytes, bytes], None, None]: + lab_name: Optional[str] = None, wait: bool = False) -> Generator[Tuple[bytes, bytes], None, None]: """Exec a command on a device in a running network scenario. Args: @@ -208,6 +208,7 @@ def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = command (List[str]): The command to exec on the device. lab_hash (Optional[str]): The hash of the network scenario where the device is deployed. lab_name (Optional[str]): The name of the network scenario where the device is deployed. + wait (bool): If True, wait the end of the startup before executing the command. Returns: Generator[Tuple[bytes, bytes]]: A generator of tuples containing the stdout and stderr in bytes. @@ -215,7 +216,7 @@ def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = Raises: InvocationError: If a running network scenario hash or name is not specified. """ - return self.manager.exec(machine_name, command, lab_hash, lab_name) + return self.manager.exec(machine_name, command, lab_hash, lab_name, wait) def copy_files(self, machine: Machine, guest_to_host: Dict[str, io.IOBase]) -> None: """Copy files on a running device in the specified paths. diff --git a/src/Kathara/manager/docker/DockerMachine.py b/src/Kathara/manager/docker/DockerMachine.py index 746b76ed..56ca80a1 100644 --- a/src/Kathara/manager/docker/DockerMachine.py +++ b/src/Kathara/manager/docker/DockerMachine.py @@ -525,50 +525,14 @@ def connect(self, lab_hash: str, machine_name: str, user: str = None, shell: str logging.debug(f"Connect to device `{machine_name}` with shell: {shell}") - def wait_user_input_linux(): - """Non-blocking input function for Linux and macOS.""" - import select - to_break, _, _ = select.select([sys.stdin], [], [], 0.1) - return to_break - - def wait_user_input_windows(): - """Return True if a keypress is waiting to be read. Only for Windows.""" - import msvcrt - return msvcrt.kbhit() - - startup_waited = True + startup_waited = False if wait: - # Wait until the startup commands are executed or until the user requests the control over the device. - logging.debug(f"Waiting startup commands execution for device {machine_name}") - exit_code = 1 - while exit_code != 0: - exec_result = self._exec_run(container, - cmd="cat /var/log/EOS", - stdout=True, - stderr=False, - privileged=False, - detach=False - ) - exit_code = exec_result['exit_code'] - - # To print the message on the same line at each loop - sys.stdout.write("\033[2J") - sys.stdout.write("\033[0;0H") - sys.stdout.write("Waiting startup commands execution. Press enter to take control of the device...") - sys.stdout.flush() - - # If the user requests the control, break the while loop - if utils.exec_by_platform(wait_user_input_linux, wait_user_input_windows, wait_user_input_linux): - startup_waited = False - break - - time.sleep(0.1) - - # Clean the terminal output - sys.stdout.write("\033[2J") - sys.stdout.write("\033[0;0H") - sys.stdout.flush() + startup_waited = self._waiting_startup_execution(container, machine_name) + # Clean the terminal output + sys.stdout.write("\033[2J") + sys.stdout.write("\033[0;0H") + sys.stdout.flush() command = "" if logs: @@ -610,8 +574,46 @@ def cmd_connect(): utils.exec_by_platform(tty_connect, cmd_connect, tty_connect) + def _waiting_startup_execution(self, container: docker.models.containers.Container, machine_name: str): + """Wait until the startup commands are executed or until the user requests the control over the device. + + Args: + container (str): TThe Docker container to wait. + machine_name (str): The name of the device to connect. + + Returns: + bool: False if the user requests the control before the ending of the startup. Else, True. + """ + logging.debug(f"Waiting startup commands execution for device {machine_name}") + exit_code = 1 + startup_waited = True + while exit_code != 0: + exec_result = self._exec_run(container, + cmd="cat /var/log/EOS", + stdout=True, + stderr=False, + privileged=False, + detach=False + ) + exit_code = exec_result['exit_code'] + + # To print the message on the same line at each loop + sys.stdout.write("\033[2J") + sys.stdout.write("\033[0;0H") + sys.stdout.write("Waiting startup commands execution. Press enter to take control of the device...") + sys.stdout.flush() + + # If the user requests the control, break the while loop + if utils.exec_by_platform(utils.wait_user_input_linux, utils.wait_user_input_windows, + utils.wait_user_input_linux): + startup_waited = False + break + + time.sleep(0.1) + return startup_waited + def exec(self, lab_hash: str, machine_name: str, command: Union[str, List], user: str = None, - tty: bool = True) -> Generator[Tuple[bytes, bytes], None, None]: + tty: bool = True, wait: bool = False) -> Generator[Tuple[bytes, bytes], None, None]: """Execute the command on the Docker container specified by the lab_hash and the machine_name. Args: @@ -620,6 +622,7 @@ def exec(self, lab_hash: str, machine_name: str, command: Union[str, List], user user (str): The name of a current user on the host. command (Union[str, List]): The command to execute. tty (bool): If True, open a new tty. + wait (bool): If True, wait the end of the startup before executing the command. Returns: Generator[Tuple[bytes, bytes]]: A generator of tuples containing the stdout and stderr in bytes. @@ -634,6 +637,14 @@ def exec(self, lab_hash: str, machine_name: str, command: Union[str, List], user raise MachineNotFoundError("The specified device `%s` is not running." % machine_name) container = containers.pop() + if wait: + startup_waited = self._waiting_startup_execution(container, machine_name) + + # Clean the terminal output + sys.stdout.write("\033[2J") + sys.stdout.write("\033[0;0H") + sys.stdout.flush() + command = shlex.split(command) if type(command) == str else command exec_result = self._exec_run(container, cmd=command, diff --git a/src/Kathara/manager/docker/DockerManager.py b/src/Kathara/manager/docker/DockerManager.py index 31cf1b66..72870454 100644 --- a/src/Kathara/manager/docker/DockerManager.py +++ b/src/Kathara/manager/docker/DockerManager.py @@ -324,7 +324,7 @@ def connect_tty(self, machine_name: str, lab_hash: Optional[str] = None, lab_nam @privileged def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = None, - lab_name: Optional[str] = None) -> Generator[Tuple[bytes, bytes], None, None]: + lab_name: Optional[str] = None, wait: bool = False) -> Generator[Tuple[bytes, bytes], None, None]: """Exec a command on a device in a running network scenario. Args: @@ -332,6 +332,7 @@ def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = command (List[str]): The command to exec on the device. lab_hash (Optional[str]): The hash of the network scenario where the device is deployed. lab_name (Optional[str]): The name of the network scenario where the device is deployed. + wait (bool): If True, wait the end of the startup before executing the command. Returns: Generator[Tuple[bytes, bytes]]: A generator of tuples containing the stdout and stderr in bytes. @@ -346,7 +347,7 @@ def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = if lab_name: lab_hash = utils.generate_urlsafe_hash(lab_name) - return self.docker_machine.exec(lab_hash, machine_name, command, user=user_name, tty=False) + return self.docker_machine.exec(lab_hash, machine_name, command, user=user_name, tty=False, wait=wait) @privileged def copy_files(self, machine: Machine, guest_to_host: Dict[str, io.IOBase]) -> None: diff --git a/src/Kathara/manager/kubernetes/KubernetesManager.py b/src/Kathara/manager/kubernetes/KubernetesManager.py index 10da678d..8dee3c0c 100644 --- a/src/Kathara/manager/kubernetes/KubernetesManager.py +++ b/src/Kathara/manager/kubernetes/KubernetesManager.py @@ -328,7 +328,7 @@ def connect_tty(self, machine_name: str, lab_hash: Optional[str] = None, lab_nam ) def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = None, - lab_name: Optional[str] = None) -> Generator[Tuple[bytes, bytes], None, None]: + lab_name: Optional[str] = None, wait: bool = False) -> Generator[Tuple[bytes, bytes], None, None]: """Exec a command on a device in a running network scenario. Args: @@ -336,6 +336,7 @@ def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = command (List[str]): The command to exec on the device. lab_hash (Optional[str]): The hash of the network scenario where the device is deployed. lab_name (Optional[str]): The name of the network scenario where the device is deployed. + wait (bool): If True, wait the end of the startup before executing the command. No effect on Megalos. Returns: Generator[Tuple[bytes, bytes]]: A generator of tuples containing the stdout and stderr in bytes. @@ -346,6 +347,9 @@ def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = if not lab_hash and not lab_name: raise InvocationError("You must specify a running network scenario hash or name.") + if wait: + logging.warning("Wait option has no effect on Megalos.") + if lab_name: lab_hash = utils.generate_urlsafe_hash(lab_name) diff --git a/src/Kathara/utils.py b/src/Kathara/utils.py index 37c284ef..7ab25baa 100644 --- a/src/Kathara/utils.py +++ b/src/Kathara/utils.py @@ -313,3 +313,16 @@ def pack_files_for_tar(guest_to_host: Dict) -> bytes: tar_data = temp_file.read() return tar_data + + +def wait_user_input_linux(): + """Non-blocking input function for Linux and macOS.""" + import select + to_break, _, _ = select.select([sys.stdin], [], [], 0.1) + return to_break + + +def wait_user_input_windows(): + """Return True if a keypress is waiting to be read. Only for Windows.""" + import msvcrt + return msvcrt.kbhit() From a423a91cd5f937f6f1688ff49a1ba08d94b29174 Mon Sep 17 00:00:00 2001 From: Tommaso Caiazzi Date: Tue, 4 Apr 2023 15:40:01 +0200 Subject: [PATCH 09/34] Add unit tests to the managers for exec and connect --- tests/cli/exec_command_test.py | 43 +++++- tests/manager/docker/docker_manager_test.py | 141 ++++++++++++++++++ .../kubernetes/kubernetes_manager_test.py | 110 ++++++++++++++ 3 files changed, 287 insertions(+), 7 deletions(-) diff --git a/tests/cli/exec_command_test.py b/tests/cli/exec_command_test.py index 54856dae..3718b9eb 100644 --- a/tests/cli/exec_command_test.py +++ b/tests/cli/exec_command_test.py @@ -27,9 +27,26 @@ def test_run_no_params(mock_stderr_write, mock_stdout_write, mock_lab, mock_pars mock_parse_lab.return_value = mock_lab mock_exec.return_value = exec_output command = ExecCommand() + command.run('.', ['pc1', ['test', 'command']]) + mock_parse_lab.assert_called_once_with(os.getcwd()) + mock_exec.assert_called_once_with("pc1", ['test', 'command'], lab_hash=mock_lab.hash, wait=False) + mock_stdout_write.assert_called_once_with('stdout') + mock_stderr_write.assert_called_once_with('stderr') + + +@mock.patch("src.Kathara.manager.Kathara.Kathara.exec") +@mock.patch("src.Kathara.parser.netkit.LabParser.LabParser.parse") +@mock.patch("src.Kathara.model.Lab.Lab") +@mock.patch('sys.stdout.write') +@mock.patch('sys.stderr.write') +def test_run_no_params_single_string_command(mock_stderr_write, mock_stdout_write, mock_lab, mock_parse_lab, mock_exec, + exec_output): + mock_parse_lab.return_value = mock_lab + mock_exec.return_value = exec_output + command = ExecCommand() command.run('.', ['pc1', 'test command']) mock_parse_lab.assert_called_once_with(os.getcwd()) - mock_exec.assert_called_once_with("pc1", ['test command'], lab_hash=mock_lab.hash) + mock_exec.assert_called_once_with("pc1", 'test command', lab_hash=mock_lab.hash, wait=False) mock_stdout_write.assert_called_once_with('stdout') mock_stderr_write.assert_called_once_with('stderr') @@ -46,7 +63,7 @@ def test_run_with_directory_absolute_path(mock_stderr_write, mock_stdout_write, command = ExecCommand() command.run('.', ['-d', '/test/path', 'pc1', 'test command']) mock_parse_lab.assert_called_once_with('/test/path') - mock_exec.assert_called_once_with("pc1", ['test command'], lab_hash=mock_lab.hash) + mock_exec.assert_called_once_with("pc1", 'test command', lab_hash=mock_lab.hash, wait=False) mock_stdout_write.assert_called_once_with('stdout') mock_stderr_write.assert_called_once_with('stderr') @@ -63,7 +80,7 @@ def test_run_with_directory_relative_path(mock_stderr_write, mock_stdout_write, command = ExecCommand() command.run('.', ['-d', 'test/path', 'pc1', 'test command']) mock_parse_lab.assert_called_once_with(os.path.join(os.getcwd(), 'test/path')) - mock_exec.assert_called_once_with("pc1", ['test command'], lab_hash=mock_lab.hash) + mock_exec.assert_called_once_with("pc1", 'test command', lab_hash=mock_lab.hash, wait=False) mock_stdout_write.assert_called_once_with('stdout') mock_stderr_write.assert_called_once_with('stderr') @@ -78,7 +95,7 @@ def test_run_with_v_option(mock_stderr_write, mock_stdout_write, mock_parse_lab, command = ExecCommand() command.run('.', ['-v', 'pc1', 'test command']) assert not mock_parse_lab.called - mock_exec.assert_called_once_with("pc1", ['test command'], lab_hash=lab.hash) + mock_exec.assert_called_once_with("pc1", 'test command', lab_hash=lab.hash, wait=False) mock_stdout_write.assert_called_once_with('stdout') mock_stderr_write.assert_called_once_with('stderr') @@ -94,7 +111,7 @@ def test_run_no_stdout(mock_stderr_write, mock_stdout_write, mock_lab, mock_pars command = ExecCommand() command.run('.', ['--no-stdout', 'pc1', 'test command']) mock_parse_lab.assert_called_once_with(os.getcwd()) - mock_exec.assert_called_once_with("pc1", ['test command'], lab_hash=mock_lab.hash) + mock_exec.assert_called_once_with("pc1", 'test command', lab_hash=mock_lab.hash, wait=False) assert not mock_stdout_write.called mock_stderr_write.assert_called_once_with('stderr') @@ -110,7 +127,7 @@ def test_run_no_stderr(mock_stderr_write, mock_stdout_write, mock_lab, mock_pars command = ExecCommand() command.run('.', ['--no-stderr', 'pc1', 'test command']) mock_parse_lab.assert_called_once_with(os.getcwd()) - mock_exec.assert_called_once_with("pc1", ['test command'], lab_hash=mock_lab.hash) + mock_exec.assert_called_once_with("pc1", 'test command', lab_hash=mock_lab.hash, wait=False) mock_stdout_write.assert_called_once_with('stdout') assert not mock_stderr_write.called @@ -127,6 +144,18 @@ def test_run_no_stdout_no_stderr(mock_stderr_write, mock_stdout_write, mock_lab, command = ExecCommand() command.run('.', ['--no-stdout', '--no-stderr', 'pc1', 'test command']) mock_parse_lab.assert_called_once_with(os.getcwd()) - mock_exec.assert_called_once_with("pc1", ['test command'], lab_hash=mock_lab.hash) + mock_exec.assert_called_once_with("pc1", 'test command', lab_hash=mock_lab.hash, wait=False) assert not mock_stdout_write.called assert not mock_stderr_write.called + + +@mock.patch("src.Kathara.manager.Kathara.Kathara.exec") +@mock.patch("src.Kathara.parser.netkit.LabParser.LabParser.parse") +@mock.patch("src.Kathara.model.Lab.Lab") +def test_run_wait(mock_lab, mock_parse_lab, mock_exec, exec_output): + mock_parse_lab.return_value = mock_lab + mock_exec.return_value = exec_output + command = ExecCommand() + command.run('.', ['--wait', 'pc1', ['test', 'command']]) + mock_parse_lab.assert_called_once_with(os.getcwd()) + mock_exec.assert_called_once_with("pc1", ['test', 'command'], lab_hash=mock_lab.hash, wait=True) diff --git a/tests/manager/docker/docker_manager_test.py b/tests/manager/docker/docker_manager_test.py index e99d1e90..2e494d01 100644 --- a/tests/manager/docker/docker_manager_test.py +++ b/tests/manager/docker/docker_manager_test.py @@ -467,6 +467,147 @@ def test_wipe_all_users_and_shared_cd(mock_setting_get_instance, mock_get_curren mock_wipe_links.assert_called_once_with(user=None) +# +# TEST: connect_tty +# +@mock.patch("src.Kathara.utils.get_current_user_name") +@mock.patch("src.Kathara.manager.docker.DockerMachine.DockerMachine.connect") +def test_connect_tty_lab_hash(mock_connect, mock_get_current_user_name, docker_manager, default_device): + mock_get_current_user_name.return_value = "kathara_user" + + docker_manager.connect_tty(default_device.name, + lab_hash=default_device.lab.hash) + + mock_connect.assert_called_once_with(lab_hash=default_device.lab.hash, + machine_name=default_device.name, + user="kathara_user", + shell=None, + logs=False) + + +@mock.patch("src.Kathara.utils.get_current_user_name") +@mock.patch("src.Kathara.manager.docker.DockerMachine.DockerMachine.connect") +def test_connect_tty_lab_name(mock_connect, mock_get_current_user_name, docker_manager, default_device): + mock_get_current_user_name.return_value = "kathara_user" + + docker_manager.connect_tty(default_device.name, + lab_name=default_device.lab.name) + + mock_connect.assert_called_once_with(lab_hash=generate_urlsafe_hash(default_device.lab.name), + machine_name=default_device.name, + user="kathara_user", + shell=None, + logs=False) + + +@mock.patch("src.Kathara.utils.get_current_user_name") +@mock.patch("src.Kathara.manager.docker.DockerMachine.DockerMachine.connect") +def test_connect_tty_with_custom_shell(mock_connect, mock_get_current_user_name, docker_manager, default_device): + mock_get_current_user_name.return_value = "kathara_user" + + docker_manager.connect_tty(default_device.name, + lab_hash=default_device.lab.hash, + shell="/usr/bin/zsh") + + mock_connect.assert_called_once_with(lab_hash=default_device.lab.hash, + machine_name=default_device.name, + user="kathara_user", + shell="/usr/bin/zsh", + logs=False) + + +@mock.patch("src.Kathara.utils.get_current_user_name") +@mock.patch("src.Kathara.manager.docker.DockerMachine.DockerMachine.connect") +def test_connect_tty_with_logs(mock_connect, mock_get_current_user_name, docker_manager, default_device): + mock_get_current_user_name.return_value = "kathara_user" + + docker_manager.connect_tty(default_device.name, + lab_hash=default_device.lab.hash, + logs=True) + + mock_connect.assert_called_once_with(lab_hash=default_device.lab.hash, + machine_name=default_device.name, + user="kathara_user", + shell=None, + logs=True) + + +@mock.patch("src.Kathara.utils.get_current_user_name") +@mock.patch("src.Kathara.manager.docker.DockerMachine.DockerMachine.connect") +def test_connect_tty_error(mock_connect, mock_get_current_user_name, docker_manager, default_device): + mock_get_current_user_name.return_value = "kathara_user" + + with pytest.raises(InvocationError): + docker_manager.connect_tty(default_device.name) + + assert not mock_connect.called + + +# +# TEST: exec +# +@mock.patch("src.Kathara.utils.get_current_user_name") +@mock.patch("src.Kathara.manager.docker.DockerMachine.DockerMachine.exec") +def test_exec_lab_hash(mock_exec, mock_get_current_user_name, docker_manager, default_device): + mock_get_current_user_name.return_value = "kathara_user" + + docker_manager.exec(default_device.name, ["test", "command"], lab_hash=default_device.lab.hash) + + mock_exec.assert_called_once_with( + default_device.lab.hash, + default_device.name, + ["test", "command"], + user="kathara_user", + tty=False, + wait=False + ) + + +@mock.patch("src.Kathara.utils.get_current_user_name") +@mock.patch("src.Kathara.manager.docker.DockerMachine.DockerMachine.exec") +def test_exec_lab_name(mock_exec, mock_get_current_user_name, docker_manager, default_device): + mock_get_current_user_name.return_value = "kathara_user" + + docker_manager.exec(default_device.name, ["test", "command"], lab_name=default_device.lab.name) + + mock_exec.assert_called_once_with( + generate_urlsafe_hash(default_device.lab.name), + default_device.name, + ["test", "command"], + user="kathara_user", + tty=False, + wait=False + ) + + +@mock.patch("src.Kathara.utils.get_current_user_name") +@mock.patch("src.Kathara.manager.docker.DockerMachine.DockerMachine.exec") +def test_exec_wait(mock_exec, mock_get_current_user_name, docker_manager, default_device): + mock_get_current_user_name.return_value = "kathara_user" + + docker_manager.exec(default_device.name, ["test", "command"], lab_hash=default_device.lab.hash, wait=True) + + mock_exec.assert_called_once_with( + default_device.lab.hash, + default_device.name, + ["test", "command"], + user="kathara_user", + tty=False, + wait=True + ) + + +@mock.patch("src.Kathara.utils.get_current_user_name") +@mock.patch("src.Kathara.manager.docker.DockerMachine.DockerMachine.exec") +def test_exec_invocation_error(mock_exec, mock_get_current_user_name, docker_manager, default_device): + mock_get_current_user_name.return_value = "kathara_user" + + with pytest.raises(InvocationError): + docker_manager.exec(default_device.name, ["test", "command"]) + + assert not mock_exec.called + + # # TEST: get_machine_api_object # diff --git a/tests/manager/kubernetes/kubernetes_manager_test.py b/tests/manager/kubernetes/kubernetes_manager_test.py index 82364bff..b2b5b7e2 100644 --- a/tests/manager/kubernetes/kubernetes_manager_test.py +++ b/tests/manager/kubernetes/kubernetes_manager_test.py @@ -452,6 +452,116 @@ def test_undeploy_lab_selected_machines(mock_undeploy_machine, mock_undeploy_lin assert not mock_namespace_undeploy.called +# +# TEST: connect_tty +# +@mock.patch("src.Kathara.manager.kubernetes.KubernetesMachine.KubernetesMachine.connect") +def test_connect_tty_lab_hash(mock_connect, kubernetes_manager, default_device): + kubernetes_manager.connect_tty(default_device.name, + lab_hash=default_device.lab.hash) + + mock_connect.assert_called_once_with(lab_hash=default_device.lab.hash.lower(), + machine_name=default_device.name, + shell=None, + logs=False) + + +@mock.patch("src.Kathara.manager.kubernetes.KubernetesMachine.KubernetesMachine.connect") +def test_connect_tty_lab_name(mock_connect, kubernetes_manager, default_device): + kubernetes_manager.connect_tty(default_device.name, + lab_name=default_device.lab.name) + + mock_connect.assert_called_once_with(lab_hash=generate_urlsafe_hash(default_device.lab.name).lower(), + machine_name=default_device.name, + shell=None, + logs=False) + + +@mock.patch("src.Kathara.manager.kubernetes.KubernetesMachine.KubernetesMachine.connect") +def test_connect_tty_custom_shell(mock_connect, kubernetes_manager, default_device): + kubernetes_manager.connect_tty(default_device.name, + lab_hash=default_device.lab.hash, + shell='/usr/bin/zsh') + + mock_connect.assert_called_once_with(lab_hash=default_device.lab.hash.lower(), + machine_name=default_device.name, + shell='/usr/bin/zsh', + logs=False) + + +@mock.patch("src.Kathara.manager.kubernetes.KubernetesMachine.KubernetesMachine.connect") +def test_connect_tty_with_logs(mock_connect, kubernetes_manager, default_device): + kubernetes_manager.connect_tty(default_device.name, + lab_hash=default_device.lab.hash, + logs=True) + + mock_connect.assert_called_once_with(lab_hash=default_device.lab.hash.lower(), + machine_name=default_device.name, + shell=None, + logs=True) + + +@mock.patch("src.Kathara.manager.kubernetes.KubernetesMachine.KubernetesMachine.connect") +def test_connect_tty_invocation_error(mock_connect, kubernetes_manager, default_device): + with pytest.raises(InvocationError): + kubernetes_manager.connect_tty(default_device.name) + + assert not mock_connect.called + + +# +# TEST: exec +# +@mock.patch("src.Kathara.manager.kubernetes.KubernetesMachine.KubernetesMachine.exec") +def test_exec_lab_hash(mock_exec, kubernetes_manager, default_device): + kubernetes_manager.exec(default_device.name, ["test", "command"], lab_hash=default_device.lab.hash) + + mock_exec.assert_called_once_with( + default_device.lab.hash.lower(), + default_device.name, + ["test", "command"], + stderr=True, + tty=False, + is_stream=True + ) + + +@mock.patch("src.Kathara.manager.kubernetes.KubernetesMachine.KubernetesMachine.exec") +def test_exec_lab_name(mock_exec, kubernetes_manager, default_device): + kubernetes_manager.exec(default_device.name, ["test", "command"], lab_name=default_device.lab.name) + + mock_exec.assert_called_once_with( + generate_urlsafe_hash(default_device.lab.name).lower(), + default_device.name, + ["test", "command"], + stderr=True, + tty=False, + is_stream=True + ) + + +@mock.patch("src.Kathara.manager.kubernetes.KubernetesMachine.KubernetesMachine.exec") +def test_exec_wait(mock_exec, kubernetes_manager, default_device): + kubernetes_manager.exec(default_device.name, ["test", "command"], lab_hash=default_device.lab.hash, wait=True) + + mock_exec.assert_called_once_with( + default_device.lab.hash.lower(), + default_device.name, + ["test", "command"], + stderr=True, + tty=False, + is_stream=True + ) + + +@mock.patch("src.Kathara.manager.kubernetes.KubernetesMachine.KubernetesMachine.exec") +def test_exec_invocation_error(mock_exec, kubernetes_manager, default_device): + with pytest.raises(InvocationError): + kubernetes_manager.exec(default_device.name, ["test", "command"]) + + assert not mock_exec.called + + # # TEST: get_machine_api_objects # From 4e1abe76877610d6b0ee475c402bceb7f008c5d0 Mon Sep 17 00:00:00 2001 From: Tommaso Caiazzi Date: Fri, 14 Apr 2023 17:34:58 +0200 Subject: [PATCH 10/34] Fix _waiting_startup_execution (#209) --- src/Kathara/manager/docker/DockerMachine.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Kathara/manager/docker/DockerMachine.py b/src/Kathara/manager/docker/DockerMachine.py index 56ca80a1..fef3daba 100644 --- a/src/Kathara/manager/docker/DockerMachine.py +++ b/src/Kathara/manager/docker/DockerMachine.py @@ -587,6 +587,10 @@ def _waiting_startup_execution(self, container: docker.models.containers.Contain logging.debug(f"Waiting startup commands execution for device {machine_name}") exit_code = 1 startup_waited = True + + sys.stdout.write("Waiting startup commands execution. Press enter to take control of the device...") + sys.stdout.flush() + while exit_code != 0: exec_result = self._exec_run(container, cmd="cat /var/log/EOS", @@ -597,12 +601,6 @@ def _waiting_startup_execution(self, container: docker.models.containers.Contain ) exit_code = exec_result['exit_code'] - # To print the message on the same line at each loop - sys.stdout.write("\033[2J") - sys.stdout.write("\033[0;0H") - sys.stdout.write("Waiting startup commands execution. Press enter to take control of the device...") - sys.stdout.flush() - # If the user requests the control, break the while loop if utils.exec_by_platform(utils.wait_user_input_linux, utils.wait_user_input_windows, utils.wait_user_input_linux): From 18f9b74609d8f3202b3e73d01d2f3ebe3cca2832 Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Sat, 8 Jul 2023 16:08:00 +0200 Subject: [PATCH 11/34] Add support for negative values in sysctls (close #226) --- src/Kathara/model/Machine.py | 4 ++-- tests/model/machine_test.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/Kathara/model/Machine.py b/src/Kathara/model/Machine.py index 5ef68923..04122643 100644 --- a/src/Kathara/model/Machine.py +++ b/src/Kathara/model/Machine.py @@ -150,7 +150,7 @@ def add_meta(self, name: str, value: Any) -> Optional[Any]: return old_value if name == "sysctl": - matches = re.search(r"^(?Pnet\.([\w-]+\.)+[\w-]+)=(?P\w+)$", value) + matches = re.search(r"^(?Pnet\.([\w-]+\.)+[\w-]+)=(?P-?\w+)$", value) # Check for valid kv-pair if matches: @@ -160,7 +160,7 @@ def add_meta(self, name: str, value: Any) -> Optional[Any]: old_value = self.meta['sysctls'][key] if key in self.meta['sysctls'] else None # Convert to int if possible - self.meta['sysctls'][key] = int(val) if val.isdigit() else val + self.meta['sysctls'][key] = int(val) if val.strip('-').isnumeric() else val else: raise MachineOptionError( "Invalid sysctl value (`%s`) on `%s`, missing `=` or value not in `net.` namespace." diff --git a/tests/model/machine_test.py b/tests/model/machine_test.py index 05032924..348f44d7 100644 --- a/tests/model/machine_test.py +++ b/tests/model/machine_test.py @@ -140,6 +140,21 @@ def test_add_meta_sysctl_not_format_exception(default_device: Machine): default_device.add_meta("sysctl", "kernel.shm_rmid_forced") +def test_add_meta_sysctl_non_numeric(default_device: Machine): + default_device.add_meta("sysctl", "net.test_sysctl.text=test") + assert default_device.meta['sysctls']['net.test_sysctl.text'] == "test" + + +def test_add_meta_sysctl_negative_number(default_device: Machine): + default_device.add_meta("sysctl", "net.test_sysctl.negative=-1") + assert default_device.meta['sysctls']['net.test_sysctl.negative'] == -1 + + +def test_add_meta_sysctl_negative_number_not_format(default_device: Machine): + with pytest.raises(MachineOptionError): + default_device.add_meta("sysctl", "net.test_sysctl.negative=-1-") + + def test_add_meta_env(default_device: Machine): default_device.add_meta("env", "MY_ENV_VAR=test") assert default_device.meta['envs']['MY_ENV_VAR'] == "test" From 08009784aad2064dd6cd3d200bc32ee3d505adeb Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Sat, 8 Jul 2023 16:24:32 +0200 Subject: [PATCH 12/34] Fix ipv6 precedence rules on device + Minor fixes on metas (close #227) --- src/Kathara/model/Machine.py | 16 ++++++++++++++-- tests/model/machine_test.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/Kathara/model/Machine.py b/src/Kathara/model/Machine.py index 04122643..95f2d7c7 100644 --- a/src/Kathara/model/Machine.py +++ b/src/Kathara/model/Machine.py @@ -252,6 +252,12 @@ def update_meta(self, args: Dict[str, Any]) -> None: for envs in args['envs']: self.add_meta("env", envs) + if 'ipv6' in args and args['ipv6'] is not None: + self.add_meta("ipv6", args['ipv6']) + + if 'shell' in args and args['shell'] is not None: + self.add_meta("shell", args['shell']) + def check(self) -> None: """Sort interfaces and check if there are missing interface numbers. @@ -460,9 +466,15 @@ def is_ipv6_enabled(self) -> bool: Raises: MachineOptionError: If the IPv6 value specified is not valid. """ + is_v6_enabled = Setting.get_instance().enable_ipv6 + try: - return strtobool(self.lab.general_options["ipv6"]) if "ipv6" in self.lab.general_options else \ - strtobool(self.meta["ipv6"]) if "ipv6" in self.meta else Setting.get_instance().enable_ipv6 + if "ipv6" in self.lab.general_options: + is_v6_enabled = self.lab.general_options["ipv6"] + elif "ipv6" in self.meta: + is_v6_enabled = self.meta["ipv6"] + + return is_v6_enabled if type(is_v6_enabled) == bool else strtobool(is_v6_enabled) except ValueError: raise MachineOptionError("IPv6 value not valid on `%s`." % self.name) diff --git a/tests/model/machine_test.py b/tests/model/machine_test.py index 348f44d7..75382b99 100644 --- a/tests/model/machine_test.py +++ b/tests/model/machine_test.py @@ -343,3 +343,36 @@ def test_get_num_terms_mix(): device2 = Machine(lab, "test_machine2") assert device1.get_num_terms() == 2 assert device2.get_num_terms() == 2 + + +# +# TEST: is_ipv6_enabled +# +def test_is_ipv6_enabled_from_lab_options(): + lab = Lab('mem_test') + lab.add_option("ipv6", True) + device = Machine(lab, "test_machine") + assert device.is_ipv6_enabled() + + +def test_is_ipv6_enabled_from_device_meta_bool(): + kwargs = {"ipv6": True} + device = Machine(Lab("test_lab"), "test_machine", **kwargs) + assert device.is_ipv6_enabled() + + +def test_is_ipv6_enabled_from_device_meta_str(): + kwargs = {"ipv6": "True"} + device = Machine(Lab("test_lab"), "test_machine", **kwargs) + assert device.is_ipv6_enabled() + + +def test_is_ipv6_enabled_mix(): + # Lab options have a greater priority than machine options + lab = Lab('mem_test') + lab.add_option("ipv6", False) + kwargs = {"ipv6": True} + device1 = Machine(lab, "test_machine1", **kwargs) + device2 = Machine(lab, "test_machine2") + assert not device1.is_ipv6_enabled() + assert not device2.is_ipv6_enabled() From 5459432260ae80831c86ede200c394425c335afc Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Sat, 8 Jul 2023 18:05:46 +0200 Subject: [PATCH 13/34] Delete container volumes on removal --- src/Kathara/manager/docker/DockerMachine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Kathara/manager/docker/DockerMachine.py b/src/Kathara/manager/docker/DockerMachine.py index 155f4ac6..e71d0a5e 100644 --- a/src/Kathara/manager/docker/DockerMachine.py +++ b/src/Kathara/manager/docker/DockerMachine.py @@ -795,4 +795,4 @@ def _delete_machine(self, container: docker.models.containers.Container) -> None f"Shutdown commands will not be executed." ) - container.remove(force=True) + container.remove(v=True, force=True) From 507dde51beb2f18169366195e574824ad6cb58e6 Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Mon, 10 Jul 2023 19:48:28 +0200 Subject: [PATCH 14/34] Disable IPv6 RA by default when IPv6 is activated --- src/Kathara/manager/docker/DockerMachine.py | 1 + src/Kathara/manager/kubernetes/KubernetesMachine.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/Kathara/manager/docker/DockerMachine.py b/src/Kathara/manager/docker/DockerMachine.py index e71d0a5e..ae8e038d 100644 --- a/src/Kathara/manager/docker/DockerMachine.py +++ b/src/Kathara/manager/docker/DockerMachine.py @@ -234,6 +234,7 @@ def create(self, machine: Machine) -> None: if machine.is_ipv6_enabled(): sysctl_parameters["net.ipv6.conf.all.forwarding"] = 1 + sysctl_parameters["net.ipv6.conf.all.accept_ra"] = 0 sysctl_parameters["net.ipv6.icmp.ratelimit"] = 0 sysctl_parameters["net.ipv6.conf.default.disable_ipv6"] = 0 sysctl_parameters["net.ipv6.conf.all.disable_ipv6"] = 0 diff --git a/src/Kathara/manager/kubernetes/KubernetesMachine.py b/src/Kathara/manager/kubernetes/KubernetesMachine.py index c4b84a7b..17f80280 100644 --- a/src/Kathara/manager/kubernetes/KubernetesMachine.py +++ b/src/Kathara/manager/kubernetes/KubernetesMachine.py @@ -262,6 +262,7 @@ def create(self, machine: Machine) -> None: if machine.is_ipv6_enabled(): sysctl_parameters["net.ipv6.conf.all.forwarding"] = 1 + sysctl_parameters["net.ipv6.conf.all.accept_ra"] = 0 sysctl_parameters["net.ipv6.icmp.ratelimit"] = 0 sysctl_parameters["net.ipv6.conf.default.disable_ipv6"] = 0 sysctl_parameters["net.ipv6.conf.all.disable_ipv6"] = 0 From c015ffefcfdf788d43d2a0c273c46dbd0d211777 Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Fri, 14 Jul 2023 12:02:34 +0200 Subject: [PATCH 15/34] Polish (WIP) (#209) --- docs/kathara-exec.1.ronn | 4 +- src/Kathara/cli/command/ExecCommand.py | 2 +- src/Kathara/foundation/manager/IManager.py | 2 +- src/Kathara/manager/Kathara.py | 2 +- src/Kathara/manager/docker/DockerMachine.py | 83 +++++++++---------- src/Kathara/manager/docker/DockerManager.py | 2 +- .../manager/kubernetes/KubernetesManager.py | 3 +- src/Kathara/utils.py | 4 +- 8 files changed, 51 insertions(+), 51 deletions(-) diff --git a/docs/kathara-exec.1.ronn b/docs/kathara-exec.1.ronn index 9dfb168f..588a8716 100644 --- a/docs/kathara-exec.1.ronn +++ b/docs/kathara-exec.1.ronn @@ -36,7 +36,9 @@ Execute a command in the Kathara device DEVICE_NAME. Disable stderr of the executed command. * `--wait`: - Wait startup commands execution. + Wait until startup commands execution finishes. + + You can override the timer by pressing `[ENTER]`. * `: Name of the device to execute the command into. diff --git a/src/Kathara/cli/command/ExecCommand.py b/src/Kathara/cli/command/ExecCommand.py index 36268d70..3fad00c9 100644 --- a/src/Kathara/cli/command/ExecCommand.py +++ b/src/Kathara/cli/command/ExecCommand.py @@ -57,7 +57,7 @@ def __init__(self) -> None: dest="wait", action="store_true", default=False, - help='Wait startup commands execution.', + help='Wait until startup commands execution finishes.', ) self.parser.add_argument( 'machine_name', diff --git a/src/Kathara/foundation/manager/IManager.py b/src/Kathara/foundation/manager/IManager.py index 4e27a479..bff63028 100644 --- a/src/Kathara/foundation/manager/IManager.py +++ b/src/Kathara/foundation/manager/IManager.py @@ -183,7 +183,7 @@ def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = command (List[str]): The command to exec on the device. lab_hash (Optional[str]): The hash of the network scenario where the device is deployed. lab_name (Optional[str]): The name of the network scenario where the device is deployed. - wait (bool): If True, wait the end of the startup before executing the command. + wait (bool): If True, wait until end of the startup commands execution before executing the command. Returns: Generator[Tuple[bytes, bytes]]: A generator of tuples containing the stdout and stderr in bytes. diff --git a/src/Kathara/manager/Kathara.py b/src/Kathara/manager/Kathara.py index d27a1082..05bbaf48 100644 --- a/src/Kathara/manager/Kathara.py +++ b/src/Kathara/manager/Kathara.py @@ -208,7 +208,7 @@ def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = command (List[str]): The command to exec on the device. lab_hash (Optional[str]): The hash of the network scenario where the device is deployed. lab_name (Optional[str]): The name of the network scenario where the device is deployed. - wait (bool): If True, wait the end of the startup before executing the command. + wait (bool): If True, wait until end of the startup commands execution before executing the command. Returns: Generator[Tuple[bytes, bytes]]: A generator of tuples containing the stdout and stderr in bytes. diff --git a/src/Kathara/manager/docker/DockerMachine.py b/src/Kathara/manager/docker/DockerMachine.py index a91da53a..955ff9f5 100644 --- a/src/Kathara/manager/docker/DockerMachine.py +++ b/src/Kathara/manager/docker/DockerMachine.py @@ -530,7 +530,7 @@ def connect(self, lab_hash: str, machine_name: str, user: str = None, shell: str startup_waited = False if wait: - startup_waited = self._waiting_startup_execution(container, machine_name) + startup_waited = self._wait_startup_execution(container) # Clean the terminal output sys.stdout.write("\033[2J") @@ -577,42 +577,6 @@ def cmd_connect(): utils.exec_by_platform(tty_connect, cmd_connect, tty_connect) - def _waiting_startup_execution(self, container: docker.models.containers.Container, machine_name: str): - """Wait until the startup commands are executed or until the user requests the control over the device. - - Args: - container (str): TThe Docker container to wait. - machine_name (str): The name of the device to connect. - - Returns: - bool: False if the user requests the control before the ending of the startup. Else, True. - """ - logging.debug(f"Waiting startup commands execution for device {machine_name}") - exit_code = 1 - startup_waited = True - - sys.stdout.write("Waiting startup commands execution. Press enter to take control of the device...") - sys.stdout.flush() - - while exit_code != 0: - exec_result = self._exec_run(container, - cmd="cat /var/log/EOS", - stdout=True, - stderr=False, - privileged=False, - detach=False - ) - exit_code = exec_result['exit_code'] - - # If the user requests the control, break the while loop - if utils.exec_by_platform(utils.wait_user_input_linux, utils.wait_user_input_windows, - utils.wait_user_input_linux): - startup_waited = False - break - - time.sleep(0.1) - return startup_waited - def exec(self, lab_hash: str, machine_name: str, command: Union[str, List], user: str = None, tty: bool = True, wait: bool = False) -> Generator[Tuple[bytes, bytes], None, None]: """Execute the command on the Docker container specified by the lab_hash and the machine_name. @@ -639,12 +603,7 @@ def exec(self, lab_hash: str, machine_name: str, command: Union[str, List], user container = containers.pop() if wait: - startup_waited = self._waiting_startup_execution(container, machine_name) - - # Clean the terminal output - sys.stdout.write("\033[2J") - sys.stdout.write("\033[0;0H") - sys.stdout.flush() + self._wait_startup_execution(container) command = shlex.split(command) if type(command) == str else command exec_result = self._exec_run(container, @@ -729,6 +688,44 @@ def _exec_run(self, container: docker.models.containers.Container, return {'exit_code': int(exit_code) if exit_code is not None else None, 'output': exec_output} + def _wait_startup_execution(self, container: docker.models.containers.Container): + """Wait until the startup commands are executed or until the user requests the control over the device. + + Args: + container (docker.models.containers.Container): The Docker container to wait. + + Returns: + bool: False if the user requests the control before the ending of the startup. Else, True. + """ + machine_name = container.labels['name'] + + sys.stdout.write("Waiting startup commands execution. Press [ENTER] to take control of the device...") + sys.stdout.flush() + + logging.debug(f"Waiting startup commands execution for device {machine_name}...") + exit_code = 1 + startup_waited = True + while exit_code != 0: + exec_result = self._exec_run(container, + cmd="cat /var/log/EOS", + stdout=True, + stderr=False, + privileged=False, + detach=False + ) + exit_code = exec_result['exit_code'] + + # If the user requests the control, break the while loop + if utils.exec_by_platform(utils.wait_user_input_linux, + utils.wait_user_input_windows, + utils.wait_user_input_linux): + startup_waited = False + break + + time.sleep(1) + + return startup_waited + @staticmethod def copy_files(machine_api_object: docker.models.containers.Container, path: str, tar_data: bytes) -> None: """Copy the files contained in tar_data in the Docker container path specified by the machine_api_object. diff --git a/src/Kathara/manager/docker/DockerManager.py b/src/Kathara/manager/docker/DockerManager.py index 72870454..799d2f11 100644 --- a/src/Kathara/manager/docker/DockerManager.py +++ b/src/Kathara/manager/docker/DockerManager.py @@ -332,7 +332,7 @@ def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = command (List[str]): The command to exec on the device. lab_hash (Optional[str]): The hash of the network scenario where the device is deployed. lab_name (Optional[str]): The name of the network scenario where the device is deployed. - wait (bool): If True, wait the end of the startup before executing the command. + wait (bool): If True, wait until end of the startup commands execution before executing the command. Returns: Generator[Tuple[bytes, bytes]]: A generator of tuples containing the stdout and stderr in bytes. diff --git a/src/Kathara/manager/kubernetes/KubernetesManager.py b/src/Kathara/manager/kubernetes/KubernetesManager.py index 8dee3c0c..72ece4e2 100644 --- a/src/Kathara/manager/kubernetes/KubernetesManager.py +++ b/src/Kathara/manager/kubernetes/KubernetesManager.py @@ -336,7 +336,8 @@ def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = command (List[str]): The command to exec on the device. lab_hash (Optional[str]): The hash of the network scenario where the device is deployed. lab_name (Optional[str]): The name of the network scenario where the device is deployed. - wait (bool): If True, wait the end of the startup before executing the command. No effect on Megalos. + wait (bool): If True, wait until end of the startup commands execution before executing the command. + No effect on Megalos. Returns: Generator[Tuple[bytes, bytes]]: A generator of tuples containing the stdout and stderr in bytes. diff --git a/src/Kathara/utils.py b/src/Kathara/utils.py index 0dbdb6fe..6cbd521d 100644 --- a/src/Kathara/utils.py +++ b/src/Kathara/utils.py @@ -315,14 +315,14 @@ def pack_files_for_tar(guest_to_host: Dict) -> bytes: return tar_data -def wait_user_input_linux(): +def wait_user_input_linux() -> list: """Non-blocking input function for Linux and macOS.""" import select to_break, _, _ = select.select([sys.stdin], [], [], 0.1) return to_break -def wait_user_input_windows(): +def wait_user_input_windows() -> bool: """Return True if a keypress is waiting to be read. Only for Windows.""" import msvcrt return msvcrt.kbhit() From 35932870c19d5947d11c06e7b04b6664016da84d Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Fri, 14 Jul 2023 14:26:15 +0200 Subject: [PATCH 16/34] Merge branch '208-terminal-crashes-if-startup-script-output-contains-text-that-cannot-be-parsed-as-utf-8' into 209-waiting-startup-on-connect --- pyproject.toml | 1 + setup.py | 1 + src/Kathara/cli/command/ExecCommand.py | 10 +++- src/Kathara/foundation/test/Test.py | 9 +++- src/Kathara/manager/docker/DockerMachine.py | 49 ++++++++++++------- .../manager/kubernetes/KubernetesMachine.py | 6 ++- src/requirements.txt | 1 + 7 files changed, 54 insertions(+), 23 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0bca11bd..8659c3a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ dependencies = [ "pyroute2>=0.5.19", "progressbar2>=1.14.0", "fs>=2.4.16", + "chardet", "libtmux>=0.8.2; platform_system == 'darwin' or platform_system == 'linux'", "appscript>=1.1.0; platform_system == 'darwin'", "pypiwin32>=223; platform_system == 'win32'", diff --git a/setup.py b/setup.py index 1b4dc94f..84756367 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ "pyroute2>=0.5.19", "progressbar2>=1.14.0", "fs>=2.4.16", + "chardet", "libtmux>=0.8.2; platform_system == 'darwin' or platform_system == 'linux'", "appscript>=1.1.0; platform_system == 'darwin'", "pypiwin32>=223; platform_system == 'win32'", diff --git a/src/Kathara/cli/command/ExecCommand.py b/src/Kathara/cli/command/ExecCommand.py index 3fad00c9..f640ba83 100644 --- a/src/Kathara/cli/command/ExecCommand.py +++ b/src/Kathara/cli/command/ExecCommand.py @@ -2,6 +2,8 @@ import sys from typing import List +import chardet + from ... import utils from ...foundation.cli.command.Command import Command from ...manager.Kathara import Kathara @@ -95,8 +97,12 @@ def run(self, current_path: str, argv: List[str]) -> None: try: while True: (stdout, stderr) = next(exec_output) - stdout = stdout.decode('utf-8') if stdout else "" - stderr = stderr.decode('utf-8') if stderr else "" + + stdout_char_encoding = chardet.detect(stdout) if stdout else None + stderr_char_encoding = chardet.detect(stderr) if stderr else None + + stdout = stdout.decode(stdout_char_encoding['encoding']) if stdout else "" + stderr = stderr.decode(stderr_char_encoding['encoding']) if stderr else "" if not args['no_stdout']: sys.stdout.write(stdout) diff --git a/src/Kathara/foundation/test/Test.py b/src/Kathara/foundation/test/Test.py index d30e91d2..7886f9fd 100644 --- a/src/Kathara/foundation/test/Test.py +++ b/src/Kathara/foundation/test/Test.py @@ -2,6 +2,7 @@ from abc import ABC, abstractmethod from typing import Dict, Union, List +import chardet from deepdiff import DeepDiff from ...manager.Kathara import Kathara @@ -42,9 +43,13 @@ def _get_machine_command_output(lab_hash, machine_name, command): try: while True: (stdout, stderr) = next(exec_output) + + stdout_char_encoding = chardet.detect(stdout) if stdout else None + stderr_char_encoding = chardet.detect(stderr) if stderr else None + if stdout: - result['stdout'] += stdout.decode('utf-8') + result['stdout'] += stdout.decode(stdout_char_encoding['encoding']) if stdout else "" if stderr: - result['stderr'] += stderr.decode('utf-8') + result['stderr'] += stderr.decode(stderr_char_encoding['encoding']) if stderr else "" except StopIteration: return result['stdout'], result['stderr'] diff --git a/src/Kathara/manager/docker/DockerMachine.py b/src/Kathara/manager/docker/DockerMachine.py index 955ff9f5..8a98f414 100644 --- a/src/Kathara/manager/docker/DockerMachine.py +++ b/src/Kathara/manager/docker/DockerMachine.py @@ -7,6 +7,7 @@ from multiprocessing.dummy import Pool from typing import List, Dict, Generator, Optional, Set, Tuple, Union, Any +import chardet import docker.models.containers from docker import DockerClient from docker.errors import APIError @@ -526,7 +527,7 @@ def connect(self, lab_hash: str, machine_name: str, user: str = None, shell: str else: shell = shlex.split(shell) - logging.debug(f"Connect to device `{machine_name}` with shell: {shell}") + logging.debug("Connect to device `%s` with shell: %s" % (machine_name, shell)) startup_waited = False if wait: @@ -537,24 +538,30 @@ def connect(self, lab_hash: str, machine_name: str, user: str = None, shell: str sys.stdout.write("\033[0;0H") sys.stdout.flush() - command = "" - if logs: - # Print the startup logs inside the container and open the shell - cat_logs_cmd = "[ -f var/log/shared.log ] && " \ - "(echo '-- Shared Commands --'; cat /var/log/shared.log; " \ - "echo '-- End Shared Commands --\n');" \ - "[ -f /var/log/startup.log ] && " \ - "(echo '-- Device Commands --'; cat /var/log/startup.log; echo '-- End Device Commands --')" - command += f"{shell[0]} -c \"echo '--- Startup Commands Log\n';" \ - f"{cat_logs_cmd};" - command += "echo '\n--- End Startup Commands Log\n';" if startup_waited else \ - f"echo '\n--- Executing other commands in background\n';" - command += f"{shell[0]}\"" - else: - command += f"{shell[0]}" + # Get the logs, if the command fails it means that the shell is not found. + cat_logs_cmd = "cat /var/log/shared.log /var/log/startup.log" + startup_command = [item for item in shell] + startup_command.extend(['-c', cat_logs_cmd]) + exec_result = self._exec_run(container, + cmd=startup_command, + stdout=True, + stderr=False, + privileged=False, + detach=False + ) + detect = chardet.detect(exec_result['output']) if exec_result['output'] else None + startup_output = exec_result['output'].decode(detect) if exec_result['output'] else None + + if startup_output and logs and Setting.get_instance().print_startup_log: + print("--- Startup Commands Log\n") + print(startup_output) + print("--- End Startup Commands Log\n") + + if not startup_waited: + print("--- Executing other commands in background\n") resp = self.client.api.exec_create(container.id, - command, + shell, stdout=True, stderr=True, stdin=True, @@ -678,7 +685,13 @@ def _exec_run(self, container: docker.models.containers.Container, exit_code = self.client.api.exec_inspect(resp['Id'])['ExitCode'] if not socket and not stream and (exit_code is not None and exit_code != 0): (stdout_out, _) = exec_output if demux else (exec_output, None) - exec_stdout = (stdout_out.decode('utf-8') if type(stdout_out) == bytes else stdout_out) if stdout else "" + exec_stdout = "" + if stdout_out: + if type(stdout_out) == bytes: + detect = chardet.detect(stdout_out) + exec_stdout = stdout_out.decode(detect['encoding']) + else: + exec_stdout = stdout_out matches = OCI_RUNTIME_RE.search(exec_stdout) if matches: raise MachineBinaryError(matches.group(3) or matches.group(4), container.labels['name']) diff --git a/src/Kathara/manager/kubernetes/KubernetesMachine.py b/src/Kathara/manager/kubernetes/KubernetesMachine.py index 17f80280..dff41950 100644 --- a/src/Kathara/manager/kubernetes/KubernetesMachine.py +++ b/src/Kathara/manager/kubernetes/KubernetesMachine.py @@ -8,6 +8,7 @@ from multiprocessing.dummy import Pool from typing import Optional, Set, List, Union, Generator, Tuple, Dict +import chardet from kubernetes import client from kubernetes.client.api import apps_v1_api from kubernetes.client.api import core_v1_api @@ -602,7 +603,10 @@ def connect(self, lab_hash: str, machine_name: str, shell: Union[str, List[str]] print("--- Startup Commands Log\n") while True: (stdout, _) = next(exec_output) - stdout = stdout.decode('utf-8') if stdout else "" + + char_encoding = chardet.detect(stdout) if stdout else None + + stdout = stdout.decode(char_encoding['encoding']) if stdout else "" sys.stdout.write(stdout) except StopIteration: print("\n--- End Startup Commands Log\n") diff --git a/src/requirements.txt b/src/requirements.txt index feebbf0d..5eb85b05 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -9,6 +9,7 @@ deepdiff==6.2.2; pyroute2==0.5.19; progressbar2>=1.14.0; fs>=2.4.16; +chardet; libtmux>=0.18.0; sys_platform == 'darwin' or sys_platform == 'linux' git+https://github.com/saghul/pyuv@master#egg=pyuv appscript>=1.1.0; sys_platform == 'darwin' From 627c30f725188dce42656c5ace3cdeedcdc36e26 Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Fri, 14 Jul 2023 14:58:25 +0200 Subject: [PATCH 17/34] Minor fixes + Fix tests (#209) --- src/Kathara/cli/command/ExecCommand.py | 10 ++-- src/Kathara/manager/docker/DockerMachine.py | 8 +-- .../manager/kubernetes/KubernetesMachine.py | 4 +- tests/cli/exec_command_test.py | 51 ++++--------------- .../kubernetes/kubernetes_machine_test.py | 1 + 5 files changed, 23 insertions(+), 51 deletions(-) diff --git a/src/Kathara/cli/command/ExecCommand.py b/src/Kathara/cli/command/ExecCommand.py index f640ba83..93762169 100644 --- a/src/Kathara/cli/command/ExecCommand.py +++ b/src/Kathara/cli/command/ExecCommand.py @@ -98,15 +98,13 @@ def run(self, current_path: str, argv: List[str]) -> None: while True: (stdout, stderr) = next(exec_output) - stdout_char_encoding = chardet.detect(stdout) if stdout else None - stderr_char_encoding = chardet.detect(stderr) if stderr else None - - stdout = stdout.decode(stdout_char_encoding['encoding']) if stdout else "" - stderr = stderr.decode(stderr_char_encoding['encoding']) if stderr else "" - if not args['no_stdout']: + stdout_char_encoding = chardet.detect(stdout) if stdout else None + stdout = stdout.decode(stdout_char_encoding['encoding']) if stdout else "" sys.stdout.write(stdout) if stderr and not args['no_stderr']: + stderr_char_encoding = chardet.detect(stderr) if stderr else None + stderr = stderr.decode(stderr_char_encoding['encoding']) if stderr else "" sys.stderr.write(stderr) except StopIteration: pass diff --git a/src/Kathara/manager/docker/DockerMachine.py b/src/Kathara/manager/docker/DockerMachine.py index 8a98f414..d4f95817 100644 --- a/src/Kathara/manager/docker/DockerMachine.py +++ b/src/Kathara/manager/docker/DockerMachine.py @@ -553,12 +553,12 @@ def connect(self, lab_hash: str, machine_name: str, user: str = None, shell: str startup_output = exec_result['output'].decode(detect) if exec_result['output'] else None if startup_output and logs and Setting.get_instance().print_startup_log: - print("--- Startup Commands Log\n") - print(startup_output) - print("--- End Startup Commands Log\n") + sys.stdout.write("--- Startup Commands Log\n") + sys.stdout.write(startup_output) + sys.stdout.write("--- End Startup Commands Log\n") if not startup_waited: - print("--- Executing other commands in background\n") + sys.stdout.write("--- Executing other commands in background\n") resp = self.client.api.exec_create(container.id, shell, diff --git a/src/Kathara/manager/kubernetes/KubernetesMachine.py b/src/Kathara/manager/kubernetes/KubernetesMachine.py index dff41950..0ebe9f1e 100644 --- a/src/Kathara/manager/kubernetes/KubernetesMachine.py +++ b/src/Kathara/manager/kubernetes/KubernetesMachine.py @@ -98,7 +98,9 @@ "route del default dev eth0 || true", # Placeholder for user commands - "{machine_commands}" + "{machine_commands}", + + "touch /var/log/EOS" ] SHUTDOWN_COMMANDS = [ diff --git a/tests/cli/exec_command_test.py b/tests/cli/exec_command_test.py index a4d1026a..2d1d2a37 100644 --- a/tests/cli/exec_command_test.py +++ b/tests/cli/exec_command_test.py @@ -30,26 +30,9 @@ def test_run_no_params(mock_stderr_write, mock_stdout_write, mock_lab, mock_pars mock_manager_get_instance.return_value = mock_docker_manager mock_docker_manager.exec.return_value = exec_output command = ExecCommand() - command.run('.', ['pc1', ['test', 'command']]) - mock_parse_lab.assert_called_once_with(os.getcwd()) - mock_exec.assert_called_once_with("pc1", ['test', 'command'], lab_hash=mock_lab.hash, wait=False) - mock_stdout_write.assert_called_once_with('stdout') - mock_stderr_write.assert_called_once_with('stderr') - - -@mock.patch("src.Kathara.manager.Kathara.Kathara.exec") -@mock.patch("src.Kathara.parser.netkit.LabParser.LabParser.parse") -@mock.patch("src.Kathara.model.Lab.Lab") -@mock.patch('sys.stdout.write') -@mock.patch('sys.stderr.write') -def test_run_no_params_single_string_command(mock_stderr_write, mock_stdout_write, mock_lab, mock_parse_lab, mock_exec, - exec_output): - mock_parse_lab.return_value = mock_lab - mock_exec.return_value = exec_output - command = ExecCommand() command.run('.', ['pc1', 'test command']) mock_parse_lab.assert_called_once_with(os.getcwd()) - mock_exec.assert_called_once_with("pc1", 'test command', lab_hash=mock_lab.hash, wait=False) + mock_docker_manager.exec.assert_called_once_with("pc1", 'test command', lab_hash=mock_lab.hash, wait=False) mock_stdout_write.assert_called_once_with('stdout') mock_stderr_write.assert_called_once_with('stderr') @@ -68,9 +51,9 @@ def test_run_with_directory_absolute_path(mock_stderr_write, mock_stdout_write, mock_manager_get_instance.return_value = mock_docker_manager mock_docker_manager.exec.return_value = exec_output command = ExecCommand() - command.run('.', ['-d', '/test/path', 'pc1', 'test command']) - mock_parse_lab.assert_called_once_with('/test/path') - mock_exec.assert_called_once_with("pc1", 'test command', lab_hash=mock_lab.hash, wait=False) + command.run('.', ['-d', os.path.join('/test', 'path'), 'pc1', 'test command']) + mock_parse_lab.assert_called_once_with(os.path.abspath(os.path.join('/test', 'path'))) + mock_docker_manager.exec.assert_called_once_with("pc1", 'test command', lab_hash=mock_lab.hash, wait=False) mock_stdout_write.assert_called_once_with('stdout') mock_stderr_write.assert_called_once_with('stderr') @@ -88,9 +71,9 @@ def test_run_with_directory_relative_path(mock_stderr_write, mock_stdout_write, mock_manager_get_instance.return_value = mock_docker_manager mock_docker_manager.exec.return_value = exec_output command = ExecCommand() - command.run('.', ['-d', 'test/path', 'pc1', 'test command']) - mock_parse_lab.assert_called_once_with(os.path.join(os.getcwd(), 'test/path')) - mock_exec.assert_called_once_with("pc1", 'test command', lab_hash=mock_lab.hash, wait=False) + command.run('.', ['-d', os.path.join('test', 'path'), 'pc1', 'test command']) + mock_parse_lab.assert_called_once_with(os.path.join(os.getcwd(), 'test', 'path')) + mock_docker_manager.exec.assert_called_once_with("pc1", 'test command', lab_hash=mock_lab.hash, wait=False) mock_stdout_write.assert_called_once_with('stdout') mock_stderr_write.assert_called_once_with('stderr') @@ -108,7 +91,7 @@ def test_run_with_v_option(mock_stderr_write, mock_stdout_write, mock_parse_lab, command = ExecCommand() command.run('.', ['-v', 'pc1', 'test command']) assert not mock_parse_lab.called - mock_exec.assert_called_once_with("pc1", 'test command', lab_hash=lab.hash, wait=False) + mock_docker_manager.exec.assert_called_once_with("pc1", 'test command', lab_hash=lab.hash, wait=False) mock_stdout_write.assert_called_once_with('stdout') mock_stderr_write.assert_called_once_with('stderr') @@ -127,7 +110,7 @@ def test_run_no_stdout(mock_stderr_write, mock_stdout_write, mock_lab, mock_pars command = ExecCommand() command.run('.', ['--no-stdout', 'pc1', 'test command']) mock_parse_lab.assert_called_once_with(os.getcwd()) - mock_exec.assert_called_once_with("pc1", 'test command', lab_hash=mock_lab.hash, wait=False) + mock_docker_manager.exec.assert_called_once_with("pc1", 'test command', lab_hash=mock_lab.hash, wait=False) assert not mock_stdout_write.called mock_stderr_write.assert_called_once_with('stderr') @@ -146,7 +129,7 @@ def test_run_no_stderr(mock_stderr_write, mock_stdout_write, mock_lab, mock_pars command = ExecCommand() command.run('.', ['--no-stderr', 'pc1', 'test command']) mock_parse_lab.assert_called_once_with(os.getcwd()) - mock_exec.assert_called_once_with("pc1", 'test command', lab_hash=mock_lab.hash, wait=False) + mock_docker_manager.exec.assert_called_once_with("pc1", 'test command', lab_hash=mock_lab.hash, wait=False) mock_stdout_write.assert_called_once_with('stdout') assert not mock_stderr_write.called @@ -166,18 +149,6 @@ def test_run_no_stdout_no_stderr(mock_stderr_write, mock_stdout_write, mock_lab, command = ExecCommand() command.run('.', ['--no-stdout', '--no-stderr', 'pc1', 'test command']) mock_parse_lab.assert_called_once_with(os.getcwd()) - mock_exec.assert_called_once_with("pc1", 'test command', lab_hash=mock_lab.hash, wait=False) + mock_docker_manager.exec.assert_called_once_with("pc1", 'test command', lab_hash=mock_lab.hash, wait=False) assert not mock_stdout_write.called assert not mock_stderr_write.called - - -@mock.patch("src.Kathara.manager.Kathara.Kathara.exec") -@mock.patch("src.Kathara.parser.netkit.LabParser.LabParser.parse") -@mock.patch("src.Kathara.model.Lab.Lab") -def test_run_wait(mock_lab, mock_parse_lab, mock_exec, exec_output): - mock_parse_lab.return_value = mock_lab - mock_exec.return_value = exec_output - command = ExecCommand() - command.run('.', ['--wait', 'pc1', ['test', 'command']]) - mock_parse_lab.assert_called_once_with(os.getcwd()) - mock_exec.assert_called_once_with("pc1", ['test', 'command'], lab_hash=mock_lab.hash, wait=True) diff --git a/tests/manager/kubernetes/kubernetes_machine_test.py b/tests/manager/kubernetes/kubernetes_machine_test.py index fafc0766..7953b655 100644 --- a/tests/manager/kubernetes/kubernetes_machine_test.py +++ b/tests/manager/kubernetes/kubernetes_machine_test.py @@ -351,6 +351,7 @@ def test_create_ipv6(mock_setting_get_instance, kubernetes_machine, default_devi 'net.ipv4.ip_forward': 1, 'net.ipv4.icmp_ratelimit': 0, 'net.ipv6.conf.all.forwarding': 1, + 'net.ipv6.conf.all.accept_ra': 0, 'net.ipv6.icmp.ratelimit': 0, 'net.ipv6.conf.default.disable_ipv6': 0, 'net.ipv6.conf.all.disable_ipv6': 0 From 1e86cdb2a386c4a6da1a7c3f309822f0497caea6 Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Fri, 14 Jul 2023 16:33:19 +0200 Subject: [PATCH 18/34] Fix tar extraction permissions + Fix docs --- docs/kathara-lab.conf.5.ronn | 5 +++-- src/Kathara/manager/docker/DockerMachine.py | 6 +++--- src/Kathara/manager/kubernetes/KubernetesMachine.py | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/kathara-lab.conf.5.ronn b/docs/kathara-lab.conf.5.ronn index ff3e7d2b..721af546 100644 --- a/docs/kathara-lab.conf.5.ronn +++ b/docs/kathara-lab.conf.5.ronn @@ -27,12 +27,13 @@ In order to establish a uniform convention, comment lines should always start wi * `cpus` (float): Limit the amount of CPU available for this device. - This option takes a positive float, ranging from 0 to max number of host logical CPUs. For instance, if the host device has two CPUs and you set `device[cpus]=1.5`, the container is guaranteed at most one and a half of the CPUs. + This option takes a positive float, ranging from 0 to max number of host logical CPUs. For instance, if the host device has two CPUs and you set `device[cpus]=1.5`, the device is guaranteed at most one and a half of the CPUs. * `port` (string): - Map localhost port HOST to the internal port GUEST of the device for the specified PROTOCOL. + Map localhost port HOST to the internal port GUEST of the device for the specified PROTOCOL. The syntax is [HOST:]GUEST[/PROTOCOL]. If HOST port is not specified, default is 3000. If PROTOCOL is not specified, default is `tcp`. Supported PROTOCOL values are: tcp, udp, or sctp. + For instance, with this command you can map host's port 8080 to device's port 80 with TCP protocol: `device[port]="8080:80/tcp"`. * `bridged` (boolean): Connect the device to the host network by adding an additional network interface. This interface will be connected to the host network through a NAT connection. diff --git a/src/Kathara/manager/docker/DockerMachine.py b/src/Kathara/manager/docker/DockerMachine.py index ae8e038d..61856b8e 100644 --- a/src/Kathara/manager/docker/DockerMachine.py +++ b/src/Kathara/manager/docker/DockerMachine.py @@ -36,7 +36,7 @@ # Copy the machine folder (if present) from the hostlab directory into the root folder of the container # In this way, files are all replaced in the container root folder "if [ -d \"/hostlab/{machine_name}\" ]; then " - "(cd /hostlab/{machine_name} && tar c .) | (cd / && tar xhf -); fi", + "(cd /hostlab/{machine_name} && tar c .) | (cd / && tar --no-same-owner --no-same-permissions xhf -); fi", # If /etc/hosts is not configured by the user, add the localhost mapping "if [ ! -s \"/etc/hosts\" ]; then " @@ -50,12 +50,12 @@ # Give proper permissions to Quagga files (if present) "if [ -d \"/etc/quagga\" ]; then " - "chown --recursive quagga:quagga /etc/quagga/", + "chown -R quagga:quagga /etc/quagga/", "chmod 640 /etc/quagga/*; fi", # Give proper permissions to FRR files (if present) "if [ -d \"/etc/frr\" ]; then " - "chown frr:frr /etc/frr/*", + "chown -R frr:frr /etc/frr/", "chmod 640 /etc/frr/*; fi", # If shared.startup file is present diff --git a/src/Kathara/manager/kubernetes/KubernetesMachine.py b/src/Kathara/manager/kubernetes/KubernetesMachine.py index 17f80280..d82e2a4f 100644 --- a/src/Kathara/manager/kubernetes/KubernetesMachine.py +++ b/src/Kathara/manager/kubernetes/KubernetesMachine.py @@ -53,7 +53,7 @@ # Copy the machine folder (if present) from the hostlab directory into the root folder of the container # In this way, files are all replaced in the container root folder "if [ -d \"/hostlab/{machine_name}\" ]; then " - "(cd /hostlab/{machine_name} && tar c .) | (cd / && tar xhf -); fi", + "(cd /hostlab/{machine_name} && tar c .) | (cd / && tar --no-same-owner --no-same-permissions xhf -); fi", # If /etc/hosts is not configured by the user, add the localhost mapping "if [ ! -s \"/etc/hosts\" ]; then " @@ -67,12 +67,12 @@ # Give proper permissions to Quagga files (if present) "if [ -d \"/etc/quagga\" ]; then " - "chown --recursive quagga:quagga /etc/quagga/", + "chown -R quagga:quagga /etc/quagga/", "chmod 640 /etc/quagga/*; fi", # Give proper permissions to FRR files (if present) "if [ -d \"/etc/frr\" ]; then " - "chown frr:frr /etc/frr/*", + "chown -R frr:frr /etc/frr/", "chmod 640 /etc/frr/*; fi", # If shared.startup file is present From ef74813fdd13739e868e862a8c447994dceb07d6 Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Fri, 14 Jul 2023 16:39:35 +0200 Subject: [PATCH 19/34] Fix Windows script --- scripts/Windows/WindowsBuild.bat | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/Windows/WindowsBuild.bat b/scripts/Windows/WindowsBuild.bat index 099a5f2e..c249b1cb 100644 --- a/scripts/Windows/WindowsBuild.bat +++ b/scripts/Windows/WindowsBuild.bat @@ -1,7 +1,11 @@ set VENV_DIR=%cd%\venv +rmdir /S /Q %VENV_DIR% + python3.10 -m venv %VENV_DIR% +if %errorlevel% neq 0 exit /b %errorlevel% CALL %VENV_DIR%\Scripts\activate +if %errorlevel% neq 0 exit /b %errorlevel% pip install win_inet_pton pip install pyinstaller From c1316a1445e57bf945cb6b7751dfe468d166b10f Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Fri, 14 Jul 2023 18:54:31 +0200 Subject: [PATCH 20/34] Add wait parameters control with tuple (#209) --- src/Kathara/foundation/manager/IManager.py | 16 ++- src/Kathara/manager/Kathara.py | 18 ++- src/Kathara/manager/docker/DockerMachine.py | 131 ++++++++++++------ src/Kathara/manager/docker/DockerManager.py | 19 ++- .../manager/kubernetes/KubernetesManager.py | 17 ++- tests/manager/docker/docker_manager_test.py | 12 +- 6 files changed, 151 insertions(+), 62 deletions(-) diff --git a/src/Kathara/foundation/manager/IManager.py b/src/Kathara/foundation/manager/IManager.py index bff63028..53958482 100644 --- a/src/Kathara/foundation/manager/IManager.py +++ b/src/Kathara/foundation/manager/IManager.py @@ -1,6 +1,6 @@ import io from abc import ABC, abstractmethod -from typing import Dict, Set, Any, Generator, Tuple, List, Optional +from typing import Dict, Set, Any, Generator, Tuple, List, Optional, Union from .stats.ILinkStats import ILinkStats from .stats.IMachineStats import IMachineStats @@ -155,7 +155,7 @@ def wipe(self, all_users: bool = False) -> None: @abstractmethod def connect_tty(self, machine_name: str, lab_hash: Optional[str] = None, lab_name: Optional[str] = None, - shell: str = None, logs: bool = False) -> None: + shell: str = None, logs: bool = False, wait: Union[bool, Tuple[int, float]] = True) -> None: """Connect to a device in a running network scenario, using the specified shell. Args: @@ -164,6 +164,10 @@ def connect_tty(self, machine_name: str, lab_hash: Optional[str] = None, lab_nam lab_name (str): The name of the network scenario where the device is deployed. shell (str): The name of the shell to use for connecting. logs (bool): If True, print startup logs on stdout. + wait (Union[bool, Tuple[int, float]]): If True, wait indefinitely until the end of the startup commands + execution before connecting. If a tuple is provided, the first value indicates the number of retries + before stopping waiting and the second value indicates the time interval to wait for each retry. + Default is True. Returns: None @@ -175,7 +179,8 @@ def connect_tty(self, machine_name: str, lab_hash: Optional[str] = None, lab_nam @abstractmethod def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = None, - lab_name: Optional[str] = None, wait: bool = False) -> Generator[Tuple[bytes, bytes], None, None]: + lab_name: Optional[str] = None, wait: Union[bool, Tuple[int, float]] = False) \ + -> Generator[Tuple[bytes, bytes], None, None]: """Exec a command on a device in a running network scenario. Args: @@ -183,7 +188,10 @@ def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = command (List[str]): The command to exec on the device. lab_hash (Optional[str]): The hash of the network scenario where the device is deployed. lab_name (Optional[str]): The name of the network scenario where the device is deployed. - wait (bool): If True, wait until end of the startup commands execution before executing the command. + wait (Union[bool, Tuple[int, float]]): If True, wait indefinitely until the end of the startup commands + execution before executing the command. If a tuple is provided, the first value indicates the + number of retries before stopping waiting and the second value indicates the time interval to wait + for each retry. Default is False. Returns: Generator[Tuple[bytes, bytes]]: A generator of tuples containing the stdout and stderr in bytes. diff --git a/src/Kathara/manager/Kathara.py b/src/Kathara/manager/Kathara.py index 05bbaf48..289e321b 100644 --- a/src/Kathara/manager/Kathara.py +++ b/src/Kathara/manager/Kathara.py @@ -1,7 +1,7 @@ from __future__ import annotations import io -from typing import Set, Dict, Generator, Any, Tuple, List, Optional +from typing import Set, Dict, Generator, Any, Tuple, List, Optional, Union from ..exceptions import InstantiationError from ..foundation.manager.IManager import IManager @@ -181,7 +181,7 @@ def wipe(self, all_users: bool = False) -> None: self.manager.wipe(all_users) def connect_tty(self, machine_name: str, lab_hash: Optional[str] = None, lab_name: Optional[str] = None, - shell: str = None, logs: bool = False) -> None: + shell: str = None, logs: bool = False, wait: Union[bool, Tuple[int, float]] = True) -> None: """Connect to a device in a running network scenario, using the specified shell. Args: @@ -190,6 +190,10 @@ def connect_tty(self, machine_name: str, lab_hash: Optional[str] = None, lab_nam lab_name (str): The name of the network scenario where the device is deployed. shell (str): The name of the shell to use for connecting. logs (bool): If True, print startup logs on stdout. + wait (Union[bool, Tuple[int, float]]): If True, wait indefinitely until the end of the startup commands + execution before connecting. If a tuple is provided, the first value indicates the number of retries + before stopping waiting and the second value indicates the time interval to wait for each retry. + Default is True. Returns: None @@ -197,10 +201,11 @@ def connect_tty(self, machine_name: str, lab_hash: Optional[str] = None, lab_nam Raises: InvocationError: If a running network scenario hash or name is not specified. """ - self.manager.connect_tty(machine_name, lab_hash, lab_name, shell, logs) + self.manager.connect_tty(machine_name, lab_hash, lab_name, shell, logs, wait) def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = None, - lab_name: Optional[str] = None, wait: bool = False) -> Generator[Tuple[bytes, bytes], None, None]: + lab_name: Optional[str] = None, wait: Union[bool, Tuple[int, float]] = False) \ + -> Generator[Tuple[bytes, bytes], None, None]: """Exec a command on a device in a running network scenario. Args: @@ -208,7 +213,10 @@ def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = command (List[str]): The command to exec on the device. lab_hash (Optional[str]): The hash of the network scenario where the device is deployed. lab_name (Optional[str]): The name of the network scenario where the device is deployed. - wait (bool): If True, wait until end of the startup commands execution before executing the command. + wait (Union[bool, Tuple[int, float]]): If True, wait indefinitely until the end of the startup commands + execution before executing the command. If a tuple is provided, the first value indicates the + number of retries before stopping waiting and the second value indicates the time interval to wait + for each retry. Default is False. Returns: Generator[Tuple[bytes, bytes]]: A generator of tuples containing the stdout and stderr in bytes. diff --git a/src/Kathara/manager/docker/DockerMachine.py b/src/Kathara/manager/docker/DockerMachine.py index d4f95817..6b65cc8e 100644 --- a/src/Kathara/manager/docker/DockerMachine.py +++ b/src/Kathara/manager/docker/DockerMachine.py @@ -500,7 +500,7 @@ def _undeploy_machine(self, machine_api_object: docker.models.containers.Contain EventDispatcher.get_instance().dispatch("machine_undeployed", item=machine_api_object) def connect(self, lab_hash: str, machine_name: str, user: str = None, shell: str = None, - logs: bool = False, wait: bool = True) -> None: + logs: bool = False, wait: Union[bool, Tuple[int, float]] = True) -> None: """Open a stream to the Docker container specified by machine_name using the specified shell. Args: @@ -509,13 +509,17 @@ def connect(self, lab_hash: str, machine_name: str, user: str = None, shell: str user (str): The name of a current user on the host. shell (str): The path to the desired shell. logs (bool): If True, print the logs of the startup command. - wait (bool): If True, wait the end of the startup commands before giving control to the user. + wait (Union[bool, Tuple[int, float]]): If True, wait indefinitely until the end of the startup commands + execution before giving control to the user. If a tuple is provided, the first value indicates the + number of retries before stopping waiting and the second value indicates the time interval to wait + for each retry. Default is True. Returns: None Raises: MachineNotFoundError: If the specified device is not running. + ValueError: If the wait values is neither a boolean nor a tuple, or an invalid tuple. """ containers = self.get_machines_api_objects_by_filters(lab_hash=lab_hash, machine_name=machine_name, user=user) if not containers: @@ -529,36 +533,52 @@ def connect(self, lab_hash: str, machine_name: str, user: str = None, shell: str logging.debug("Connect to device `%s` with shell: %s" % (machine_name, shell)) + if isinstance(wait, tuple): + if len(wait) != 2: + raise ValueError("Invalid `wait` value.") + + n_retries, retry_interval = wait + should_wait = True + elif isinstance(wait, bool): + n_retries = None + retry_interval = 1 + should_wait = wait + else: + raise ValueError("Invalid `wait` value.") + startup_waited = False - if wait: - startup_waited = self._wait_startup_execution(container) + if should_wait: + startup_waited = self._wait_startup_execution(container, n_retries=n_retries, retry_interval=retry_interval) # Clean the terminal output sys.stdout.write("\033[2J") sys.stdout.write("\033[0;0H") sys.stdout.flush() - # Get the logs, if the command fails it means that the shell is not found. - cat_logs_cmd = "cat /var/log/shared.log /var/log/startup.log" - startup_command = [item for item in shell] - startup_command.extend(['-c', cat_logs_cmd]) - exec_result = self._exec_run(container, - cmd=startup_command, - stdout=True, - stderr=False, - privileged=False, - detach=False - ) - detect = chardet.detect(exec_result['output']) if exec_result['output'] else None - startup_output = exec_result['output'].decode(detect) if exec_result['output'] else None + if logs and Setting.get_instance().print_startup_log: + # Get the logs, if the command fails it means that the shell is not found. + cat_logs_cmd = "cat /var/log/shared.log /var/log/startup.log" + startup_command = [item for item in shell] + startup_command.extend(['-c', cat_logs_cmd]) + exec_result = self._exec_run(container, + cmd=startup_command, + stdout=True, + stderr=False, + privileged=False, + detach=False + ) + char_encoding = chardet.detect(exec_result['output']) if exec_result['output'] else None + startup_output = exec_result['output'].decode(char_encoding['encoding']) if exec_result['output'] else None + + if startup_output: + sys.stdout.write("--- Startup Commands Log\n") + sys.stdout.write(startup_output) + sys.stdout.write("--- End Startup Commands Log\n") - if startup_output and logs and Setting.get_instance().print_startup_log: - sys.stdout.write("--- Startup Commands Log\n") - sys.stdout.write(startup_output) - sys.stdout.write("--- End Startup Commands Log\n") + if not startup_waited: + sys.stdout.write("--- Executing other commands in background\n") - if not startup_waited: - sys.stdout.write("--- Executing other commands in background\n") + sys.stdout.flush() resp = self.client.api.exec_create(container.id, shell, @@ -585,7 +605,8 @@ def cmd_connect(): utils.exec_by_platform(tty_connect, cmd_connect, tty_connect) def exec(self, lab_hash: str, machine_name: str, command: Union[str, List], user: str = None, - tty: bool = True, wait: bool = False) -> Generator[Tuple[bytes, bytes], None, None]: + tty: bool = True, wait: Union[bool, Tuple[int, float]] = False) \ + -> Generator[Tuple[bytes, bytes], None, None]: """Execute the command on the Docker container specified by the lab_hash and the machine_name. Args: @@ -594,13 +615,17 @@ def exec(self, lab_hash: str, machine_name: str, command: Union[str, List], user user (str): The name of a current user on the host. command (Union[str, List]): The command to execute. tty (bool): If True, open a new tty. - wait (bool): If True, wait the end of the startup before executing the command. + wait (Union[bool, Tuple[int, float]]): If True, wait indefinitely until the end of the startup commands + execution before executing the command. If a tuple is provided, the first value indicates the + number of retries before stopping waiting and the second value indicates the time interval to + wait for each retry. Default is False. Returns: Generator[Tuple[bytes, bytes]]: A generator of tuples containing the stdout and stderr in bytes. Raises: MachineNotFoundError: If the specified device is not running. + ValueError: If the wait values is neither a boolean nor a tuple, or an invalid tuple. """ logging.debug("Executing command `%s` to device with name: %s" % (command, machine_name)) @@ -609,8 +634,21 @@ def exec(self, lab_hash: str, machine_name: str, command: Union[str, List], user raise MachineNotFoundError("The specified device `%s` is not running." % machine_name) container = containers.pop() - if wait: - self._wait_startup_execution(container) + if isinstance(wait, tuple): + if len(wait) != 2: + raise ValueError("Invalid `wait` value.") + + n_retries, retry_interval = wait + should_wait = True + elif isinstance(wait, bool): + n_retries = None + retry_interval = 1 + should_wait = wait + else: + raise ValueError("Invalid `wait` value.") + + if should_wait: + self._wait_startup_execution(container, n_retries=n_retries, retry_interval=retry_interval) command = shlex.split(command) if type(command) == str else command exec_result = self._exec_run(container, @@ -688,8 +726,8 @@ def _exec_run(self, container: docker.models.containers.Container, exec_stdout = "" if stdout_out: if type(stdout_out) == bytes: - detect = chardet.detect(stdout_out) - exec_stdout = stdout_out.decode(detect['encoding']) + char_encoding = chardet.detect(stdout_out) + exec_stdout = stdout_out.decode(char_encoding['encoding']) else: exec_stdout = stdout_out matches = OCI_RUNTIME_RE.search(exec_stdout) @@ -701,24 +739,28 @@ def _exec_run(self, container: docker.models.containers.Container, return {'exit_code': int(exit_code) if exit_code is not None else None, 'output': exec_output} - def _wait_startup_execution(self, container: docker.models.containers.Container): + def _wait_startup_execution(self, container: docker.models.containers.Container, + n_retries: Optional[int] = None, retry_interval: float = 1) -> bool: """Wait until the startup commands are executed or until the user requests the control over the device. Args: container (docker.models.containers.Container): The Docker container to wait. + n_retries (Optional[int]): Number of retries before stopping waiting. Default is None, waits indefinitely. + retry_interval (float): The time interval in seconds to wait for each retry. Default is 1. Returns: bool: False if the user requests the control before the ending of the startup. Else, True. """ - machine_name = container.labels['name'] + logging.debug(f"Waiting startup commands execution for device {container.labels['name']}...") - sys.stdout.write("Waiting startup commands execution. Press [ENTER] to take control of the device...") - sys.stdout.flush() + retry_interval = retry_interval if retry_interval >= 0 else 1 + n_retries = n_retries if n_retries is None or n_retries >= 0 else abs(n_retries) - logging.debug(f"Waiting startup commands execution for device {machine_name}...") - exit_code = 1 + retries = 0 + is_cmd_success = False startup_waited = True - while exit_code != 0: + printed = False + while not is_cmd_success: exec_result = self._exec_run(container, cmd="cat /var/log/EOS", stdout=True, @@ -726,16 +768,27 @@ def _wait_startup_execution(self, container: docker.models.containers.Container) privileged=False, detach=False ) - exit_code = exec_result['exit_code'] + is_cmd_success = exec_result['exit_code'] == 0 + + if not printed and not is_cmd_success: + sys.stdout.write("Waiting startup commands execution. Press [ENTER] to override...") + sys.stdout.flush() + printed = True # If the user requests the control, break the while loop if utils.exec_by_platform(utils.wait_user_input_linux, utils.wait_user_input_windows, utils.wait_user_input_linux): - startup_waited = False + startup_waited = False or is_cmd_success break - time.sleep(1) + if not is_cmd_success: + if n_retries is not None: + retries += 1 + if retries == n_retries: + break + + time.sleep(retry_interval) return startup_waited diff --git a/src/Kathara/manager/docker/DockerManager.py b/src/Kathara/manager/docker/DockerManager.py index 799d2f11..4eebb136 100644 --- a/src/Kathara/manager/docker/DockerManager.py +++ b/src/Kathara/manager/docker/DockerManager.py @@ -1,6 +1,6 @@ import io import logging -from typing import Set, Dict, Generator, Tuple, List, Optional +from typing import Set, Dict, Generator, Tuple, List, Optional, Union import docker import docker.models.containers @@ -291,7 +291,7 @@ def wipe(self, all_users: bool = False) -> None: @privileged def connect_tty(self, machine_name: str, lab_hash: Optional[str] = None, lab_name: Optional[str] = None, - shell: str = None, logs: bool = False) -> None: + shell: str = None, logs: bool = False, wait: Union[bool, Tuple[int, float]] = True) -> None: """Connect to a device in a running network scenario, using the specified shell. Args: @@ -300,6 +300,10 @@ def connect_tty(self, machine_name: str, lab_hash: Optional[str] = None, lab_nam lab_name (str): The name of the network scenario where the device is deployed. shell (str): The name of the shell to use for connecting. logs (bool): If True, print startup logs on stdout. + wait (Union[bool, Tuple[int, float]]): If True, wait indefinitely until the end of the startup commands + execution before connecting. If a tuple is provided, the first value indicates the number of retries + before stopping waiting and the second value indicates the time interval to wait for each retry. + Default is True. Returns: None @@ -319,12 +323,14 @@ def connect_tty(self, machine_name: str, lab_hash: Optional[str] = None, lab_nam machine_name=machine_name, user=user_name, shell=shell, - logs=logs + logs=logs, + wait=wait ) @privileged def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = None, - lab_name: Optional[str] = None, wait: bool = False) -> Generator[Tuple[bytes, bytes], None, None]: + lab_name: Optional[str] = None, wait: Union[bool, Tuple[int, float]] = False) \ + -> Generator[Tuple[bytes, bytes], None, None]: """Exec a command on a device in a running network scenario. Args: @@ -332,7 +338,10 @@ def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = command (List[str]): The command to exec on the device. lab_hash (Optional[str]): The hash of the network scenario where the device is deployed. lab_name (Optional[str]): The name of the network scenario where the device is deployed. - wait (bool): If True, wait until end of the startup commands execution before executing the command. + wait (Union[bool, Tuple[int, float]]): If True, wait indefinitely until the end of the startup commands + execution before executing the command. If a tuple is provided, the first value indicates the + number of retries before stopping waiting and the second value indicates the time interval to wait + for each retry. Default is False. Returns: Generator[Tuple[bytes, bytes]]: A generator of tuples containing the stdout and stderr in bytes. diff --git a/src/Kathara/manager/kubernetes/KubernetesManager.py b/src/Kathara/manager/kubernetes/KubernetesManager.py index 72ece4e2..1883602f 100644 --- a/src/Kathara/manager/kubernetes/KubernetesManager.py +++ b/src/Kathara/manager/kubernetes/KubernetesManager.py @@ -1,7 +1,7 @@ import io import json import logging -from typing import Set, Dict, Generator, Any, List, Tuple, Optional +from typing import Set, Dict, Generator, Any, List, Tuple, Optional, Union from kubernetes import client from kubernetes.client.rest import ApiException @@ -297,7 +297,7 @@ def wipe(self, all_users: bool = False) -> None: self.k8s_namespace.wipe() def connect_tty(self, machine_name: str, lab_hash: Optional[str] = None, lab_name: Optional[str] = None, - shell: str = None, logs: bool = False) -> None: + shell: str = None, logs: bool = False, wait: Union[bool, Tuple[int, float]] = True) -> None: """Connect to a device in a running network scenario, using the specified shell. Args: @@ -306,6 +306,10 @@ def connect_tty(self, machine_name: str, lab_hash: Optional[str] = None, lab_nam lab_name (str): The name of the network scenario where the device is deployed. shell (str): The name of the shell to use for connecting. logs (bool): If True, print startup logs on stdout. + wait (Union[bool, Tuple[int, float]]): If True, wait indefinitely until the end of the startup commands + execution before connecting. If a tuple is provided, the first value indicates the number of retries + before stopping waiting and the second value indicates the time interval to wait for each retry. + Default is True. No effect on Kubernetes. Returns: None @@ -328,7 +332,8 @@ def connect_tty(self, machine_name: str, lab_hash: Optional[str] = None, lab_nam ) def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = None, - lab_name: Optional[str] = None, wait: bool = False) -> Generator[Tuple[bytes, bytes], None, None]: + lab_name: Optional[str] = None, wait: Union[bool, Tuple[int, float]] = False) \ + -> Generator[Tuple[bytes, bytes], None, None]: """Exec a command on a device in a running network scenario. Args: @@ -336,8 +341,10 @@ def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = command (List[str]): The command to exec on the device. lab_hash (Optional[str]): The hash of the network scenario where the device is deployed. lab_name (Optional[str]): The name of the network scenario where the device is deployed. - wait (bool): If True, wait until end of the startup commands execution before executing the command. - No effect on Megalos. + wait (Union[bool, Tuple[int, float]]): If True, wait indefinitely until the end of the startup commands + execution before executing the command. If a tuple is provided, the first value indicates the + number of retries before stopping waiting and the second value indicates the time interval to wait + for each retry. Default is False. No effect on Kubernetes. Returns: Generator[Tuple[bytes, bytes]]: A generator of tuples containing the stdout and stderr in bytes. diff --git a/tests/manager/docker/docker_manager_test.py b/tests/manager/docker/docker_manager_test.py index 6fcae2c3..061fbe1d 100644 --- a/tests/manager/docker/docker_manager_test.py +++ b/tests/manager/docker/docker_manager_test.py @@ -482,7 +482,8 @@ def test_connect_tty_lab_hash(mock_connect, mock_get_current_user_name, docker_m machine_name=default_device.name, user="kathara_user", shell=None, - logs=False) + logs=False, + wait=True) @mock.patch("src.Kathara.utils.get_current_user_name") @@ -497,7 +498,8 @@ def test_connect_tty_lab_name(mock_connect, mock_get_current_user_name, docker_m machine_name=default_device.name, user="kathara_user", shell=None, - logs=False) + logs=False, + wait=True) @mock.patch("src.Kathara.utils.get_current_user_name") @@ -513,7 +515,8 @@ def test_connect_tty_with_custom_shell(mock_connect, mock_get_current_user_name, machine_name=default_device.name, user="kathara_user", shell="/usr/bin/zsh", - logs=False) + logs=False, + wait=True) @mock.patch("src.Kathara.utils.get_current_user_name") @@ -529,7 +532,8 @@ def test_connect_tty_with_logs(mock_connect, mock_get_current_user_name, docker_ machine_name=default_device.name, user="kathara_user", shell=None, - logs=True) + logs=True, + wait=True) @mock.patch("src.Kathara.utils.get_current_user_name") From 62097ddbe6c3df7eea3510338cf8df35e9744029 Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Fri, 14 Jul 2023 19:00:14 +0200 Subject: [PATCH 21/34] Swap retries increment --- src/Kathara/manager/docker/DockerMachine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Kathara/manager/docker/DockerMachine.py b/src/Kathara/manager/docker/DockerMachine.py index 6b65cc8e..1941211a 100644 --- a/src/Kathara/manager/docker/DockerMachine.py +++ b/src/Kathara/manager/docker/DockerMachine.py @@ -784,9 +784,9 @@ def _wait_startup_execution(self, container: docker.models.containers.Container, if not is_cmd_success: if n_retries is not None: - retries += 1 if retries == n_retries: break + retries += 1 time.sleep(retry_interval) From cf9b894d0cc3756e7c6de0892ab06d1809228bf1 Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Mon, 17 Jul 2023 17:14:41 +0200 Subject: [PATCH 22/34] Wait output is managed in Event so it is disabled on API (#209) --- ...neTerminal.py => HandleMachineTerminal.py} | 23 ++++++- src/Kathara/cli/ui/event/register.py | 9 ++- src/Kathara/manager/docker/DockerMachine.py | 68 +++++++++---------- 3 files changed, 62 insertions(+), 38 deletions(-) rename src/Kathara/cli/ui/event/{OpenMachineTerminal.py => HandleMachineTerminal.py} (53%) diff --git a/src/Kathara/cli/ui/event/OpenMachineTerminal.py b/src/Kathara/cli/ui/event/HandleMachineTerminal.py similarity index 53% rename from src/Kathara/cli/ui/event/OpenMachineTerminal.py rename to src/Kathara/cli/ui/event/HandleMachineTerminal.py index bf146c79..6edb3673 100644 --- a/src/Kathara/cli/ui/event/OpenMachineTerminal.py +++ b/src/Kathara/cli/ui/event/HandleMachineTerminal.py @@ -1,9 +1,11 @@ +import sys + from ..utils import open_machine_terminal from ....model import Machine as MachinePackage from ....setting.Setting import Setting -class OpenMachineTerminal(object): +class HandleMachineTerminal(object): """Listener fired when a device is deployed and started.""" def run(self, item: 'MachinePackage.Machine') -> None: @@ -18,3 +20,22 @@ def run(self, item: 'MachinePackage.Machine') -> None: if Setting.get_instance().open_terminals: for i in range(0, item.get_num_terms()): open_machine_terminal(item) + + def flush(self) -> None: + """Clean the stdout buffer. + + Returns: + None + """ + sys.stdout.write("\033[2J") + sys.stdout.write("\033[0;0H") + sys.stdout.flush() + + def print_wait_msg(self) -> None: + """Print the startup commands waiting message. + + Returns: + None + """ + sys.stdout.write("Waiting startup commands execution. Press [ENTER] to override...") + sys.stdout.flush() diff --git a/src/Kathara/cli/ui/event/register.py b/src/Kathara/cli/ui/event/register.py index 6d866e36..25cf0990 100644 --- a/src/Kathara/cli/ui/event/register.py +++ b/src/Kathara/cli/ui/event/register.py @@ -1,6 +1,6 @@ -from .UpdateDockerImage import UpdateDockerImage -from .OpenMachineTerminal import OpenMachineTerminal +from .HandleMachineTerminal import HandleMachineTerminal from .HandleProgressBar import HandleProgressBar +from .UpdateDockerImage import UpdateDockerImage from ....event.EventDispatcher import EventDispatcher @@ -39,4 +39,7 @@ def _register_machine_events() -> None: EventDispatcher.get_instance().register("machine_undeployed", machine_undeploy_progress_bar_handler, "update") EventDispatcher.get_instance().register("machines_undeploy_ended", machine_undeploy_progress_bar_handler, "finish") - EventDispatcher.get_instance().register("machine_deployed", OpenMachineTerminal()) + machine_terminal_handler = HandleMachineTerminal() + EventDispatcher.get_instance().register("machine_deployed", machine_terminal_handler, "run") + EventDispatcher.get_instance().register("machine_startup_wait_started", machine_terminal_handler, "print_wait_msg") + EventDispatcher.get_instance().register("machine_startup_wait_ended", machine_terminal_handler, "flush") diff --git a/src/Kathara/manager/docker/DockerMachine.py b/src/Kathara/manager/docker/DockerMachine.py index 1941211a..41840583 100644 --- a/src/Kathara/manager/docker/DockerMachine.py +++ b/src/Kathara/manager/docker/DockerMachine.py @@ -550,10 +550,7 @@ def connect(self, lab_hash: str, machine_name: str, user: str = None, shell: str if should_wait: startup_waited = self._wait_startup_execution(container, n_retries=n_retries, retry_interval=retry_interval) - # Clean the terminal output - sys.stdout.write("\033[2J") - sys.stdout.write("\033[0;0H") - sys.stdout.flush() + EventDispatcher.get_instance().dispatch("machine_startup_wait_ended") if logs and Setting.get_instance().print_startup_log: # Get the logs, if the command fails it means that the shell is not found. @@ -576,7 +573,7 @@ def connect(self, lab_hash: str, machine_name: str, user: str = None, shell: str sys.stdout.write("--- End Startup Commands Log\n") if not startup_waited: - sys.stdout.write("--- Executing other commands in background\n") + sys.stdout.write("!!! Executing other commands in background !!!\n") sys.stdout.flush() @@ -753,42 +750,45 @@ def _wait_startup_execution(self, container: docker.models.containers.Container, """ logging.debug(f"Waiting startup commands execution for device {container.labels['name']}...") - retry_interval = retry_interval if retry_interval >= 0 else 1 n_retries = n_retries if n_retries is None or n_retries >= 0 else abs(n_retries) + retry_interval = retry_interval if retry_interval >= 0 else 1 retries = 0 is_cmd_success = False startup_waited = True printed = False while not is_cmd_success: - exec_result = self._exec_run(container, - cmd="cat /var/log/EOS", - stdout=True, - stderr=False, - privileged=False, - detach=False - ) - is_cmd_success = exec_result['exit_code'] == 0 - - if not printed and not is_cmd_success: - sys.stdout.write("Waiting startup commands execution. Press [ENTER] to override...") - sys.stdout.flush() - printed = True - - # If the user requests the control, break the while loop - if utils.exec_by_platform(utils.wait_user_input_linux, - utils.wait_user_input_windows, - utils.wait_user_input_linux): - startup_waited = False or is_cmd_success - break - - if not is_cmd_success: - if n_retries is not None: - if retries == n_retries: - break - retries += 1 - - time.sleep(retry_interval) + try: + exec_result = self._exec_run(container, + cmd="cat /var/log/EOS", + stdout=True, + stderr=False, + privileged=False, + detach=False + ) + is_cmd_success = exec_result['exit_code'] == 0 + + if not printed and not is_cmd_success: + EventDispatcher.get_instance().dispatch("machine_startup_wait_started") + printed = True + + # If the user requests the control, break the while loop + if utils.exec_by_platform(utils.wait_user_input_linux, + utils.wait_user_input_windows, + utils.wait_user_input_linux): + startup_waited = False or is_cmd_success + break + + if not is_cmd_success: + if n_retries is not None: + if retries == n_retries: + break + retries += 1 + + time.sleep(retry_interval) + except KeyboardInterrupt: + # Disable the CTRL+C interrupt while waiting for startup, otherwise terminal will close. + pass return startup_waited From c6ec65df032699846aacae93af295fe6703c0537 Mon Sep 17 00:00:00 2001 From: Tommaso Caiazzi Date: Tue, 18 Jul 2023 12:04:51 +0200 Subject: [PATCH 23/34] Fix windows waiting (#209) --- src/Kathara/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Kathara/utils.py b/src/Kathara/utils.py index 6cbd521d..9a06f5dc 100644 --- a/src/Kathara/utils.py +++ b/src/Kathara/utils.py @@ -325,4 +325,4 @@ def wait_user_input_linux() -> list: def wait_user_input_windows() -> bool: """Return True if a keypress is waiting to be read. Only for Windows.""" import msvcrt - return msvcrt.kbhit() + return b'\r' in msvcrt.getch() if msvcrt.kbhit() else False From 2922279422b921cf8f704d9fa2df5d16cf5530c3 Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Tue, 18 Jul 2023 13:39:34 +0200 Subject: [PATCH 24/34] Rename "startup_commands" meta in "exec_commands" (closes #228) --- src/Kathara/manager/docker/DockerMachine.py | 18 ++++++++---------- .../manager/kubernetes/KubernetesMachine.py | 6 +++--- src/Kathara/model/Machine.py | 10 +++++----- tests/model/machine_test.py | 2 +- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/Kathara/manager/docker/DockerMachine.py b/src/Kathara/manager/docker/DockerMachine.py index 715b5209..ae319aaa 100644 --- a/src/Kathara/manager/docker/DockerMachine.py +++ b/src/Kathara/manager/docker/DockerMachine.py @@ -29,7 +29,7 @@ ) # Known commands that each container should execute -# Run order: shared.startup, machine.startup and machine.meta['startup_commands'] +# Run order: shared.startup, machine.startup and machine.meta['exec_commands'] STARTUP_COMMANDS = [ # Unmount the /etc/resolv.conf and /etc/hosts files, automatically mounted by Docker inside the container. # In this way, they can be overwritten by custom user files. @@ -82,7 +82,7 @@ # Placeholder for user commands "{machine_commands}", - "touch /var/log/EOS" + "touch /tmp/EOS" ] SHUTDOWN_COMMANDS = [ @@ -402,19 +402,17 @@ def start(self, machine: Machine) -> None: bridge_link.connect(machine.api_object) # Append executed machine startup commands inside the /var/log/startup.log file - if machine.meta['startup_commands']: + if machine.meta['exec_commands']: new_commands = [] - for command in machine.meta['startup_commands']: + for command in machine.meta['exec_commands']: new_commands.append("echo \"++ %s\" &>> /var/log/startup.log" % command) new_commands.append(command) - machine.meta['startup_commands'] = new_commands + machine.meta['exec_commands'] = new_commands # Build the final startup commands string - startup_commands_string = "; ".join( - STARTUP_COMMANDS if machine.meta['startup_commands'] else STARTUP_COMMANDS[:-2] + STARTUP_COMMANDS[-1:] - ).format( + startup_commands_string = "; ".join(STARTUP_COMMANDS).format( machine_name=machine.name, - machine_commands="; ".join(machine.meta['startup_commands']) + machine_commands="; ".join(machine.meta['exec_commands']) if machine.meta['exec_commands'] else ":" ) logging.debug(f"Executing startup command on `{machine.name}`: {startup_commands_string}") @@ -760,7 +758,7 @@ def _wait_startup_execution(self, container: docker.models.containers.Container, while not is_cmd_success: try: exec_result = self._exec_run(container, - cmd="cat /var/log/EOS", + cmd="cat /tmp/EOS", stdout=True, stderr=False, privileged=False, diff --git a/src/Kathara/manager/kubernetes/KubernetesMachine.py b/src/Kathara/manager/kubernetes/KubernetesMachine.py index 49803dd2..a9a947e5 100644 --- a/src/Kathara/manager/kubernetes/KubernetesMachine.py +++ b/src/Kathara/manager/kubernetes/KubernetesMachine.py @@ -30,7 +30,7 @@ MAX_RESTART_COUNT = 3 # Known commands that each container should execute -# Run order: shared.startup, machine.startup and machine.meta['startup_commands'] +# Run order: shared.startup, machine.startup and machine.meta['exec_commands'] STARTUP_COMMANDS = [ # If execution flag file is found, abort (this means that postStart has been called again) # If not flag the startup execution with a file @@ -100,7 +100,7 @@ # Placeholder for user commands "{machine_commands}", - "touch /var/log/EOS" + "touch /tmp/EOS" ] SHUTDOWN_COMMANDS = [ @@ -345,7 +345,7 @@ def _build_definition(self, machine: Machine, config_map: client.V1ConfigMap) -> startup_commands_string = "; ".join(STARTUP_COMMANDS) \ .format(machine_name=machine.name, sysctl_commands=sysctl_commands, - machine_commands="; ".join(machine.meta['startup_commands']) + machine_commands="; ".join(machine.meta['exec_commands']) ) post_start = client.V1LifecycleHandler( diff --git a/src/Kathara/model/Machine.py b/src/Kathara/model/Machine.py index 95f2d7c7..115d118a 100644 --- a/src/Kathara/model/Machine.py +++ b/src/Kathara/model/Machine.py @@ -64,7 +64,7 @@ def __init__(self, lab: 'LabPackage.Lab', name: str, **kwargs) -> None: self.interfaces: OrderedDict[int, Link] = collections.OrderedDict() self.meta: Dict[str, Any] = { - 'startup_commands': [], + 'exec_commands': [], 'sysctls': {}, 'envs': {}, 'ports': {}, @@ -141,7 +141,7 @@ def add_meta(self, name: str, value: Any) -> Optional[Any]: MachineOptionError: If the specified value is not valid for the specified property. """ if name == "exec": - self.meta['startup_commands'].append(value) + self.meta['exec_commands'].append(value) return None if name == "bridged": @@ -316,13 +316,13 @@ def pack_data(self) -> Optional[bytes]: # If no machine files are found, return None. return None - def get_startup_commands(self) -> List[str]: - """Get the additional device startup commands. + def get_exec_commands(self) -> List[str]: + """Get the device exec commands. Returns: List[str]: The list containing the additional commands. """ - return self.meta['startup_commands'] + return self.meta['exec_commands'] def is_bridged(self) -> bool: """Return True if the device is bridged, else return False. diff --git a/tests/model/machine_test.py b/tests/model/machine_test.py index 75382b99..a37e894b 100644 --- a/tests/model/machine_test.py +++ b/tests/model/machine_test.py @@ -25,7 +25,7 @@ def test_default_device_parameters(default_device: Machine): assert default_device.name == "test_machine" assert len(default_device.interfaces) == 0 assert default_device.meta == { - 'startup_commands': [], + 'exec_commands': [], 'sysctls': {}, 'envs': {}, 'ports': {}, From d22139aafeaff4475873b5bb860f69c0882bab2a Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Tue, 18 Jul 2023 13:40:17 +0200 Subject: [PATCH 25/34] Fix Machine tar extraction parameters --- src/Kathara/manager/docker/DockerMachine.py | 2 +- src/Kathara/manager/kubernetes/KubernetesMachine.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Kathara/manager/docker/DockerMachine.py b/src/Kathara/manager/docker/DockerMachine.py index ae319aaa..3508f2e5 100644 --- a/src/Kathara/manager/docker/DockerMachine.py +++ b/src/Kathara/manager/docker/DockerMachine.py @@ -39,7 +39,7 @@ # Copy the machine folder (if present) from the hostlab directory into the root folder of the container # In this way, files are all replaced in the container root folder "if [ -d \"/hostlab/{machine_name}\" ]; then " - "(cd /hostlab/{machine_name} && tar c .) | (cd / && tar --no-same-owner --no-same-permissions xhf -); fi", + "(cd /hostlab/{machine_name} && tar c .) | (cd / && tar xhf - --no-same-owner --no-same-permissions); fi", # If /etc/hosts is not configured by the user, add the localhost mapping "if [ ! -s \"/etc/hosts\" ]; then " diff --git a/src/Kathara/manager/kubernetes/KubernetesMachine.py b/src/Kathara/manager/kubernetes/KubernetesMachine.py index a9a947e5..affb2ebf 100644 --- a/src/Kathara/manager/kubernetes/KubernetesMachine.py +++ b/src/Kathara/manager/kubernetes/KubernetesMachine.py @@ -54,7 +54,7 @@ # Copy the machine folder (if present) from the hostlab directory into the root folder of the container # In this way, files are all replaced in the container root folder "if [ -d \"/hostlab/{machine_name}\" ]; then " - "(cd /hostlab/{machine_name} && tar c .) | (cd / && tar --no-same-owner --no-same-permissions xhf -); fi", + "(cd /hostlab/{machine_name} && tar c .) | (cd / && tar xhf - --no-same-owner --no-same-permissions); fi", # If /etc/hosts is not configured by the user, add the localhost mapping "if [ ! -s \"/etc/hosts\" ]; then " From c6244764067f08dee655e52ea605f5a0bf670a36 Mon Sep 17 00:00:00 2001 From: Tommaso Caiazzi Date: Tue, 18 Jul 2023 13:42:19 +0200 Subject: [PATCH 26/34] Fix `wait_user_input_windows` pydoc --- src/Kathara/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Kathara/utils.py b/src/Kathara/utils.py index 9a06f5dc..9923d9e5 100644 --- a/src/Kathara/utils.py +++ b/src/Kathara/utils.py @@ -323,6 +323,6 @@ def wait_user_input_linux() -> list: def wait_user_input_windows() -> bool: - """Return True if a keypress is waiting to be read. Only for Windows.""" + """Return True if an Enter keypress is waiting to be read. Only for Windows.""" import msvcrt return b'\r' in msvcrt.getch() if msvcrt.kbhit() else False From db15c579d4a9625ad1ac1dfb0e5560bc39a8e2dd Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Tue, 18 Jul 2023 15:56:40 +0200 Subject: [PATCH 27/34] Add help methods in Lab and Machine (closes #229) --- src/Kathara/manager/docker/DockerManager.py | 2 +- .../manager/kubernetes/KubernetesManager.py | 2 +- src/Kathara/model/Lab.py | 85 +++++++++++++++++-- src/Kathara/model/Machine.py | 8 +- tests/model/lab_test.py | 10 +-- 5 files changed, 89 insertions(+), 18 deletions(-) diff --git a/src/Kathara/manager/docker/DockerManager.py b/src/Kathara/manager/docker/DockerManager.py index 4eebb136..44f3f99e 100644 --- a/src/Kathara/manager/docker/DockerManager.py +++ b/src/Kathara/manager/docker/DockerManager.py @@ -124,7 +124,7 @@ def deploy_lab(self, lab: Lab, selected_machines: Set[str] = None) -> None: Raises: MachineNotFoundError: If the specified devices are not in the network scenario. """ - if selected_machines and not lab.find_machines(selected_machines): + if selected_machines and not lab.has_machines(selected_machines): machines_not_in_lab = selected_machines - set(lab.machines.keys()) raise MachineNotFoundError(f"The following devices are not in the network scenario: {machines_not_in_lab}.") diff --git a/src/Kathara/manager/kubernetes/KubernetesManager.py b/src/Kathara/manager/kubernetes/KubernetesManager.py index 1883602f..46de9204 100644 --- a/src/Kathara/manager/kubernetes/KubernetesManager.py +++ b/src/Kathara/manager/kubernetes/KubernetesManager.py @@ -90,7 +90,7 @@ def deploy_lab(self, lab: Lab, selected_machines: Set[str] = None) -> None: LabAlreadyExistsError: If a network scenario is deployed while it is terminating its execution. ApiError: If the Kubernetes APIs throw an exception. """ - if selected_machines and not lab.find_machines(selected_machines): + if selected_machines and not lab.has_machines(selected_machines): machines_not_in_lab = selected_machines - set(lab.machines.keys()) raise MachineNotFoundError(f"The following devices are not in the network scenario: {machines_not_in_lab}.") diff --git a/src/Kathara/model/Lab.py b/src/Kathara/model/Lab.py index 6ecc53aa..83005fee 100644 --- a/src/Kathara/model/Lab.py +++ b/src/Kathara/model/Lab.py @@ -106,6 +106,29 @@ def connect_machine_to_link(self, machine_name: str, link_name: str, machine_ifa return machine, link + def connect_machine_obj_to_link(self, machine: 'MachinePackage.Machine', + link_name: str, machine_iface_number: int = None) -> Tuple[Link, Optional[int]]: + """Connect the specified device object to the specified collision domain. + + Args: + machine (Kathara.model.Machine): The device object. + link_name (str): The collision domain name. + machine_iface_number (int): The number of the device interface to connect. If it is None, the first free + number is used. + + Returns: + Tuple[Kathara.model.Link, Optional[int]]: A tuple containing the collision domain and + the assigned interface number (if machine_iface_number is None). + + Raises: + Exception: If an already used interface number is specified. + """ + link = self.get_or_new_link(link_name) + + machine_iface_number = machine.add_interface(link, number=machine_iface_number) + + return link, machine_iface_number + def assign_meta_to_machine(self, machine_name: str, meta_name: str, meta_value: str) -> Optional[Any]: """Assign meta information to the specified device. @@ -153,19 +176,41 @@ def check_integrity(self) -> None: for machine in self.machines: self.machines[machine].check() - def get_links_from_machines(self, selected_machines: Union[List[str], Set[str]]) -> Set[str]: - """Return the name of the collision domains connected to the selected devices. + def get_links_from_machines(self, machines: Union[List[str], Set[str]]) -> Set[str]: + """Return the name of the collision domains connected to the devices. Args: - selected_machines (Set[str]): A set with selected devices names. + machines (Union[List[str], Set[str]]): A set or a list with selected devices names. Returns: - Set[str]: A set of names of collision domains to deploy. + Set[str]: A set of names of collision domains. """ # Intersect selected machines names with self.machines keys - selected_machines = set(self.machines.keys()) & set(selected_machines) + machines = set(self.machines.keys()) & set(machines) # Apply filtering - machines = [v for (k, v) in self.machines.items() if k in selected_machines] + machines = [v for (k, v) in self.machines.items() if k in machines] + + # Get only selected machines Link objects. + selected_links = set(chain.from_iterable([machine.interfaces.values() for machine in machines])) + selected_links = {link.name for link in selected_links} + + return selected_links + + def get_links_from_machine_objs(self, + machines: Union[List['MachinePackage.Machine'], Set['MachinePackage.Machine']]) -> \ + Set[str]: + """Return the name of the collision domains connected to the devices. + + Args: + machines (Union[List[str], Set[str]]): A set or a list with selected devices names. + + Returns: + Set[str]: A set of names of collision domains. + """ + # Intersect selected machines names with self.machines keys + machines = set(self.machines.keys()) & set(map(lambda x: x.name, machines)) + # Apply filtering + machines = [v for (k, v) in self.machines.items() if k in machines] # Get only selected machines Link objects. selected_links = set(chain.from_iterable([machine.interfaces.values() for machine in machines])) @@ -338,7 +383,7 @@ def add_option(self, name: str, value: Any) -> None: if value is not None: self.general_options[name] = value - def find_machine(self, machine_name: str) -> bool: + def has_machine(self, machine_name: str) -> bool: """Check if the specified device is in the network scenario. Args: @@ -349,7 +394,7 @@ def find_machine(self, machine_name: str) -> bool: """ return machine_name in self.machines.keys() - def find_machines(self, machine_names: Set[str]) -> bool: + def has_machines(self, machine_names: Set[str]) -> bool: """Check if the specified devices are in the network scenario. Args: @@ -358,7 +403,29 @@ def find_machines(self, machine_names: Set[str]) -> bool: Returns: bool: True if the devices are all in the network scenario, else False. """ - return all(map(lambda x: self.find_machine(x), machine_names)) + return all(map(lambda x: self.has_machine(x), machine_names)) + + def has_link(self, link_name: str) -> bool: + """Check if the specified collision domain is in the network scenario. + + Args: + link_name (str): The name of the collision domain to search. + + Returns: + bool: True if the collision domain is in the network scenario, else False. + """ + return link_name in self.links.keys() + + def has_links(self, link_names: Set[str]) -> bool: + """Check if the specified collision domains are in the network scenario. + + Args: + link_names (Set[str]): A set of strings containing the names of the collision domains to search. + + Returns: + bool: True if the collision domains are all in the network scenario, else False. + """ + return all(map(lambda x: self.has_link(x), link_names)) def __repr__(self) -> str: return "Lab(%s, %s, %s, %s)" % (self.fs, self.hash, self.machines, self.links) diff --git a/src/Kathara/model/Machine.py b/src/Kathara/model/Machine.py index 115d118a..9dbd5730 100644 --- a/src/Kathara/model/Machine.py +++ b/src/Kathara/model/Machine.py @@ -77,7 +77,7 @@ def __init__(self, lab: 'LabPackage.Lab', name: str, **kwargs) -> None: self.update_meta(kwargs) - def add_interface(self, link: 'LinkPackage.Link', number: int = None) -> None: + def add_interface(self, link: 'LinkPackage.Link', number: int = None) -> Optional[int]: """Add an interface to the device attached to the specified collision domain. Args: @@ -85,14 +85,16 @@ def add_interface(self, link: 'LinkPackage.Link', number: int = None) -> None: number (int): The number of the new interface. If it is None, the first free number is selected. Returns: - None + Optional[int]: The number of the assigned interface if not passed, else None. Raises: MachineCollisionDomainConflictError: If the interface number specified is already used on the device. MachineCollisionDomainConflictError: If the device is already connected to the collision domain. """ + had_number = True if number is None: number = len(self.interfaces.keys()) + had_number = False if number in self.interfaces: raise MachineCollisionDomainError(f"Interface {number} already set on device `{self.name}`.") @@ -105,6 +107,8 @@ def add_interface(self, link: 'LinkPackage.Link', number: int = None) -> None: self.interfaces[number] = link link.machines[self.name] = self + return number if not had_number else None + def remove_interface(self, link: 'LinkPackage.Link') -> None: """Disconnect the device from the specified collision domain. diff --git a/tests/model/lab_test.py b/tests/model/lab_test.py index f4cc6e00..15e10203 100644 --- a/tests/model/lab_test.py +++ b/tests/model/lab_test.py @@ -260,7 +260,7 @@ def test_intersect_machines(default_scenario: Lab): default_scenario.connect_machine_to_link("pc2", "A") default_scenario.connect_machine_to_link("pc2", "B") assert len(default_scenario.machines) == 2 - links = default_scenario.get_links_from_machines(selected_machines=["pc1"]) + links = default_scenario.get_links_from_machines(machines=["pc1"]) assert len(default_scenario.machines) == 2 assert 'pc1' in default_scenario.machines assert 'pc2' in default_scenario.machines @@ -292,13 +292,13 @@ def test_apply_dependencies(default_scenario: Lab): def test_find_machine_true(default_scenario: Lab): default_scenario.get_or_new_machine("pc1") - assert default_scenario.find_machine("pc1") + assert default_scenario.has_machine("pc1") def test_find_machine_false(default_scenario: Lab): default_scenario.get_or_new_machine("pc1") - assert not default_scenario.find_machine("pc2") + assert not default_scenario.has_machine("pc2") def test_find_machines_true(default_scenario: Lab): @@ -306,11 +306,11 @@ def test_find_machines_true(default_scenario: Lab): default_scenario.get_or_new_machine("pc2") default_scenario.get_or_new_machine("pc3") - assert default_scenario.find_machines({"pc1", "pc2", "pc3"}) + assert default_scenario.has_machines({"pc1", "pc2", "pc3"}) def test_find_machines_false(default_scenario: Lab): default_scenario.get_or_new_machine("pc1") default_scenario.get_or_new_machine("pc2") - assert not default_scenario.find_machines({"pc1", "pc2", "pc3"}) + assert not default_scenario.has_machines({"pc1", "pc2", "pc3"}) From 90b5f0a187cc00dbf35a090f014188700f4e96c7 Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Tue, 18 Jul 2023 16:10:49 +0200 Subject: [PATCH 28/34] Add tests (#229) --- tests/model/external_link_test.py | 4 +- tests/model/lab_test.py | 79 +++++++++++++++++++++++++++++++ tests/model/machine_test.py | 10 +++- 3 files changed, 90 insertions(+), 3 deletions(-) diff --git a/tests/model/external_link_test.py b/tests/model/external_link_test.py index 266e21f9..2b2147f1 100644 --- a/tests/model/external_link_test.py +++ b/tests/model/external_link_test.py @@ -25,7 +25,7 @@ def test_external_link_creation(external_link_vlan): def test_get_name_and_vlan(external_link_vlan): interface, vlan = external_link_vlan.get_name_and_vlan() assert interface == "eth0" - assert vlan is 1 + assert vlan == 1 def test_get_name_and_vlan_no_vlan(external_link_no_vlan): @@ -40,7 +40,7 @@ def test_get_name_and_vlan_long_name(): # If the length of interface name + vlan tag is more than 15 chars, we truncate the interface name to # 15 - VLAN_NAME_LENGTH in order to fit the whole string in 15 chars assert interface == "long-interfac" - assert vlan is 1 + assert vlan == 1 def test_full_name(external_link_vlan): diff --git a/tests/model/lab_test.py b/tests/model/lab_test.py index 15e10203..234bc90f 100644 --- a/tests/model/lab_test.py +++ b/tests/model/lab_test.py @@ -199,6 +199,72 @@ def test_connect_machine_to_two_links(default_scenario: Lab): assert result_2 == (default_scenario.machines['pc1'], default_scenario.links['B']) +def test_connect_machine_to_link_iface_numbers(default_scenario: Lab): + result_1 = default_scenario.connect_machine_to_link("pc1", "A", machine_iface_number=2) + assert len(default_scenario.machines) == 1 + assert default_scenario.machines['pc1'] + assert len(default_scenario.links) == 1 + assert default_scenario.links['A'] + assert default_scenario.machines['pc1'].interfaces[2].name == 'A' + assert result_1 == (default_scenario.machines['pc1'], default_scenario.links['A']) + + +def test_connect_one_machine_obj_to_link(default_scenario: Lab): + pc1 = default_scenario.new_machine("pc1") + result_1 = default_scenario.connect_machine_obj_to_link(pc1, "A") + assert len(default_scenario.machines) == 1 + assert default_scenario.machines['pc1'] + assert len(default_scenario.links) == 1 + assert default_scenario.links['A'] + assert default_scenario.machines['pc1'].interfaces[0].name == 'A' + assert result_1 == (default_scenario.links['A'], 0) + + +def test_connect_two_machine_obj_to_link(default_scenario: Lab): + pc1 = default_scenario.new_machine("pc1") + result_1 = default_scenario.connect_machine_obj_to_link(pc1, "A") + assert len(default_scenario.machines) == 1 + assert default_scenario.machines['pc1'] + assert len(default_scenario.links) == 1 + assert default_scenario.links['A'] + pc2 = default_scenario.new_machine("pc2") + result_2 = default_scenario.connect_machine_obj_to_link(pc2, "A") + assert len(default_scenario.machines) == 2 + assert default_scenario.machines['pc2'] + assert len(default_scenario.links) == 1 + assert default_scenario.links['A'] + assert default_scenario.machines['pc1'].interfaces[0].name == 'A' + assert default_scenario.machines['pc2'].interfaces[0].name == 'A' + assert result_1 == (default_scenario.links['A'], 0) + assert result_2 == (default_scenario.links['A'], 0) + + +def test_connect_machine_obj_to_two_links(default_scenario: Lab): + pc1 = default_scenario.new_machine("pc1") + result_1 = default_scenario.connect_machine_obj_to_link(pc1, "A") + result_2 = default_scenario.connect_machine_obj_to_link(pc1, "B") + assert len(default_scenario.machines) == 1 + assert default_scenario.machines['pc1'] + assert len(default_scenario.links) == 2 + assert default_scenario.links['A'] + assert default_scenario.links['B'] + assert default_scenario.machines['pc1'].interfaces[0].name == 'A' + assert default_scenario.machines['pc1'].interfaces[1].name == 'B' + assert result_1 == (default_scenario.links['A'], 0) + assert result_2 == (default_scenario.links['B'], 1) + + +def test_connect_machine_obj_to_link_iface_numbers(default_scenario: Lab): + pc1 = default_scenario.new_machine("pc1") + result_1 = default_scenario.connect_machine_obj_to_link(pc1, "A", machine_iface_number=2) + assert len(default_scenario.machines) == 1 + assert default_scenario.machines['pc1'] + assert len(default_scenario.links) == 1 + assert default_scenario.links['A'] + assert default_scenario.machines['pc1'].interfaces[2].name == 'A' + assert result_1 == (default_scenario.links['A'], None) + + def test_assign_meta_to_machine(default_scenario: Lab): default_scenario.get_or_new_machine("pc1") result = default_scenario.assign_meta_to_machine("pc1", "test_meta", "test_value") @@ -268,6 +334,19 @@ def test_intersect_machines(default_scenario: Lab): assert 'B' not in links +def test_intersect_machines_objs(default_scenario: Lab): + default_scenario.connect_machine_to_link("pc1", "A") + default_scenario.connect_machine_to_link("pc2", "A") + default_scenario.connect_machine_to_link("pc2", "B") + assert len(default_scenario.machines) == 2 + links = default_scenario.get_links_from_machine_objs(machines=[default_scenario.get_machine("pc1")]) + assert len(default_scenario.machines) == 2 + assert 'pc1' in default_scenario.machines + assert 'pc2' in default_scenario.machines + assert 'A' in links + assert 'B' not in links + + def test_create_shared_folder(directory_scenario: Lab): directory_scenario.create_shared_folder() assert directory_scenario.fs.isdir('shared') diff --git a/tests/model/machine_test.py b/tests/model/machine_test.py index a37e894b..360a75b4 100644 --- a/tests/model/machine_test.py +++ b/tests/model/machine_test.py @@ -39,9 +39,17 @@ def test_default_device_parameters(default_device: Machine): # TEST: add_interface # def test_add_interface(default_device: Machine): - default_device.add_interface(Link(default_device.lab, "A")) + result = default_device.add_interface(Link(default_device.lab, "A")) assert len(default_device.interfaces) == 1 assert default_device.interfaces[0].name == "A" + assert result == 0 + + +def test_add_interface_with_number(default_device: Machine): + result = default_device.add_interface(Link(default_device.lab, "A"), number=2) + assert len(default_device.interfaces) == 1 + assert default_device.interfaces[2].name == "A" + assert result is None def test_add_interface_exception(default_device: Machine): From 05827aeb27db05eba2eb512731a7eba7e2e86dc0 Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Wed, 19 Jul 2023 10:40:03 +0200 Subject: [PATCH 29/34] Minor polish --- docs/kathara-exec.1.ronn | 4 +- src/Kathara/cli/command/WipeCommand.py | 4 -- src/Kathara/foundation/manager/IManager.py | 4 +- src/Kathara/manager/Kathara.py | 4 +- src/Kathara/manager/docker/DockerManager.py | 4 +- .../manager/kubernetes/KubernetesManager.py | 4 +- src/Kathara/utils.py | 48 ++++++------------- tests/cli/wipe_command_test.py | 35 ++------------ 8 files changed, 29 insertions(+), 78 deletions(-) diff --git a/docs/kathara-exec.1.ronn b/docs/kathara-exec.1.ronn index 588a8716..6161aafc 100644 --- a/docs/kathara-exec.1.ronn +++ b/docs/kathara-exec.1.ronn @@ -5,7 +5,7 @@ kathara-exec(1) -- Execute a command in a Kathara device ## SYNOPSIS `kathara exec` [`-h`] [`-d` \| `-v`] -[`--no-stdout`] [`--no-stderr`] +[`--no-stdout`] [`--no-stderr`] [`--wait`] [ ...] ## DESCRIPTION @@ -38,7 +38,7 @@ Execute a command in the Kathara device DEVICE_NAME. * `--wait`: Wait until startup commands execution finishes. - You can override the timer by pressing `[ENTER]`. + You can override the wait by pressing `[ENTER]`. * `: Name of the device to execute the command into. diff --git a/src/Kathara/cli/command/WipeCommand.py b/src/Kathara/cli/command/WipeCommand.py index ab01d8fa..d6915987 100644 --- a/src/Kathara/cli/command/WipeCommand.py +++ b/src/Kathara/cli/command/WipeCommand.py @@ -1,5 +1,4 @@ import argparse -import shutil import sys from typing import List @@ -67,6 +66,3 @@ def run(self, current_path: str, argv: List[str]) -> None: raise PrivilegeError("You must be root in order to wipe all Kathara devices of all users.") Kathara.get_instance().wipe(all_users=bool(args['all'])) - - vlab_dir = utils.get_vlab_temp_path(force_creation=False) - shutil.rmtree(vlab_dir, ignore_errors=True) diff --git a/src/Kathara/foundation/manager/IManager.py b/src/Kathara/foundation/manager/IManager.py index 53958482..7edac597 100644 --- a/src/Kathara/foundation/manager/IManager.py +++ b/src/Kathara/foundation/manager/IManager.py @@ -178,14 +178,14 @@ def connect_tty(self, machine_name: str, lab_hash: Optional[str] = None, lab_nam raise NotImplementedError("You must implement `connect_tty` method.") @abstractmethod - def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = None, + def exec(self, machine_name: str, command: Union[List[str], str], lab_hash: Optional[str] = None, lab_name: Optional[str] = None, wait: Union[bool, Tuple[int, float]] = False) \ -> Generator[Tuple[bytes, bytes], None, None]: """Exec a command on a device in a running network scenario. Args: machine_name (str): The name of the device to connect. - command (List[str]): The command to exec on the device. + command (Union[List[str], str]): The command to exec on the device. lab_hash (Optional[str]): The hash of the network scenario where the device is deployed. lab_name (Optional[str]): The name of the network scenario where the device is deployed. wait (Union[bool, Tuple[int, float]]): If True, wait indefinitely until the end of the startup commands diff --git a/src/Kathara/manager/Kathara.py b/src/Kathara/manager/Kathara.py index 289e321b..2bfb51ac 100644 --- a/src/Kathara/manager/Kathara.py +++ b/src/Kathara/manager/Kathara.py @@ -203,14 +203,14 @@ def connect_tty(self, machine_name: str, lab_hash: Optional[str] = None, lab_nam """ self.manager.connect_tty(machine_name, lab_hash, lab_name, shell, logs, wait) - def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = None, + def exec(self, machine_name: str, command: Union[List[str], str], lab_hash: Optional[str] = None, lab_name: Optional[str] = None, wait: Union[bool, Tuple[int, float]] = False) \ -> Generator[Tuple[bytes, bytes], None, None]: """Exec a command on a device in a running network scenario. Args: machine_name (str): The name of the device to connect. - command (List[str]): The command to exec on the device. + command (Union[List[str], str]): The command to exec on the device. lab_hash (Optional[str]): The hash of the network scenario where the device is deployed. lab_name (Optional[str]): The name of the network scenario where the device is deployed. wait (Union[bool, Tuple[int, float]]): If True, wait indefinitely until the end of the startup commands diff --git a/src/Kathara/manager/docker/DockerManager.py b/src/Kathara/manager/docker/DockerManager.py index 44f3f99e..652074c1 100644 --- a/src/Kathara/manager/docker/DockerManager.py +++ b/src/Kathara/manager/docker/DockerManager.py @@ -328,14 +328,14 @@ def connect_tty(self, machine_name: str, lab_hash: Optional[str] = None, lab_nam ) @privileged - def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = None, + def exec(self, machine_name: str, command: Union[List[str], str], lab_hash: Optional[str] = None, lab_name: Optional[str] = None, wait: Union[bool, Tuple[int, float]] = False) \ -> Generator[Tuple[bytes, bytes], None, None]: """Exec a command on a device in a running network scenario. Args: machine_name (str): The name of the device to connect. - command (List[str]): The command to exec on the device. + command (Union[List[str], str]): The command to exec on the device. lab_hash (Optional[str]): The hash of the network scenario where the device is deployed. lab_name (Optional[str]): The name of the network scenario where the device is deployed. wait (Union[bool, Tuple[int, float]]): If True, wait indefinitely until the end of the startup commands diff --git a/src/Kathara/manager/kubernetes/KubernetesManager.py b/src/Kathara/manager/kubernetes/KubernetesManager.py index 46de9204..01b5054d 100644 --- a/src/Kathara/manager/kubernetes/KubernetesManager.py +++ b/src/Kathara/manager/kubernetes/KubernetesManager.py @@ -331,14 +331,14 @@ def connect_tty(self, machine_name: str, lab_hash: Optional[str] = None, lab_nam logs=logs ) - def exec(self, machine_name: str, command: List[str], lab_hash: Optional[str] = None, + def exec(self, machine_name: str, command: Union[List[str], str], lab_hash: Optional[str] = None, lab_name: Optional[str] = None, wait: Union[bool, Tuple[int, float]] = False) \ -> Generator[Tuple[bytes, bytes], None, None]: """Exec a command on a device in a running network scenario. Args: machine_name (str): The name of the device to connect. - command (List[str]): The command to exec on the device. + command (Union[List[str], str]): The command to exec on the device. lab_hash (Optional[str]): The hash of the network scenario where the device is deployed. lab_name (Optional[str]): The name of the network scenario where the device is deployed. wait (Union[bool, Tuple[int, float]]): If True, wait indefinitely until the end of the startup commands diff --git a/src/Kathara/utils.py b/src/Kathara/utils.py index 9923d9e5..b8904bb2 100644 --- a/src/Kathara/utils.py +++ b/src/Kathara/utils.py @@ -15,8 +15,8 @@ from multiprocessing import cpu_count from platform import node, machine from sys import platform as _platform -from typing import Any, Optional, Match, Generator, List, Callable, Union, Dict, Iterable from types import ModuleType +from typing import Any, Optional, Match, Generator, List, Callable, Union, Dict, Iterable from binaryornot.check import is_binary from slug import slug @@ -141,6 +141,19 @@ def import_pywintypes() -> ModuleType: return exec_by_platform(pywintypes_import_stub, pywintypes_import_win, pywintypes_import_stub) +def wait_user_input_linux() -> list: + """Non-blocking input function for Linux and macOS.""" + import select + to_break, _, _ = select.select([sys.stdin], [], [], 0.1) + return to_break + + +def wait_user_input_windows() -> bool: + """Return True if an Enter keypress is waiting to be read. Only for Windows.""" + import msvcrt + return b'\r' in msvcrt.getch() if msvcrt.kbhit() else False + + # Architecture Test def get_architecture() -> str: architecture = machine().lower() @@ -261,26 +274,6 @@ def human_readable_bytes(size_bytes: int) -> str: # Lab Functions -def get_lab_temp_path(lab_name: str, force_creation: bool = True) -> str: - def windows_path(): - import win32file - return win32file.GetLongPathName(tempfile.gettempdir()) - - tempdir = exec_by_platform(tempfile.gettempdir, - windows_path, - lambda: re.sub(r"/+", "/", "/%s" % get_absolute_path("/tmp")) - ) - lab_temp_directory = os.path.join(tempdir, lab_name) - if not os.path.isdir(lab_temp_directory) and force_creation: - os.mkdir(lab_temp_directory) - - return lab_temp_directory - - -def get_vlab_temp_path(force_creation: bool = True) -> str: - return get_lab_temp_path("kathara_vlab", force_creation=force_creation) - - def pack_file_for_tar(file_obj: Union[str, io.IOBase], arc_name: str) -> (tarfile.TarInfo, bytes): if isinstance(file_obj, str): file_content_patched = convert_win_2_linux(file_obj) @@ -313,16 +306,3 @@ def pack_files_for_tar(guest_to_host: Dict) -> bytes: tar_data = temp_file.read() return tar_data - - -def wait_user_input_linux() -> list: - """Non-blocking input function for Linux and macOS.""" - import select - to_break, _, _ = select.select([sys.stdin], [], [], 0.1) - return to_break - - -def wait_user_input_windows() -> bool: - """Return True if an Enter keypress is waiting to be read. Only for Windows.""" - import msvcrt - return b'\r' in msvcrt.getch() if msvcrt.kbhit() else False diff --git a/tests/cli/wipe_command_test.py b/tests/cli/wipe_command_test.py index 5611b148..97cac8e4 100644 --- a/tests/cli/wipe_command_test.py +++ b/tests/cli/wipe_command_test.py @@ -9,84 +9,59 @@ from src.Kathara.exceptions import PrivilegeError -@mock.patch("shutil.rmtree") -@mock.patch("src.Kathara.utils.get_vlab_temp_path") @mock.patch("src.Kathara.cli.command.WipeCommand.confirmation_prompt") @mock.patch("src.Kathara.manager.Kathara.Kathara.get_instance") @mock.patch("src.Kathara.manager.docker.DockerManager.DockerManager") -def test_run_no_params(mock_docker_manager, mock_manager_get_instance, - mock_confirmation_prompt, mock_get_vlab_temp_path, mock_rm_tree): +def test_run_no_params(mock_docker_manager, mock_manager_get_instance, mock_confirmation_prompt): mock_manager_get_instance.return_value = mock_docker_manager - mock_get_vlab_temp_path.return_value = '/vlab/path' command = WipeCommand() command.run('.', []) mock_confirmation_prompt.assert_called_once() mock_docker_manager.wipe.assert_called_once_with(all_users=False) - mock_rm_tree.assert_called_once_with('/vlab/path', ignore_errors=True) -@mock.patch("shutil.rmtree") -@mock.patch("src.Kathara.utils.get_vlab_temp_path") @mock.patch("src.Kathara.cli.command.WipeCommand.confirmation_prompt") @mock.patch("src.Kathara.manager.Kathara.Kathara.get_instance") @mock.patch("src.Kathara.manager.docker.DockerManager.DockerManager") -def test_run_with_force(mock_docker_manager, mock_manager_get_instance, mock_confirmation_prompt, - mock_get_vlab_temp_path, mock_rm_tree): +def test_run_with_force(mock_docker_manager, mock_manager_get_instance, mock_confirmation_prompt): mock_manager_get_instance.return_value = mock_docker_manager - mock_get_vlab_temp_path.return_value = '/vlab/path' command = WipeCommand() command.run('.', ['-f']) assert not mock_confirmation_prompt.called mock_docker_manager.wipe.assert_called_once_with(all_users=False) - mock_rm_tree.assert_called_once_with('/vlab/path', ignore_errors=True) -@mock.patch("shutil.rmtree") -@mock.patch("src.Kathara.utils.get_vlab_temp_path") @mock.patch("src.Kathara.setting.Setting.Setting.wipe_from_disk") @mock.patch("src.Kathara.manager.Kathara.Kathara.wipe") @mock.patch("src.Kathara.cli.command.WipeCommand.confirmation_prompt") -def test_run_with_setting(mock_confirmation_prompt, mock_wipe, mock_wipe_from_disk, mock_get_vlab_temp_path, - mock_rm_tree): - mock_get_vlab_temp_path.return_value = '/vlab/path' +def test_run_with_setting(mock_confirmation_prompt, mock_wipe, mock_wipe_from_disk): command = WipeCommand() command.run('.', ['-s']) mock_confirmation_prompt.assert_called_once() mock_wipe_from_disk.assert_called_once() assert not mock_wipe.called - assert not mock_rm_tree.called -@mock.patch("shutil.rmtree") -@mock.patch("src.Kathara.utils.get_vlab_temp_path") @mock.patch("src.Kathara.utils.is_admin") @mock.patch("src.Kathara.cli.command.WipeCommand.confirmation_prompt") @mock.patch("src.Kathara.manager.Kathara.Kathara.get_instance") @mock.patch("src.Kathara.manager.docker.DockerManager.DockerManager") -def test_run_with_all(mock_docker_manager, mock_manager_get_instance, mock_confirmation_prompt, mock_is_admin, - mock_get_vlab_temp_path, mock_rm_tree): - mock_get_vlab_temp_path.return_value = '/vlab/path' +def test_run_with_all(mock_docker_manager, mock_manager_get_instance, mock_confirmation_prompt, mock_is_admin): mock_manager_get_instance.return_value = mock_docker_manager mock_is_admin.return_value = True command = WipeCommand() command.run('.', ['-a']) mock_confirmation_prompt.assert_called_once() mock_docker_manager.wipe.assert_called_once_with(all_users=True) - mock_rm_tree.assert_called_once_with('/vlab/path', ignore_errors=True) -@mock.patch("shutil.rmtree") -@mock.patch("src.Kathara.utils.get_vlab_temp_path") @mock.patch("src.Kathara.manager.Kathara.Kathara.wipe") @mock.patch("src.Kathara.utils.is_admin") @mock.patch("src.Kathara.cli.command.WipeCommand.confirmation_prompt") -def test_run_with_all_no_root(mock_confirmation_prompt, mock_is_admin, mock_wipe, mock_get_vlab_temp_path, - mock_rm_tree): - mock_get_vlab_temp_path.return_value = '/vlab/path' +def test_run_with_all_no_root(mock_confirmation_prompt, mock_is_admin, mock_wipe): mock_is_admin.return_value = False command = WipeCommand() with pytest.raises(PrivilegeError): command.run('.', ['-a']) mock_confirmation_prompt.assert_called_once() assert not mock_wipe.called - assert not mock_rm_tree.called From 5a058a3251a728e642ee25b95dd3169906f04270 Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Wed, 19 Jul 2023 14:45:05 +0200 Subject: [PATCH 30/34] Add Lab object parameter in undeploy_lab --- src/Kathara/foundation/manager/IManager.py | 12 +++-- src/Kathara/manager/Kathara.py | 14 +++--- src/Kathara/manager/docker/DockerManager.py | 20 +++++--- .../manager/kubernetes/KubernetesManager.py | 20 +++++--- tests/manager/docker/docker_manager_test.py | 50 ++++++++++++++++++- 5 files changed, 87 insertions(+), 29 deletions(-) diff --git a/src/Kathara/foundation/manager/IManager.py b/src/Kathara/foundation/manager/IManager.py index 7edac597..15f41703 100644 --- a/src/Kathara/foundation/manager/IManager.py +++ b/src/Kathara/foundation/manager/IManager.py @@ -121,15 +121,17 @@ def undeploy_link(self, link: Link) -> None: raise NotImplementedError("You must implement `undeploy_link` method.") @abstractmethod - def undeploy_lab(self, lab_hash: Optional[str] = None, lab_name: Optional[str] = None, + def undeploy_lab(self, lab_hash: Optional[str] = None, lab_name: Optional[str] = None, lab: Optional[Lab] = None, selected_machines: Optional[Set[str]] = None) -> None: """Undeploy a Kathara network scenario. Args: - lab_hash (Optional[str]): The hash of the network scenario. Can be used as an alternative to lab_name. - If None, lab_name should be set. - lab_name (Optional[str]): The name of the network scenario. Can be used as an alternative to lab_hash. - If None, lab_hash should be set. + lab_hash (Optional[str]): The hash of the network scenario. + Can be used as an alternative to lab_name and lab. If None, lab_name or lab should be set. + lab_name (Optional[str]): The name of the network scenario. + Can be used as an alternative to lab_hash and lab. If None, lab_hash or lab should be set. + lab (Optional[Kathara.model.Lab]): The network scenario object. + Can be used as an alternative to lab_hash and lab_name. If None, lab_hash or lab_name should be set. selected_machines (Optional[Set[str]]): If not None, undeploy only the specified devices. Returns: diff --git a/src/Kathara/manager/Kathara.py b/src/Kathara/manager/Kathara.py index 2bfb51ac..f86412bd 100644 --- a/src/Kathara/manager/Kathara.py +++ b/src/Kathara/manager/Kathara.py @@ -149,15 +149,17 @@ def undeploy_link(self, link: Link) -> None: """ self.manager.undeploy_link(link) - def undeploy_lab(self, lab_hash: Optional[str] = None, lab_name: Optional[str] = None, + def undeploy_lab(self, lab_hash: Optional[str] = None, lab_name: Optional[str] = None, lab: Optional[Lab] = None, selected_machines: Optional[Set[str]] = None) -> None: """Undeploy a Kathara network scenario. Args: - lab_hash (Optional[str]): The hash of the network scenario. Can be used as an alternative to lab_name. - If None, lab_name should be set. - lab_name (Optional[str]): The name of the network scenario. Can be used as an alternative to lab_hash. - If None, lab_hash should be set. + lab_hash (Optional[str]): The hash of the network scenario. + Can be used as an alternative to lab_name and lab. If None, lab_name or lab should be set. + lab_name (Optional[str]): The name of the network scenario. + Can be used as an alternative to lab_hash and lab. If None, lab_hash or lab should be set. + lab (Optional[Kathara.model.Lab]): The network scenario object. + Can be used as an alternative to lab_hash and lab_name. If None, lab_hash or lab_name should be set. selected_machines (Optional[Set[str]]): If not None, undeploy only the specified devices. Returns: @@ -166,7 +168,7 @@ def undeploy_lab(self, lab_hash: Optional[str] = None, lab_name: Optional[str] = Raises: InvocationError: If a running network scenario hash or name is not specified. """ - self.manager.undeploy_lab(lab_hash, lab_name, selected_machines) + self.manager.undeploy_lab(lab_hash, lab_name, lab, selected_machines) def wipe(self, all_users: bool = False) -> None: """Undeploy all the running network scenarios. diff --git a/src/Kathara/manager/docker/DockerManager.py b/src/Kathara/manager/docker/DockerManager.py index 652074c1..91cfa444 100644 --- a/src/Kathara/manager/docker/DockerManager.py +++ b/src/Kathara/manager/docker/DockerManager.py @@ -240,15 +240,17 @@ def undeploy_link(self, link: Link) -> None: self.docker_link.undeploy(link.lab.hash, selected_links={link.name}) @privileged - def undeploy_lab(self, lab_hash: Optional[str] = None, lab_name: Optional[str] = None, + def undeploy_lab(self, lab_hash: Optional[str] = None, lab_name: Optional[str] = None, lab: Optional[Lab] = None, selected_machines: Optional[Set[str]] = None) -> None: """Undeploy a Kathara network scenario. Args: - lab_hash (Optional[str]): The hash of the network scenario. Can be used as an alternative to lab_name. - If None, lab_name should be set. - lab_name (Optional[str]): The name of the network scenario. Can be used as an alternative to lab_hash. - If None, lab_hash should be set. + lab_hash (Optional[str]): The hash of the network scenario. + Can be used as an alternative to lab_name and lab. If None, lab_name or lab should be set. + lab_name (Optional[str]): The name of the network scenario. + Can be used as an alternative to lab_hash and lab. If None, lab_hash or lab should be set. + lab (Optional[Kathara.model.Lab]): The network scenario object. + Can be used as an alternative to lab_hash and lab_name. If None, lab_hash or lab_name should be set. selected_machines (Optional[Set[str]]): If not None, undeploy only the specified devices. Returns: @@ -257,10 +259,12 @@ def undeploy_lab(self, lab_hash: Optional[str] = None, lab_name: Optional[str] = Raises: InvocationError: If a running network scenario hash or name is not specified. """ - if not lab_hash and not lab_name: - raise InvocationError("You must specify a running network scenario hash or name.") + if not lab_hash and not lab_name and not lab: + raise InvocationError("You must specify a running network scenario hash, name or object.") - if lab_name: + if lab: + lab_hash = lab.hash + elif lab_name: lab_hash = utils.generate_urlsafe_hash(lab_name) self.docker_machine.undeploy(lab_hash, selected_machines=selected_machines) diff --git a/src/Kathara/manager/kubernetes/KubernetesManager.py b/src/Kathara/manager/kubernetes/KubernetesManager.py index 01b5054d..9fe81efa 100644 --- a/src/Kathara/manager/kubernetes/KubernetesManager.py +++ b/src/Kathara/manager/kubernetes/KubernetesManager.py @@ -217,15 +217,17 @@ def undeploy_link(self, link: Link) -> None: self.k8s_link.undeploy(link.lab.hash, selected_links={network_name}) - def undeploy_lab(self, lab_hash: Optional[str] = None, lab_name: Optional[str] = None, + def undeploy_lab(self, lab_hash: Optional[str] = None, lab_name: Optional[str] = None, lab: Optional[Lab] = None, selected_machines: Optional[Set[str]] = None) -> None: """Undeploy a Kathara network scenario. Args: - lab_hash (Optional[str]): The hash of the network scenario. Can be used as an alternative to lab_name. - If None, lab_name should be set. - lab_name (Optional[str]): The name of the network scenario. Can be used as an alternative to lab_hash. - If None, lab_hash should be set. + lab_hash (Optional[str]): The hash of the network scenario. + Can be used as an alternative to lab_name and lab. If None, lab_name or lab should be set. + lab_name (Optional[str]): The name of the network scenario. + Can be used as an alternative to lab_hash and lab. If None, lab_hash or lab should be set. + lab (Optional[Kathara.model.Lab]): The network scenario object. + Can be used as an alternative to lab_hash and lab_name. If None, lab_hash or lab_name should be set. selected_machines (Optional[Set[str]]): If not None, undeploy only the specified devices. Returns: @@ -234,10 +236,12 @@ def undeploy_lab(self, lab_hash: Optional[str] = None, lab_name: Optional[str] = Raises: InvocationError: If a running network scenario hash or name is not specified. """ - if not lab_hash and not lab_name: - raise InvocationError("You must specify a running network scenario hash or name.") + if not lab_hash and not lab_name and not lab: + raise InvocationError("You must specify a running network scenario hash, name or object.") - if lab_name: + if lab: + lab_hash = lab.hash + elif lab_name: lab_hash = utils.generate_urlsafe_hash(lab_name) lab_hash = lab_hash.lower() diff --git a/tests/manager/docker/docker_manager_test.py b/tests/manager/docker/docker_manager_test.py index 061fbe1d..a65c7cd7 100644 --- a/tests/manager/docker/docker_manager_test.py +++ b/tests/manager/docker/docker_manager_test.py @@ -401,7 +401,7 @@ def test_undeploy_link_no_lab(docker_manager, default_link): @mock.patch("src.Kathara.manager.docker.DockerLink.DockerLink.undeploy") @mock.patch("src.Kathara.manager.docker.DockerMachine.DockerMachine.undeploy") def test_undeploy_lab(mock_undeploy_machine, mock_undeploy_link, docker_manager): - docker_manager.undeploy_lab('lab_hash') + docker_manager.undeploy_lab(lab_hash='lab_hash') mock_undeploy_machine.assert_called_once_with('lab_hash', selected_machines=None) mock_undeploy_link.assert_called_once_with('lab_hash') @@ -409,11 +409,57 @@ def test_undeploy_lab(mock_undeploy_machine, mock_undeploy_link, docker_manager) @mock.patch("src.Kathara.manager.docker.DockerLink.DockerLink.undeploy") @mock.patch("src.Kathara.manager.docker.DockerMachine.DockerMachine.undeploy") def test_undeploy_lab_selected_machines(mock_undeploy_machine, mock_undeploy_link, docker_manager): - docker_manager.undeploy_lab('lab_hash', selected_machines={'pc1', 'pc2'}) + docker_manager.undeploy_lab(lab_hash='lab_hash', selected_machines={'pc1', 'pc2'}) mock_undeploy_machine.assert_called_once_with('lab_hash', selected_machines={'pc1', 'pc2'}) mock_undeploy_link.assert_called_once_with('lab_hash') +@mock.patch("src.Kathara.manager.docker.DockerLink.DockerLink.undeploy") +@mock.patch("src.Kathara.manager.docker.DockerMachine.DockerMachine.undeploy") +@mock.patch("src.Kathara.utils.generate_urlsafe_hash") +def test_undeploy_lab_lab_name(mock_generate_urlsafe_hash, mock_undeploy_machine, mock_undeploy_link, docker_manager): + mock_generate_urlsafe_hash.return_value = "lab_hash" + + docker_manager.undeploy_lab(lab_name='lab_name') + mock_undeploy_machine.assert_called_once_with('lab_hash', selected_machines=None) + mock_undeploy_link.assert_called_once_with('lab_hash') + mock_generate_urlsafe_hash.assert_called_once_with("lab_name") + + +@mock.patch("src.Kathara.manager.docker.DockerLink.DockerLink.undeploy") +@mock.patch("src.Kathara.manager.docker.DockerMachine.DockerMachine.undeploy") +@mock.patch("src.Kathara.utils.generate_urlsafe_hash") +def test_undeploy_lab_lab_name_selected_machines(mock_generate_urlsafe_hash, + mock_undeploy_machine, mock_undeploy_link, docker_manager): + mock_generate_urlsafe_hash.return_value = "lab_hash" + + docker_manager.undeploy_lab(lab_name='lab_name', selected_machines={'pc1', 'pc2'}) + mock_undeploy_machine.assert_called_once_with('lab_hash', selected_machines={'pc1', 'pc2'}) + mock_undeploy_link.assert_called_once_with('lab_hash') + mock_generate_urlsafe_hash.assert_called_once_with("lab_name") + + +@mock.patch("src.Kathara.manager.docker.DockerLink.DockerLink.undeploy") +@mock.patch("src.Kathara.manager.docker.DockerMachine.DockerMachine.undeploy") +def test_undeploy_lab_lab_obj(mock_undeploy_machine, mock_undeploy_link, docker_manager, two_device_scenario): + expected_hash = two_device_scenario.hash + + docker_manager.undeploy_lab(lab=two_device_scenario) + mock_undeploy_machine.assert_called_once_with(expected_hash, selected_machines=None) + mock_undeploy_link.assert_called_once_with(expected_hash) + + +@mock.patch("src.Kathara.manager.docker.DockerLink.DockerLink.undeploy") +@mock.patch("src.Kathara.manager.docker.DockerMachine.DockerMachine.undeploy") +def test_undeploy_lab_lab_obj_selected_machines(mock_undeploy_machine, mock_undeploy_link, docker_manager, + two_device_scenario): + expected_hash = two_device_scenario.hash + + docker_manager.undeploy_lab(lab=two_device_scenario, selected_machines={'pc1', 'pc2'}) + mock_undeploy_machine.assert_called_once_with(expected_hash, selected_machines={'pc1', 'pc2'}) + mock_undeploy_link.assert_called_once_with(expected_hash) + + # # TEST: wipe # From 416fe05424b31358fbeab89e563d66a61e4b31dd Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Wed, 19 Jul 2023 14:58:35 +0200 Subject: [PATCH 31/34] Bump version number and changelog --- pyproject.toml | 2 +- scripts/Linux-Deb/Makefile | 2 +- scripts/Linux-Deb/debian/changelog | 10 +++++++--- scripts/Linux-Pkg/Makefile | 2 +- scripts/Linux-Pkg/pkginfo/kathara.changelog | 10 +++++++--- scripts/Linux-Rpm/Makefile | 2 +- scripts/Linux-Rpm/rpm/kathara.spec | 10 +++++++--- scripts/OSX/Makefile | 2 +- scripts/Windows/installer.iss | 2 +- setup.cfg | 2 +- setup.py | 4 ++-- src/Kathara/version.py | 2 +- 12 files changed, 31 insertions(+), 19 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8659c3a7..3b5dcb71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "kathara" -version = "3.6.2" +version = "3.6.3" description = "A lightweight container-based emulation system." readme = "README.md" requires-python = ">=3.9" diff --git a/scripts/Linux-Deb/Makefile b/scripts/Linux-Deb/Makefile index c72dd019..244989ec 100644 --- a/scripts/Linux-Deb/Makefile +++ b/scripts/Linux-Deb/Makefile @@ -1,6 +1,6 @@ #!/usr/bin/make -f -VERSION=3.6.2 +VERSION=3.6.3 DEBIAN_PACKAGE_VERSION=1 LAUNCHPAD_NAME=user NO_BINARY_PACKAGES=pyroute2|pyuv|deepdiff diff --git a/scripts/Linux-Deb/debian/changelog b/scripts/Linux-Deb/debian/changelog index 39bd2647..96be5572 100644 --- a/scripts/Linux-Deb/debian/changelog +++ b/scripts/Linux-Deb/debian/changelog @@ -1,8 +1,12 @@ kathara (__VERSION__-__DEBIAN_PACKAGE_VERSION____UBUNTU_VERSION__) __UBUNTU_VERSION__; urgency=low - * Fix FilesystemMixin APIs for file modifications - * Add a warning if a meta is repeated on the same device - * Add support for Docker Images tags in "kathara settings" + * Use "chardet" to parse all byte strings in order to correctly encode non-UTF8 characters + * Wait for startup commands execution while opening a connection to a device (the wait can be overridden by pressing [ENTER]) + * Keep correct folders/files permissions when copying files inside the device + * Fix "sysctl" metadata parsing for negative values + * Fix "machine.is_ipv6_enabled()"" method to correctly follow priority of "ipv6" meta + * Rename "startup_commands" meta to "exec_commands" to avoid ambiguity with ".startup" files + * Add new helper methods to "model.Lab" * Minor fixes -- Kathara Team __DATE__ diff --git a/scripts/Linux-Pkg/Makefile b/scripts/Linux-Pkg/Makefile index 8f492c01..b2c6afca 100644 --- a/scripts/Linux-Pkg/Makefile +++ b/scripts/Linux-Pkg/Makefile @@ -1,6 +1,6 @@ #!/usr/bin/make -f -VERSION=3.6.2 +VERSION=3.6.3 PACKAGE_VERSION=1 AUR_NAME=user AUR_MAIL=contact@kathara.org diff --git a/scripts/Linux-Pkg/pkginfo/kathara.changelog b/scripts/Linux-Pkg/pkginfo/kathara.changelog index 5a393585..7c61ab5d 100644 --- a/scripts/Linux-Pkg/pkginfo/kathara.changelog +++ b/scripts/Linux-Pkg/pkginfo/kathara.changelog @@ -1,7 +1,11 @@ __DATE__ Kathara Team <******@kathara.org> * Release v__VERSION__ - * Fix FilesystemMixin APIs for file modifications - * Add a warning if a meta is repeated on the same device - * Add support for Docker Images tags in "kathara settings" + * Use "chardet" to parse all byte strings in order to correctly encode non-UTF8 characters + * Wait for startup commands execution while opening a connection to a device (the wait can be overridden by pressing [ENTER]) + * Keep correct folders/files permissions when copying files inside the device + * Fix "sysctl" metadata parsing for negative values + * Fix "machine.is_ipv6_enabled()"" method to correctly follow priority of "ipv6" meta + * Rename "startup_commands" meta to "exec_commands" to avoid ambiguity with ".startup" files + * Add new helper methods to "model.Lab" * Minor fixes \ No newline at end of file diff --git a/scripts/Linux-Rpm/Makefile b/scripts/Linux-Rpm/Makefile index c3f26e2b..b1d3c9d5 100644 --- a/scripts/Linux-Rpm/Makefile +++ b/scripts/Linux-Rpm/Makefile @@ -1,6 +1,6 @@ #!/usr/bin/make -f -VERSION=3.6.2 +VERSION=3.6.3 PACKAGE_VERSION=1 .PHONY: all clean docker-build-image prepare-source prepare-man-pages prepare-bash-completion pack-source build diff --git a/scripts/Linux-Rpm/rpm/kathara.spec b/scripts/Linux-Rpm/rpm/kathara.spec index fbbdc3b3..58a905e5 100644 --- a/scripts/Linux-Rpm/rpm/kathara.spec +++ b/scripts/Linux-Rpm/rpm/kathara.spec @@ -68,7 +68,11 @@ chmod g+s %{_libdir}/kathara/kathara %changelog * __DATE__ Kathara Team <******@kathara.org> - __VERSION__-__PACKAGE_VERSION__ -- Fix FilesystemMixin APIs for file modifications -- Add a warning if a meta is repeated on the same device -- Add support for Docker Images tags in "kathara settings" +- Use "chardet" to parse all byte strings in order to correctly encode non-UTF8 characters +- Wait for startup commands execution while opening a connection to a device (the wait can be overridden by pressing [ENTER]) +- Keep correct folders/files permissions when copying files inside the device +- Fix "sysctl" metadata parsing for negative values +- Fix "machine.is_ipv6_enabled()"" method to correctly follow priority of "ipv6" meta +- Rename "startup_commands" meta to "exec_commands" to avoid ambiguity with ".startup" files +- Add new helper methods to "model.Lab" - Minor fixes \ No newline at end of file diff --git a/scripts/OSX/Makefile b/scripts/OSX/Makefile index bbbe1871..1d8d18bb 100644 --- a/scripts/OSX/Makefile +++ b/scripts/OSX/Makefile @@ -1,7 +1,7 @@ #!/usr/bin/make -s PRODUCT=Kathara -VERSION=3.6.2 +VERSION=3.6.3 TARGET_DIRECTORY=Output APPLE_DEVELOPER_CERTIFICATE_ID=FakeID ROFF_DIR=../../docs/Roff diff --git a/scripts/Windows/installer.iss b/scripts/Windows/installer.iss index 34bb7423..76aaf2f8 100644 --- a/scripts/Windows/installer.iss +++ b/scripts/Windows/installer.iss @@ -2,7 +2,7 @@ ; SEE THE DOCUMENTATION FOR DETAILS ON CREATING INNO SETUP SCRIPT FILES! #define MyAppName "Kathara" -#define MyAppVersion "3.6.2" +#define MyAppVersion "3.6.3" #define MyAppPublisher "Kathara Team" #define MyAppURL "https://www.kathara.org" #define MyAppExeName "kathara.exe" diff --git a/setup.cfg b/setup.cfg index 76140b55..c103c18e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = kathara -version = 3.6.2 +version = 3.6.3 author = Kathara Framework author_email = contact@kathara.org description = A lightweight container based emulation system diff --git a/setup.py b/setup.py index 84756367..4fe626c6 100644 --- a/setup.py +++ b/setup.py @@ -7,13 +7,13 @@ package_dir={'': 'src'}, packages=find_packages('src'), py_modules=['kathara'], - version='3.6.2', + version='3.6.3', license='gpl-3.0', description='A lightweight container based emulation system.', author='Kathara Framework', author_email='contact@kathara.org', url='https://www.kathara.org', - download_url='https://github.com/KatharaFramework/Kathara/archive/refs/tags/3.6.2.tar.gz', + download_url='https://github.com/KatharaFramework/Kathara/archive/refs/tags/3.6.3.tar.gz', keywords=['NETWORK-EMULATION', 'CONTAINERS', 'NFV'], install_requires=[ "binaryornot>=0.4.4", diff --git a/src/Kathara/version.py b/src/Kathara/version.py index b8f5e393..f9cd44d4 100644 --- a/src/Kathara/version.py +++ b/src/Kathara/version.py @@ -1,6 +1,6 @@ from typing import Tuple -CURRENT_VERSION = "3.6.2" +CURRENT_VERSION = "3.6.3" def parse(version: str) -> Tuple: From 83bd4312c7e7d27928e6fc85f5618d575da12f94 Mon Sep 17 00:00:00 2001 From: Tommaso Caiazzi Date: Thu, 20 Jul 2023 17:52:22 +0200 Subject: [PATCH 32/34] Remove Kinetic from `.deb` build --- scripts/Linux-Deb/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/Linux-Deb/Makefile b/scripts/Linux-Deb/Makefile index 244989ec..2acb376c 100644 --- a/scripts/Linux-Deb/Makefile +++ b/scripts/Linux-Deb/Makefile @@ -7,7 +7,7 @@ NO_BINARY_PACKAGES=pyroute2|pyuv|deepdiff .PHONY: allSigned docker-build-image prepare-deb-source unpack-deb-source clean-output-folder copy-debian-folder download-pip build-man build-deb-unsigned build-deb-signed ppa clean venv autocompletion -allSigned: clean prepare-deb-source docker-signed_focal docker-signed_jammy docker-signed_kinetic +allSigned: clean prepare-deb-source docker-signed_focal docker-signed_jammy docker-unsigned_%: clean prepare-deb-source docker-build-image_% docker run -ti --rm -v `pwd`/../../:/opt/kathara kathara/linux-build-deb:$* /bin/bash -c \ From cdaf635df0803c977c3a3c04b5228d1125e2fe14 Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Thu, 20 Jul 2023 19:02:38 +0200 Subject: [PATCH 33/34] Update vstart man --- docs/kathara-vstart.1.ronn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/kathara-vstart.1.ronn b/docs/kathara-vstart.1.ronn index abb13182..ed5566c1 100644 --- a/docs/kathara-vstart.1.ronn +++ b/docs/kathara-vstart.1.ronn @@ -94,9 +94,10 @@ Notice: unless differently stated, command line arguments (DEVICE_NAME) and opti Connect the device to the host network by adding an additional network interface (will be the last one). This interface will be connected to the host network through a NAT connection and will receive its IP configuration automatically via DHCP. * `--port` <[HOST\:]GUEST[/PROTOCOL]> [<[HOST\:]GUEST[/PROTOCOL]> ...]: - Map localhost port HOST to the internal port GUEST of the device for the specified PROTOCOL. + Map localhost port HOST to the internal port GUEST of the device for the specified PROTOCOL. The syntax is [HOST:]GUEST[/PROTOCOL]. If HOST port is not specified, default is 3000. If PROTOCOL is not specified, default is `tcp`. Supported PROTOCOL values are: tcp, udp, or sctp. + For instance, with this command you can map host's port 8080 to device's port 80 with TCP protocol: `--port "8080:80/tcp"`. * `--sysctl` [ ...]: Set a sysctl option for the device. Only the `net.` namespace is allowed to be set. From 04010e1fca0f62e613d84471a42bcc249a6320f9 Mon Sep 17 00:00:00 2001 From: Tommaso Caiazzi Date: Thu, 20 Jul 2023 21:56:12 +0200 Subject: [PATCH 34/34] Fix pydoc generation --- scripts/pydoc/Makefile | 18 +++++------------- scripts/pydoc/generate_doc.py | 14 +++++++++----- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/scripts/pydoc/Makefile b/scripts/pydoc/Makefile index fcce2de0..1da54f7b 100644 --- a/scripts/pydoc/Makefile +++ b/scripts/pydoc/Makefile @@ -2,23 +2,15 @@ .PHONY: all doc verify-doc clean -all: clean doc verify-doc +all: doc doc: clean verify-doc python3 generate_doc.py cd docs && rm Kathara.model.md Kathara.setting.md Kathara.manager.md - sed '/manager.md/d' ./docs/Kathara-API-Docs.md - sed '/setting.md/d' ./docs/Kathara-API-Docs.md - sed '/docker.md/d' ./docs/Kathara-API-Docs.md - sed '/kubernetes.md/d' ./docs/Kathara-API-Docs.md - sed '/KubernetesSettingsAddon.md/d' ./docs/Kathara-API-Docs.md - sed '/DockerSettingsAddon.md/d' ./docs/Kathara-API-Docs.md - sed '/model.md/d' ./docs/Kathara-API-Docs.md - sed 's/\.\///g' ./docs/Kathara-API-Docs.md - sed 's/\.md//g' ./docs/Kathara-API-Docs.md - sed '/Modules/,/Classes/d' ./docs/Kathara-API-Docs.md - sed '/Functions/q' ./docs/Kathara-API-Docs.md - sed '/Functions/d' ./docs/Kathara-API-Docs.md + sed -i 's/\.\///' docs/Kathara-API-Docs.md + sed -i 's/\.md//' docs/Kathara-API-Docs.md + sed -i '/Modules/,/Classes/{/Classes/!d;}' docs/Kathara-API-Docs.md + rm docs/.pages verify-doc: python3 -m pip install lazydocs pydocstyle diff --git a/scripts/pydoc/generate_doc.py b/scripts/pydoc/generate_doc.py index 7bf4fe1d..e241c962 100644 --- a/scripts/pydoc/generate_doc.py +++ b/scripts/pydoc/generate_doc.py @@ -1,12 +1,16 @@ from lazydocs import generate_docs ignored_modules = [ - "Kathara.version", "Kathara.auth", "Kathara.cli", "Kathara.event", "Kathara.foundation", - "Kathara.manager.docker.terminal", "Kathara.manager.kubernetes.terminal", "Kathara.os", - "Kathara.parser", "Kathara.test", "Kathara.trdparty", "Kathara.validator", "Kathara.kathara", "Kathara.decorators", - "Kathara.exceptions", "Kathara.strings", "Kathara.utils", "Kathara.webhooks", "kathara" + "Kathara.version", "Kathara.auth", "Kathara.cli", "Kathara.trdparty", "Kathara.foundation.cli", + "Kathara.foundation.test", "Kathara.exceptions", "Kathara.webhooks", "Kathara.validator", "Kathara.test", "kathara", + "Kathara.manager.kubernetes.terminal.KubernetesWSTerminal", "Kathara.foundation.factory", + "Kathara.manager.docker.terminal.DockerTTYTerminal", + "Kathara.foundation.manager.terminal.Terminal", "Kathara.foundation.manager.ManagerFactory", + "Kathara.foundation.setting.SettingsAddon", "Kathara.foundation.setting.SettingsAddonFactory", + "Kathara.setting.addon.DockerSettingsAddon", "Kathara.setting.addon.KubernetesSettingsAddon" ] -generate_docs(["../../src"], src_base_url="https://github.com/KatharaFramework/Kathara/tree/master", output_path="./docs", +generate_docs(["../../src"], + src_base_url="https://github.com/KatharaFramework/Kathara/tree/main", output_path="./docs", ignored_modules=ignored_modules, overview_file="Kathara-API-Docs.md", remove_package_prefix=False)