From d020eb3b896b2fde79e74d212e4c47e66d1e1a98 Mon Sep 17 00:00:00 2001 From: Mathieu Broillet Date: Mon, 19 Jun 2023 14:01:44 +0200 Subject: [PATCH] replaced pct by ssh when possible for speed --- src/lxc/lxc.py | 44 +++++++++++++++++++++++++++++--------------- src/lxc/lxc_utils.py | 7 ++++--- src/utils/machine.py | 25 ++++++++++++++----------- src/utils/proxmox.py | 31 ++++++++++++++++--------------- 4 files changed, 63 insertions(+), 44 deletions(-) diff --git a/src/lxc/lxc.py b/src/lxc/lxc.py index a9a9dd1..eedfbd6 100644 --- a/src/lxc/lxc.py +++ b/src/lxc/lxc.py @@ -1,4 +1,5 @@ import logging +import time from . import creation_utils, lxc_utils from ..utils.machine import LinuxMachine @@ -171,25 +172,28 @@ class LXC(LinuxMachine): """ return self.bridge - def get_ipv4(self, netmask: bool = False): + def get_ipv4(self, netmask: bool = False, use_ssh: bool = False): """ Get IPv4 :return: ipv4 """ if self.ipv4 == "dhcp": if self.is_running(): - if self.has_program("ip"): + if self.has_program("ip", use_ssh=use_ssh): if netmask: ip = self.run_command( - """ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 """) + """ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 """, + use_ssh=use_ssh) return ip ip = self.run_command( - """ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 -d'/'""") + """ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 -d'/'""", + use_ssh=use_ssh) return ip - elif self.has_program("ifconfig"): - return self.run_command(command="ifconfig eth0 | awk '/inet addr/{print substr($2,6)}'") + elif self.has_program("ifconfig", use_ssh=use_ssh): + return self.run_command(command="ifconfig eth0 | awk '/inet addr/{print substr($2,6)}'", + use_ssh=use_ssh) return self.ipv4 @@ -328,11 +332,12 @@ class LXC(LinuxMachine): self.start() # Make sure bash is installed for later use - if not self.has_program("bash"): - self.install_package("bash") + if not self.has_program("bash", use_ssh=False): + self.install_package("bash", use_ssh=False) logging.info("Setting up SSH for LXC") lxc_utils.run_protected_script(lxc=self, script_path="protected/scripts/install-config-ssh.sh") + self.pve.run_command(f"ssh-keygen -f '/root/.ssh/known_hosts' -R {self.get_ipv4(use_ssh=False)}") def run_creation(self): """ @@ -355,19 +360,19 @@ class LXC(LinuxMachine): def deploy(self): pass - def run_script(self, script_path): + def run_script(self, script_path, use_ssh: bool = True): """ Run script on LXC filesystem using bash :param script_path: :return: """ - return self.run_command(command=f"bash {script_path}") + return self.run_command(command=f"bash {script_path}", use_ssh=use_ssh) def run_command(self, command: str, return_status_code: bool = False, exception_on_exit: bool = False, exception_on_empty_stdout: bool = False, - working_directory: str = None, shell: bool = False): + working_directory: str = None, use_ssh: bool = True): """ Run command on LXC :param command: command to run @@ -384,7 +389,16 @@ class LXC(LinuxMachine): if working_directory: command = f"cd {working_directory} && {command}" - return self.pve.run_command(command=f"pct exec {self.lxc_id} -- {command}", - return_status_code=return_status_code, - exception_on_exit=exception_on_exit, - exception_on_empty_stdout=exception_on_empty_stdout) + + # Using pct exec works every time but is 8x slower than using ssh + if use_ssh: + return self.pve.run_command( + command=f"ssh -o StrictHostKeyChecking=no root@{self.get_ipv4()} -- {command}", + return_status_code=return_status_code, + exception_on_exit=exception_on_exit, + exception_on_empty_stdout=exception_on_empty_stdout) + else: + return self.pve.run_command(command=f"pct exec {self.lxc_id} -- {command}", + return_status_code=return_status_code, + exception_on_exit=exception_on_exit, + exception_on_empty_stdout=exception_on_empty_stdout) diff --git a/src/lxc/lxc_utils.py b/src/lxc/lxc_utils.py index 6173b01..8a3238f 100644 --- a/src/lxc/lxc_utils.py +++ b/src/lxc/lxc_utils.py @@ -156,9 +156,10 @@ def run_protected_script(lxc: LXC, script_path: str): script_path = get_path(lxc, script_path) utils.copy_local_file_to_pve(lxc.pve, script_path, f"/tmp/pdj-temp/{script_path.name}") - lxc.pve.copy_file_to_lxc(lxc, f"/tmp/pdj-temp/{script_path.name}", f"/tmp/pdj-temp/{script_path.name}") - lxc.run_script(f"/tmp/pdj-temp/{script_path.name}") - lxc.delete_file(f"/tmp/pdj-temp/{script_path.name}") + lxc.pve.copy_file_to_lxc(lxc, f"/tmp/pdj-temp/{script_path.name}", f"/tmp/pdj-temp/{script_path.name}", + use_ssh=False) + lxc.run_script(f"/tmp/pdj-temp/{script_path.name}", use_ssh=False) + lxc.delete_file(f"/tmp/pdj-temp/{script_path.name}", use_ssh=False) lxc.pve.delete_file(f"/tmp/pdj-temp/{script_path.name}") diff --git a/src/utils/machine.py b/src/utils/machine.py index 8225061..ac3cdba 100644 --- a/src/utils/machine.py +++ b/src/utils/machine.py @@ -8,6 +8,9 @@ class LinuxMachine(): def __init__(self): pass + def update(self): + pass + def get_hostname(self): return self.run_command("hostname") @@ -51,16 +54,16 @@ class LinuxMachine(): def reboot(self): pass - def has_program(self, program: str): + def has_program(self, program: str, use_ssh: bool = True): """Check if program is installed on LXC :param program: program executable name :return: boolean """ if type(program) == str: - return self.run_command("which " + program, return_status_code=True) == 0 + return self.run_command("which " + program, return_status_code=True, use_ssh=use_ssh) == 0 elif type(program) == list: - return all((self.run_command("which " + p, return_status_code=True) == 0) for p in program) + return all((self.run_command("which " + p, return_status_code=True, use_ssh=use_ssh) == 0) for p in program) def has_file(self, file: str or Path): """Check if file exists on LXC @@ -103,21 +106,21 @@ class LinuxMachine(): if permission != 644: self.run_command(f"chmod {permission} {file}", return_status_code=True) - def create_directory(self, directory: str or Path, permission: int = 755): + def create_directory(self, directory: str or Path, permission: int = 755, use_ssh: bool = True): """Create directory""" if isinstance(directory, Path): directory = str(directory.as_posix()) - self.run_command(f"mkdir -p {directory}", return_status_code=True) + self.run_command(f"mkdir -p {directory}", return_status_code=True, use_ssh=use_ssh) if permission != 755: - self.run_command(f"chmod -R {permission} {directory}", return_status_code=True) + self.run_command(f"chmod -R {permission} {directory}", return_status_code=True, use_ssh=use_ssh) - def delete_file(self, file: str or Path): + def delete_file(self, file: str or Path, use_ssh: bool = True): """Delete file""" if isinstance(file, Path): file = str(file.as_posix()) - self.run_command(f"rm {file}", return_status_code=True) + self.run_command(f"rm {file}", return_status_code=True, use_ssh=use_ssh) def delete_directory(self, directory: str or Path): """Delete directory""" @@ -219,7 +222,7 @@ class LinuxMachine(): self.run_command(f"mkdir -p {destination} && tar -xzf {path} --directory {destination}", return_status_code=True) - def install_package(self, package: str or list): + def install_package(self, package: str or list, use_ssh: bool = True): """Install a package in the Linux Machine Parameters @@ -236,10 +239,10 @@ class LinuxMachine(): if type(package) is list: for p in package: self.run_command(f"{utils.get_install_package_command(self.get_os_name())} {package}", - return_status_code=True) + return_status_code=True, use_ssh=use_ssh) else: self.run_command(f"{utils.get_install_package_command(self.get_os_name())} {package}", - return_status_code=True) + return_status_code=True, use_ssh=use_ssh) def remove_package(self, package: str or list): """Remove a package in the Linux Machine diff --git a/src/utils/proxmox.py b/src/utils/proxmox.py index c7f6729..01921cf 100644 --- a/src/utils/proxmox.py +++ b/src/utils/proxmox.py @@ -2,6 +2,7 @@ from __future__ import annotations import fnmatch import subprocess +import time from pathlib import Path from typing import TYPE_CHECKING @@ -55,7 +56,7 @@ class ProxmoxHost(LinuxMachine): return self.connection def run_command(self, command: str, return_status_code: bool = False, exception_on_exit: bool = True, - exception_on_empty_stdout: bool = False): + exception_on_empty_stdout: bool = False, **kwargs): """Run a command on the Proxmox VE host The default behavior is as follows : - runs the command on the Proxmox VE host @@ -69,7 +70,7 @@ class ProxmoxHost(LinuxMachine): Parameters ---------- - command: str + command_result: str command to run return_status_code: bool, optional should it return the exit code and not the stdout, disables exception_on_exit @@ -100,31 +101,31 @@ class ProxmoxHost(LinuxMachine): # Check if host is None, if it is, run the command locally, else run it on the host via SSH if self.host is None: - command = subprocess.run(command, shell=True, capture_output=True, encoding="utf-8") + command_result = subprocess.run(command, shell=True, capture_output=True, encoding="utf-8") elif self.connection is not None: - command = self.connection.run(command, hide=True, warn=True, encoding="utf-8") + command_result = self.connection.run(command, hide=True, warn=True, encoding="utf-8") else: raise Exception("No host or connection provided") # If return code is not 0 and that exception_on_exit is True and return_status_code is False, throw an exception - if command.return_code != 0 and exception_on_exit and not return_status_code: - raise Exception(f"Error while running command: \n{command.stderr}") + if command_result.return_code != 0 and exception_on_exit and not return_status_code: + raise Exception(f"Error while running command: \n{command_result.stderr}") if return_status_code: - return command.return_code + return command_result.return_code # Check if stdout is empty, throw an exception or return empty string depending on exception_on_empty_stdout - if (command.stdout is None or command.stdout == "") and exception_on_empty_stdout: + if (command_result.stdout is None or command_result.stdout == "") and exception_on_empty_stdout: raise Exception( - f"Error, no output from command, try using the command with return_status_code instead: \n{command.stderr}") - elif command.stdout is None or command.stdout == "": + f"Error, no output from command, try using the command with return_status_code instead: \n{command_result.stderr}") + elif command_result.stdout is None or command_result.stdout == "": return "" # Decode stdout if it's bytes - if type(command.stdout) == bytes: - return command.stdout.decode().rstrip() + if type(command_result.stdout) == bytes: + return command_result.stdout.decode().rstrip() - return command.stdout.rstrip() + return command_result.stdout.rstrip() def get_version(self): """Get the version of the Proxmox host.""" @@ -208,14 +209,14 @@ class ProxmoxHost(LinuxMachine): self.run_command(f"qm reboot {vm_id}") - def copy_file_to_lxc(self, lxc: LXC, source: str or Path, destination: str or Path): + def copy_file_to_lxc(self, lxc: LXC, source: str or Path, destination: str or Path, use_ssh: bool = True): """Copy the given file to the given LXC.""" if isinstance(source, Path): source = str(source.as_posix()) if isinstance(destination, str): destination = Path(destination) - lxc.create_directory(destination.parent.as_posix()) + lxc.create_directory(destination.parent.as_posix(), use_ssh=use_ssh) self.run_command(f"pct push {lxc.get_id()} {source} {destination.as_posix()}") def copy_folder_to_lxc(self, lxc: LXC, source: str or Path, destination: str or Path):