From 00f14332e01dec77d21ba9516d4c0b8a04443d72 Mon Sep 17 00:00:00 2001 From: Mathieu Broillet Date: Tue, 13 Jun 2023 15:29:22 +0200 Subject: [PATCH] improve documentation and fixed/improved path handling and added copy files from local->pve(->lxc) --- src/lxc/__init__.py | 12 +- src/lxc/commands_utils.py | 193 +++++++++++++++++++------- src/lxc/creation_utils.py | 24 ++-- src/lxc/utils.py | 17 +-- src/main.py | 14 +- src/utils/proxmox_utils.py | 258 ++++++++++++++++++++++++++++------- src/utils/resources_utils.py | 41 +++++- 7 files changed, 426 insertions(+), 133 deletions(-) diff --git a/src/lxc/__init__.py b/src/lxc/__init__.py index 4c99427..d18d7b6 100644 --- a/src/lxc/__init__.py +++ b/src/lxc/__init__.py @@ -7,7 +7,8 @@ from ..utils import proxmox_utils class LXC: """LXC object""" - def __init__(self, lxc_id, lxc_hostname, os, resources, network, options, creation, deploy): + def __init__(self, lxc_id: int, lxc_hostname: str, os: dict, resources: dict, network: dict, options: dict, + creation: dict, deploy: dict): self.lxc_id = lxc_id self.lxc_hostname = lxc_hostname @@ -52,9 +53,12 @@ class LXC: return hash(self.lxc_id) def get_id(self): - """ - Get LXC ID - :return: lxc id + """Get LXC ID + + Returns + ------- + int + LXC ID """ return self.lxc_id diff --git a/src/lxc/commands_utils.py b/src/lxc/commands_utils.py index 9286cbb..b9238be 100644 --- a/src/lxc/commands_utils.py +++ b/src/lxc/commands_utils.py @@ -1,10 +1,11 @@ from pathlib import Path +from . import LXC from ..utils import proxmox_utils from ..utils.resources_utils import get_path -def run_script(lxc, step): +def run_script(lxc: LXC, step: dict): """Method use to dispatch the script to the correct method depending on the type of script, being local (on the machine running this program), remote (url) or in the LXC. @@ -13,7 +14,7 @@ def run_script(lxc, step): lxc : LXC The LXC object used to run the script step: dict - Dictionary containing the step information about the script to run + Dictionary containing the step information and configuration about the script to run Typically read from the "creation/steps/" in JSON file """ @@ -23,8 +24,9 @@ def run_script(lxc, step): install_package(lxc, "bash") # Run local script - if "path" in step: - _run_local_script_on_lxc(lxc, step["path"]) + if "local_path" in step: + path = get_path(lxc, step["local_path"]) + _run_local_script_on_lxc(lxc, path) # Run remote script elif "url" in step: @@ -32,33 +34,78 @@ def run_script(lxc, step): # Run script in LXC elif "lxc_path" in step: - _run_script_on_lxc(lxc, step["lxc_path"]) + path = get_path(lxc, step["lxc_path"]) + _run_script_on_lxc(lxc, path) -def _run_script_on_lxc(lxc, path: "Path inside the LXC"): - """Run a script in the LXC""" +def _run_script_on_lxc(lxc: LXC, path: Path): + """Runs a script present inside the LXC storage, without copying or downloading it - lxc.run_command(f"bash {path}") + Parameters + ---------- + lxc: LXC + The LXC object used to run the script + path: pathlib.Path + Path to the script inside the LXC storage + """ + + lxc.run_command(f"bash {str(path)}") -def _run_local_script_on_lxc(lxc, path: "Path to the script on the machine running this program"): - """Run a local script in the LXC""" +def _run_local_script_on_lxc(lxc: LXC, path: Path): + """Runs a script present on the machine running this program, inside the LXC + It works by copying the script from the local machine to the PVE using SCP and + then using "pct push" to send it to the LXC. - path = get_path(lxc, path) - with open(path, "r") as file: - script = file.read() - lxc.run_command("bash <>> run_docker_command(lxc, "", "") """ + lxc.run_command(f"docker exec -it {container} {command}") -def run_docker_compose_command(lxc, command, working_directory=None): +def run_docker_compose_command(lxc: LXC, command: str, working_directory: str = None): """Run a docker-compose command in the LXC Parameters @@ -130,13 +207,14 @@ def run_docker_compose_command(lxc, command, working_directory=None): The LXC object of the LXC to run the docker-compose command in command: str The docker-compose command to run - working_directory: str + working_directory: str, optional The working directory to run the command in Examples -------- >>> run_docker_compose_command(lxc, "up -d", "/home/user/traefik") """ + docker_compose_exec = "docker-compose" if not lxc.has_program(docker_compose_exec): docker_compose_exec = "docker compose" @@ -147,16 +225,33 @@ def run_docker_compose_command(lxc, command, working_directory=None): lxc.run_command(f"{docker_compose_exec} {command}") -def download_file(lxc, url, destination): +def download_file(lxc: LXC, url: str, destination: str): + """Download a file from a URL to the LXC and save it to the destination + + Parameters + ---------- + lxc: LXC + The LXC object of the LXC to download the file to + url: str + URL of the file to download + destination: str + Path to the destination to save the file to + Needs to end with a trailing slash + + Examples + -------- + >>> download_file(lxc, "https://example.com/file.zip", "/home/user/") + + """ """Download a file from a URL to the LXC""" if type(url) is list: for u in url: - lxc.run_command(f"wget {u} -O {destination}") + lxc.run_command(f"wget {u} --directory-prefix={destination}") else: - lxc.run_command(f"wget {url} -O {destination}") + lxc.run_command(f"wget {url} --directory-prefix={destination}") -def unzip_file(lxc, path, destination=None): +def unzip_file(lxc: LXC, path: str, destination: str = None): """Unzip a file in the LXC Parameters @@ -165,13 +260,15 @@ def unzip_file(lxc, path, destination=None): The LXC object of the LXC to unzip the file in path: str Path to the file to unzip - destination: str + destination: str, optional Path to the destination to unzip the file to + If not specified, it will unzip the file in the same directory as the file + Needs to end with a trailing slash Examples -------- - >>> unzip_file(lxc, "/home/user/file.zip", "/home/user/destination") - >>> unzip_file(lxc, "/home/user/file.tar.gz", "/home/user/destination") + >>> unzip_file(lxc, "/home/user/file.zip", "/home/user/extracted_files/") + >>> unzip_file(lxc, "/home/user/file.tar.gz", "/home/user/extracted_files/") """ if destination is None: @@ -180,10 +277,10 @@ def unzip_file(lxc, path, destination=None): if ".zip" in path: lxc.run_command(f"unzip {path} -d {destination}") elif ".tar.gz" in path: - lxc.run_command(f"tar -xzf {path} -C {destination}") + lxc.run_command(f"mkdir -p {destination} && tar -xzf {path} --directory {destination}") -def install_package(lxc, package): +def install_package(lxc: LXC, package: str or list): """Install a package in the LXC Parameters @@ -191,7 +288,7 @@ def install_package(lxc, package): lxc: LXC The LXC object of the LXC to install the package in package: str or list - Name of the package to install + Name(s) of the package(s) to install Examples -------- @@ -206,7 +303,7 @@ def install_package(lxc, package): lxc.run_command(f"{proxmox_utils.get_install_package_command(lxc.get_os_name())} {package}") -def remove_package(lxc, package): +def remove_package(lxc: LXC, package: str or list): """Remove a package in the LXC Parameters @@ -214,7 +311,7 @@ def remove_package(lxc, package): lxc: LXC The LXC object of the LXC to remove the package in package: str or list - Name of the package to remove + Name(s) of the package(s) to remove Examples -------- @@ -232,20 +329,20 @@ def remove_package(lxc, package): lxc.run_command(f"{proxmox_utils.get_remove_package_command(lxc.get_os_name())} {package}") -def replace_in_files(lxc, path, search, replace, case_sensitive=False): +def replace_in_files(lxc: LXC, path: str or list, search: str, replace: str, case_sensitive: bool = False): """Replace a string in one or multiples files in the LXC Parameters ---------- lxc : LXC The LXC object of the LXC to replace the string in - path : str or list + path : str or list of str Path to the file(s) to replace the string in search : str String to search for replace : str String to replace the search string with - case_sensitive : bool + case_sensitive : bool, optional Whether the search should be case sensitive or not Examples @@ -258,6 +355,6 @@ def replace_in_files(lxc, path, search, replace, case_sensitive=False): if type(path) is list: for p in path: - lxc.run_command(f"sed -i {'-i' if case_sensitive else ''} 's/{search}/{replace}/g' {p}") + lxc.run_command(f"sed {'-i' if case_sensitive else ''} 's/{search}/{replace}/g' {p}") else: - lxc.run_command(f"sed -i {'-i' if case_sensitive else ''} 's/{search}/{replace}/g' {path}") + lxc.run_command(f"sed {'-i' if case_sensitive else ''} 's/{search}/{replace}/g' {path}") diff --git a/src/lxc/creation_utils.py b/src/lxc/creation_utils.py index 56ab5a9..84212d5 100644 --- a/src/lxc/creation_utils.py +++ b/src/lxc/creation_utils.py @@ -1,10 +1,11 @@ import logging from pathlib import Path -from . import commands_utils +from . import commands_utils, LXC +from ..utils.resources_utils import get_path -def are_all_conditions_met(lxc): +def are_all_conditions_met(lxc: LXC): """ Check conditions for running the creation steps for an LXC. @@ -43,28 +44,29 @@ def are_all_conditions_met(lxc): return all(result) -def optional(var, placeholder): +def optional(var: any, placeholder: any): """Return a placeholder if the given variable is None, otherwise return the variable Parameters ---------- - var + var: any Variable to check if it is None - placeholder + placeholder: any Placeholder to return if the variable is None Returns ------- - var or placeholder - + any[any] + The variable if it is not None, otherwise the placeholder """ + if var is None: return placeholder else: return var -def run_steps(lxc): +def run_steps(lxc: LXC): """Run the creation steps for the given LXC Parameters @@ -85,18 +87,18 @@ def run_steps(lxc): case "file_create": commands_utils.create_file_in_lxc(lxc, step["path"], optional(step["permission"], 644)) case "file_copy": - commands_utils.copy_local_file_to_lxc(lxc, step["path"], step["destination"]) + commands_utils.copy_local_file_to_lxc(lxc, get_path(lxc, step["path"]), step["destination"]) case "folder": commands_utils.create_folder_in_lxc(lxc, step["path"], optional(step["permission"], 755)) case "folder_copy": - commands_utils.copy_local_folder_to_lxc(lxc, step["path"], step["destination"]) + commands_utils.copy_local_folder_to_lxc(lxc, get_path(lxc, step["path"]), step["destination"]) case "command": lxc.run_command(command=step["command"], working_directory=optional(step["working_directory"], None)) case "docker": commands_utils.run_docker_command(lxc, step["container"], step["command"]) case "docker_compose": commands_utils.run_docker_compose_command(lxc, step["command"], - optional(step["working_directory"], None)) + optional(step["working_directory"], None)) case "git": lxc.run_command(command=f"git clone {step['url']} {step['destination']}") case "download": diff --git a/src/lxc/utils.py b/src/lxc/utils.py index ba82f83..2202856 100644 --- a/src/lxc/utils.py +++ b/src/lxc/utils.py @@ -1,4 +1,5 @@ import json +from pathlib import Path from . import LXC @@ -17,12 +18,12 @@ def get_all_lxcs(): return lxcs -def get_lxc(lxc_id): +def get_lxc(lxc_id: int): """Get LXC by ID Parameters ---------- - lxc_id : str + lxc_id : int ID of the LXC to get Returns @@ -38,23 +39,23 @@ def get_lxc(lxc_id): return None -def load_lxc(file, lxc_id): +def load_lxc(lxc_id: int, filepath: Path): """Load LXC from JSON file Parameters ---------- - file : str - Path to the JSON file of the LXC to load - lxc_id : str + lxc_id : int ID of the LXC to load + filepath : pathlib.Path + Path object to the JSON file of the LXC to load Examples -------- - >>> load_lxc("./resources/lxc/100/config.json", "100") + >>> load_lxc(100, Path("/resources/lxc/100/config.json")) """ # Load JSON data - data = json.loads(file) + data = json.loads(filepath.read_text()) # Extract values from JSON lxc_id = lxc_id diff --git a/src/main.py b/src/main.py index 3dd8fb3..8e3add9 100644 --- a/src/main.py +++ b/src/main.py @@ -1,5 +1,5 @@ import logging -import os +from pathlib import Path from . import project_path from .lxc.utils import load_lxc, get_all_lxcs @@ -7,19 +7,17 @@ from .lxc.utils import load_lxc, get_all_lxcs def run(): # Read all files in the resources directory - resources = os.listdir(project_path / "resources") + resources = Path(project_path).joinpath("resources").glob("*") # Go through each LXC file for resource in resources: if resource == "lxc": # Read all files in the LXC directory - lxc_folders = os.listdir(project_path / "resources" / "lxc") + lxc_folders = Path(project_path).joinpath("resources", "lxc").glob("*") for lxc_folder in lxc_folders: - lxc_file = os.path.join(project_path / "resources" / "lxc", lxc_folder, "config.json") - # Open the file - with open(lxc_file, "r") as file: - # Load the LXC - load_lxc(file.read(), lxc_id=lxc_folder) + lxc_file = Path(project_path).joinpath("resources", "lxc", lxc_folder, "config.json") + + load_lxc(filepath=lxc_file, lxc_id=int(lxc_folder.name)) for lxc in get_all_lxcs(): logging.info(f"Loading LXC {lxc.lxc_id}") diff --git a/src/utils/proxmox_utils.py b/src/utils/proxmox_utils.py index 981925c..04b8bf9 100644 --- a/src/utils/proxmox_utils.py +++ b/src/utils/proxmox_utils.py @@ -1,56 +1,84 @@ import json import logging import os +import pathlib import subprocess +from pathlib import Path from .. import project_path def get_pve_version(): - """ - Get PVE version - :return: pve version + """Get PVE version + + Returns + ------- + str + PVE version """ return run_command_on_pve(command="pveversion", warn_exit_status=True) def get_pve_hostname(): - """ - Get PVE hostname - :return: pve hostname + """Get PVE hostname + + Returns + ------- + str + PVE hostname + """ return run_command_on_pve(command="hostname", warn_exit_status=True) -def does_lxc_exist(lxc_id): - """ - Check if LXC exists +def does_lxc_exist(lxc_id: int): + """Check if an LXC exists with the given ID - :param lxc_id: lxc id - :return: does lxc exists + Parameters + ---------- + lxc_id: int + + Returns + ------- + bool + Does LXC exist? """ + # TODO: only check in VMID column - return lxc_id in run_command_on_pve(command=f"pct list | awk '/{lxc_id}/'") + return str(lxc_id) in run_command_on_pve(command=f"pct list | awk '/{lxc_id}/'") -def does_qemu_vm_exist(vm_id): +def does_qemu_vm_exist(vm_id: int): + """Check if a QEMU VM exists with the given ID + + Parameters + ---------- + vm_id: int + + Returns + ------- + bool + Does QEMU VM exist? """ - Check if QEMU VM exists - :param vm_id: vm id - :return: does qemu vm exists - """ # TODO: only check in VMID column - return vm_id in run_command_on_pve(command=f"qm list {vm_id}") + return str(vm_id) in run_command_on_pve(command=f"qm list {vm_id}") -def execute_tteck_script(script_url, env_variables): - """ - Execute TTECK script with already filled environment variables to run silently (non-interactive) +def execute_tteck_script(script_url: str, env_variables: list): + """Execute TTECK script with already filled environment variables to run silently (non-interactive) - :param script_url: script url (github or other) - :param env_variables: list of environment variables - :return: status code + Parameters + ---------- + script_url: str + script url (github or other) + env_variables: list + list of environment variables + + Returns + ------- + int + status code """ env_variables = " ".join(env_variables) @@ -59,20 +87,46 @@ def execute_tteck_script(script_url, env_variables): def get_config(): + """Get the JSON content of config.json file as a dict + + Returns + ------- + dict + JSON content of config.json file + """ + config_file = os.path.join(project_path / "resources" / "config.json") with open(config_file, "r") as file: return json.loads(file.read()) -def run_command_on_pve(command, warn_exit_status=False, only_code=False, local=False): - """ - Run command on PVE +def run_command_on_pve(command: str, warn_exit_status: bool = False, only_code: bool = False, local: bool = False): + """Run command on the Proxmox VE host + + + Parameters + ---------- + command: str + command to run + warn_exit_status: bool, optional + should an exception be thrown if the exit status is not 0 + only_code: bool, optional + should it only return the exit code and not the stdout + local: bool, optional + should it be run locally not via ssh even with local mode off in the pve section of config.json + + Returns + ------- + str + command output by default + int + command exit code if only_code is True + + Raises + ------ + Exception + if the exit status is not 0 and warn_exit_status is True - :param local: should be run locally not via ssh even with local mode off - :param only_code: should we only return the exit code and not the stdout - :param warn_exit_status: should an exception be thrown if the exit status is not 0 - :param command: command - :return: command """ # Get config @@ -117,12 +171,45 @@ def run_command_on_pve(command, warn_exit_status=False, only_code=False, local=F return command.stdout.rstrip() -def run_command_locally(command, warn_exit_status=False, only_code=False): +def run_command_locally(command: str, warn_exit_status: bool = False, only_code: bool = False): + """Force running command locally even if local mode is off in config.json + + See Also + -------- + run_command_on_pve + """ + run_command_on_pve(command=command, warn_exit_status=warn_exit_status, only_code=only_code, local=True) -def run_command_ssh(command, host, username, port=22, warn_exit_status=False, only_code=False): - # Run command on PVE via SSH and return output +def run_command_ssh(command: str, host: str, username: str, port: int = 22, warn_exit_status: bool = False, + only_code: bool = False): + """Run command on a remote host via SSH + + Parameters + ---------- + command: str + command to run + host: str + host to run the command on + username: str + username to use to connect to the host (usually root for lxc and vm) + port: int, optional + port to use to connect to the host (default: 22) + warn_exit_status: bool, optional + should an exception be thrown if the exit status is not 0 + only_code: bool, optional + should it only return the exit code and not the stdout + + Returns + ------- + str + command output by default + int + command exit code if only_code is True + + """ + logging.debug(f"Running command on host {host} (ssh): {command}") # catch errors code @@ -139,12 +226,24 @@ def run_command_ssh(command, host, username, port=22, warn_exit_status=False, on return command.stdout.rstrip() -def get_install_package_command(distribution): - """ - Get the install package without interaction command based on the distribution. +def get_install_package_command(distribution: str): + """Retrieve the correct command to install a package based on the distribution specified + It supports all the distribution supported by Proxmox VE (from the pct command documentation). + Debian, Ubuntu, CentOS, Fedora, Gentoo, Alpine, ArchLinux, Devuan, NixOS, OpenSUSE - :param distribution: Distribution name - :return: Install package command + Parameters + ---------- + distribution: str + Name of the distribution as specific in the proxmox pct command documentation + + See Also + -------- + https://pve.proxmox.com/pve-docs/pct.1.html + + Returns + ------- + str + Beginning of the command to install a package, the package name should be appended to it """ distribution = distribution.lower() @@ -173,12 +272,24 @@ def get_install_package_command(distribution): raise Exception(f"Unsupported distribution: {distribution}") -def get_remove_package_command(distribution): - """ - Get the remove package without interaction command based on the distribution. +def get_remove_package_command(distribution: str): + """Retrieve the correct command to uninstall a package based on the distribution specified + It supports all the distribution supported by Proxmox VE (from the pct command documentation). + Debian, Ubuntu, CentOS, Fedora, Gentoo, Alpine, ArchLinux, Devuan, NixOS, OpenSUSE - :param distribution: Distribution name - :return: Remove package command + Parameters + ---------- + distribution: str + Name of the distribution as specific in the proxmox pct command documentation + + See Also + -------- + https://pve.proxmox.com/pve-docs/pct.1.html + + Returns + ------- + str + Beginning of the command to uninstall a package, the package name should be appended to it """ distribution = distribution.lower() @@ -208,8 +319,59 @@ def get_remove_package_command(distribution): def get_ssh_public_key(): + """ Retrieve the SSH public key of the machine running this script + + Returns + ------- + str + SSH public key """ - Get SSH public key - :return: ssh public key - """ + + # TODO: maybe implement custom path for the key return run_command_on_pve(command="cat ~/.ssh/id_rsa.pub", warn_exit_status=True) + + +def copy_file_to_pve(path: Path, destination: str): + """Copy a local file (on the machine running this script) to the PVE + + Parameters + ---------- + path: pathlib.Path + Path object to file on local machine + destination: str + Destination on PVE + + Examples + -------- + >>> copy_file_to_pve(Path("/home/user/file.txt"), "/root/file.txt") + """ + + config = get_config() + run_command_locally(f"scp {str(path)} {config['pve']['root']}@{config['pve']['host']}:{destination}") + + +def copy_file_to_lxc(lxc_id: int, path: Path, destination: str): + """Copy a local file (on the machine running this script) to the PVE and finally to the LXC + It works by copying the script from the local machine to the PVE using SCP and + then using "pct push" to send it to the LXC. + Sometimes SSH is not installed on freshs LXC, so by using pct push we are sure that it will work. + + Parameters + ---------- + lxc_id: int + LXC ID + path: pathlib.Path + Path object to file on local machine + destination: str + Destination on LXC + + Examples + -------- + >>> copy_file_to_lxc(100, Path("/home/user/file.txt"), "/root/") + """ + + file = path.name + temp_path = f"/tmp/proxmoxdeployscripts/{file}" + + copy_file_to_pve(path, temp_path) + run_command_on_pve(f"pct push {lxc_id} {temp_path} {destination}") diff --git a/src/utils/resources_utils.py b/src/utils/resources_utils.py index c276bf6..843fd14 100644 --- a/src/utils/resources_utils.py +++ b/src/utils/resources_utils.py @@ -1,15 +1,44 @@ -import os +import pathlib from .. import project_path +from ..lxc import LXC -def get_path(lxc, path): - parent = os.path.join(project_path / "resources") +def get_path(lxc: LXC, path: str): + """Returns the complete path to a global, protected or LXC-relative resource file. - if path.startswith("/global/"): + For example, if the path is "global/test.txt", the function will return the complete path to + the file in the resources/ folder. + Another example, if the path is "protected/test.txt", the function will return the complete path to + the file in the protected_resources/ folder. + Other folders can be added like "global/scripts/test.sh" or "protected/scripts/test.sh". + + If none of the above conditions are met, the function will return the path relative to the LXC folder + depending on the given LXC object. + + Parameters + ---------- + lxc: LXC + LXC object to use to get the relative path + Can be omitted if you know what you are doing and the path is either global or protected. + path: str + Path to the resource file + + Returns + ------- + pathlib.Path + Complete path to the resource file + + """ + parent = pathlib.Path(project_path).joinpath("resources") + + if path.startswith("global/"): # Use global folder - return parent + "/scripts/" + path[len("/global/"):] + return parent.joinpath(path[len("global/"):]) + elif path.startswith("protected/"): + # Use protected folder + return parent.parent.joinpath("protected_resources", path[len("protected/"):]) else: # Use VM/LXC folder lxc_id = lxc.get_id() - return parent + f"/resources/lxc/{lxc_id}/{path}" + return parent.joinpath("lxc", lxc_id, path)