From 099638570fbafbefd62bc7953dda2fbf59c96af7 Mon Sep 17 00:00:00 2001 From: Mathieu Broillet Date: Thu, 22 Jun 2023 15:43:35 +0200 Subject: [PATCH] remade the steps runner and improved some functions --- src/lxc/creation_utils.py | 58 +------------ src/machine/machine.py | 12 ++- src/main.py | 6 +- src/proxmox/proxmox.py | 26 +++++- src/utils/git_utils.py | 4 +- src/utils/steps_utils.py | 177 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 216 insertions(+), 67 deletions(-) create mode 100644 src/utils/steps_utils.py diff --git a/src/lxc/creation_utils.py b/src/lxc/creation_utils.py index 09fc401..75117b6 100644 --- a/src/lxc/creation_utils.py +++ b/src/lxc/creation_utils.py @@ -1,11 +1,8 @@ from __future__ import annotations -import logging from typing import TYPE_CHECKING -from . import lxc_utils -from ..utils import conditions_utils -from ..utils.resources_utils import get_path +from ..utils import steps_utils if TYPE_CHECKING: from .lxc import LXC @@ -20,54 +17,7 @@ def run_steps(lxc: LXC): The LXC object used to run the creation steps """ - creation = lxc.creation - creation_steps = creation["steps"] + creation_steps = lxc.creation["steps"] - for index, step in enumerate(creation_steps): - - if conditions_utils.verify_step_conditions(lxc, step): - logging.info(f"Conditions already met for step {index + 1}/{len(creation_steps)} for LXC {lxc.id}, " - f"skipping...") - continue - - logging.info(f"Running step {index + 1}/{len(creation_steps)} for LXC {lxc.id}...") - - match step["type"]: - - # Support for scripts - case "script": - lxc_utils.run_script_step_parser(lxc, step) - case "file_create": - lxc.create_file(step["path"], step.get("permission", 644), step.get("owner", "root")) - case "file_copy": - lxc.pve.copy_file_to_lxc(lxc, get_path(lxc, step["path"]), step["destination"]) - case "folder_create": - lxc.create_directory(step["path"], step.get("permission", 755), step.get("owner", "root")) - case "folder_copy": - lxc.pve.copy_folder_to_lxc(lxc, get_path(lxc, step["path"]), step["destination"]) - case "command": - lxc.run_command(command=step["command"], working_directory=step.get("working_directory"), - return_status_code=True) - case "docker": - lxc.run_docker_command(step["container"], step["command"]) - case "docker_compose": - lxc.run_docker_compose_command(command=step["command"], - working_directory=step.get("working_directory")) - case "git": - lxc.run_command(command=f"git clone {step['url']} {step['destination']}", return_status_code=True) - case "download": - lxc.download_file(step["url"], step["destination"]) - case "unzip": - lxc.unzip_file(step["path"], step.get("destination")) - case "install-package": - lxc.install_package(step["package"]) - case "remove-package": - lxc.remove_package(step["package"]) - case "start": - lxc.start() - case "stop": - lxc.stop() - case "reboot": - lxc.reboot() - case "replace-in-files": - lxc.replace_in_files(step["files"], step["search"], step["replace"]) + # Run the steps + steps_utils.run_steps(lxc, creation_steps) diff --git a/src/machine/machine.py b/src/machine/machine.py index 3ecbe85..a6e43d4 100644 --- a/src/machine/machine.py +++ b/src/machine/machine.py @@ -7,6 +7,7 @@ class LinuxMachine: def __init__(self): self.known_programs = [] + self.id = None pass def update(self): @@ -144,7 +145,7 @@ class LinuxMachine: self.run_command(f"rm -rf {directory}", return_status_code=True) - def run_docker_command(self, container, command): + def run_docker_command(self, container, command: str or list): """Run a command inside a docker container on a linux host Parameters @@ -159,6 +160,9 @@ class LinuxMachine: >>> self.run_docker_command(linux_machine, "", "") """ + if type(command) == list: + command = " && ".join(command) + if not self.has_program("docker"): raise Exception(f"Docker is not installed on this machine {self.get_hostname()}") @@ -188,12 +192,12 @@ class LinuxMachine: else: self.run_command(f"{docker_compose_exec} {command}", return_status_code=True) - def download_file(self, url: str, destination: str): + def download_file(self, url: str or list, destination: str): """Download a file from a URL to the Linux Machine and save it to the destination Parameters ---------- - url: str + url: str or list URL of the file to download destination: str Path to the destination to save the file to @@ -206,7 +210,7 @@ class LinuxMachine: if type(url) is list: for u in url: - self.run_command(f"wget {u} --directory-prefix={destination}", return_status_code=True) + self.download_file(u, destination) else: self.run_command(f"wget {url} --directory-prefix={destination}", return_status_code=True) diff --git a/src/main.py b/src/main.py index 184919b..3da7653 100644 --- a/src/main.py +++ b/src/main.py @@ -2,8 +2,8 @@ import logging from pathlib import Path from .lxc.lxc_utils import load_lxc, get_all_lxcs -from .utils import git_utils from .proxmox.proxmox import ProxmoxHost +from .utils import git_utils def run(args): @@ -20,11 +20,11 @@ def run(args): # Check if the repo is already cloned if not pve.has_directory(pve.repo_path): logging.info(f"Cloning repository {args.repo} to {args.path}...") - git_utils.clone_repo(url=args.repo, path=pve.repo_path, linux_machine=pve) + git_utils.clone_repo(linux_machine=pve, url=args.repo, path=pve.repo_path) if pve.has_directory(Path(pve.repo_path).joinpath(".git")): logging.info(f"Repository already cloned at {pve.repo_path}, updating...") - git_utils.update_repo(path=pve.repo_path, linux_machine=pve) + git_utils.update_repo(linux_machine=pve, path=pve.repo_path) if not pve.has_directory(pve.repo_path): raise FileNotFoundError(f"Repo not found at {pve.repo_path}") diff --git a/src/proxmox/proxmox.py b/src/proxmox/proxmox.py index 8420353..0a71d7b 100644 --- a/src/proxmox/proxmox.py +++ b/src/proxmox/proxmox.py @@ -78,7 +78,7 @@ class ProxmoxHost(LinuxMachine): self.connection = Connection(host=self.host, user=self.user, port=self.port) def run_command(self, command: str, return_status_code: bool = False, exception_on_exit: bool = True, - exception_on_empty_stdout: bool = False, **kwargs): + exception_on_empty_stdout: bool = False, working_directory=None, **kwargs): """Run a command on the Proxmox VE host The default behavior is as follows : - runs the command on the Proxmox VE host @@ -92,7 +92,7 @@ class ProxmoxHost(LinuxMachine): Parameters ---------- - command_result: str + command: str command to run return_status_code: bool, optional should it return the exit code and not the stdout, disables exception_on_exit @@ -121,6 +121,12 @@ class ProxmoxHost(LinuxMachine): if the stdout is empty and exception_on_empty_stdout is True """ + if type(command) == list: + command = ' && '.join(command) + + if working_directory: + command = f"cd {working_directory} && {command}" + start_time = time.time() # Check if host is None, if it is, run the command locally, else run it on the host via SSH @@ -242,7 +248,8 @@ class ProxmoxHost(LinuxMachine): self.run_command(f"qm reboot {vm_id}") - def copy_file_to_lxc(self, lxc: LXC, source: str or Path, destination: str or Path, use_ssh: bool = True): + def copy_file_to_lxc(self, lxc: LXC, source: str or Path, destination: str or Path, permission: int = 644, + owner: str = "root", use_ssh: bool = True): """Copy the given file to the given LXC.""" if isinstance(source, Path): source = str(source.as_posix()) @@ -252,7 +259,13 @@ class ProxmoxHost(LinuxMachine): lxc.create_directory(destination.parent.as_posix(), use_ssh=use_ssh) self.run_command(f"pct push {lxc.id} {source} {destination.as_posix()}") - def copy_folder_to_lxc(self, lxc: LXC, source: str or Path, destination: str or Path): + if permission != 755: + lxc.run_command(f"chmod {permission} {destination}", return_status_code=True, use_ssh=use_ssh) + if owner != "root": + lxc.run_command(f"chown {owner} {destination}", return_status_code=True) + + def copy_folder_to_lxc(self, lxc: LXC, source: str or Path, destination: str or Path, permission: int = 755, + owner: str = "root"): """Copy the given folder to the given LXC.""" if isinstance(source, Path): source = str(source.as_posix()) @@ -261,6 +274,11 @@ class ProxmoxHost(LinuxMachine): self.run_command(f"scp -o StrictHostKeyChecking=no -B -r {source} {lxc.get_ssh_string()}:{destination}") + if permission != 755: + lxc.run_command(f"chmod -R {permission} {destination}", return_status_code=True) + if owner != "root": + lxc.run_command(f"chown -R {owner} {destination}", return_status_code=True) + def list_dir(self, directory: str or Path, glob_filter: str = '*'): """List the given directory.""" if isinstance(directory, Path): diff --git a/src/utils/git_utils.py b/src/utils/git_utils.py index e45283d..c5d135e 100644 --- a/src/utils/git_utils.py +++ b/src/utils/git_utils.py @@ -3,13 +3,13 @@ import logging from ..machine.machine import LinuxMachine -def clone_repo(url, path, linux_machine: LinuxMachine): +def clone_repo(linux_machine: LinuxMachine, url, path): """Clone the given repository to the given path""" logging.info(f"Cloning repository {url} to {path}...") linux_machine.run_command(f"git clone {url} {path}") -def update_repo(path, linux_machine: LinuxMachine): +def update_repo(linux_machine: LinuxMachine, path): """Update the given repository""" logging.info(f"Updating repository at {path}...") linux_machine.run_command(f"git -C {path} pull") diff --git a/src/utils/steps_utils.py b/src/utils/steps_utils.py new file mode 100644 index 0000000..3aca0b4 --- /dev/null +++ b/src/utils/steps_utils.py @@ -0,0 +1,177 @@ +import logging +import time + +from . import git_utils +from .resources_utils import get_path +from ..lxc import lxc_utils +from ..lxc.lxc import LXC +from ..machine.machine import LinuxMachine +from ..utils import conditions_utils + + +def _run_script_step(linux_machine, step): + if isinstance(linux_machine, LXC): + lxc_utils.run_script_step_parser(linux_machine, step) + else: + logging.warning(f"Script step only supported on LXCs") + pass + + +def _run_file_create_step(linux_machine, step): + linux_machine.create_file(step["path"], step.get("permission", 644), step.get("owner", "root")) + + +def _run_file_copy_step(linux_machine, step): + if isinstance(linux_machine, LXC): + linux_machine.pve.copy_file_to_lxc(linux_machine, get_path(linux_machine, step["path"]), step["destination"], + step.get("permission", 644), step.get("owner", "root")) + else: + logging.warning(f"File copy step only supported on LXCs") + + +def _run_folder_create_step(linux_machine, step): + linux_machine.create_directory(step["path"], step.get("permission", 755), step.get("owner", "root")) + + +def _run_folder_copy_step(linux_machine, step): + linux_machine.pve.copy_folder_to_lxc(linux_machine, get_path(linux_machine, step["path"]), step["destination"], + step.get("permission", 644), step.get("owner", "root")) + + +def _run_command_step(linux_machine, step): + linux_machine.run_command(command=step["command"], working_directory=step.get("workdir"), + return_status_code=True) + + +def _run_docker_step(linux_machine, step): + linux_machine.run_docker_command(step["container"], step["command"]) + + +def _run_docker_compose_step(linux_machine, step): + linux_machine.run_docker_compose_command(command=step["command"], working_directory=step.get("working_directory")) + + +def _run_git_clone_step(linux_machine, step): + git_utils.clone_repo(linux_machine=linux_machine, url=step["url"], path=step["destination"]) + + +def _run_git_pull_step(linux_machine, step): + git_utils.update_repo(linux_machine=linux_machine, path=step["path"]) + + +def _run_download_step(linux_machine, step): + linux_machine.download_file(step["url"], step["destination"]) + + +def _run_install_package_step(linux_machine, step): + linux_machine.install_package(step["package"]) + + +def _run_remove_package_step(linux_machine, step): + linux_machine.remove_package(step["package"]) + + +def _run_start_step(linux_machine, step): + linux_machine.start() + + +def _run_stop_step(linux_machine, step): + linux_machine.stop() + + +def _run_reboot_step(linux_machine, step): + linux_machine.reboot() + + +def _run_service_start_step(linux_machine, step): + pass + + +def _run_service_stop_step(linux_machine, step): + pass + + +def _run_service_restart_step(linux_machine, step): + pass + + +def _run_service_enable_step(linux_machine, step): + pass + + +def _run_service_disable_step(linux_machine, step): + pass + + +def _run_replace_in_file_step(linux_machine, step): + pass + + +def _run_unzip_step(linux_machine, step): + linux_machine.unzip_file(step["path"], step.get("destination")) + + +def _run_wait_step(step): + time.sleep(step["seconds"]) + + +def run_steps(linux_machine: LinuxMachine, steps: dict): + for index, step in enumerate(steps): + + if conditions_utils.verify_step_conditions(linux_machine, step): + logging.info(f"Conditions already met for step {index + 1}/{len(steps)} for LXC {linux_machine.id}, " + f"skipping...") + continue + + logging.info(f"Running step {index + 1}/{len(steps)} for LXC {linux_machine.id}...") + + # Run command step + match step['type']: + case "script": + _run_script_step(linux_machine, step) + case "file_create": + _run_file_create_step(linux_machine, step) + case "file_copy": + _run_file_copy_step(linux_machine, step) + case "folder_create": + _run_folder_create_step(linux_machine, step) + case "folder_copy": + _run_folder_copy_step(linux_machine, step) + case "command": + _run_command_step(linux_machine, step) + case "docker": + _run_docker_step(linux_machine, step) + case "docker_compose": + _run_docker_compose_step(linux_machine, step) + case "git_clone": + _run_git_clone_step(linux_machine, step) + case "git_pull": + _run_git_pull_step(linux_machine, step) + case "download": + _run_download_step(linux_machine, step) + case "install_package": + _run_install_package_step(linux_machine, step) + case "remove_package": + _run_remove_package_step(linux_machine, step) + case "start": + _run_start_step(linux_machine, step) + case "stop": + _run_stop_step(linux_machine, step) + case "reboot": + _run_reboot_step(linux_machine, step) + case "service_start": + _run_service_start_step(linux_machine, step) + case "service_stop": + _run_service_stop_step(linux_machine, step) + case "service_restart": + _run_service_restart_step(linux_machine, step) + case "service_enable": + _run_service_enable_step(linux_machine, step) + case "service_disable": + _run_service_disable_step(linux_machine, step) + case "replace_in_file": + _run_replace_in_file_step(linux_machine, step) + case "unzip": + _run_unzip_step(linux_machine, step) + case "wait": + _run_wait_step(step)