From fb4154c7e5479fa09100e494324f82b4910827c9 Mon Sep 17 00:00:00 2001 From: Mathieu Broillet Date: Mon, 12 Jun 2023 15:15:51 +0200 Subject: [PATCH] implemented most creations steps and global improvements --- README.md | 44 +++++++++++++----- resources/config.json | 3 ++ resources/lxc/100/config.json | 1 - src/utils/creation_utils.py | 77 +++++++++++++------------------- src/utils/lxc_commands_utils.py | 76 +++++++++++++++++++++++++++++++ src/utils/lxc_utils.py | 57 +++++++++++++++++------- src/utils/proxmox_utils.py | 79 ++++++++++++++++++++------------- src/utils/resources_utils.py | 8 ++++ 8 files changed, 240 insertions(+), 105 deletions(-) create mode 100644 src/utils/lxc_commands_utils.py create mode 100644 src/utils/resources_utils.py diff --git a/README.md b/README.md index 29d8c1e..fcccaed 100644 --- a/README.md +++ b/README.md @@ -167,28 +167,50 @@ The script step will execute a script. ``` ### File -The file step will copy a file to the VM/LXC. +The file step will create / copy a file to the VM/LXC. ```json "steps": [ { - "type": "file", - "path": "traefik.toml", // local path (here: resources/lxc//traefik.toml) - "destination": "/var/data/traefikv2/traefik.toml" // lxc/vm path + "type": "file_create", + "path": "/var/data/traefikv2/traefik.toml", // lxc/vm path + "permissions": "644" // (optional) permissions of the file } ] ``` +```json +"steps": [ + { + "type": "file_copy", + "path": "traefik.toml", // local path (here: resources/lxc//traefik.toml) + "destination": "/var/data/traefikv2/traefik.toml", // lxc/vm path + "permissions": "644" // (optional) permissions of the file + } +] +``` +*Note: `file_copy` creates the folder if it doesn't exist* ### Folder The folder step will copy a folder to the VM/LXC. ```json "steps": [ { - "type": "folder", - "path": "/data/", // local path (here: resources/lxc//data/) - "destination": "/var/data/traefikv2" // lxc/vm path + "type": "folder_create", + "path": "/var/data/traefikv2", // lxc/vm path + "permissions": "755" // (optional) permissions of the folder } ] ``` +```json +"steps": [ + { + "type": "folder_copy", + "path": "/data/", // local path (here: resources/lxc//data/) + "destination": "/var/data/traefikv2", // lxc/vm path + "permissions": "755" // (optional) permissions of the folder + } +] +``` +*Note: `folder_copy` creates the folder if it doesn't exist* ### Command The command step will execute a command on the VM/LXC. @@ -197,19 +219,19 @@ The command step will execute a command on the VM/LXC. { "type": "command", "command": "whoami", // command to execute - "working_directory": "/var/data/config/traefikv2" // lxc/vm path + "working_directory": "/var/data/config/traefikv2" // (optional) lxc/vm path } ] ``` ### Docker -The docker step will execute a docker command on the VM/LXC. +The docker step will execute a command inside a docker container running on the VM/LXC. ```json "steps": [ { "type": "docker", - "command": "run -d --name=traefikv2 --restart=always -p 80:80 -p 443:443 -p 8080:8080 -v /var/data/traefikv2:/etc/traefik -v /var/data/config/traefikv2:/config -v /var/run/docker.sock:/var/run/docker.sock traefik:latest", // docker command to execute - "working_directory": "/var/data/config/traefikv2" // lxc/vm path + "container": "", // docker container name + "command": "" // docker command to execute } ] ``` diff --git a/resources/config.json b/resources/config.json index 3f0ca86..c0fb520 100644 --- a/resources/config.json +++ b/resources/config.json @@ -4,5 +4,8 @@ "user": "root", "port": 22, "local": false + }, + "settings": { + "setAuthorizedHostsSSH": true } } \ No newline at end of file diff --git a/resources/lxc/100/config.json b/resources/lxc/100/config.json index 03c2c09..11b2e3f 100644 --- a/resources/lxc/100/config.json +++ b/resources/lxc/100/config.json @@ -25,7 +25,6 @@ "start_on_boot": "false", "startup_order": 2, "password": "qwertz1234", - "ssh": false, "tags": "2-proxy+auth" }, "creation": { diff --git a/src/utils/creation_utils.py b/src/utils/creation_utils.py index 3c13f46..ebf27f7 100644 --- a/src/utils/creation_utils.py +++ b/src/utils/creation_utils.py @@ -1,7 +1,8 @@ import logging -import os from pathlib import Path +from src.utils import lxc_commands_utils + def are_all_conditions_met(lxc): """ @@ -10,7 +11,7 @@ def are_all_conditions_met(lxc): If all conditions are met, the deploy/updates steps are run Otherwise, we run the creation steps - :param lxc_id: lxc id + :param lxc: :return: are conditions met """ @@ -34,6 +35,13 @@ def are_all_conditions_met(lxc): return all(result) +def optional(var, placeholder): + if var is None: + return placeholder + else: + return var + + def run_steps(lxc): """ Run creation steps for an LXC @@ -49,48 +57,23 @@ def run_steps(lxc): # Support for scripts case "script": - run_script(step, lxc) - - -def run_script(step, lxc): - # Run local script - if "path" in step: - path = step["path"] - - with open(path, "r") as file: - script = file.read() - lxc.run_command("'bash -s' <<'ENDSSH'\n" + script) - - # Run remote script - elif "url" in step: - url = step["url"] - lxc.has - pass - - # Run script in LXC - elif "lxc_path" in step: - lxc_path = step["lxc_path"] - pass - - path = step["path"] - if path.startswith("/global/"): - # Use global folder - full_path = "resources/scripts" + path[len("/global/"):] - else: - # Use VM/LXC folder - lxc_id = lxc.get_id() - full_path = f"resources/lxc/{lxc_id}/{path}" - - if os.path.isfile(full_path): - lxc.run_command(f"bash {full_path}") - - -def has_program(lxc, program): - """ - Check if program is installed in LXC - - :param lxc: lxc - :param program: program - :return: is program installed - """ - return lxc.run_command("which " + program, only_code=True) == 0 + lxc_commands_utils.run_script(step, lxc) + case "file_create": + lxc_commands_utils.create_file_in_lxc(lxc, step["path"], optional(step["permission"], 644)) + case "file_copy": + lxc_commands_utils.copy_local_file_to_lxc(lxc, step["path"], step["destination"]) + case "folder": + lxc_commands_utils.create_folder_in_lxc(lxc, step["path"], optional(step["permission"], 755)) + case "folder_copy": + lxc_commands_utils.copy_local_folder_to_lxc(lxc, step["path"], step["destination"]) + case "command": + lxc.run_command(step["command"], optional(step["working_directory"], None)) + case "docker": + lxc_commands_utils.run_docker_command(lxc, step["container"], step["command"]) + case "docker_compose": + lxc_commands_utils.run_docker_compose_command(lxc, step["command"], + optional(step["working_directory"], None)) + case "git": + lxc.run_command(f"git clone {step['url']} {step['destination']}") + case "download": + lxc_commands_utils.download_file(lxc, step["url"], step["destination"]) diff --git a/src/utils/lxc_commands_utils.py b/src/utils/lxc_commands_utils.py new file mode 100644 index 0000000..52a5ddd --- /dev/null +++ b/src/utils/lxc_commands_utils.py @@ -0,0 +1,76 @@ +from src.utils import proxmox_utils +from src.utils.resources_utils import get_path + + +def run_script(lxc, step): + # Run local script + if "path" in step: + _run_local_script_on_lxc(lxc, step["path"]) + + # Run remote script + elif "url" in step: + _run_remote_script_on_lxc(lxc, step["url"]) + + # Run script in LXC + elif "lxc_path" in step: + _run_script_on_lxc(lxc, step["lxc_path"]) + + +def _run_script_on_lxc(lxc, path): + lxc.run_command(f"bash {path}") + + +def _run_local_script_on_lxc(lxc, path): + path = get_path(lxc, path) + with open(path, "r") as file: + script = file.read() + lxc.run_command("'bash -s' <<'ENDSSH'\n" + script) + + +def _run_remote_script_on_lxc(lxc, url): + if lxc.has_program("curl"): + lxc.run_command(f"curl -sSL {url} | bash") + + +def create_folder_in_lxc(lxc, path, permission=755): + lxc.run_command(f"mkdir -p {path}") + + if permission != 755: + lxc.run_command(f"chmod -R {permission} {path}") + + +def create_file_in_lxc(lxc, path, permission=644): + lxc.run_command(f"touch {path}") + if permission != 644: + lxc.run_command(f"chmod {permission} {path}") + + +def copy_local_file_to_lxc(lxc, path, destination): + proxmox_utils.run_command_locally(f"scp {path} {lxc.get_ssh_string()}:{destination}") + + +def copy_local_folder_to_lxc(lxc, path, destination): + proxmox_utils.run_command_locally(f"scp -r {path} {lxc.get_ssh_string()}:{destination}") + + +def run_docker_command(lxc, container, command): + lxc.run_command(f"docker exec -it {container} {command}") + + +def run_docker_compose_command(lxc, command, working_directory=None): + docker_compose_exec = "docker-compose" + if not lxc.has_program(docker_compose_exec): + docker_compose_exec = "docker compose" + + if working_directory is not None: + lxc.run_command(f"cd {working_directory} && {docker_compose_exec} {command}") + else: + lxc.run_command(f"{docker_compose_exec} {command}") + + +def download_file(lxc, url, destination): + if type(url) is list: + for u in url: + lxc.run_command(f"wget {u} -O {destination}") + else: + lxc.run_command(f"wget {url} -O {destination}") diff --git a/src/utils/lxc_utils.py b/src/utils/lxc_utils.py index 279255a..91df21b 100644 --- a/src/utils/lxc_utils.py +++ b/src/utils/lxc_utils.py @@ -1,7 +1,7 @@ import json import logging -from src.utils import proxmox_utils, creation_utils +from src.utils import proxmox_utils, creation_utils, lxc_commands_utils lxcs = [] @@ -86,7 +86,6 @@ class LXC: self.privileged = options["privileged"] self.start_on_boot = options["start_on_boot"] self.password = options["password"] - self.ssh = options["ssh"] self.creation = creation self.deploy = deploy @@ -283,12 +282,9 @@ class LXC: """ return self.password - def is_ssh_enabled(self): - """ - Is SSH enabled - :return: ssh - """ - return self.ssh + def get_ssh_string(self): + # TODO: implement hostname and maybe ipv6? + return "root@" + self.get_ipv4() def get_deploy(self): """ @@ -311,6 +307,13 @@ class LXC: """ return proxmox_utils.run_command_on_pve(f"pct list | awk '/running/ && /{self.lxc_id}/'") != "" + def does_exist(self): + """ + Does exist + :return: does lxc exist? (boolean) + """ + return proxmox_utils.does_lxc_exist(self.lxc_id) + def start(self): """ Start LXC @@ -326,13 +329,17 @@ class LXC: """ Create LXC """ - if proxmox_utils.does_lxc_exist(self.lxc_id): + if self.does_exist(): logging.info(f"LXC {self.lxc_id} already exists, skipping creation") proxmox_utils.run_command_on_pve(self.get_pct_command(create=False), True) else: logging.info(f"Creating LXC {self.lxc_id}") proxmox_utils.run_command_on_pve(self.get_pct_command(create=True), True) + if proxmox_utils.get_config()['settings']['setAuthorizedHostsSSH']: + pve_public_key = proxmox_utils.get_ssh_public_key() + self.run_command(f"echo '{pve_public_key}' >> /root/.ssh/authorized_keys") + def creation(self): """ Run the creations checks and steps @@ -343,21 +350,37 @@ class LXC: self.start() # Check if all creation conditions are met - if not creation_utils.creation_conditions_met(self): + if not creation_utils.are_all_conditions_met(self): logging.info(f"Not all creation conditions met for LXC {self.lxc_id}, running creation steps...") # Run creation steps - creation_utils.run_creation_steps(self) - - + creation_utils.run_steps(self) def deploy(self): pass def has_program(self, program): - return creation_utils.has_program(self, program) + """ + Check if program is installed on LXC + :param program: program executable name - def run_command(self, command, warn_exit_status=False, only_code=False): + :return: boolean + """ + if type(program) == str: + return self.run_command("which " + program, only_code=True) == 0 + elif type(program) == list: + return all((self.run_command("which " + p, only_code=True) == 0) for p in program) + + def run_script(self, script_path): + """ + Run script on LXC filesystem using bash + + :param script_path: + :return: + """ + return lxc_commands_utils.run_script(self, {"lxc_path": script_path}) + + def run_command(self, command, warn_exit_status=False, only_code=False, working_directory=None): """ Run command on LXC :param command: command to run @@ -365,6 +388,10 @@ class LXC: """ logging.debug(f"Running command {command} on LXC {self.lxc_id}") + + if working_directory: + command = f"cd {working_directory} && {command}" + if only_code: return proxmox_utils.run_command_on_pve(f"pct exec {self.lxc_id} -- {command}", warn_exit_status, only_code) diff --git a/src/utils/proxmox_utils.py b/src/utils/proxmox_utils.py index 7de00d7..f88c056 100644 --- a/src/utils/proxmox_utils.py +++ b/src/utils/proxmox_utils.py @@ -58,55 +58,72 @@ def execute_tteck_script(script_url, env_variables): run_command_on_pve(f"{env_variables} && bash -c \"$(wget -qLO - {script_url}\"", True) -def run_command_on_pve(command, warn_exit_status=False, only_code=False): +def get_config(): + config_file = os.path.join(project_path / "resources" / "config.json") + with open(config_file, "r") as file: + return json.loads(file.read()) + + +def run_command_on_pve(command, warn_exit_status=False, only_code=False, local=False): """ Run command on PVE + :param local: should be run locally not via ssh even with local mode off :param only_code: should we only return the exit code and not the stdout :param warn_exit_status: should an exception be thrown if the exit status is not 0 :param command: command :return: command """ - config_file = os.path.join(project_path / "resources" / "config.json") - with open(config_file, "r") as file: + # Get config + config = get_config() - data = json.loads(file.read()) + # Check if PVE is local or remote + if config['pve']['local'] or local: + # Run command and return output (not as bytes) + logging.debug(f"Running command on PVE (locally): \n{command}") - # Check if PVE is local or remote - if data['pve']['local']: - # Run command and return output (not as bytes) - logging.debug(f"Running command on PVE (locally): \n{command}") + command = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + encoding="utf-8") - command = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding="utf-8") + if command.returncode != 0 and warn_exit_status: + logging.error(f"Error while running command on PVE: \n{command.stderr}") + raise Exception(f"Error while running command on PVE: \n{command.stderr}") - if command.returncode != 0 and warn_exit_status: - logging.error(f"Error while running command on PVE: \n{command.stderr}") - raise Exception(f"Error while running command on PVE: \n{command.stderr}") + if only_code: + return command.returncode - if only_code: - return command.returncode + return command.stdout.decode().rstrip() - return command.stdout.decode().rstrip() + else: + host = config['pve']['host'] + username = config['pve']['user'] + port = config['pve']['port'] - else: - host = data['pve']['host'] - username = data['pve']['user'] - port = data['pve']['port'] + # Run command on PVE via SSH and return output + logging.debug(f"Running command on PVE (ssh): \n{command}") - # Run command on PVE via SSH and return output - logging.debug(f"Running command on PVE (ssh): \n{command}") + # catch errors code + command = subprocess.run(f"ssh {username}@{host} -p {port} \"{command}\"", shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8") - # catch errors code - command = subprocess.run(f"ssh {username}@{host} -p {port} \"{command}\"", shell=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8") + if command.returncode != 0 and warn_exit_status: + logging.error(f"Error while running command on PVE: \n{command.stderr}") + raise Exception(f"Error while running command on PVE: \n{command.stderr}") - if command.returncode != 0 and warn_exit_status: - logging.error(f"Error while running command on PVE: \n{command.stderr}") - raise Exception(f"Error while running command on PVE: \n{command.stderr}") + if only_code: + return command.returncode - if only_code: - return command.returncode + return command.stdout.rstrip() - return command.stdout.rstrip() \ No newline at end of file + +def run_command_locally(command, warn_exit_status=False, only_code=False): + run_command_on_pve(command, warn_exit_status, only_code, local=True) + + +def get_ssh_public_key(): + """ + Get SSH public key + :return: ssh public key + """ + return run_command_on_pve("cat ~/.ssh/id_rsa.pub", True) diff --git a/src/utils/resources_utils.py b/src/utils/resources_utils.py new file mode 100644 index 0000000..d48aa87 --- /dev/null +++ b/src/utils/resources_utils.py @@ -0,0 +1,8 @@ +def get_path(lxc, path): + if path.startswith("/global/"): + # Use global folder + return "resources/scripts" + path[len("/global/"):] + else: + # Use VM/LXC folder + lxc_id = lxc.get_id() + return f"resources/lxc/{lxc_id}/{path}" \ No newline at end of file