From e6dd3d81cea5d550bf3b42463e7ee483d316d031 Mon Sep 17 00:00:00 2001 From: Mathieu Broillet Date: Tue, 20 Jun 2023 10:01:13 +0200 Subject: [PATCH] sped up the soft by about 3x using basic caching and threads --- src/lxc/lxc.py | 11 +++++------ src/utils/machine.py | 11 +++++++++-- src/utils/proxmox.py | 41 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/src/lxc/lxc.py b/src/lxc/lxc.py index eedfbd6..af1518c 100644 --- a/src/lxc/lxc.py +++ b/src/lxc/lxc.py @@ -1,5 +1,4 @@ import logging -import time from . import creation_utils, lxc_utils from ..utils.machine import LinuxMachine @@ -177,7 +176,7 @@ class LXC(LinuxMachine): Get IPv4 :return: ipv4 """ - if self.ipv4 == "dhcp": + if self.ipv4 == "dhcp" or self.ipv4 == "auto": if self.is_running(): if self.has_program("ip", use_ssh=use_ssh): if netmask: @@ -304,6 +303,7 @@ class LXC(LinuxMachine): Start LXC """ self.pve.start_lxc(self.lxc_id) + self.ipv4 = self.get_ipv4() def stop(self): """ @@ -389,7 +389,6 @@ class LXC(LinuxMachine): if working_directory: command = f"cd {working_directory} && {command}" - # Using pct exec works every time but is 8x slower than using ssh if use_ssh: return self.pve.run_command( @@ -399,6 +398,6 @@ class LXC(LinuxMachine): exception_on_empty_stdout=exception_on_empty_stdout) else: return self.pve.run_command(command=f"pct exec {self.lxc_id} -- {command}", - return_status_code=return_status_code, - exception_on_exit=exception_on_exit, - exception_on_empty_stdout=exception_on_empty_stdout) + return_status_code=return_status_code, + exception_on_exit=exception_on_exit, + exception_on_empty_stdout=exception_on_empty_stdout) diff --git a/src/utils/machine.py b/src/utils/machine.py index ac3cdba..8d3870e 100644 --- a/src/utils/machine.py +++ b/src/utils/machine.py @@ -6,6 +6,7 @@ from src.utils import utils class LinuxMachine(): def __init__(self): + self.known_programs = [] pass def update(self): @@ -60,10 +61,16 @@ class LinuxMachine(): :return: boolean """ + if program in self.known_programs: + return True + if type(program) == str: - return self.run_command("which " + program, return_status_code=True, use_ssh=use_ssh) == 0 + result = self.run_command("which " + program, return_status_code=True, use_ssh=use_ssh) == 0 + if result: + self.known_programs.append(program) + return result elif type(program) == list: - return all((self.run_command("which " + p, return_status_code=True, use_ssh=use_ssh) == 0) for p in program) + return all(self.has_program(program=p, use_ssh=use_ssh) for p in program) def has_file(self, file: str or Path): """Check if file exists on LXC diff --git a/src/utils/proxmox.py b/src/utils/proxmox.py index 01921cf..8baea8c 100644 --- a/src/utils/proxmox.py +++ b/src/utils/proxmox.py @@ -2,6 +2,7 @@ from __future__ import annotations import fnmatch import subprocess +import threading import time from pathlib import Path from typing import TYPE_CHECKING @@ -40,6 +41,31 @@ class ProxmoxHost(LinuxMachine): self.connection = None self.repo_path = path.as_posix() + self.running_lxcs = [] + + threading.Thread(target=self.update, daemon=True).start() + + def update(self): + update_connection = Connection(host=self.host, user=self.user, port=self.port) + while True: + pct_output = update_connection.run("pct list", hide=True, warn=True, encoding="utf-8").stdout.split("\n")[ + 1:] + for line in pct_output: + if line.strip(): + if len(line.split()) == 3: + vm_id, status, name = line.split() + else: + vm_id, status, lock, name = line.split() + + if "running" in status: + if vm_id not in self.running_lxcs: + self.running_lxcs.append(int(vm_id)) + else: + if vm_id in self.running_lxcs: + self.running_lxcs.remove(int(vm_id)) + + time.sleep(0.5) + def __str__(self): return f"ProxmoxHost({self.host}, {self.user}, {self.port})" @@ -99,6 +125,8 @@ class ProxmoxHost(LinuxMachine): if the stdout is empty and exception_on_empty_stdout is True """ + start_time = time.time() + # Check if host is None, if it is, run the command locally, else run it on the host via SSH if self.host is None: command_result = subprocess.run(command, shell=True, capture_output=True, encoding="utf-8") @@ -107,6 +135,9 @@ class ProxmoxHost(LinuxMachine): else: raise Exception("No host or connection provided") + end_time = time.time() + print(f"Command {command} took {end_time - start_time} seconds to run") + # If return code is not 0 and that exception_on_exit is True and return_status_code is False, throw an exception if command_result.return_code != 0 and exception_on_exit and not return_status_code: raise Exception(f"Error while running command: \n{command_result.stderr}") @@ -149,8 +180,14 @@ class ProxmoxHost(LinuxMachine): def is_lxc_running(self, lxc_id): """Check if the given LXC is running on the Proxmox host.""" - pct_status_output = self.run_command(f"pct status {lxc_id}", exception_on_exit=False) - return "running" in pct_status_output + if lxc_id in self.running_lxcs: + return True + else: + return False + + # Not used anymore because it's too slow, now using an update thread to keep track of running LXCs + # pct_status_output = self.run_command(f"pct status {lxc_id}", exception_on_exit=False) + # return "running" in pct_status_output def start_lxc(self, lxc_id): """Start the given LXC on the Proxmox host."""