From 957df53cd5a16bfad899a01f8716b45480c487c9 Mon Sep 17 00:00:00 2001 From: Mathieu Broillet Date: Mon, 19 Jun 2023 16:38:55 +0200 Subject: [PATCH] started implementing caching --- src/lxc/lxc.py | 22 ++++++++++++---- src/utils/machine.py | 62 +++++++++++++++++++++++++++++++++++++------- src/utils/proxmox.py | 26 +++++++++++++++++-- 3 files changed, 94 insertions(+), 16 deletions(-) diff --git a/src/lxc/lxc.py b/src/lxc/lxc.py index eedfbd6..3b32720 100644 --- a/src/lxc/lxc.py +++ b/src/lxc/lxc.py @@ -1,5 +1,4 @@ import logging -import time from . import creation_utils, lxc_utils from ..utils.machine import LinuxMachine @@ -177,6 +176,20 @@ class LXC(LinuxMachine): Get IPv4 :return: ipv4 """ + + if netmask: + ipv4 = self.get_in_cache('network', 'ipv4_netmask') + if ipv4 is None: + self.set_in_cache('network', 'ipv4_netmask', self.retrieve_ipv4(netmask=netmask, use_ssh=use_ssh)) + return self.get_in_cache('network', 'ipv4_netmask') + + else: + ipv4 = self.get_in_cache('network', 'ipv4') + if ipv4 is None: + self.set_in_cache('network', 'ipv4', self.retrieve_ipv4(netmask=netmask, use_ssh=use_ssh)) + return self.get_in_cache('network', 'ipv4') + + def retrieve_ipv4(self, netmask: bool = False, use_ssh: bool = False): if self.ipv4 == "dhcp": if self.is_running(): if self.has_program("ip", use_ssh=use_ssh): @@ -389,7 +402,6 @@ class LXC(LinuxMachine): if working_directory: command = f"cd {working_directory} && {command}" - # Using pct exec works every time but is 8x slower than using ssh if use_ssh: return self.pve.run_command( @@ -399,6 +411,6 @@ class LXC(LinuxMachine): exception_on_empty_stdout=exception_on_empty_stdout) else: return self.pve.run_command(command=f"pct exec {self.lxc_id} -- {command}", - return_status_code=return_status_code, - exception_on_exit=exception_on_exit, - exception_on_empty_stdout=exception_on_empty_stdout) + return_status_code=return_status_code, + exception_on_exit=exception_on_exit, + exception_on_empty_stdout=exception_on_empty_stdout) diff --git a/src/utils/machine.py b/src/utils/machine.py index ac3cdba..f2bab45 100644 --- a/src/utils/machine.py +++ b/src/utils/machine.py @@ -6,7 +6,7 @@ from src.utils import utils class LinuxMachine(): def __init__(self): - pass + self.cache = {} def update(self): pass @@ -29,21 +29,38 @@ class LinuxMachine(): def get_memory(self): """Get memory""" - return self.run_command("free -m | grep Mem | awk '{print $2}'") + + memory = self.get_in_cache('system', 'memory') + if memory is None: + self.set_in_cache('system', 'memory', self.run_command("free -m | grep Mem | awk '{print $2}'")) + + return self.get_in_cache('system', 'memory') def get_ipv4(self): """Get IPv4 address""" - if self.has_program("ip"): - return self.run_command("""ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 -d'/'""") - elif self.has_program("ifconfig"): - return self.run_command(command="ifconfig eth0 | awk '/inet addr/{print substr($2,6)}'") + ipv4 = self.get_in_cache('system', 'ipv4') + if ipv4 is None: + if self.has_program("ip"): + self.set_in_cache('system', 'ipv4', self.run_command(command= + """ip addr | grep 'state UP' -A2 | tail -n1 | + awk '{print $2}' | cut -f1 -d'/'""")) + elif self.has_program("ifconfig"): + self.set_in_cache('system', 'ipv4', + self.run_command(command="ifconfig eth0 | awk '/inet addr/{print substr($2,6)}'")) + + return self.get_in_cache('system', 'ipv4') def get_ipv6(self): pass def get_mac(self): """Get MAC address""" - return self.run_command("""cat /sys/class/net/$(ip route show default | awk '/default/ {print $5}')/address""") + mac = self.get_in_cache('system', 'mac') + if mac is None: + self.set_in_cache('system', 'mac', self.run_command( + """cat /sys/class/net/$(ip route show default | awk '/default/ {print $5}')/address""")) + + return self.get_in_cache('system', 'mac') def start(self): pass @@ -61,9 +78,17 @@ class LinuxMachine(): :return: boolean """ if type(program) == str: - return self.run_command("which " + program, return_status_code=True, use_ssh=use_ssh) == 0 + program_cache = self.get_in_cache('known_programs', program) + if program_cache is None: + self.set_in_cache('known_programs', program, str(self.run_command("which " + program, + return_status_code=True, + use_ssh=use_ssh) == 0)) + return bool(self.get_in_cache('known_programs', program)) elif type(program) == list: - return all((self.run_command("which " + p, return_status_code=True, use_ssh=use_ssh) == 0) for p in program) + programs = [] + for p in program: + programs.append(self.has_program(p)) + return all(programs) def has_file(self, file: str or Path): """Check if file exists on LXC @@ -301,3 +326,22 @@ class LinuxMachine(): else: self.run_command(f"sed {'-i' if case_sensitive else ''} 's/{search}/{replace}/g' {path}", return_status_code=True) + + def get_in_cache(self, category: str, property: str): + if category in self.cache: + if property in self.cache[category]: + return self.cache[category][property] + else: + return None + + def set_in_cache(self, category: str, property: str = None, value: str = None): + if category not in self.cache: + self.cache[category] = {} + + if property is not None and value is not None: + if property not in self.cache[category]: + self.cache[category] = {} + + self.cache[category] = {property: value} + else: + raise ValueError("Key and value must be specified to save in the cache") diff --git a/src/utils/proxmox.py b/src/utils/proxmox.py index 01921cf..06a1725 100644 --- a/src/utils/proxmox.py +++ b/src/utils/proxmox.py @@ -2,6 +2,7 @@ from __future__ import annotations import fnmatch import subprocess +import threading import time from pathlib import Path from typing import TYPE_CHECKING @@ -40,12 +41,26 @@ class ProxmoxHost(LinuxMachine): self.connection = None self.repo_path = path.as_posix() + threading.Thread(target=self.update).start() + def __str__(self): return f"ProxmoxHost({self.host}, {self.user}, {self.port})" def __repr__(self): return f"ProxmoxHost({self.host}, {self.user}, {self.port})" + def update(self): + update_connection = Connection(host=self.host, user=self.user, port=self.port) + while True: + pct_output = update_connection.run("pct list", hide=True, warn=True, encoding="utf-8").stdout.split("\n")[ + 1:] + for line in pct_output: + if line.strip(): + vmid, status, name = line.split() + self.set_in_cache('lxc', vmid.strip(), str("running" in status.strip())) + + time.sleep(1) + def connect(self): """Connect to the Proxmox host.""" if self.host is not None: @@ -100,12 +115,15 @@ class ProxmoxHost(LinuxMachine): """ # Check if host is None, if it is, run the command locally, else run it on the host via SSH + start_time = time.time() + if self.host is None: command_result = subprocess.run(command, shell=True, capture_output=True, encoding="utf-8") elif self.connection is not None: command_result = self.connection.run(command, hide=True, warn=True, encoding="utf-8") else: raise Exception("No host or connection provided") + print(f"Time taken for {command}: {time.time() - start_time}") # If return code is not 0 and that exception_on_exit is True and return_status_code is False, throw an exception if command_result.return_code != 0 and exception_on_exit and not return_status_code: @@ -149,8 +167,12 @@ class ProxmoxHost(LinuxMachine): def is_lxc_running(self, lxc_id): """Check if the given LXC is running on the Proxmox host.""" - pct_status_output = self.run_command(f"pct status {lxc_id}", exception_on_exit=False) - return "running" in pct_status_output + lxc = self.get_in_cache('lxc', lxc_id) + if lxc is None: + self.set_in_cache('lxc', lxc_id, + str('running' in self.run_command(f"pct status {lxc_id}", exception_on_exit=False))) + + return self.get_in_cache('lxc', lxc_id) def start_lxc(self, lxc_id): """Start the given LXC on the Proxmox host."""