started implementing caching

src/lxc/lxc.py

@@ -171,25 +171,42 @@ class LXC(LinuxMachine):
         """
         return self.bridge
 
-    def get_ipv4(self, netmask: bool = False):
+    def get_ipv4(self, netmask: bool = False, use_ssh: bool = False):
         """
         Get IPv4
         :return: ipv4
         """
+
+        if netmask:
+            ipv4 = self.get_in_cache('network', 'ipv4_netmask')
+            if ipv4 is None:
+                self.set_in_cache('network', 'ipv4_netmask', self.retrieve_ipv4(netmask=netmask, use_ssh=use_ssh))
+            return self.get_in_cache('network', 'ipv4_netmask')
+
+        else:
+            ipv4 = self.get_in_cache('network', 'ipv4')
+            if ipv4 is None:
+                self.set_in_cache('network', 'ipv4', self.retrieve_ipv4(netmask=netmask, use_ssh=use_ssh))
+            return self.get_in_cache('network', 'ipv4')
+
+    def retrieve_ipv4(self, netmask: bool = False, use_ssh: bool = False):
         if self.ipv4 == "dhcp":
             if self.is_running():
-                if self.has_program("ip"):
+                if self.has_program("ip", use_ssh=use_ssh):
                     if netmask:
                         ip = self.run_command(
-                            """ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 """)
+                            """ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 """,
+                            use_ssh=use_ssh)
                         return ip
 
                     ip = self.run_command(
-                        """ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 -d'/'""")
+                        """ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 -d'/'""",
+                        use_ssh=use_ssh)
                     return ip
 
-                elif self.has_program("ifconfig"):
+                elif self.has_program("ifconfig", use_ssh=use_ssh):
-                    return self.run_command(command="ifconfig eth0 | awk '/inet addr/{print substr($2,6)}'")
+                    return self.run_command(command="ifconfig eth0 | awk '/inet addr/{print substr($2,6)}'",
+                                            use_ssh=use_ssh)
 
         return self.ipv4
 
@@ -328,11 +345,12 @@ class LXC(LinuxMachine):
             self.start()
 
             # Make sure bash is installed for later use
-            if not self.has_program("bash"):
+            if not self.has_program("bash", use_ssh=False):
-                self.install_package("bash")
+                self.install_package("bash", use_ssh=False)
 
             logging.info("Setting up SSH for LXC")
             lxc_utils.run_protected_script(lxc=self, script_path="protected/scripts/install-config-ssh.sh")
+            self.pve.run_command(f"ssh-keygen -f '/root/.ssh/known_hosts' -R {self.get_ipv4(use_ssh=False)}")
 
     def run_creation(self):
         """
@@ -355,19 +373,19 @@ class LXC(LinuxMachine):
     def deploy(self):
         pass
 
-    def run_script(self, script_path):
+    def run_script(self, script_path, use_ssh: bool = True):
         """
         Run script on LXC filesystem using bash
 
         :param script_path:
         :return:
         """
-        return self.run_command(command=f"bash {script_path}")
+        return self.run_command(command=f"bash {script_path}", use_ssh=use_ssh)
 
     def run_command(self, command: str, return_status_code: bool = False,
                     exception_on_exit: bool = False,
                     exception_on_empty_stdout: bool = False,
-                    working_directory: str = None, shell: bool = False):
+                    working_directory: str = None, use_ssh: bool = True):
         """
         Run command on LXC
         :param command: command to run
@@ -384,7 +402,15 @@ class LXC(LinuxMachine):
         if working_directory:
             command = f"cd {working_directory} && {command}"
 
-        return self.pve.run_command(command=f"pct exec {self.lxc_id} -- {command}",
+        # Using pct exec works every time but is 8x slower than using ssh
-                                    return_status_code=return_status_code,
+        if use_ssh:
-                                    exception_on_exit=exception_on_exit,
+            return self.pve.run_command(
-                                    exception_on_empty_stdout=exception_on_empty_stdout)
+                command=f"ssh -o StrictHostKeyChecking=no root@{self.get_ipv4()} -- {command}",
+                return_status_code=return_status_code,
+                exception_on_exit=exception_on_exit,
+                exception_on_empty_stdout=exception_on_empty_stdout)
+        else:
+            return self.pve.run_command(command=f"pct exec {self.lxc_id} -- {command}",
+                                        return_status_code=return_status_code,
+                                        exception_on_exit=exception_on_exit,
+                                        exception_on_empty_stdout=exception_on_empty_stdout)

src/lxc/lxc_utils.py

@@ -156,9 +156,10 @@ def run_protected_script(lxc: LXC, script_path: str):
     script_path = get_path(lxc, script_path)
 
     utils.copy_local_file_to_pve(lxc.pve, script_path, f"/tmp/pdj-temp/{script_path.name}")
-    lxc.pve.copy_file_to_lxc(lxc, f"/tmp/pdj-temp/{script_path.name}", f"/tmp/pdj-temp/{script_path.name}")
+    lxc.pve.copy_file_to_lxc(lxc, f"/tmp/pdj-temp/{script_path.name}", f"/tmp/pdj-temp/{script_path.name}",
-    lxc.run_script(f"/tmp/pdj-temp/{script_path.name}")
+                             use_ssh=False)
-    lxc.delete_file(f"/tmp/pdj-temp/{script_path.name}")
+    lxc.run_script(f"/tmp/pdj-temp/{script_path.name}", use_ssh=False)
+    lxc.delete_file(f"/tmp/pdj-temp/{script_path.name}", use_ssh=False)
     lxc.pve.delete_file(f"/tmp/pdj-temp/{script_path.name}")
 
 

src/utils/machine.py

@@ -6,6 +6,9 @@ from src.utils import utils
 class LinuxMachine():
 
     def __init__(self):
+        self.cache = {}
+
+    def update(self):
         pass
 
     def get_hostname(self):
@@ -26,21 +29,38 @@ class LinuxMachine():
 
     def get_memory(self):
         """Get memory"""
-        return self.run_command("free -m | grep Mem | awk '{print $2}'")
+
+        memory = self.get_in_cache('system', 'memory')
+        if memory is None:
+            self.set_in_cache('system', 'memory', self.run_command("free -m | grep Mem | awk '{print $2}'"))
+
+        return self.get_in_cache('system', 'memory')
 
     def get_ipv4(self):
         """Get IPv4 address"""
-        if self.has_program("ip"):
+        ipv4 = self.get_in_cache('system', 'ipv4')
-            return self.run_command("""ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1  -d'/'""")
+        if ipv4 is None:
-        elif self.has_program("ifconfig"):
+            if self.has_program("ip"):
-            return self.run_command(command="ifconfig eth0 | awk '/inet addr/{print substr($2,6)}'")
+                self.set_in_cache('system', 'ipv4', self.run_command(command=
+                                                                     """ip addr | grep 'state UP' -A2 | tail -n1 | 
+                                                                     awk '{print $2}' | cut -f1  -d'/'"""))
+            elif self.has_program("ifconfig"):
+                self.set_in_cache('system', 'ipv4',
+                                  self.run_command(command="ifconfig eth0 | awk '/inet addr/{print substr($2,6)}'"))
+
+        return self.get_in_cache('system', 'ipv4')
 
     def get_ipv6(self):
         pass
 
     def get_mac(self):
         """Get MAC address"""
-        return self.run_command("""cat /sys/class/net/$(ip route show default | awk '/default/ {print $5}')/address""")
+        mac = self.get_in_cache('system', 'mac')
+        if mac is None:
+            self.set_in_cache('system', 'mac', self.run_command(
+                """cat /sys/class/net/$(ip route show default | awk '/default/ {print $5}')/address"""))
+
+        return self.get_in_cache('system', 'mac')
 
     def start(self):
         pass
@@ -51,16 +71,24 @@ class LinuxMachine():
     def reboot(self):
         pass
 
-    def has_program(self, program: str):
+    def has_program(self, program: str, use_ssh: bool = True):
         """Check if program is installed on LXC
         :param program: program executable name
 
         :return: boolean
         """
         if type(program) == str:
-            return self.run_command("which " + program, return_status_code=True) == 0
+            program_cache = self.get_in_cache('known_programs', program)
+            if program_cache is None:
+                self.set_in_cache('known_programs', program, str(self.run_command("which " + program,
+                                                                                  return_status_code=True,
+                                                                                  use_ssh=use_ssh) == 0))
+            return bool(self.get_in_cache('known_programs', program))
         elif type(program) == list:
-            return all((self.run_command("which " + p, return_status_code=True) == 0) for p in program)
+            programs = []
+            for p in program:
+                programs.append(self.has_program(p))
+            return all(programs)
 
     def has_file(self, file: str or Path):
         """Check if file exists on LXC
@@ -103,21 +131,21 @@ class LinuxMachine():
         if permission != 644:
             self.run_command(f"chmod {permission} {file}", return_status_code=True)
 
-    def create_directory(self, directory: str or Path, permission: int = 755):
+    def create_directory(self, directory: str or Path, permission: int = 755, use_ssh: bool = True):
         """Create directory"""
         if isinstance(directory, Path):
             directory = str(directory.as_posix())
 
-        self.run_command(f"mkdir -p {directory}", return_status_code=True)
+        self.run_command(f"mkdir -p {directory}", return_status_code=True, use_ssh=use_ssh)
         if permission != 755:
-            self.run_command(f"chmod -R {permission} {directory}", return_status_code=True)
+            self.run_command(f"chmod -R {permission} {directory}", return_status_code=True, use_ssh=use_ssh)
 
-    def delete_file(self, file: str or Path):
+    def delete_file(self, file: str or Path, use_ssh: bool = True):
         """Delete file"""
         if isinstance(file, Path):
             file = str(file.as_posix())
 
-        self.run_command(f"rm {file}", return_status_code=True)
+        self.run_command(f"rm {file}", return_status_code=True, use_ssh=use_ssh)
 
     def delete_directory(self, directory: str or Path):
         """Delete directory"""
@@ -219,7 +247,7 @@ class LinuxMachine():
             self.run_command(f"mkdir -p {destination} && tar -xzf {path} --directory {destination}",
                              return_status_code=True)
 
-    def install_package(self, package: str or list):
+    def install_package(self, package: str or list, use_ssh: bool = True):
         """Install a package in the Linux Machine
 
         Parameters
@@ -236,10 +264,10 @@ class LinuxMachine():
         if type(package) is list:
             for p in package:
                 self.run_command(f"{utils.get_install_package_command(self.get_os_name())} {package}",
-                                 return_status_code=True)
+                                 return_status_code=True, use_ssh=use_ssh)
         else:
             self.run_command(f"{utils.get_install_package_command(self.get_os_name())} {package}",
-                             return_status_code=True)
+                             return_status_code=True, use_ssh=use_ssh)
 
     def remove_package(self, package: str or list):
         """Remove a package in the Linux Machine
@@ -298,3 +326,22 @@ class LinuxMachine():
         else:
             self.run_command(f"sed {'-i' if case_sensitive else ''} 's/{search}/{replace}/g' {path}",
                              return_status_code=True)
+
+    def get_in_cache(self, category: str, property: str):
+        if category in self.cache:
+            if property in self.cache[category]:
+                return self.cache[category][property]
+            else:
+                return None
+
+    def set_in_cache(self, category: str, property: str = None, value: str = None):
+        if category not in self.cache:
+            self.cache[category] = {}
+
+        if property is not None and value is not None:
+            if property not in self.cache[category]:
+                self.cache[category] = {}
+
+            self.cache[category] = {property: value}
+        else:
+            raise ValueError("Key and value must be specified to save in the cache")

src/utils/proxmox.py

@@ -2,6 +2,8 @@ from __future__ import annotations
 
 import fnmatch
 import subprocess
+import threading
+import time
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -39,12 +41,26 @@ class ProxmoxHost(LinuxMachine):
         self.connection = None
         self.repo_path = path.as_posix()
 
+        threading.Thread(target=self.update).start()
+
     def __str__(self):
         return f"ProxmoxHost({self.host}, {self.user}, {self.port})"
 
     def __repr__(self):
         return f"ProxmoxHost({self.host}, {self.user}, {self.port})"
 
+    def update(self):
+        update_connection = Connection(host=self.host, user=self.user, port=self.port)
+        while True:
+            pct_output = update_connection.run("pct list", hide=True, warn=True, encoding="utf-8").stdout.split("\n")[
+                         1:]
+            for line in pct_output:
+                if line.strip():
+                    vmid, status, name = line.split()
+                    self.set_in_cache('lxc', vmid.strip(), str("running" in status.strip()))
+
+            time.sleep(1)
+
     def connect(self):
         """Connect to the Proxmox host."""
         if self.host is not None:
@@ -55,7 +71,7 @@ class ProxmoxHost(LinuxMachine):
         return self.connection
 
     def run_command(self, command: str, return_status_code: bool = False, exception_on_exit: bool = True,
-                    exception_on_empty_stdout: bool = False):
+                    exception_on_empty_stdout: bool = False, **kwargs):
         """Run a command on the Proxmox VE host
         The default behavior is as follows :
         - runs the command on the Proxmox VE host
@@ -69,7 +85,7 @@ class ProxmoxHost(LinuxMachine):
 
         Parameters
         ----------
-        command: str
+        command_result: str
             command to run
         return_status_code: bool, optional
             should it return the exit code and not the stdout, disables exception_on_exit
@@ -99,32 +115,35 @@ class ProxmoxHost(LinuxMachine):
         """
 
         # Check if host is None, if it is, run the command locally, else run it on the host via SSH
+        start_time = time.time()
+
         if self.host is None:
-            command = subprocess.run(command, shell=True, capture_output=True, encoding="utf-8")
+            command_result = subprocess.run(command, shell=True, capture_output=True, encoding="utf-8")
         elif self.connection is not None:
-            command = self.connection.run(command, hide=True, warn=True, encoding="utf-8")
+            command_result = self.connection.run(command, hide=True, warn=True, encoding="utf-8")
         else:
             raise Exception("No host or connection provided")
+        print(f"Time taken for {command}: {time.time() - start_time}")
 
         # If return code is not 0 and that exception_on_exit is True and return_status_code is False, throw an exception
-        if command.return_code != 0 and exception_on_exit and not return_status_code:
+        if command_result.return_code != 0 and exception_on_exit and not return_status_code:
-            raise Exception(f"Error while running command: \n{command.stderr}")
+            raise Exception(f"Error while running command: \n{command_result.stderr}")
 
         if return_status_code:
-            return command.return_code
+            return command_result.return_code
 
         # Check if stdout is empty, throw an exception or return empty string depending on exception_on_empty_stdout
-        if (command.stdout is None or command.stdout == "") and exception_on_empty_stdout:
+        if (command_result.stdout is None or command_result.stdout == "") and exception_on_empty_stdout:
             raise Exception(
-                f"Error, no output from command, try using the command with return_status_code instead: \n{command.stderr}")
+                f"Error, no output from command, try using the command with return_status_code instead: \n{command_result.stderr}")
-        elif command.stdout is None or command.stdout == "":
+        elif command_result.stdout is None or command_result.stdout == "":
             return ""
 
         # Decode stdout if it's bytes
-        if type(command.stdout) == bytes:
+        if type(command_result.stdout) == bytes:
-            return command.stdout.decode().rstrip()
+            return command_result.stdout.decode().rstrip()
 
-        return command.stdout.rstrip()
+        return command_result.stdout.rstrip()
 
     def get_version(self):
         """Get the version of the Proxmox host."""
@@ -148,8 +167,12 @@ class ProxmoxHost(LinuxMachine):
     def is_lxc_running(self, lxc_id):
         """Check if the given LXC is running on the Proxmox host."""
 
-        pct_status_output = self.run_command(f"pct status {lxc_id}", exception_on_exit=False)
+        lxc = self.get_in_cache('lxc', lxc_id)
-        return "running" in pct_status_output
+        if lxc is None:
+            self.set_in_cache('lxc', lxc_id,
+                              str('running' in self.run_command(f"pct status {lxc_id}", exception_on_exit=False)))
+
+        return self.get_in_cache('lxc', lxc_id)
 
     def start_lxc(self, lxc_id):
         """Start the given LXC on the Proxmox host."""
@@ -208,14 +231,14 @@ class ProxmoxHost(LinuxMachine):
 
         self.run_command(f"qm reboot {vm_id}")
 
-    def copy_file_to_lxc(self, lxc: LXC, source: str or Path, destination: str or Path):
+    def copy_file_to_lxc(self, lxc: LXC, source: str or Path, destination: str or Path, use_ssh: bool = True):
         """Copy the given file to the given LXC."""
         if isinstance(source, Path):
             source = str(source.as_posix())
         if isinstance(destination, str):
             destination = Path(destination)
 
-        lxc.create_directory(destination.parent.as_posix())
+        lxc.create_directory(destination.parent.as_posix(), use_ssh=use_ssh)
         self.run_command(f"pct push {lxc.get_id()} {source} {destination.as_posix()}")
 
     def copy_folder_to_lxc(self, lxc: LXC, source: str or Path, destination: str or Path):
@@ -225,7 +248,7 @@ class ProxmoxHost(LinuxMachine):
         if isinstance(destination, Path):
             destination = str(destination.as_posix())
 
-        self.run_command(f"scp -B -r {source} {lxc.get_ssh_string()}:{destination}")
+        self.run_command(f"scp -o StrictHostKeyChecking=no -B -r {source} {lxc.get_ssh_string()}:{destination}")
 
     def list_dir(self, directory: str or Path, glob_filter: str = '*'):
         """List the given directory."""