mathieu/ProxmoxDeploy
mathieu/ProxmoxDeploy
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

started creations conditions and steps checking/running
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source
This commit is contained in:
Mathieu Broillet 2023-06-11 21:06:22 +02:00
parent 6bbec3c95f
commit be285606e2
Signed by: mathieu
GPG Key ID: A08E484FE95074C1
9 changed files with 335 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.idea/ProxmoxDeploy.iml
View File

@ -4,7 +4,7 @@
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/venv" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="jdk" jdkName="Python 3.11 (ProxmoxDeploy)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

144
.idea/workspace.xml
View File

@ -1,16 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="AutoImportSettings">
<option name="autoReloadType" value="SELECTIVE" />
</component>
<component name="ChangeListManager">
<list default="true" id="ef90a940-975e-45ac-b0cb-e18c5b09ff29" name="Changes" comment="base">
<change afterPath="$PROJECT_DIR$/README.md" afterDir="false" />
<change afterPath="$PROJECT_DIR$/requirements.txt" afterDir="false" />
<change afterPath="$PROJECT_DIR$/run.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/src/get_path_file.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/src/main.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/src/utils/__init__.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/src/utils/lxc_utils.py" afterDir="false" />
<list default="true" id="ef90a940-975e-45ac-b0cb-e18c5b09ff29" name="Changes" comment="update readme">
<change afterPath="$PROJECT_DIR$/resources/scripts/install-docker.sh" afterDir="false" />
<change afterPath="$PROJECT_DIR$/src/utils/creation_utils.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/ProxmoxDeploy.iml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/ProxmoxDeploy.iml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/main.py" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/resources/config.json" beforeDir="false" afterPath="$PROJECT_DIR$/resources/config.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/resources/lxc/0000/config.json" beforeDir="false" afterPath="$PROJECT_DIR$/resources/lxc/100/config.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/run.py" beforeDir="false" afterPath="$PROJECT_DIR$/run.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/src/main.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/utils/lxc_utils.py" beforeDir="false" afterPath="$PROJECT_DIR$/src/utils/lxc_utils.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/utils/proxmox_utils.py" beforeDir="false" afterPath="$PROJECT_DIR$/src/utils/proxmox_utils.py" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
@ -43,7 +47,13 @@
&quot;RunOnceActivity.OpenProjectViewOnStart&quot;: &quot;true&quot;,
&quot;RunOnceActivity.ShowReadmeOnStart&quot;: &quot;true&quot;,
&quot;WebServerToolWindowFactoryState&quot;: &quot;false&quot;,
&quot;settings.editor.selected.configurable&quot;: &quot;preferences.pluginManager&quot;
&quot;last_opened_file_path&quot;: &quot;/home/mathieu/Documents/Local/Developpement/PycharmProjects/ProxmoxDeploy&quot;,
&quot;node.js.detected.package.eslint&quot;: &quot;true&quot;,
&quot;node.js.detected.package.tslint&quot;: &quot;true&quot;,
&quot;node.js.selected.package.eslint&quot;: &quot;(autodetect)&quot;,
&quot;node.js.selected.package.tslint&quot;: &quot;(autodetect)&quot;,
&quot;settings.editor.selected.configurable&quot;: &quot;vcs.Git&quot;,
&quot;vue.rearranger.settings.migration&quot;: &quot;true&quot;
}
}</component>
<component name="RecentsManager">
@ -51,29 +61,7 @@
<recent name="C:\Users\lmbbrm3\PycharmProjects\ProxmoxDeploy\resources\lxc" />
</key>
</component>
<component name="RunManager" selected="Python.run">
<configuration name="main" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true">
<module name="ProxmoxDeploy" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/main.py" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<option name="MODULE_MODE" value="false" />
<option name="REDIRECT_INPUT" value="false" />
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
<component name="RunManager">
<configuration name="run" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
<module name="ProxmoxDeploy" />
<option name="INTERPRETER_OPTIONS" value="" />
@ -112,6 +100,9 @@
<updated>1686293135999</updated>
<workItem from="1686293144782" duration="176000" />
<workItem from="1686293341823" duration="6751000" />
<workItem from="1686342148216" duration="3872000" />
<workItem from="1686406521171" duration="618000" />
<workItem from="1686498465624" duration="11804000" />
</task>
<task id="LOCAL-00001" summary="base">
<created>1686293500295</created>
@ -120,7 +111,84 @@
<option name="project" value="LOCAL" />
<updated>1686293500295</updated>
</task>
<option name="localTasksCounter" value="2" />
<task id="LOCAL-00002" summary="update drone">
<created>1686342404046</created>
<option name="number" value="00002" />
<option name="presentableId" value="LOCAL-00002" />
<option name="project" value="LOCAL" />
<updated>1686342404046</updated>
</task>
<task id="LOCAL-00003" summary="update drone">
<created>1686342511585</created>
<option name="number" value="00003" />
<option name="presentableId" value="LOCAL-00003" />
<option name="project" value="LOCAL" />
<updated>1686342511585</updated>
</task>
<task id="LOCAL-00004" summary="update drone">
<created>1686342681849</created>
<option name="number" value="00004" />
<option name="presentableId" value="LOCAL-00004" />
<option name="project" value="LOCAL" />
<updated>1686342681849</updated>
</task>
<task id="LOCAL-00005" summary="update drone">
<created>1686342794207</created>
<option name="number" value="00005" />
<option name="presentableId" value="LOCAL-00005" />
<option name="project" value="LOCAL" />
<updated>1686342794207</updated>
</task>
<task id="LOCAL-00006" summary="update drone">
<created>1686342966430</created>
<option name="number" value="00006" />
<option name="presentableId" value="LOCAL-00006" />
<option name="project" value="LOCAL" />
<updated>1686342966430</updated>
</task>
<task id="LOCAL-00007" summary="update drone">
<created>1686344421245</created>
<option name="number" value="00007" />
<option name="presentableId" value="LOCAL-00007" />
<option name="project" value="LOCAL" />
<updated>1686344421245</updated>
</task>
<task id="LOCAL-00008" summary="update drone">
<created>1686345032503</created>
<option name="number" value="00008" />
<option name="presentableId" value="LOCAL-00008" />
<option name="project" value="LOCAL" />
<updated>1686345032503</updated>
</task>
<task id="LOCAL-00009" summary="update drone to debian">
<created>1686346024833</created>
<option name="number" value="00009" />
<option name="presentableId" value="LOCAL-00009" />
<option name="project" value="LOCAL" />
<updated>1686346024833</updated>
</task>
<task id="LOCAL-00010" summary="update drone to debian">
<created>1686406538341</created>
<option name="number" value="00010" />
<option name="presentableId" value="LOCAL-00010" />
<option name="project" value="LOCAL" />
<updated>1686406538341</updated>
</task>
<task id="LOCAL-00011" summary="ssh/local for commands and lxc creation">
<created>1686503708967</created>
<option name="number" value="00011" />
<option name="presentableId" value="LOCAL-00011" />
<option name="project" value="LOCAL" />
<updated>1686503708967</updated>
</task>
<task id="LOCAL-00012" summary="update readme">
<created>1686504338382</created>
<option name="number" value="00012" />
<option name="presentableId" value="LOCAL-00012" />
<option name="project" value="LOCAL" />
<updated>1686504338382</updated>
</task>
<option name="localTasksCounter" value="13" />
<servers />
</component>
<component name="TypeScriptGeneratedFilesManager">
@ -139,10 +207,14 @@
</component>
<component name="VcsManagerConfiguration">
<MESSAGE value="base" />
<option name="LAST_COMMIT_MESSAGE" value="base" />
<MESSAGE value="update drone" />
<MESSAGE value="update drone to debian" />
<MESSAGE value="ssh/local for commands and lxc creation" />
<MESSAGE value="update readme" />
<option name="LAST_COMMIT_MESSAGE" value="update readme" />
</component>
<component name="com.intellij.coverage.CoverageDataManagerImpl">
<SUITE FILE_PATH="coverage/ProxmoxDeploy$run.coverage" NAME="run Coverage Results" MODIFIED="1686510225291" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
<SUITE FILE_PATH="coverage/ProxmoxDeploy$main.coverage" NAME="main Coverage Results" MODIFIED="1686295586466" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
<SUITE FILE_PATH="coverage/ProxmoxDeploy$run.coverage" NAME="run Coverage Results" MODIFIED="1686304048372" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
</component>
</project>

2
resources/config.json
View File

@ -1,6 +1,6 @@
{
"pve":{
"host": "192.168.10.99",
"host": "192.168.11.99",
"user": "root",
"port": 22,
"local": false

31
resources/lxc/0000/config.json
View File

@ -1,31 +0,0 @@
{
"lxc_hostname": "test",
"os": {
"name": "alpine",
"release": "3.17"
},
"resources": {
"cpu": "1",
"memory": "512",
"swap": "512",
"disk": "10",
"storage": "local-zfs"
},
"network": {
"bridge": "vmbr0",
"ipv4": "dhcp",
"ipv6": "auto",
"mac": "00:00:00:00:00:00",
"gateway4": "",
"gateway6": "",
"vlan": ""
},
"options": {
"privileged": "false",
"start_on_boot": "false",
"password": "qwertz1234",
"ssh": false
},
"creation": {},
"deploy": []
}

50
resources/lxc/100/config.json Normal file
View File

@ -0,0 +1,50 @@
{
"lxc_hostname": "traefik",
"os": {
"name": "alpine",
"release": "3.17"
},
"resources": {
"cpu": "2",
"memory": "1024",
"swap": "256",
"disk": "8",
"storage": "local-lvm"
},
"network": {
"bridge": "vmbr0",
"ipv4": "dhcp",
"ipv6": "auto",
"mac": "92:A6:71:77:8E:D8",
"gateway4": "",
"gateway6": "",
"vlan": ""
},
"options": {
"privileged": "false",
"start_on_boot": "false",
"startup_order": 2,
"password": "qwertz1234",
"ssh": false,
"tags": "2-proxy+auth"
},
"creation": {
"conditions": [
{
"type": "program",
"program": "docker"
},
{
"type": "folder",
"path": "/var/data/traefik"
}
],
"steps":[
{
"type": "script",
"path": "/global/install-docker.sh"
}
]
},
"deploy": []
}

14
src/main.py
View File

@ -8,25 +8,21 @@ from src.utils.lxc_utils import load_lxc, get_all_lxcs
def run():
# Read all files in the resources directory
resources = os.listdir(project_path / "resources")
logging.info(f"Resources found: {resources}")
# Go through each LXC file
for resource in resources:
if resource == "lxc":
logging.info("LXC folder found")
# Read all files in the LXC directory
lxc_folders = os.listdir(project_path / "resources" / "lxc")
for lxc_folder in lxc_folders:
lxc_file = os.path.join(project_path / "resources" / "lxc", lxc_folder, "config.json")
logging.info(f"Reading LXC ID {lxc_folder}")
# Open the file
with open(lxc_file, "r") as file:
# Load the LXC
load_lxc(file.read(), lxc_id=lxc_folder)
print(get_all_lxcs())
# print(get_all_lxcs()[0].get_tteck_env_variables())
print(get_all_lxcs()[0].get_pct_command())
for lxc in get_all_lxcs():
logging.info(f"Loading LXC {lxc.lxc_id}")
lxc.create()
lxc.start()
lxc.check_creation_conditions()

57
src/utils/creation_utils.py Normal file
View File

@ -0,0 +1,57 @@
import logging
import os
from pathlib import Path
def are_all_conditions_met(lxc):
"""
Check conditions for running the creations steps for an LXC
The conditions are for checking if the LXC has already been configured or not
If all conditions are met, the deploy/updates steps are run
Otherwise, we run the creation steps
:param lxc_id: lxc id
:return: are conditions met
"""
creation = lxc.get_creation()
conditions = creation["conditions"]
result = []
for condition in conditions:
if condition["type"] == "file":
result.append(Path(condition["path"]).is_file())
elif condition["type"] == "folder":
result.append(Path(condition["path"]).is_dir())
elif condition["type"] == "program":
result.append(lxc.run_command("which " + condition["program"], only_code=True) == 0)
else:
raise Exception(f"Unknown condition type {condition['type']}")
if all(result):
logging.info(f"All creations conditions met for LXC {lxc.lxc_id}, running deploy steps...")
else:
logging.info(f"Not all creations conditions met for LXC {lxc.lxc_id}, running creation steps...")
return all(result)
def run_creations_steps(lxc):
"""
Run creation steps for an LXC
:param lxc: lxc
:return: None
"""
creation = lxc.get_creation()
creation_steps = creation["creation_steps"]
for step in creation_steps:
if step["type"] == "command":
lxc.run_command(step["command"])
elif step["type"] == "script":
lxc.run_script(step["script"])
else:
raise Exception(f"Unknown creation step type {step['type']}")

103
src/utils/lxc_utils.py
View File

@ -1,7 +1,7 @@
import json
import logging
from src.utils import proxmox_utils
from src.utils import proxmox_utils, creation_utils
lxcs = []
@ -144,15 +144,18 @@ class LXC:
:return: os template
"""
wanted_template = proxmox_utils.run_command_on_pve(
f"pveam available --section system | grep {self.os_name}-{self.os_release} | awk '{{print \\$2}}'")
# TODO: might have to run "pveam update" before running this command on fresh install of PVE
if wanted_template == "":
logging.warning(f"Template {self.os_name}-{self.os_release} not found, downloading it...")
proxmox_utils.run_command_on_pve(f"pveam download local {wanted_template}")
return None
template_name = proxmox_utils.run_command_on_pve(
f"pveam available --section system | awk /'{self.os_name}-{self.os_release}/' | awk '{{print \\$2}}'")
return f"local:vztmpl/{wanted_template}"
is_template_downloaded = proxmox_utils.run_command_on_pve(f"pveam list local | awk /'{template_name}/'")
if is_template_downloaded == "":
logging.info(f"Template {template_name} not found, downloading it...")
proxmox_utils.run_command_on_pve(f"pveam download local {template_name}", True)
return f"local:vztmpl/{template_name}"
def get_resources(self):
"""
@ -294,6 +297,20 @@ class LXC:
"""
return self.deploy
def get_creation(self):
"""
Get creation
:return: creation
"""
return self.creation
def is_running(self):
"""
Is running
:return: is lxc running? (boolean)
"""
return proxmox_utils.run_command_on_pve(f"pct list | awk '/running/ && /{self.lxc_id}/'") != ""
def create(self):
"""
Create LXC
@ -302,33 +319,71 @@ class LXC:
"""
if proxmox_utils.does_lxc_exist(self.lxc_id):
logging.info(f"LXC {self.lxc_id} already exists, skipping creation")
proxmox_utils.run_command_on_pve(self.get_pct_command(create=False), True)
return
else:
logging.info(f"Creating LXC {self.lxc_id}")
if self.creation['type'] == "tteck":
proxmox_utils.execute_tteck_script(self.creation['script'], self.get_tteck_env_variables())
proxmox_utils.run_command_on_pve(self.get_pct_command(create=True), True)
def start(self):
"""
Start LXC
"""
if self.is_running():
logging.info(f"LXC {self.lxc_id} already running, skipping start")
return
else:
logging.info(f"Starting LXC {self.lxc_id}")
proxmox_utils.run_command_on_pve(f"pct start {self.lxc_id}", True)
def run_command(self, command, warn_exit_status=False, only_code=False):
"""
Run command on LXC
:param command: command to run
:return: command output
"""
logging.debug(f"Running command {command} on LXC {self.lxc_id}")
if only_code:
return proxmox_utils.run_command_on_pve(f"pct exec {self.lxc_id} -- {command}", warn_exit_status, only_code)
return proxmox_utils.run_command_on_pve(f"pct exec {self.lxc_id} -- {command}", warn_exit_status)
def deploy(self):
pass
def get_pct_command(self):
def check_creation_conditions(self):
return creation_utils.are_all_conditions_met(self)
def get_pct_command(self, create=True):
"""
Get pct command to create LXC
Get pct command to create/edit LXC
:return: pct command
"""
pct_command = f"pct create {self.lxc_id} {self.get_os_template()} " \
f"--hostname {self.lxc_hostname} " \
f"--cores {self.cpu} " \
f"--memory {self.memory} " \
f"--swap {self.memory} " \
f"--net0 name=eth0,bridge={self.bridge},ip={self.ipv4},hwaddr={self.mac} " \
f"--onboot {int(self.start_on_boot == 0)} " \
f"--ostype {self.os_name} " \
f"--password {self.password} " \
f"--storage {self.storage} " \
f"--rootfs volume={self.storage}:{self.disk},size={self.disk} " \
f"--unprivileged {not self.privileged} "
if create:
pct_command = f"pct create {self.lxc_id} {self.get_os_template()} " \
f"--hostname {self.lxc_hostname} " \
f"--cores {self.cpu} " \
f"--memory {self.memory} " \
f"--swap {self.memory} " \
f"--net0 name=eth0,bridge={self.bridge},ip={self.ipv4},hwaddr={self.mac},type=veth " \
f"--onboot {int(self.start_on_boot == 0)} " \
f"--ostype {self.os_name} " \
f"--password {self.password} " \
f"--storage {self.storage} " \
f"--unprivileged {not self.privileged} " \
f"--rootfs volume={self.storage}:{self.disk},size={self.disk} " \
f"--unprivileged {not self.privileged}"
else:
pct_command = f"pct set {self.lxc_id} " \
f"--hostname {self.lxc_hostname} " \
f"--cores {self.cpu} " \
f"--memory {self.memory} " \
f"--swap {self.memory} " \
f"--net0 name=eth0,bridge={self.bridge},ip={self.ipv4},hwaddr={self.mac},type=veth " \
f"--onboot {int(self.start_on_boot == 0)} " \
f"--ostype {self.os_name} "
# TODO: add gateway4
# f"ip6={self.ipv6},gw6={self.gateway6},trunks={self.vlan} " \ # TODO

45
src/utils/proxmox_utils.py
View File

@ -11,7 +11,7 @@ def get_pve_version():
Get PVE version
:return: pve version
"""
return run_command_on_pve("pveversion")
return run_command_on_pve("pveversion", True)
def get_pve_hostname():
@ -19,7 +19,7 @@ def get_pve_hostname():
Get PVE hostname
:return: pve hostname
"""
return run_command_on_pve("hostname")
return run_command_on_pve("hostname", True)
def does_lxc_exist(lxc_id):
@ -30,7 +30,7 @@ def does_lxc_exist(lxc_id):
:return: does lxc exists
"""
# TODO: only check in VMID column
return lxc_id in run_command_on_pve(f"pct list {lxc_id}")
return lxc_id in run_command_on_pve(f"pct list | awk '/{lxc_id}/'")
def does_qemu_vm_exist(vm_id):
@ -55,13 +55,15 @@ def execute_tteck_script(script_url, env_variables):
env_variables = " ".join(env_variables)
run_command_on_pve(f"{env_variables} && bash -c \"$(wget -qLO - {script_url}\"")
run_command_on_pve(f"{env_variables} && bash -c \"$(wget -qLO - {script_url}\"", True)
def run_command_on_pve(command):
def run_command_on_pve(command, warn_exit_status=False, only_code=False):
"""
Run command on PVE
:param only_code: should we only return the exit code and not the stdout
:param warn_exit_status: should an exception be thrown if the exit status is not 0
:param command: command
:return: command
"""
@ -74,9 +76,19 @@ def run_command_on_pve(command):
# Check if PVE is local or remote
if data['pve']['local']:
# Run command and return output (not as bytes)
logging.debug(f"Running command on PVE (ssh): \n{command}")
return subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
encoding="utf-8").stdout.decode().rstrip()
logging.debug(f"Running command on PVE (locally): \n{command}")
command = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
encoding="utf-8")
if command.returncode != 0 and warn_exit_status:
logging.error(f"Error while running command on PVE: \n{command.stderr}")
raise Exception(f"Error while running command on PVE: \n{command.stderr}")
if only_code:
return command.returncode
return command.stdout.decode().rstrip()
else:
host = data['pve']['host']
@ -84,6 +96,17 @@ def run_command_on_pve(command):
port = data['pve']['port']
# Run command on PVE via SSH and return output
logging.debug(f"Running command on PVE (locally): \n{command}")
return subprocess.run(f"ssh {username}@{host} -p {port} \"{command}\"", shell=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, encoding="utf-8").stdout.rstrip()
logging.debug(f"Running command on PVE (ssh): \n{command}")
# catch errors code
command = subprocess.run(f"ssh {username}@{host} -p {port} \"{command}\"", shell=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8")
if command.returncode != 0 and warn_exit_status:
logging.error(f"Error while running command on PVE: \n{command.stderr}")
raise Exception(f"Error while running command on PVE: \n{command.stderr}")
if only_code:
return command.returncode
return command.stdout.rstrip()