qwc-open
/
pybackive
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Working prototype 

All the minimal features work now.
- describing a device
- describing a backup
- link a backup to a device
- schedule backups at day intervals
- mount the device
- put stuff on it
- umount the device

Downsides:
- executed as root, fully!
- all scripts are executed as root, dangerous!
- Bad and undocumented code.. :)
- No tests and only minimal checking (lotta TODOs)
This commit is contained in:
Marcel Otte 2020-03-19 21:16:38 +01:00
parent 3c4f9a59ac
commit d1dfb809e3
5 changed files with 163 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
backive/backive_service
View File

@ -10,6 +10,7 @@ import logging
import json
from backive.core.events import EventInterface
from backive.core.scheduler import Scheduler
from backive.core.backup import Backup
from backive.config.config import Config
@ -26,22 +27,44 @@ logging.info("Backive starting.")
class Backive:
def __init__(self):
self._scheduler = Scheduler()
self._config = Config()
self._events = None
self.initialize_scheduler()
def initialize_scheduler(self):
backups = self._config.get_backups()
for backup in backups:
self._scheduler.register_backup(backup.name, backup.get_frequency())
async def callback(self, data=None):
data_dict = json.loads(data)
uuid = data_dict.get("ID_FS_UUID", None)
logging.info("UUID: %s", uuid)
logging.debug(json.dumps(data_dict, indent=4))
if uuid and data_dict.get("ACTION") == "add":
logging.debug(json.dumps(data_dict, indent=4))
backups = await self._config.get_backups_by_device(uuid)
device = await self._config.get_uuid_device(uuid)
prefs = await self._config.get_preferences()
if backups:
logging.info("Mounting device '%s'", uuid)
mount_available = await device.mount(prefs.get("mount_root"))
if mount_available:
for backup in backups:
if await self._scheduler.should_run(backup.name):
logging.info("Running backup '%s'", backup.name)
result = await backup.run()
logging.debug("Result: %s", str(result))
await self._scheduler.register_run(backup.name)
else:
logging.info(
"Backup '%s' next run interval has not been reached.",
backup.name
)
logging.info("Unmounting device '%s'", uuid)
await device.unmount()
else:
logging.error("Device %s could not be mounted...", uuid)
def serve(self):
loop = asyncio.get_event_loop()

54
backive/config/config.py
View File

@ -1,4 +1,3 @@
import logging
import os
import pwd
import json
@ -6,9 +5,8 @@ from ruamel.yaml import YAML
import logging
import jsonschema
from backive.core.backup import Backup
from backive.core.device import Device
class RegularUserException(Exception):
pass
class Config:
__shared_state = dict()
@ -20,7 +18,7 @@ class Config:
logging.info("Loading configuration...")
self._schema = dict()
self._backups = list()
self._devices = list()
self._devices = dict()
file_path = os.path.realpath(__file__)
schema_path = os.path.join(
os.path.dirname(
@ -42,6 +40,13 @@ class Config:
if uid == 0:
config_file = "/etc/backive.yml"
else:
raise RegularUserException(
"""
It is planned to add functionality to use this service
as a regular user, but for the time being it is advised to
execute this service as root, because this feature is still
planned and needs more development time.
""")
config_file = os.path.join(
os.path.expanduser("~"),
".config",
@ -51,24 +56,39 @@ class Config:
with open(config_file, "r") as cfg:
self._config = YAML().load(cfg)
logging.debug("Found config: %s", json.dumps(self._config, indent=4))
logging.debug(
"Found config: %s\n%s",
config_file,
json.dumps(self._config, indent=4)
)
jsonschema.validate(self._config, self._schema)
except RegularUserException as e:
raise e
except Exception as e:
logging.error(e)
def get_devices(self):
from backive.core.device import Device
if self._config.get("devices", None) and not self._devices:
data = self._config.get("devices")
for device in data:
self._devices.append(
for device, values in data.items():
self._devices.update({
device:
Device.instance(
device,
data.get(device)
)
values
)
})
return self._devices
def get_backups(self):
async def get_device(self, name):
for device, value in self.get_devices().items():
if device == name:
return value
return None
def get_backups(self) -> list:
from backive.core.backup import Backup
if self._config.get("backups", None) and not self._backups:
data = self._config.get("backups")
for name in data:
@ -80,6 +100,16 @@ class Config:
)
return self._backups
async def get_uuid_device(self, uuid):
logging.debug("get device %s", uuid)
for device, value in self.get_devices().items():
logging.debug(
"device %s, config %s", device, json.dumps(value.config)
)
if value.config.get("uuid") == uuid:
return value
return None
async def get_backups_by_device(self, uuid):
name = None
if not self._config.get("devices"):
@ -98,7 +128,7 @@ class Config:
backups.append(backup)
return backups
def get_preferences(self):
async def get_preferences(self):
if self._config.get("preferences", None):
return self._config.get("preferences")
return {}

31
backive/core/backup.py
View File

@ -2,6 +2,7 @@ import os
import logging
from subprocess import Popen
import asyncio
from backive.config.config import Config
class Backup:
@ -19,13 +20,37 @@ class Backup:
async def run(self):
logging.debug("Running backup %s", self.name)
if self.config.get("scripts", None) is not None:
if self.config.get("script", None) is not None:
logging.debug("Executing script..")
backup_env = os.environ.copy()
backup_env["BACKIVE_FROM"] = self.config.get("from")
backup_env["BACKIVE_TO"] = self.config.get("to")
backup_env["BACKIVE_MOUNT"] = os.path.join(
(await Config().get_preferences()).get("mount_root"),
(await Config().get_device(
self.config.get("target_device")
)).config.get("mountname")
)
proc = await asyncio.create_subprocess_shell(
self.config.get("script"),
"""mkdir -p {}""".format(
os.path.join(
backup_env["BACKIVE_MOUNT"],
backup_env["BACKIVE_TO"]
)
),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
stderr=asyncio.subprocess.PIPE,
)
stdout, stderr = await proc.communicate()
logging.debug("stdout: %s", stdout)
logging.debug("stderr: %s", stderr.decode())
proc = await asyncio.create_subprocess_shell(
self.config.get("script"),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
env=backup_env
)
stdout, stderr = await proc.communicate()
logging.debug("stdout: %s", stdout.decode())
logging.debug("stderr: %s", stderr.decode())
return stdout

39
backive/core/device.py
View File

@ -1,5 +1,6 @@
import os
import logging
import asyncio
import backive.config.config as cfg
@ -9,6 +10,7 @@ class Device:
def __init__(self, name, config=None):
self.name = name
self.config = config
self._mount_dir = None
@classmethod
def instance(cls, name, config=None):
@ -22,9 +24,36 @@ class Device:
return uuids
return []
def mount(self, path):
pass
def unmount(self):
pass
async def mount(self, path):
self._mount_dir = os.path.join(path, self.config.get("mountname"))
dev_path = os.path.join(self.disks_by_uuid, self.config.get("uuid"))
logging.debug("dev: %s ;; mount: %s", dev_path, self._mount_dir)
# TODO: use mkdir as indicator for correct access rights (when backive
# is run as user!)
proc = await asyncio.create_subprocess_shell(
"""mkdir -p {mountpoint}
sudo mount -v -o users {dev_path} {mountpoint}""".format(
mountpoint=self._mount_dir,
dev_path=dev_path
),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, stderr = await proc.communicate()
logging.debug("stdout: %s", stdout)
# TODO: Also add a touch operation in the target mount if the correct
# access rights are given! (when backive is run as user)
return True # on success, False on failure
async def unmount(self):
if not self._mount_dir:
return
proc = await asyncio.create_subprocess_shell(
"""sync
sudo umount -v %s
""" % self._mount_dir,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, stderr = await proc.communicate()
logging.debug("stdout: %s", stdout)

32
backive/core/scheduler.py
View File

@ -26,6 +26,8 @@ class Scheduler():
if not os.path.exists(os.path.dirname(self._data_file)):
os.makedirs(os.path.dirname(self._data_file))
self.save()
else:
self.load()
def save(self):
with open(self._data_file, "w") as stream:
@ -36,6 +38,7 @@ class Scheduler():
self.__data = json.load(stream)
def register_backup(self, name, frequency):
logging.debug("Registering %s, freq %s in Scheduler", name, frequency)
backups = self.__data.get("backups", dict())
if not backups:
self.__data["backups"] = backups
@ -46,7 +49,8 @@ class Scheduler():
backups[name] = frequency
self.save()
def register_run(self, name):
async def register_run(self, name):
logging.info("Registered run of backup '%s'", name)
runs = self.__data.get("runs", dict())
if not runs:
self.__data["runs"] = runs
@ -56,22 +60,42 @@ class Scheduler():
runs[name].append(datetime.now().isoformat())
self.save()
def should_run(self, name):
async def should_run(self, name):
logging.debug("Checking if %s may run...", name)
runs = self.__data.get("runs", dict())
if name not in runs:
logging.debug("Not registered, so YES")
return True
frequency = 0
if name in runs:
logging.debug("Registered, checking...")
backups = self.__data.get("backups", dict())
if name in backups:
logging.debug("retrieving frequency")
frequency = backups[name]
last_ts = runs[name][-1]
now = datetime.now()
last = datetime.fromisoformat(last_ts)
diff = now - last
days = diff.days
if days > frequency and days >= 1:
if days >= frequency and days >= 1 or frequency == 0:
logging.debug("YES, should run.")
return True
logging.debug("No should not run.")
return False
def get_overtimed(self):
return list()
overtime = list()
now = datetime.now()
runs = self.__data.get("runs", dict())
for bkp_name, freq in self.__data.get("backups").items():
if bkp_name not in runs.keys():
overtime.append(bkp_name)
else:
last_ts = runs[bkp_name][-1]
last = datetime.fromisoformat(last_ts)
diff = now - last
days = diff.days
if days > freq and freq != 0:
overtime.append(bkp_name)
return overtime