Latest WIP

- UnixSocket working
- Converted complicated Singleton to Borg pattern
- Corrected some implementations
- Introduced pipenv
This commit is contained in:
Marcel Otte 2020-01-05 13:17:46 +01:00
parent 88c83cc1d8
commit b12d85e7f1
10 changed files with 431 additions and 69 deletions

21
Pipfile Normal file
View File

@ -0,0 +1,21 @@
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
setuptools = "*"
pytest = "*"
pytest_runner = "*"
pytest_cov = "*"
pytest_pylint = "*"
pytest-pep8 = "*"
pylint = "*"
coverage = "*"
[packages]
jsonschema = "*"
"ruamel.yaml" = "*"
[requires]
python_version = "3.8"

279
Pipfile.lock generated Normal file
View File

@ -0,0 +1,279 @@
{
"_meta": {
"hash": {
"sha256": "4be395776086a6418e10dd27cb606a4dd9d5dcc15f3b875692880a7873c647db"
},
"pipfile-spec": 6,
"requires": {
"python_version": "3.8"
},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {
"attrs": {
"hashes": [
"sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
"sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
],
"version": "==19.3.0"
},
"jsonschema": {
"hashes": [
"sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163",
"sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"
],
"index": "pypi",
"version": "==3.2.0"
},
"pyrsistent": {
"hashes": [
"sha256:f3b280d030afb652f79d67c5586157c5c1355c9a58dfc7940566e28d28f3df1b"
],
"version": "==0.15.6"
},
"ruamel.yaml": {
"hashes": [
"sha256:0db639b1b2742dae666c6fc009b8d1931ef15c9276ef31c0673cc6dcf766cf40",
"sha256:412a6f5cfdc0525dee6a27c08f5415c7fd832a7afcb7a0ed7319628aed23d408"
],
"index": "pypi",
"version": "==0.16.5"
},
"six": {
"hashes": [
"sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd",
"sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"
],
"version": "==1.13.0"
}
},
"develop": {
"apipkg": {
"hashes": [
"sha256:37228cda29411948b422fae072f57e31d3396d2ee1c9783775980ee9c9990af6",
"sha256:58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c"
],
"version": "==1.5"
},
"astroid": {
"hashes": [
"sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a",
"sha256:840947ebfa8b58f318d42301cf8c0a20fd794a33b61cc4638e28e9e61ba32f42"
],
"version": "==2.3.3"
},
"attrs": {
"hashes": [
"sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
"sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
],
"version": "==19.3.0"
},
"coverage": {
"hashes": [
"sha256:0101888bd1592a20ccadae081ba10e8b204d20235d18d05c6f7d5e904a38fc10",
"sha256:04b961862334687549eb91cd5178a6fbe977ad365bddc7c60f2227f2f9880cf4",
"sha256:1ca43dbd739c0fc30b0a3637a003a0d2c7edc1dd618359d58cc1e211742f8bd1",
"sha256:1cbb88b34187bdb841f2599770b7e6ff8e259dc3bb64fc7893acf44998acf5f8",
"sha256:232f0b52a5b978288f0bbc282a6c03fe48cd19a04202df44309919c142b3bb9c",
"sha256:24bcfa86fd9ce86b73a8368383c39d919c497a06eebb888b6f0c12f13e920b1a",
"sha256:25b8f60b5c7da71e64c18888f3067d5b6f1334b9681876b2fb41eea26de881ae",
"sha256:2714160a63da18aed9340c70ed514973971ee7e665e6b336917ff4cca81a25b1",
"sha256:2ca2cd5264e84b2cafc73f0045437f70c6378c0d7dbcddc9ee3fe192c1e29e5d",
"sha256:2cc707fc9aad2592fc686d63ef72dc0031fc98b6fb921d2f5395d9ab84fbc3ef",
"sha256:348630edea485f4228233c2f310a598abf8afa5f8c716c02a9698089687b6085",
"sha256:40fbfd6b044c9db13aeec1daf5887d322c710d811f944011757526ef6e323fd9",
"sha256:46c9c6a1d1190c0b75ec7c0f339088309952b82ae8d67a79ff1319eb4e749b96",
"sha256:591506e088901bdc25620c37aec885e82cc896528f28c57e113751e3471fc314",
"sha256:5ac71bba1e07eab403b082c4428f868c1c9e26a21041436b4905c4c3d4e49b08",
"sha256:5f622f19abda4e934938e24f1d67599249abc201844933a6f01aaa8663094489",
"sha256:65bead1ac8c8930cf92a1ccaedcce19a57298547d5d1db5c9d4d068a0675c38b",
"sha256:7362a7f829feda10c7265b553455de596b83d1623b3d436b6d3c51c688c57bf6",
"sha256:7f2675750c50151f806070ec11258edf4c328340916c53bac0adbc465abd6b1e",
"sha256:960d7f42277391e8b1c0b0ae427a214e1b31a1278de6b73f8807b20c2e913bba",
"sha256:a50b0888d8a021a3342d36a6086501e30de7d840ab68fca44913e97d14487dc1",
"sha256:b7dbc5e8c39ea3ad3db22715f1b5401cd698a621218680c6daf42c2f9d36e205",
"sha256:bb3d29df5d07d5399d58a394d0ef50adf303ab4fbf66dfd25b9ef258effcb692",
"sha256:c0fff2733f7c2950f58a4fd09b5db257b00c6fec57bf3f68c5bae004d804b407",
"sha256:c792d3707a86c01c02607ae74364854220fb3e82735f631cd0a345dea6b4cee5",
"sha256:c90bda74e16bcd03861b09b1d37c0a4158feda5d5a036bb2d6e58de6ff65793e",
"sha256:cfce79ce41cc1a1dc7fc85bb41eeeb32d34a4cf39a645c717c0550287e30ff06",
"sha256:eeafb646f374988c22c8e6da5ab9fb81367ecfe81c70c292623373d2a021b1a1",
"sha256:f425f50a6dd807cb9043d15a4fcfba3b5874a54d9587ccbb748899f70dc18c47",
"sha256:fcd4459fe35a400b8f416bc57906862693c9f88b66dc925e7f2a933e77f6b18b",
"sha256:ff3936dd5feaefb4f91c8c1f50a06c588b5dc69fba4f7d9c79a6617ad80bb7df"
],
"index": "pypi",
"version": "==5.0.1"
},
"execnet": {
"hashes": [
"sha256:cacb9df31c9680ec5f95553976c4da484d407e85e41c83cb812aa014f0eddc50",
"sha256:d4efd397930c46415f62f8a31388d6be4f27a91d7550eb79bc64a756e0056547"
],
"version": "==1.7.1"
},
"isort": {
"hashes": [
"sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1",
"sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd"
],
"version": "==4.3.21"
},
"lazy-object-proxy": {
"hashes": [
"sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d",
"sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449",
"sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08",
"sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a",
"sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50",
"sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd",
"sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239",
"sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb",
"sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea",
"sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e",
"sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156",
"sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142",
"sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442",
"sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62",
"sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db",
"sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531",
"sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383",
"sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a",
"sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357",
"sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4",
"sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0"
],
"version": "==1.4.3"
},
"mccabe": {
"hashes": [
"sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
"sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
],
"version": "==0.6.1"
},
"more-itertools": {
"hashes": [
"sha256:b84b238cce0d9adad5ed87e745778d20a3f8487d0f0cb8b8a586816c7496458d",
"sha256:c833ef592a0324bcc6a60e48440da07645063c453880c9477ceb22490aec1564"
],
"version": "==8.0.2"
},
"packaging": {
"hashes": [
"sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47",
"sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108"
],
"version": "==19.2"
},
"pep8": {
"hashes": [
"sha256:b22cfae5db09833bb9bd7c8463b53e1a9c9b39f12e304a8d0bba729c501827ee",
"sha256:fe249b52e20498e59e0b5c5256aa52ee99fc295b26ec9eaa85776ffdb9fe6374"
],
"version": "==1.7.1"
},
"pluggy": {
"hashes": [
"sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
"sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
],
"version": "==0.13.1"
},
"py": {
"hashes": [
"sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa",
"sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0"
],
"version": "==1.8.1"
},
"pylint": {
"hashes": [
"sha256:3db5468ad013380e987410a8d6956226963aed94ecb5f9d3a28acca6d9ac36cd",
"sha256:886e6afc935ea2590b462664b161ca9a5e40168ea99e5300935f6591ad467df4"
],
"index": "pypi",
"version": "==2.4.4"
},
"pyparsing": {
"hashes": [
"sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f",
"sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec"
],
"version": "==2.4.6"
},
"pytest": {
"hashes": [
"sha256:6b571215b5a790f9b41f19f3531c53a45cf6bb8ef2988bc1ff9afb38270b25fa",
"sha256:e41d489ff43948babd0fad7ad5e49b8735d5d55e26628a58673c39ff61d95de4"
],
"index": "pypi",
"version": "==5.3.2"
},
"pytest-cache": {
"hashes": [
"sha256:be7468edd4d3d83f1e844959fd6e3fd28e77a481440a7118d430130ea31b07a9"
],
"version": "==1.0"
},
"pytest-cov": {
"hashes": [
"sha256:cc6742d8bac45070217169f5f72ceee1e0e55b0221f54bcf24845972d3a47f2b",
"sha256:cdbdef4f870408ebdbfeb44e63e07eb18bb4619fae852f6e760645fa36172626"
],
"index": "pypi",
"version": "==2.8.1"
},
"pytest-pep8": {
"hashes": [
"sha256:032ef7e5fa3ac30f4458c73e05bb67b0f036a8a5cb418a534b3170f89f120318"
],
"index": "pypi",
"version": "==1.0.6"
},
"pytest-pylint": {
"hashes": [
"sha256:8c38ea779e540e27ec4378b0820d906006e09f4ac834defbd886abbf57c7d2ec",
"sha256:a4f5e5007f88c2095dcac799e9f7eed3d7e7a2e657596e26093814980ff5fa20",
"sha256:a574c246535308f8f6ceac10fa82f8fffffa837071f7985b22515895185700c1"
],
"index": "pypi",
"version": "==0.14.1"
},
"pytest-runner": {
"hashes": [
"sha256:5534b08b133ef9a5e2c22c7886a8f8508c95bb0b0bdc6cc13214f269c3c70d51",
"sha256:96c7e73ead7b93e388c5d614770d2bae6526efd997757d3543fe17b557a0942b"
],
"index": "pypi",
"version": "==5.2"
},
"six": {
"hashes": [
"sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd",
"sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"
],
"version": "==1.13.0"
},
"wcwidth": {
"hashes": [
"sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603",
"sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8"
],
"version": "==0.1.8"
},
"wrapt": {
"hashes": [
"sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1"
],
"version": "==1.11.2"
}
}
}

View File

@ -3,6 +3,7 @@
""" """
Service startup script. Service startup script.
""" """
import asyncio
from backive.core.events import EventInterface from backive.core.events import EventInterface
from backive.config.config import Config from backive.config.config import Config
@ -12,7 +13,13 @@ class Backive:
self._config = Config() self._config = Config()
self._events = None self._events = None
async def callback(self, data=None):
print("Callback: {}".format(str(data)))
def serve(self): def serve(self):
loop = asyncio.get_event_loop()
self._events = EventInterface(self.callback, None, loop)
loop.run_forever()
pass pass

View File

@ -9,12 +9,21 @@ from backive.core.device import Device
class Config: class Config:
__shared_state = dict
def __init__(self): def __init__(self):
self.__dict__ = self.__shared_state
self._config = dict() self._config = dict()
self._schema = dict() self._schema = dict()
self._backups = list()
self._devices = list()
file_path = os.path.realpath(__file__) file_path = os.path.realpath(__file__)
schema_path = os.path.join(file_path, "schema.yml") schema_path = os.path.join(
os.path.dirname(
file_path
),
"schema.yml"
)
with open(schema_path, "r") as stream: with open(schema_path, "r") as stream:
self._schema = YAML().load(stream) self._schema = YAML().load(stream)
@ -41,37 +50,42 @@ class Config:
logging.error(e) logging.error(e)
def get_devices(self): def get_devices(self):
devices = [] if self._config.get("devices", None) and not self._devices:
if self._config.get("devices", None):
data = self._config.get("devices") data = self._config.get("devices")
for device in data: for device in data:
devices.append( self._devices.append(
Device.instance( Device.instance(
device, device,
data.get(device) data.get(device)
) )
) )
return devices return self._devices
def get_backups(self): def get_backups(self):
backups = [] if self._config.get("backups", None) and not self._backups:
if self._config.get("backups", None):
data = self._config.get("backups") data = self._config.get("backups")
for name in data: for name in data:
backups.append( self._backups.append(
Backup.instance( Backup.instance(
name, name,
data.get(name) data.get(name)
) )
) )
return backups return self._backups
def get_device_backups(self, device): def get_backups_by_device(self, uuid):
uuid = device name = None
device_name = self._config.get("devices").get(uuid).get("name") for k, v in self._config.get("devices").items():
backups = [] if v.get("uuid") == uuid:
name = k
if name:
return self.get_device_backups(name)
return None
def get_device_backups(self, name):
backups = list()
for backup in self.get_backups(): for backup in self.get_backups():
if backup.target == uuid or backup.target == device_name: if backup.target == name:
backups.append(backup) backups.append(backup)
return backups return backups

View File

@ -1,22 +1,22 @@
### backive config schema ### backive config schema
# #
# #
definitions: definitions:
device_section: device_section:
type: object type: object
patternProperties: patternProperties:
"^[^ \t/\\]+$": "^[^ \t/\\]+$":
# The pattern can be any name...
# "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$": # "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$":
type: object type: object
properties: properties:
name: uuid:
type: string type: string
mountname: mountname:
type: string type: string
required: required:
- mountname - mountname
- uuid
additionalProperties: false additionalProperties: false
backup_section: backup_section:
type: object type: object
@ -28,38 +28,51 @@ definitions:
type: string type: string
from: from:
type: string type: string
from_remote: # from directory
type: object # from_remote:
properties: # type: object
user: # properties:
type: string # user:
password: # type: string
type: string # password:
ssh_key_path: # type: string
type: string # ssh_key_path:
# type: string
to: to:
type: string type: string
# to directory
target_device: target_device:
type: string type: string
# target device name
frequency: frequency:
type: string type: string
scripts: # in numbers? days? weeks?
type: array # or just strings like
items: # weekly, biweekly, monthly, yearly
type: object # 7d, 2w, 1m, 1y (this needs parser)
properties: #
target: # No frequency means no reminder and the backup is always started when the device is available, but not more than once a day!
enum: [ "local", "remote" ]
script: script:
type: string type: string
execute: [ "before", "after" ] # MVP just executes a script, nothing else
additionalProperties: false # scripts:
tool: # type: array
type: object # items:
# type: object
# properties:
# target:
# enum: [ "local", "remote" ]
# script:
# type: string
# execute: [ "before", "after" ]
# additionalProperties: false
# tool:
# type: object
required: required:
- from - from
- to - to
- target_device - target_device
- script
additionalProperties: false additionalProperties: false
preferences_section: preferences_section:
type: object type: object

View File

@ -6,6 +6,10 @@ class Backup:
self.name = name self.name = name
self.config = cfg self.config = cfg
@classmethod
def instance(cls, name, config=None):
return Backup(name, config)
def get_frequency(self): def get_frequency(self):
return self.config.get("frequency", None) return self.config.get("frequency", None)

View File

@ -5,10 +5,14 @@ import backive.config.config as cfg
class Device: class Device:
disks_by_uuid = "/dev/disk/by-uuid" disks_by_uuid = "/dev/disk/by-uuid"
def __init__(self, uuid, config=None): def __init__(self, name, config=None):
self.uuid = uuid self.name = name
self.config = config self.config = config
@classmethod
def instance(cls, name, config=None):
return Device(name, config)
@classmethod @classmethod
def get_list(cls): def get_list(cls):
if os.path.exists(cls.disks_by_uuid): if os.path.exists(cls.disks_by_uuid):
@ -16,7 +20,7 @@ class Device:
return uuids return uuids
return [] return []
def mount(self): def mount(self, path):
pass pass
def unmount(self): def unmount(self):

View File

@ -14,13 +14,13 @@ class EventInterface:
except OSError: except OSError:
pass pass
if not loop: if not loop:
loop = asyncio.get_running_loop() loop = asyncio.get_event_loop()
loop.create_task(asyncio.start_unix_server(self.client_connected, unix_socket_path)) loop.create_task(asyncio.start_unix_server(self.client_connected, unix_socket_path))
async def client_connected(self, reader, writer): async def client_connected(self, reader, writer):
print("client_connected") print("client_connected")
data = None data = None
data = (await reader.read()).decode('utf8') data = (await reader.read()).decode('utf8')
self.event_callback(data) await self.event_callback(data)

View File

@ -1,18 +1,14 @@
import os import os
import json import json
from datetime import datetime
class Singleton(type): class Scheduler():
_instances = {} __shared_state = dict()
def __call__(cls, *args, **kwargs): __data = dict()
if cls not in cls._instances:
cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
return cls._instances[cls]
class Scheduler(metaclass=Singleton):
def __init__(self): def __init__(self):
self._data = dict() self.__dict__ = self.__shared_state
if not self.__data:
# who are we? # who are we?
uid = os.getuid() uid = os.getuid()
if uid == 0: if uid == 0:
@ -30,17 +26,32 @@ class Scheduler(metaclass=Singleton):
def save(self): def save(self):
with open(self._data_file, "w") as stream: with open(self._data_file, "w") as stream:
json.dump(self._data, stream, indent=2) json.dump(self.__data, stream, indent=2)
def load(self): def load(self):
with open(self._data_file, "r") as stream: with open(self._data_file, "r") as stream:
self._data = json.load(stream) self.__data = json.load(stream)
def register_backup(self, name, frequency): def register_backup(self, name, frequency):
pass backups = self.__data.get("backups", dict())
if not backups:
self.__data["backups"] = backups
if (
name not in backups.keys() or
backups[name] != frequency
):
backups[name] = frequency
self.save()
def register_run(self, name): def register_run(self, name):
pass runs = self.__data.get("runs", dict())
if not runs:
self.__data["runs"] = runs
if name not in runs.keys():
runs[name] = [datetime.now().isoformat()]
else:
runs[name].append(datetime.now().isoformat())
self.save()
def should_run(self, name): def should_run(self, name):
pass pass

View File

@ -14,7 +14,10 @@ setup_info = dict(
license="BSD", license="BSD",
classifiers=[ classifiers=[
], ],
scripts=[], scripts=[
"backive/backive_service",
"backive/backive_udev"
],
packages=find_packages(), packages=find_packages(),
setup_requires=[ setup_requires=[
"setuptools>=40.4.3", "setuptools>=40.4.3",
@ -30,7 +33,13 @@ setup_info = dict(
"pytest-pep8>=1.0.6", "pytest-pep8>=1.0.6",
"pylint>=2.1.1", "pylint>=2.1.1",
"coverage>=4.5.1", "coverage>=4.5.1",
],
package_data={
"backive.config": [
"schema.yml"
] ]
},
include_package_data=True
) )
setup(**setup_info) setup(**setup_info)