qwc-open
/
pybackive
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Stream asyncio process output instead of retrieving as a whole

This commit is contained in:
Marcel Otte 2021-05-25 21:32:32 +02:00
parent 8d1be779f5
commit 2d765d6bf2
2 changed files with 54 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backive/backive_service
View File

@ -54,7 +54,7 @@ class Backive:
if await self._scheduler.should_run(backup.name):
logging.info("Running backup '%s'", backup.name)
result = await backup.run()
logging.debug("Result: %s", result.decode())
logging.debug("Result: %s", result)
await self._scheduler.register_run(backup.name)
else:
logging.info(

69
backive/core/backup.py
View File

@ -18,6 +18,28 @@ class Backup:
def get_frequency(self):
return self.config.get("frequency", None)
async def _read_stream(self, stream, cb):
while True:
line = await stream.readline()
if line:
cb(line)
else:
break
async def stream_subprocess(self, cmd, outcb, errcb):
proc = await asyncio.create_subprocess_shell(
cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
await asyncio.wait([
self._read_stream(proc.stdout, outcb),
self._read_stream(proc.stderr, errcb)
])
return await proc.wait()
async def run(self):
logging.debug("Running backup %s", self.name)
if self.config.get("script", None) is not None:
@ -31,31 +53,40 @@ class Backup:
self.config.get("target_device")
)).config.get("mountname")
)
proc = await asyncio.create_subprocess_shell(
"""mkdir -p {}""".format(
os.path.join(
backup_env["BACKIVE_TARGET_DIR"] = os.path.join(
backup_env["BACKIVE_MOUNT"],
backup_env["BACKIVE_TO"]
)
),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
proc = await self.stream_subprocess(
"""mkdir -p {}""".format( backup_env["BACKIVE_TARGET_DIR"]),
lambda x: logging.debug("STDOUT: %s", x),
lambda x: logging.debug("STDERR: %s", x),
)
stdout, stderr = await proc.communicate()
logging.debug("stdout: %s", stdout.decode())
logging.debug("stderr: %s", stderr.decode())
# proc = await asyncio.create_subprocess_shell(
# """mkdir -p {}""".format( backup_env["BACKIVE_TARGET_DIR"]
# ),
# stdout=asyncio.subprocess.PIPE,
# stderr=asyncio.subprocess.PIPE,
# )
# stdout, stderr = await proc.communicate()
# logging.debug("stdout: %s", stdout.decode())
# logging.debug("stderr: %s", stderr.decode())
user = self.config.get("user")
proc = await asyncio.create_subprocess_shell(
# proc = await asyncio.create_subprocess_shell(
proc = await self.stream_subprocess(
# "set -x; chown -R {} ${{BACKIVE_MOUNT}}/${{BACKIVE_TO}};".format(user) +
# "sudo -E -u {} sh -c '".format(user) +
self.config.get("script"),
# "'",
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
shell=True,
env=backup_env
lambda x: logging.debug("STDOUT: %s", x),
lambda x: logging.debug("STDERR: %s", x),
)
stdout, stderr = await proc.communicate()
logging.debug("stdout: %s", stdout.decode())
logging.debug("stderr: %s", stderr.decode())
return stdout
# "'",
# stdout=asyncio.subprocess.PIPE,
# stderr=asyncio.subprocess.PIPE,
# shell=True,
# env=backup_env
# )
# stdout, stderr = await proc.communicate()
# logging.debug("stdout: %s", stdout.decode())
# logging.debug("stderr: %s", stderr.decode())
return "done"