qwc-open
/
pybackive
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Stream asyncio process output instead of retrieving as a whole

This commit is contained in:
Marcel Otte 2021-05-25 21:32:32 +02:00
parent 8d1be779f5
commit 2d765d6bf2
2 changed files with 54 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backive/backive_service
View File

@ -54,7 +54,7 @@ class Backive:
if await self._scheduler.should_run(backup.name): if await self._scheduler.should_run(backup.name):
logging.info("Running backup '%s'", backup.name) logging.info("Running backup '%s'", backup.name)
result = await backup.run() result = await backup.run()
logging.debug("Result: %s", result.decode()) logging.debug("Result: %s", result)
await self._scheduler.register_run(backup.name) await self._scheduler.register_run(backup.name)
else: else:
logging.info( logging.info(

75
backive/core/backup.py
View File

@ -17,6 +17,28 @@ class Backup:
def get_frequency(self): def get_frequency(self):
return self.config.get("frequency", None) return self.config.get("frequency", None)
async def _read_stream(self, stream, cb):
while True:
line = await stream.readline()
if line:
cb(line)
else:
break
async def stream_subprocess(self, cmd, outcb, errcb):
proc = await asyncio.create_subprocess_shell(
cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
await asyncio.wait([
self._read_stream(proc.stdout, outcb),
self._read_stream(proc.stderr, errcb)
])
return await proc.wait()
async def run(self): async def run(self):
logging.debug("Running backup %s", self.name) logging.debug("Running backup %s", self.name)
@ -31,31 +53,40 @@ class Backup:
self.config.get("target_device") self.config.get("target_device")
)).config.get("mountname") )).config.get("mountname")
) )
proc = await asyncio.create_subprocess_shell( backup_env["BACKIVE_TARGET_DIR"] = os.path.join(
"""mkdir -p {}""".format( backup_env["BACKIVE_MOUNT"],
os.path.join( backup_env["BACKIVE_TO"]
backup_env["BACKIVE_MOUNT"], )
backup_env["BACKIVE_TO"] proc = await self.stream_subprocess(
) """mkdir -p {}""".format( backup_env["BACKIVE_TARGET_DIR"]),
), lambda x: logging.debug("STDOUT: %s", x),
stdout=asyncio.subprocess.PIPE, lambda x: logging.debug("STDERR: %s", x),
stderr=asyncio.subprocess.PIPE,
) )
stdout, stderr = await proc.communicate() # proc = await asyncio.create_subprocess_shell(
logging.debug("stdout: %s", stdout.decode()) # """mkdir -p {}""".format( backup_env["BACKIVE_TARGET_DIR"]
logging.debug("stderr: %s", stderr.decode()) # ),
# stdout=asyncio.subprocess.PIPE,
# stderr=asyncio.subprocess.PIPE,
# )
# stdout, stderr = await proc.communicate()
# logging.debug("stdout: %s", stdout.decode())
# logging.debug("stderr: %s", stderr.decode())
user = self.config.get("user") user = self.config.get("user")
proc = await asyncio.create_subprocess_shell( # proc = await asyncio.create_subprocess_shell(
proc = await self.stream_subprocess(
# "set -x; chown -R {} ${{BACKIVE_MOUNT}}/${{BACKIVE_TO}};".format(user) + # "set -x; chown -R {} ${{BACKIVE_MOUNT}}/${{BACKIVE_TO}};".format(user) +
# "sudo -E -u {} sh -c '".format(user) + # "sudo -E -u {} sh -c '".format(user) +
self.config.get("script"), self.config.get("script"),
# "'", lambda x: logging.debug("STDOUT: %s", x),
stdout=asyncio.subprocess.PIPE, lambda x: logging.debug("STDERR: %s", x),
stderr=asyncio.subprocess.PIPE,
shell=True,
env=backup_env
) )
stdout, stderr = await proc.communicate() # "'",
logging.debug("stdout: %s", stdout.decode()) # stdout=asyncio.subprocess.PIPE,
logging.debug("stderr: %s", stderr.decode()) # stderr=asyncio.subprocess.PIPE,
return stdout # shell=True,
# env=backup_env
# )
# stdout, stderr = await proc.communicate()
# logging.debug("stdout: %s", stdout.decode())
# logging.debug("stderr: %s", stderr.decode())
return "done"