So i've got a long-running process. Once it finishes I need the output. But the user should be informed as its running. With the logging
module I get timestamps and whatnot preceding. So let's combine all 3:
Prelude:
import os
import logging
import subprocess
from io import IOBase
from sys import stdout
from select import select
from threading import Thread
from time import sleep
from cStringIO import StringIO
Init:
# Some other file, like __init__.py
logging.basicConfig(format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
level='INFO')
handler = logging.root.handlers.pop()
assert logging.root.handlers == [], "root logging handlers aren't empty"
handler.stream.close()
handler.stream = stdout
logging.root.addHandler(handler)
# Some other file, like __init__.py
log = logging.getLogger(__name__)
Class (based on http://stackoverflow.com/a/4838875):
class StreamLogger(IOBase):
_run = None
def __init__(self, logger_obj, level):
super(StreamLogger, self).__init__()
self.logger_obj = logger_obj
self.level = level
self.pipe = os.pipe()
self.thread = Thread(target=self._flusher)
self.thread.start()
def __call__(self): return self
def _flusher(self):
self._run = True
buf = b''
while self._run:
for fh in select([self.pipe[0]], [], [], 1)[0]:
buf += os.read(fh, 1024)
while b'\n' in buf:
data, buf = buf.split(b'\n', 1)
self.write(data.decode())
sleep(1)
self._run = None
def write(self, data): return self.logger_obj.log(self.level, data)
def fileno(self): return self.pipe[1]
def close(self):
if self._run:
self._run = False
while self._run is not None:
sleep(1)
os.close(self.pipe[0])
os.close(self.pipe[1])
self.thread.join(1)
Usage:
stderr_tee = logging.StreamHandler(StringIO())
log.addHandler(stderr_tee)
log.setLevel(logging.ERROR)
stdout_tee = logging.StreamHandler(StringIO())
log.addHandler(stdout_tee)
log.setLevel(logging.INFO)
with StreamLogger(log, logging.INFO) as out, StreamLogger(log,
logging.ERROR) as err:
subprocess.Popen('ls 1>&2', stderr=err, shell=True)
print 'stderr_tee =', stderr_tee.stream.getvalue()
print 'stdout_tee =', stdout_tee.stream.getvalue()
With a cleanup like so:
# finally block
for handler in log.handlers:
log.removeHandler(handler)
handler.stream.close()
handler.close()
stderr_tee.stream.close()
stdout_tee.stream.close()