So i've got a long-running process. Once it finishes I need the output. But the user should be informed as its running. With the logging module I get timestamps and whatnot preceding. So let's combine all 3:

Prelude:

import os
import logging
import subprocess

from io import IOBase
from sys import stdout
from select import select
from threading import Thread
from time import sleep

from cStringIO import StringIO

Init:

# Some other file, like __init__.py
logging.basicConfig(format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
                    level='INFO')
handler = logging.root.handlers.pop()
assert logging.root.handlers == [], "root logging handlers aren't empty"
handler.stream.close()
handler.stream = stdout
logging.root.addHandler(handler)
# Some other file, like __init__.py

log = logging.getLogger(__name__)

Class (based on http://stackoverflow.com/a/4838875):

class StreamLogger(IOBase):
    _run = None
    def __init__(self, logger_obj, level):
        super(StreamLogger, self).__init__()
        self.logger_obj = logger_obj
        self.level = level
        self.pipe = os.pipe()
        self.thread = Thread(target=self._flusher)
        self.thread.start()

    def __call__(self): return self

    def _flusher(self):
        self._run = True
        buf = b''
        while self._run:
            for fh in select([self.pipe[0]], [], [], 1)[0]:
                buf += os.read(fh, 1024)
                while b'\n' in buf:
                    data, buf = buf.split(b'\n', 1)
                    self.write(data.decode())
            sleep(1)
        self._run = None

    def write(self, data): return self.logger_obj.log(self.level, data)
    def fileno(self): return self.pipe[1]

    def close(self):
        if self._run:
            self._run = False
            while self._run is not None:
                sleep(1)
            os.close(self.pipe[0])
            os.close(self.pipe[1])
            self.thread.join(1)

Usage:

stderr_tee = logging.StreamHandler(StringIO())
log.addHandler(stderr_tee)
log.setLevel(logging.ERROR)

stdout_tee = logging.StreamHandler(StringIO())
log.addHandler(stdout_tee)
log.setLevel(logging.INFO)

with StreamLogger(log, logging.INFO) as out, StreamLogger(log,
                                                          logging.ERROR) as err:
    subprocess.Popen('ls 1>&2', stderr=err, shell=True)

print 'stderr_tee =', stderr_tee.stream.getvalue()
print 'stdout_tee =', stdout_tee.stream.getvalue()

With a cleanup like so:

# finally block
for handler in log.handlers:
    log.removeHandler(handler)
    handler.stream.close()
    handler.close()
stderr_tee.stream.close()
stdout_tee.stream.close()
share|improve this question

This question has an open bounty worth +50 reputation from A T ending in 5 days.

This question has not received enough attention.

Your Answer

 
discard

By posting your answer, you agree to the privacy policy and terms of service.

Browse other questions tagged or ask your own question.