python logging does not release file till all program finish

Question

I am working on python logging and I came across this problem. I could not release the file unless I stopped the python code.

Basically, I'd like to accomplish that writing a line of log data then releasing the file then opening the file again and appending new log data.

Can anyone shed some light on that how can I do that?

Cheers!

if __name__ == '__main__':
    # Main logger
    main_logger = logging.getLogger()
    main_logger.setLevel(logging.INFO)
    file_handlerWrite = RotatingFileHandler('{}'.format(pth), mode='w')
    old_factory = logging.getLogRecordFactory()
    isThisFirstTime = True

    if isThisFirstTime:
        add_logformatter(file_handlerWrite, log_title_format, logging.WARNING)
        logging.getLogger().warning(
            'Num;        Logging Time;                   MotorFreq;      MotorSpeed;     PWMVal;')
        print("write:")
        try:
            file_handlerWrite.acquire()
            file_handlerWrite.flush()
            file_handlerWrite.close()
        except (OSError, ValueError):
            pass
        finally:
            file_handlerWrite.release()
            main_logger.removeHandler(file_handlerWrite)
    else:
            pass
    logging.shutdown()

    file_handlerAppend = RotatingFileHandler('{}'.format(pth), mode='a', backupCount=1)

    add_logformatter(file_handlerAppend, log_data_format, logging.INFO)
    print("appers")
    for x in range(100):
        mtrFreq = random.randint(4000, 5000)
        mtSod = random.randint(4000, 5000)
        pwm = random.randint(4000, 5000)

        logging.setLogRecordFactory(record_factory_factory(x))

        logging.getLogger().info("    %d;          %d;            %d;" % (mtrFreq, mtSod, pwm))
    print("doneeee")
    try:
        file_handlerAppend.acquire()
        file_handlerAppend.flush()
        file_handlerAppend.close()
    except (OSError, ValueError):
        pass
    finally:
        file_handlerAppend.release()
        main_logger.removeHandler(file_handlerAppend)

    logging.shutdown()
    print("shottdown")
    while(1):
        pass

Answer 1

You would need to write your own Handler class, I'd imagine.

Basing on the default FileHandler and StreamHandler, maybe something like this.

filename_getter must be a function that returns the filename for a given log record; it can of course just be a function that always returns the same thing, if you don't need rotation or anything...

import logging
import time
from logging import StreamHandler


class QuickReleaseFileHandler(StreamHandler):
    def __init__(self, filename_getter):
        super().__init__()
        self.stream = None
        self.filename_getter = filename_getter

    def setStream(self, stream):
        raise NotImplementedError("You can't call setStream on this handler")

    def flush(self) -> None:
        pass  # we don't need to flush, since we close the stream after every message

    def emit(self, record) -> None:
        with open(self.filename_getter(record), 'a') as self.stream:
            super().emit(record)
        self.stream = None


fh = QuickReleaseFileHandler(
    lambda _: f'foo_{int(time.time() // 2)}.log',
)

logging.basicConfig(level=logging.INFO, handlers=[fh])

for x in range(10):
    print(x)
    logging.info(f"Hello {x}")
    time.sleep(1)