Doing import ray breaks my custom logging formatter

Question

I am using ray inside modin to accelerate reading some huge datasets.

In the whole project there is a custom logging formatter. From the reading I have done i think ray's log format (you can find it under ray.ray_constants called LOGGER_FORMAT) overrides my format.

So when i do import ray my logging breaks.

reproducible example

import os
import logging
import logging.config


def initialise_logger(config):
    """
    Args:
        section (str): the log_configuration section for the service.

    """
    logging.config.fileConfig(fname=config, disable_existing_loggers=False)
    old_factory = logging.getLogRecordFactory()

    def my_record_factory(*args, **kwargs):

        record = old_factory(*args, **kwargs)
        record.project = "alex"
        record.microservice = "alex"
        record.hostname = "alex"
        record.local_ip = "alex"

        # all Mesos containers in production have this environment variable.
        # if it is not present, then we are not in production
       record.mesos_task_id = "alex"

    return record

    logging.setLogRecordFactory(my_record_factory)


def main():
    text = """[loggers]
    keys=root,test_logger,Test_Class

    [handlers]
    keys=consoleHandler

    [formatters]
    keys=consoleFormatter

    [logger_root]
    level=INFO
    handlers=consoleHandler

    [logger_test_logger]
    level=INFO
    handlers=consoleHandler
    qualname=test_logger
    propagate=0

    [logger_Test_Class]
    level=INFO
    handlers=consoleHandler
    qualname=Test_Class
    propagate=0

    [handler_consoleHandler]
    class=StreamHandler
    level=INFO
    formatter=consoleFormatter
    args=(sys.stdout,)

    [formatter_consoleFormatter]
    format = %(asctime)s.%(msecs)03d %(project)s %(microservice)s %(levelname)s %(hostname)s %(local_ip)s %(name)s %(filename)s %(lineno)d %(funcName)s %(module)s %(processName)s %(process)d %(threadName)s %(thread)d  %(message)s
    datefmt= %Y-%m-%d %H:%M:%S   
    """

    with open(os.path.join(os.getcwd(), "config.ini"), "w") as conffile:
        conffile.write(text)

    initialise_logger(config="config.ini")

    logging.info("RAY NOT IMPORTED")

    import ray

    logging.info("RAY IMPORTED")


if __name__ == '__main__':
    main()

Answer 1

Instead of using the global logging module, try instantiating a logger (e.g. logger = logging.getLogger(__name__)) and then set the formatters and handlers for that logger instance. That way, even if a 3rd party library is messing with the base configuration, it shouldn't mess with your logger instance.