MongoDB "Writing to log file failed, aborting application"

Question

After save about 80G data, mongodb just crash.

journalctl -u mongod shows the error message below

mongod[4948]: {"t":{"$date":"2022-03-31T10:38:44.089+00:00"},"s":"F",  "c":"CONTROL",  "id":4522200, "ctx":"ftdc","msg":"Writing to log file failed, aborting application","attr":{"files":["/var/log/mongodb/mongod.log"]}}
mongod[4948]: BACKTRACE: {"backtrace":[{"a":"55C58B1FAF55","b":"55C58737F000","o":"3E7BF55","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.357","s+":"215"},{"a":"55C58B1FDA3A","b":"55C58737F000","o":"3E7EA3A","s":"_ZN5mongo15printStackTraceERSo","s+":"3A"},{"a":"55C58B1B48E2","b":"55C58737F000","o":"3E358E2","s":"_ZN5mongo5logv214FileRotateSink7consumeERKN5boost3log12v2s_mt_posix11record_viewERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE","s+":"512"},{"a":"55C58B1D1D93","b":"55C58737F000","o":"3E52D93","s":"_ZN5boost3log12v2s_mt_posix5sinks13unlocked_sinkIN5mongo5logv216CompositeBackendIJNS5_14FileRotateSinkENS5_10RamLogSinkES8_NS5_14UserAssertSinkEEEEE7consumeERKNS1_11record_viewE","s+":"213"},{"a":"55C58B30BF33","b":"55C58737F000","o":"3F8CF33","s":"_ZN5boost3log12v2s_mt_posix4core16push_record_moveERNS1_6recordE","s+":"1D3"},{"a":"55C58B1C2BD9","b":"55C58737F000","o":"3E43BD9","s":"_ZN5mongo5logv26detail9doLogImplEiRKNS0_11LogSeverityERKNS0_10LogOptionsENS_10StringDataERKNS0_26TypeErasedAttributeStorageE","s+":"799"},{"a":"55C58B1F7D43","b":"55C58737F000","o":"3E78D43","s":"_ZN5mongo18stack_trace_detail18logBacktraceObjectERKNS_7BSONObjEPNS_14StackTraceSinkEb","s+":"4F3"},{"a":"55C58B1FBAF3","b":"55C58737F000","o":"3E7CAF3","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.357","s+":"DB3"},{"a":"55C58B1FD9E9","b":"55C58737F000","o":"3E7E9E9","s":"_ZN5mongo15printStackTraceEv","s+":"29"},{"a":"55C58B1F6347","b":"55C58737F000","o":"3E77347","s":"_ZN5mongo12_GLOBAL__N_111myTerminateEv","s+":"D7"},{"a":"55C58B384286","b":"55C58737F000","o":"4005286","s":"_ZN10__cxxabiv111__terminateEPFvvE","s+":"6"},{"a":"55C58B418C09","b":"55C58737F000","o":"4099C09","s":"__cxa_call_terminate","s+":"39"},{"a":"55C58B383C75","b":"55C58737F000","o":"4004C75","s":"__gxx_personality_v0","s+":"275"},{"a":"7FD5BA505573","b":"7FD5BA4F5000","o":"10573","s":"_Unwind_GetTextRelBase","s+":"1E03"},{"a":"7FD5BA505AD1","b":"7FD5BA4F5000","o":"10AD1","s":"_Unwind_RaiseException","s+":"2B1"},{"a":"55C58B3843E7","b":"55C58737F000","o":"40053E7","s":"__cxa_throw","s+":"37"},{"a":"55C588722ABB","b":"55C58737F000","o":"13A3ABB","s":"_ZN5mongo13error_details23throwExceptionForStatusERKNS_6StatusE","s+":"1FDF"},{"a":"55C588739BB1","b":"55C58737F000","o":"13BABB1","s":"_ZN5mongo21uassertedWithLocationERKNS_6StatusEPKcj","s+":"337"},{"a":"55C5882A99DD","b":"55C58737F000","o":"F2A9DD","s":"_ZN5mongo14FTDCController6doLoopEv.cold.437","s+":"2D"},{"a":"55C588D96A1C","b":"55C58737F000","o":"1A17A1C","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_14FTDCController5startEvEUlvE0_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","s+":"5C"},{"a":"55C58B3A051F","b":"55C58737F000","o":"402151F","s":"execute_native_thread_routine","s+":"F"},{"a":"7FD5BA2DD6DB","b":"7FD5BA2D6000","o":"76DB","s":"start_thread","s+":"DB"},{"a":"7FD5BA00661F","b":"7FD5B9EE5000","o":"12161F","s":"clone","s+":"3F"}],"processInfo":{"mongodbVersion":"5.0.6","gitVersion":"212a8dbb47f07427dae194a9c75baec1d81d9259","compiledModules":[],"uname":{"sysname":"Linux","release":"4.15.0-173-generic","version":"#182-Ubuntu SMP Fri Mar 18 15:53:46 UTC 2022","machine":"x86_64"},"somap":[{"b":"55C58737F000","elfType":3,"buildId":"DDE3587FBCA9EFB4CB2E3A046C99B6247E65A66B"},{"b":"7FD5BA4F5000","path":"/lib/x86_64-linux-gnu/libgcc_s.so.1","elfType":3,"buildId":"679F3AE11120EC7C483BC9295345D836F5C104F7"},{"b":"7FD5BA2D6000","path":"/lib/x86_64-linux-gnu/libpthread.so.0","elfType":3,"buildId":"8B60B9F8D9A27F2264A0D8E6C0768B3B02C29926"},{"b":"7FD5B9EE5000","path":"/lib/x86_64-linux-gnu/libc.so.6","elfType":3,"buildId":"71F0F3074A929E519E85F6A5C03A7D1FD976BFE4"}]}}
mongod[4948]:   Frame: {"a":"55C58B1FAF55","b":"55C58737F000","o":"3E7BF55","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.357","s+":"215"}
mongod[4948]:   Frame: {"a":"55C58B1FDA3A","b":"55C58737F000","o":"3E7EA3A","s":"_ZN5mongo15printStackTraceERSo","s+":"3A"}
mongod[4948]:   Frame: {"a":"55C58B1B48E2","b":"55C58737F000","o":"3E358E2","s":"_ZN5mongo5logv214FileRotateSink7consumeERKN5boost3log12v2s_mt_posix11record_viewERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE","s+":"512"}
mongod[4948]:   Frame: {"a":"55C58B1D1D93","b":"55C58737F000","o":"3E52D93","s":"_ZN5boost3log12v2s_mt_posix5sinks13unlocked_sinkIN5mongo5logv216CompositeBackendIJNS5_14FileRotateSinkENS5_10RamLogSinkES8_NS5_14UserAssertSinkEEEEE7consumeERKNS1_11record_viewE","s+":"213"}
mongod[4948]:   Frame: {"a":"55C58B30BF33","b":"55C58737F000","o":"3F8CF33","s":"_ZN5boost3log12v2s_mt_posix4core16push_record_moveERNS1_6recordE","s+":"1D3"}
mongod[4948]:   Frame: {"a":"55C58B1C2BD9","b":"55C58737F000","o":"3E43BD9","s":"_ZN5mongo5logv26detail9doLogImplEiRKNS0_11LogSeverityERKNS0_10LogOptionsENS_10StringDataERKNS0_26TypeErasedAttributeStorageE","s+":"799"}
mongod[4948]:   Frame: {"a":"55C58B1F7D43","b":"55C58737F000","o":"3E78D43","s":"_ZN5mongo18stack_trace_detail18logBacktraceObjectERKNS_7BSONObjEPNS_14StackTraceSinkEb","s+":"4F3"}
mongod[4948]:   Frame: {"a":"55C58B1FBAF3","b":"55C58737F000","o":"3E7CAF3","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.357","s+":"DB3"}
mongod[4948]:   Frame: {"a":"55C58B1FD9E9","b":"55C58737F000","o":"3E7E9E9","s":"_ZN5mongo15printStackTraceEv","s+":"29"}
mongod[4948]:   Frame: {"a":"55C58B1F6347","b":"55C58737F000","o":"3E77347","s":"_ZN5mongo12_GLOBAL__N_111myTerminateEv","s+":"D7"}
mongod[4948]:   Frame: {"a":"55C58B384286","b":"55C58737F000","o":"4005286","s":"_ZN10__cxxabiv111__terminateEPFvvE","s+":"6"}
mongod[4948]:   Frame: {"a":"55C58B418C09","b":"55C58737F000","o":"4099C09","s":"__cxa_call_terminate","s+":"39"}
mongod[4948]:   Frame: {"a":"55C58B383C75","b":"55C58737F000","o":"4004C75","s":"__gxx_personality_v0","s+":"275"}
mongod[4948]:   Frame: {"a":"7FD5BA505573","b":"7FD5BA4F5000","o":"10573","s":"_Unwind_GetTextRelBase","s+":"1E03"}
mongod[4948]:   Frame: {"a":"7FD5BA505AD1","b":"7FD5BA4F5000","o":"10AD1","s":"_Unwind_RaiseException","s+":"2B1"}
mongod[4948]:   Frame: {"a":"55C58B3843E7","b":"55C58737F000","o":"40053E7","s":"__cxa_throw","s+":"37"}
mongod[4948]:   Frame: {"a":"55C588722ABB","b":"55C58737F000","o":"13A3ABB","s":"_ZN5mongo13error_details23throwExceptionForStatusERKNS_6StatusE","s+":"1FDF"}
mongod[4948]:   Frame: {"a":"55C588739BB1","b":"55C58737F000","o":"13BABB1","s":"_ZN5mongo21uassertedWithLocationERKNS_6StatusEPKcj","s+":"337"}
mongod[4948]:   Frame: {"a":"55C5882A99DD","b":"55C58737F000","o":"F2A9DD","s":"_ZN5mongo14FTDCController6doLoopEv.cold.437","s+":"2D"}
mongod[4948]:   Frame: {"a":"55C588D96A1C","b":"55C58737F000","o":"1A17A1C","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_14FTDCController5startEvEUlvE0_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","s+":"5C"}
mongod[4948]:   Frame: {"a":"55C58B3A051F","b":"55C58737F000","o":"402151F","s":"execute_native_thread_routine","s+":"F"}
mongod[4948]:   Frame: {"a":"7FD5BA2DD6DB","b":"7FD5BA2D6000","o":"76DB","s":"start_thread","s+":"DB"}
mongod[4948]:   Frame: {"a":"7FD5BA00661F","b":"7FD5B9EE5000","o":"12161F","s":"clone","s+":"3F"}
systemd[1]: mongod.service: Main process exited, code=exited, status=1/FAILURE
systemd[1]: mongod.service: Failed with result 'exit-code'.

The main message should be Writing to log file failed, aborting application

But I have found nothing in the Internet, include mongodb github issue and mongodb official forum.

Checked the log file permission, it owned by mongodb.

Checked the disk available size, it still has space.

For now, I can not start mongod service.

Does anyone have similar experience?

Answer 1

Had the issue and solved it by

> mongod --dbpath /var/lib/mongodb/ --repair
> chown -R mongodb: /var/{log,lib}/mongodb/

OS: ArchLinux
MongoDB version 4.44

Answer 2

you can use bash script to restart cli command.

#!/bin/bash
until cli_command
do
    echo "Restarting"
    sleep 2
done