Hi ,
Can anyone please help me to find out the reason based on this log, a get crashed mongo db but after restarted online again.
Issue description : Mongodb crash issue
MongoDB version v5.0
OS details - Redhat7
Topology - 3 nodes a Replica set
There is nothing suspicious in messages and dmesg logs .
{"t":{"$date":"2024-05-27T12:18:14.028+07:00"},"s":"I", "c":"ACCESS", "id":20250, "ctx":"conn8982125","msg":"Authentication succeeded","attr":{"mechanism":"SCRAM-SHA-256","speculative":false,"principalName":"__system","authenticationDatabase":"local","remote":"10.22.17.68:58190","extraInfo":{}}}
{"t":{"$date":"2024-05-27T12:18:14.031+07:00"},"s":"W", "c":"ACCESS", "id":5626700, "ctx":"conn8982125","msg":"Client has attempted to reauthenticate as a single user","attr":{"user":{"user":"__system","db":"local"}}}
{"t":{"$date":"2024-05-27T12:18:14.031+07:00"},"s":"I", "c":"ACCESS", "id":20250, "ctx":"conn8982125","msg":"Authentication succeeded","attr":{"mechanism":"SCRAM-SHA-256","speculative":false,"principalName":"__system","authenticationDatabase":"local","remote":"10.22.17.68:58190","extraInfo":{}}}
{"t":{"$date":"2024-05-27T12:18:14.033+07:00"},"s":"W", "c":"ACCESS", "id":5626700, "ctx":"conn8982125","msg":"Client has attempted to reauthenticate as a single user","attr":{"user":{"user":"__system","db":"local"}}}
{"t":{"$date":"2024-05-27T12:18:14.033+07:00"},"s":"I", "c":"ACCESS", "id":20250, "ctx":"conn8982125","msg":"Authentication succeeded","attr":{"mechanism":"SCRAM-SHA-256","speculative":false,"principalName":"__system","authenticationDatabase":"local","remote":"10.22.17.68:58190","extraInfo":{}}}
{"t":{"$date":"2024-05-27T12:18:15.331+07:00"},"s":"F", "c":"CONTROL", "id":4757800, "ctx":"conn8989571","msg":"Writing fatal message","attr":{"message":"Invalid access at address: 0x555c059f5000"}}
{"t":{"$date":"2024-05-27T12:18:15.331+07:00"},"s":"F", "c":"CONTROL", "id":4757800, "ctx":"conn8989571","msg":"Writing fatal message","attr":{"message":"Got signal: 11 (Segmentation fault).\n"}}
{"t":{"$date":"2024-05-27T12:18:15.459+07:00"},"s":"I", "c":"COMMAND", "id":51803, "ctx":"conn8991152","msg":"Slow query","attr":{"type":"command","ns":"todo_list.todo_list","command":{"find":"todo_list","filter":{"$and":[{"screening_1":{"$ne":""}},{"screening_1":{"$ne":"AUTO REJECT"}},{"screening_2":""},{"screening_3":""},{"detail.status_todolist.konfirmasi_data_entry":{"$ne":"Y"}}],"detail.identitas_order.internal_sales_force.internal_sales_force_id":"24026048","detail.cancelation_status.cancel_code":"","detail.status_todolist.invalid_status":{"$ne":"Y"},"applicant_type_code":"PER"},"sort":{"order_id":-1,"prioritas":1},"projection":{"source_order_desc":1,"prioritas":1,"status_assign_desc":1,"order_created_date":1,"screening_1":1,"screening_2":1,"screening_3":1,"current_form_desc":1,"detail.identitas_order":1,"detail.debitur.personal.no_hp_1":1,"customer_name":1,"order_id":1,"detail.debitur.personal.alamat_debitur":1,"outlet_channel_desc":1},"$db":"todo_list","$clusterTime":{"clusterTime":{"$timestamp":{"t":1716787080,"i":1}},"signature":{"hash":{"$binary":{"base64":"LGH2AmzYHhfehjNfIJg6sKsQe3k=","subType":"0"}},"keyId":7336601416347156481}},"lsid":{"id":{"$uuid":"c1835085-cea2-4a14-ae40-ad0ecf89f4e6"}},"$readPreference":{"mode":"primaryPreferred"}},"planSummary":"IXSCAN { detail.cancelation_status.cancel_code: 1 }","keysExamined":988289,"docsExamined":988289,"hasSortStage":true,"cursorExhausted":true,"numYields":1086,"nreturned":0,"queryHash":"D4800CFE","planCacheKey":"DA345023","reslen":233,"locks":{"Global":{"acquireCount":{"r":1087}},"Mutex":{"acquireCount":{"r":1}}},"readConcern":{"level":"local","provenance":"implicitDefault"},"storage":{"data":{"bytesRead":3626079171,"timeReadingMicros":3929525},"timeWaitingMicros":{"cache":16}},"remote":"10.22.17.178:50648","protocol":"op_msg","durationMillis":12087}}
{"t":{"$date":"2024-05-27T12:18:15.601+07:00"},"s":"I", "c":"COMMAND", "id":51803, "ctx":"conn8988404","msg":"Slow query","attr":{"type":"command","ns":"todo_list.todo_list","command":{"find":"todo_list","filter":{"$and":[{"screening_1":{"$ne":""}},{"screening_1":{"$ne":"AUTO REJECT"}},{"screening_2":""},{"screening_3":""},{"detail.status_todolist.konfirmasi_data_entry":{"$ne":"Y"}}],"detail.identitas_order.internal_sales_force.internal_sales_force_id":"22019694","detail.cancelation_status.cancel_code":"","detail.status_todolist.invalid_status":{"$ne":"Y"},"applicant_type_code":"PER"},"sort":{"order_id":-1,"prioritas":1},"projection":{"source_order_desc":1,"prioritas":1,"status_assign_desc":1,"order_created_date":1,"screening_1":1,"screening_2":1,"screening_3":1,"current_form_desc":1,"detail.identitas_order":1,"detail.debitur.personal.no_hp_1":1,"customer_name":1,"order_id":1,"detail.debitur.personal.alamat_debitur":1,"outlet_channel_desc":1},"$db":"todo_list","$clusterTime":{"clusterTime":{"$timestamp":{"t":1716787080,"i":1}},"signature":{"hash":{"$binary":{"base64":"LGH2AmzYHhfehjNfIJg6sKsQe3k=","subType":"0"}},"keyId":7336601416347156481}},"lsid":{"id":{"$uuid":"7d76f163-34e0-4dbc-8d98-ceb6bc57764f"}},"$readPreference":{"mode":"primaryPreferred"}},"planSummary":"IXSCAN { detail.cancelation_status.cancel_code: 1 }","cursorid":4032157102139689225,"keysExamined":988289,"docsExamined":988289,"hasSortStage":true,"numYields":1114,"nreturned":101,"queryHash":"D4800CFE","planCacheKey":"DA345023","reslen":160297,"locks":{"Global":{"acquireCount":{"r":1115}},"Mutex":{"acquireCount":{"r":1}}},"readConcern":{"level":"local","provenance":"implicitDefault"},"storage":{"data":{"bytesRead":2801670726,"timeReadingMicros":3268785},"timeWaitingMicros":{"cache":2}},"remote":"10.22.17.184:43408","protocol":"op_msg","durationMillis":12022}}
{"t":{"$date":"2024-05-27T12:18:16.066+07:00"},"s":"I", "c":"CONTROL", "id":31380, "ctx":"conn8989571","msg":"BACKTRACE","attr":{"bt":{"backtrace":[{"a":"5558CFA65E75","b":"5558CBBA8000","o":"3EBDE75","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.357","s+":"215"},{"a":"5558CFA68909","b":"5558CBBA8000","o":"3EC0909","s":"_ZN5mongo15printStackTraceEv","s+":"29"},{"a":"5558CFA6109C","b":"5558CBBA8000","o":"3EB909C","s":"abruptQuitWithAddrSignal","s+":"EC"},{"a":"5558CF08DE14","b":"5558CBBA8000","o":"34E5E14","s":"_ZL16WasmFaultHandleriP9siginfo_tPv","s+":"A4"},{"a":"7F6811942D80","b":"7F6811930000","o":"12D80","s":"funlockfile","s+":"50"},{"a":"5558CFBD2E00","b":"5558CBBA8000","o":"402AE00","s":"_ZN4absl12lts_2021032413hash_internal6WyhashEPKvmmPKm","s+":"50"},{"a":"5558CFBD2C5E","b":"5558CBBA8000","o":"402AC5E","s":"_ZN4absl12lts_2021032413hash_internal9HashState28CombineLargeContiguousImpl64EmPKhm","s+":"7E"},{"a":"5558CD11CC9E","b":"5558CBBA8000","o":"1574C9E","s":"_ZN4absl12lts_2021032413hash_internal13AbslHashValueINS1_9HashStateEEET_S4_St17basic_string_viewIcSt11char_traitsIcEE","s+":"10E"},{"a":"5558CE7E4F89","b":"5558CBBA8000","o":"2C3CF89","s":"_ZNK5mongo19projection_executor29FastPathEligibleInclusionNode17_applyProjectionsENS_7BSONObjEPNS_14BSONObjBuilderE","s+":"129"},{"a":"5558CE7E59AE","b":"5558CBBA8000","o":"2C3D9AE","s":"_ZNK5mongo19projection_executor29FastPathEligibleInclusionNode15applyToDocumentERKNS_8DocumentE","s+":"16E"},{"a":"5558CDD875E1","b":"5558CBBA8000","o":"21DF5E1","s":"_ZN5mongo19projection_executor18ProjectionExecutor19applyTransformationERKNS_8DocumentE","s+":"51"},{"a":"5558CE191B98","b":"5558CBBA8000","o":"25E9B98","s":"_ZNK5mongo22ProjectionStageDefault9transformEPNS_16WorkingSetMemberE","s+":"E8"},{"a":"5558CE190540","b":"5558CBBA8000","o":"25E8540","s":"_ZN5mongo15ProjectionStage6doWorkEPm","s+":"90"},{"a":"5558CE19B6AD","b":"5558CBBA8000","o":"25F36AD","s":"_ZN5mongo9SortStage6doWorkEPm","s+":"9D"},{"a":"5558CE1627AD","b":"5558CBBA8000","o":"25BA7AD","s":"_ZN5mongo9PlanStage4workEPm","s+":"3D"},{"a":"5558CE161574","b":"5558CBBA8000","o":"25B9574","s":"_ZN5mongo15CachedPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE","s+":"154"},{"a":"5558CE1EE1CA","b":"5558CBBA8000","o":"26461CA","s":"_ZN5mongo16PlanExecutorImpl13_pickBestPlanEv","s+":"BA"},{"a":"5558CE1EF4B5","b":"5558CBBA8000","o":"26474B5","s":"_ZN5mongo16PlanExecutorImplC1EPNS_16OperationContextESt10unique_ptrINS_10WorkingSetESt14default_deleteIS4_EES3_INS_9PlanStageES5_IS8_EES3_INS_13QuerySolutionES5_ISB_EES3_INS_14CanonicalQueryES5_ISE_EERKN5boost13intrusive_ptrINS_17ExpressionContextEEERKNS_13CollectionPtrEbNS_15NamespaceStringENS_15PlanYieldPolicy11YieldPolicyE","s+":"445"},{"a":"5558CE1EA7E4","b":"5558CBBA8000","o":"26427E4","s":"_ZN5mongo21plan_executor_factory4makeEPNS_16OperationContextESt10unique_ptrINS_10WorkingSetESt14default_deleteIS4_EES3_INS_9PlanStageES5_IS8_EES3_INS_13QuerySolutionES5_ISB_EES3_INS_14CanonicalQueryES5_ISE_EERKN5boost13intrusive_ptrINS_17ExpressionContextEEEPKNS_13CollectionPtrEmNS_15NamespaceStringENS_15PlanYieldPolicy11YieldPolicyE","s+":"104"},{"a":"5558CE1EAF03","b":"5558CBBA8000","o":"2642F03","s":"_ZN5mongo21plan_executor_factory4makeESt10unique_ptrINS_14CanonicalQueryESt14default_deleteIS2_EES1_INS_10WorkingSetES3_IS6_EES1_INS_9PlanStageES3_IS9_EEPKNS_13CollectionPtrENS_15PlanYieldPolicy11YieldPolicyEmNS_15NamespaceStringES1_INS_13QuerySolutionES3_ISI_EE","s+":"113"},{"a":"5558CE1DF962","b":"5558CBBA8000","o":"2637962","s":"_ZN5mongo11getExecutorEPNS_16OperationContextEPKNS_13CollectionPtrESt10unique_ptrINS_14CanonicalQueryESt14default_deleteIS6_EENS_15PlanYieldPolicy11YieldPolicyEm","s+":"3D2"},{"a":"5558CE1DFC2F","b":"5558CBBA8000","o":"2637C2F","s":"_ZN5mongo12_GLOBAL__N_116_getExecutorFindEPNS_16OperationContextEPKNS_13CollectionPtrESt10unique_ptrINS_14CanonicalQueryESt14default_deleteIS7_EENS_15PlanYieldPolicy11YieldPolicyEm","s+":"6F"},{"a":"5558CE1DFCC1","b":"5558CBBA8000","o":"2637CC1","s":"_ZN5mongo15getExecutorFindEPNS_16OperationContextEPKNS_13CollectionPtrESt10unique_ptrINS_14CanonicalQueryESt14default_deleteIS6_EEbm","s+":"51"},{"a":"5558CDC40F68","b":"5558CBBA8000","o":"2098F68","s":"_ZN5mongo12_GLOBAL__N_17FindCmd10Invocation3runEPNS_16OperationContextEPNS_3rpc21ReplyBuilderInterfaceE","s+":"A08"},{"a":"5558CE88774F","b":"5558CBBA8000","o":"2CDF74F","s":"_ZN5mongo14CommandHelpers20runCommandInvocationEPNS_16OperationContextERKNS_12OpMsgRequestEPNS_17CommandInvocationEPNS_3rpc21ReplyBuilderInterfaceE","s+":"7F"},{"a":"5558CE88C8BE","b":"5558CBBA8000","o":"2CE48BE","s":"_ZN5mongo14CommandHelpers20runCommandInvocationESt10shared_ptrINS_23RequestExecutionContextEES1_INS_17CommandInvocationEENS_9transport15ServiceExecutor14ThreadingModelE","s+":"1BE"},{"a":"5558CD126907","b":"5558CBBA8000","o":"157E907","s":"_ZN5mongo12_GLOBAL__N_120runCommandInvocationESt10shared_ptrINS_23RequestExecutionContextEES1_INS_17CommandInvocationEE","s+":"97"},{"a":"5558CD138F56","b":"5558CBBA8000","o":"1590F56","s":"_ZN5mongo12_GLOBAL__N_114RunCommandImpl11_runCommandEv","s+":"146"},{"a":"5558CD139FBB","b":"5558CBBA8000","o":"1591FBB","s":"_ZN5mongo12_GLOBAL__N_114RunCommandImpl8_runImplEv","s+":"8B"},{"a":"5558CD12AF4C","b":"5558CBBA8000","o":"1582F4C","s":"_ZN5mongo12_GLOBAL__N_114RunCommandImpl3runEv","s+":"13C"},{"a":"5558CD12D15E","b":"5558CBBA8000","o":"158515E","s":"_ZN5mongo12_GLOBAL__N_119ExecCommandDatabase12_commandExecEv","s+":"33E"},{"a":"5558CD135C00","b":"5558CBBA8000","o":"158DC00","s":"_ZZN5mongo12_GLOBAL__N_114executeCommandESt10shared_ptrINS0_13HandleRequest16ExecutionContextEEENUlvE0_clEv","s+":"450"},{"a":"5558CD137DF0","b":"5558CBBA8000","o":"158FDF0","s":"_ZZN5mongo15unique_functionIFvPNS_14future_details15SharedStateBaseEEE8makeImplIZNS1_10FutureImplINS1_8FakeVoidEE16makeContinuationIvZZNOS9_4thenIZNS_12_GLOBAL__N_114executeCommandESt10shared_ptrINSC_13HandleRequest16ExecutionContextEEEUlvE0_EEDaOT_ENKUlvE1_clEvEUlPNS1_15SharedStateImplIS8_EESN_E_EENS7_ISI_EEOT0_EUlS3_E_EEDaSJ_EN12SpecificImpl4callEOS3_","s+":"80"},{"a":"5558CD0F2967","b":"5558CBBA8000","o":"154A967","s":"_ZN5mongo14future_details15SharedStateBase20transitionToFinishedEv","s+":"147"},{"a":"5558CD13C1A2","b":"5558CBBA8000","o":"15941A2","s":"_ZN5mongo14future_details10FutureImplINS0_8FakeVoidEE11generalImplIZNOS3_17propagateResultToEPNS0_15SharedStateImplIS2_EEEUlOS2_E_ZNOS3_17propagateResultToES7_EUlONS_6StatusEE0_ZNOS3_17propagateResultToES7_EUlvE1_EEDaOT_OT0_OT1_","s+":"152"},{"a":"5558CD12F5FC","b":"5558CBBA8000","o":"15875FC","s":"_ZZN5mongo15unique_functionIFvPNS_14future_details15SharedStateBaseEEE8makeImplIZNS1_10FutureImplINS1_8FakeVoidEE16makeContinuationIvZZNOS9_4thenIZNS_12_GLOBAL__N_114executeCommandESt10shared_ptrINSC_13HandleRequest16ExecutionContextEEEUlvE_EEDaOT_ENKUlvE1_clEvEUlPNS1_15SharedStateImplIS8_EESN_E_EENS7_ISI_EEOT0_EUlS3_E_EEDaSJ_EN12SpecificImpl4callEOS3_","s+":"AC"},{"a":"5558CD0F2967","b":"5558CBBA8000","o":"154A967","s":"_ZN5mongo14future_details15SharedStateBase20transitionToFinishedEv","s+":"147"},{"a":"5558CD1365EA","b":"5558CBBA8000","o":"158E5EA","s":"_ZN5mongo12_GLOBAL__N_114executeCommandESt10shared_ptrINS0_13HandleRequest16ExecutionContextEE","s+":"22A"},{"a":"5558CD13701E","b":"5558CBBA8000","o":"158F01E","s":"_ZN5mongo12_GLOBAL__N_116receivedCommandsESt10shared_ptrINS0_13HandleRequest16ExecutionContextEE","s+":"1BE"},{"a":"5558CD137D36","b":"5558CBBA8000","o":"158FD36","s":"_ZN5mongo12_GLOBAL__N_115CommandOpRunner3runEv","s+":"36"},{"a":"5558CD131FE7","b":"5558CBBA8000","o":"1589FE7","s":"_ZN5mongo23ServiceEntryPointCommon13handleRequestEPNS_16OperationContextERKNS_7MessageESt10unique_ptrIKNS0_5HooksESt14default_deleteIS8_EE","s+":"307"},{"a":"5558CD121E4F","b":"5558CBBA8000","o":"1579E4F","s":"_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE","s+":"4F"},{"a":"5558CD179097","b":"5558CBBA8000","o":"15D1097","s":"_ZN5mongo9transport19ServiceStateMachine4Impl14processMessageEv","s+":"127"},{"a":"5558CD1793C7","b":"5558CBBA8000","o":"15D13C7","s":"_ZZNO5mongo14future_details10FutureImplINS0_8FakeVoidEE4thenIZNS_9transport19ServiceStateMachine4Impl12startNewLoopERKNS_6StatusEEUlvE0_EEDaOT_ENKUlOS2_E_clESE_.isra.717","s+":"27"},{"a":"5558CD17BD28","b":"5558CBBA8000","o":"15D3D28","s":"_ZN5mongo9transport19ServiceStateMachine4Impl12startNewLoopERKNS_6StatusE","s+":"1A8"},{"a":"5558CD17C22F","b":"5558CBBA8000","o":"15D422F","s":"_ZZN5mongo15unique_functionIFvNS_6StatusEEE8makeImplIZNS_9transport19ServiceStateMachine4Impl15scheduleNewLoopES1_EUlS1_E_EEDaOT_EN12SpecificImpl4callEOS1_","s+":"7F"},{"a":"5558CF2179C1","b":"5558CBBA8000","o":"366F9C1","s":"_ZZN5mongo15unique_functionIFvNS_6StatusEEE8makeImplIZNS_9transport26ServiceExecutorSynchronous18runOnDataAvailableERKSt10shared_ptrINS5_7SessionEES3_EUlS1_E_EEDaOT_EN12SpecificImpl4callEOS1_","s+":"41"},{"a":"5558CD85BED3","b":"5558CBBA8000","o":"1CB3ED3","s":"_ZZN5mongo15unique_functionIFvvEE8makeImplIZNS_9transport15ServiceExecutor8scheduleENS0_IFvNS_6StatusEEEEEUlvE_EEDaOT_EN12SpecificImpl4callEv","s+":"33"},{"a":"5558CF217B7B","b":"5558CBBA8000","o":"366FB7B","s":"_ZZN5mongo15unique_functionIFvvEE8makeImplIZNS_9transport26ServiceExecutorSynchronous12scheduleTaskES2_NS4_15ServiceExecutor13ScheduleFlagsEEUlvE0_EEDaOT_EN12SpecificImpl4callEv","s+":"BB"},{"a":"5558CF21BA2C","b":"5558CBBA8000","o":"3673A2C","s":"_ZZN5mongo15unique_functionIFvvEE8makeImplIZNS_25launchServiceWorkerThreadES2_EUlvE2_EEDaOT_EN12SpecificImpl4callEv","s+":"5C"},{"a":"5558CF21BA9C","b":"5558CBBA8000","o":"3673A9C","s":"_ZN5mongo12_GLOBAL__N_17runFuncEPv","s+":"1C"},{"a":"7F68119382DE","b":"7F6811930000","o":"82DE","s":"start_thread","s+":"FE"},{"a":"7F6811668A63","b":"7F681156C000","o":"FCA63","s":"clone","s+":"43"}],"processInfo":{"mongodbVersion":"5.0.9","gitVersion":"6f7dae919422dcd7f4892c10ff20cdc721ad00e6","compiledModules":[],"uname":{"sysname":"Linux","release":"4.18.0-80.el8.x86_64","version":"#1 SMP Wed Mar 13 12:02:46 UTC 2019","machine":"x86_64"},"somap":[{"b":"5558CBBA8000","elfType":3,"buildId":"3BED389EA1C9CD318D4E429D1725CF09CF9A6761"},{"b":"7F6811930000","path":"/lib64/libpthread.so.0","elfType":3,"buildId":"5326B8728FA01B7149DAC943100F1405533E76CE"}]}}}}
{"t":{"$date":"2024-05-27T12:18:16.067+07:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"conn8989571","msg":"Frame","attr":{"frame":{"a":"5558CFA65E75","b":"5558CBBA8000","o":"3EBDE75","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.357","s+":"215"}}}
{"t":{"$date":"2024-05-27T12:18:16.067+07:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"conn8989571","msg":"Frame","attr":{"frame":{"a":"5558CFA68909","b":"5558CBBA8000","o":"3EC0909","s":"_ZN5mongo15printStackTraceEv","s+":"29"}}}
{"t":{"$date":"2024-05-27T12:18:16.067+07:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"conn8989571","msg":"Frame","attr":{"frame":{"a":"5558CFA6109C","b":"5558CBBA8000","o":"3EB909C","s":"abruptQuitWithAddrSignal","s+":"EC"}}}
{"t":{"$date":"2024-05-27T12:18:16.067+07:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"conn8989571","msg":"Frame","attr":{"frame":{"a":"5558CF08DE14","b":"5558CBBA8000","o":"34E5E14","s":"_ZL16WasmFaultHandleriP9siginfo_tPv","s+":"A4"}}}
{"t":{"$date":"2024-05-27T12:18:16.067+07:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"conn8989571","msg":"Frame","attr":{"frame":{"a":"7F6811942D80","b":"7F6811930000","o":"12D80","s":"funlockfile","s+":"50"}}}