Repair database failed due to "Yielding a strong global MODE_X/MODE_S lock is forbidden"

Hi team,

I run the command “mongod --dbpath /data/db --repair” and the repair failed. The error log is below

mongo:6.0.10 in docker container

{
    "t": {
        "$date": "2024-01-19T00:03:37.010+00:00"
    },
    "s": "I",
    "c": "-",
    "id": 51773,
    "ctx": "initandlisten",
    "msg": "progress meter",
    "attr": {
        "name": "Index Build: scanning collection",
        "done": 309100,
        "total": 319320,
        "percent": 96
    }
}

{
    "t": {
        "$date": "2024-01-19T00:03:37.693+00:00"
    },
    "s": "F",
    "c": "ASSERT",
    "id": 23081,
    "ctx": "initandlisten",
    "msg": "Invariant failure",
    "attr": {
        "expr": "!(_modeForTicket == MODE_S || _modeForTicket == MODE_X)",
        "msg": "Yielding a strong global MODE_X/MODE_S lock is forbidden",
        "file": "src/mongo/db/concurrency/lock_state.cpp",
        "line": 743
    }
}

{
    "t": {
        "$date": "2024-01-19T00:03:37.693+00:00"
    },
    "s": "F",
    "c": "ASSERT",
    "id": 23082,
    "ctx": "initandlisten",
    "msg": "\n\n***aborting after invariant() failure\n\n"
}

{
    "t": {
        "$date": "2024-01-19T00:03:37.693+00:00"
    },
    "s": "F",
    "c": "CONTROL",
    "id": 6384300,
    "ctx": "initandlisten",
    "msg": "Writing fatal message",
    "attr": {
        "message": "Got signal: 6 (Aborted).\n"
    }
}

{
    "t": {
        "$date": "2024-01-19T00:03:37.834+00:00"
    },
    "s": "I",
    "c": "CONTROL",
    "id": 31380,
    "ctx": "initandlisten",
    "msg": "BACKTRACE",
    "attr": {
        "bt": {
            "backtrace": [
                {
                    "a": "5579496600E4",
                    "b": "557944827000",
                    "o": "4E390E4",
                    "s": "_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.362",
                    "C": "mongo::stack_trace_detail::(anonymous namespace)::printStackTraceImpl(mongo::stack_trace_detail::(anonymous namespace)::Options const&, mongo::StackTraceSink*) [clone .constprop.362]",
                    "s+": "1F4"
                },
                {
                    "a": "557949662629",
                    "b": "557944827000",
                    "o": "4E3B629",
                    "s": "_ZN5mongo15printStackTraceEv",
                    "C": "mongo::printStackTrace()",
                    "s+": "29"
                },
                {
                    "a": "55794965C306",
                    "b": "557944827000",
                    "o": "4E35306",
                    "s": "abruptQuit",
                    "s+": "66"
                },
                {
                    "a": "7F54C6869520",
                    "b": "7F54C6827000",
                    "o": "42520",
                    "s": "__sigaction",
                    "s+": "50"
                },
                {
                    "a": "7F54C68BDA7C",
                    "b": "7F54C6827000",
                    "o": "96A7C",
                    "s": "pthread_kill",
                    "s+": "12C"
                },
                {
                    "a": "7F54C6869476",
                    "b": "7F54C6827000",
                    "o": "42476",
                    "s": "raise",
                    "s+": "16"
                },
                {
                    "a": "7F54C684F7F3",
                    "b": "7F54C6827000",
                    "o": "287F3",
                    "s": "abort",
                    "s+": "D3"
                },
                {
                    "a": "55794673B610",
                    "b": "557944827000",
                    "o": "1F14610",
                    "s": "_ZN5mongo22invariantFailedWithMsgEPKcRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES1_j",
                    "C": "mongo::invariantFailedWithMsg(char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, char const*, unsigned int)",
                    "s+": "113"
                },
                {
                    "a": "55794900DCFE",
                    "b": "557944827000",
                    "o": "47E6CFE",
                    "s": "_ZN5mongo10LockerImpl22saveLockStateAndUnlockEPNS_6Locker12LockSnapshotE",
                    "C": "mongo::LockerImpl::saveLockStateAndUnlock(mongo::Locker::LockSnapshot*)",
                    "s+": "43E"
                },
                {
                    "a": "5579482A5A52",
                    "b": "557944827000",
                    "o": "3A7EA52",
                    "s": "_ZNK5mongo27SortedDataIndexAccessMethod15BulkBuilderImpl6_yieldEPNS_16OperationContextEPKNS_9YieldableERKNS_15NamespaceStringE",
                    "C": "mongo::SortedDataIndexAccessMethod::BulkBuilderImpl::_yield(mongo::OperationContext*, mongo::Yieldable const*, mongo::NamespaceString const&) const",
                    "s+": "82"
                },
                {
                    "a": "5579482A6162",
                    "b": "557944827000",
                    "o": "3A7F162",
                    "s": "_ZN5mongo27SortedDataIndexAccessMethod15BulkBuilderImpl6commitEPNS_16OperationContextERKNS_13CollectionPtrEbiRKSt8functionIFNS_6StatusERKNS_9KeyString5ValueEEERKS7_IFS8_RKNS_8RecordIdEEE",
                    "C": "mongo::SortedDataIndexAccessMethod::BulkBuilderImpl::commit(mongo::OperationContext*, mongo::CollectionPtr const&, bool, int, std::function<mongo::Status (mongo::KeyString::Value const&)> const&, std::function<mongo::Status (mongo::RecordId const&)> const&)",
                    "s+": "642"
                },
                {
                    "a": "557947D684E4",
                    "b": "557944827000",
                    "o": "35414E4",
                    "s": "_ZN5mongo15MultiIndexBlock19dumpInsertsFromBulkEPNS_16OperationContextERKNS_13CollectionPtrERKSt8functionIFNS_6StatusERKNS_8RecordIdEEE",
                    "C": "mongo::MultiIndexBlock::dumpInsertsFromBulk(mongo::OperationContext*, mongo::CollectionPtr const&, std::function<mongo::Status (mongo::RecordId const&)> const&)",
                    "s+": "354"
                },
                {
                    "a": "557947A4C883",
                    "b": "557944827000",
                    "o": "3225883",
                    "s": "_ZN5mongo18IndexBuildsManager29startBuildingIndexForRecoveryEPNS_16OperationContextERKNS_13CollectionPtrERKNS_4UUIDENS_10RepairDataE",
                    "C": "mongo::IndexBuildsManager::startBuildingIndexForRecovery(mongo::OperationContext*, mongo::CollectionPtr const&, mongo::UUID const&, mongo::RepairData)",
                    "s+": "6E3"
                },
                {
                    "a": "5579479B5B53",
                    "b": "557944827000",
                    "o": "318EB53",
                    "s": "_ZN5mongo22IndexBuildsCoordinator27_runIndexRebuildForRecoveryEPNS_16OperationContextERNS_16CollectionWriterERKNS_4UUIDENS_10RepairDataE",
                    "C": "mongo::IndexBuildsCoordinator::_runIndexRebuildForRecovery(mongo::OperationContext*, mongo::CollectionWriter&, mongo::UUID const&, mongo::RepairData)",
                    "s+": "253"
                },
                {
                    "a": "5579479B606C",
                    "b": "557944827000",
                    "o": "318F06C",
                    "s": "_ZN5mongo22IndexBuildsCoordinator25rebuildIndexesForRecoveryEPNS_16OperationContextERKNS_15NamespaceStringERKSt6vectorINS_7BSONObjESaIS7_EERKNS_4UUIDENS_10RepairDataE",
                    "C": "mongo::IndexBuildsCoordinator::rebuildIndexesForRecovery(mongo::OperationContext*, mongo::NamespaceString const&, std::vector<mongo::BSONObj, std::allocator<mongo::BSONObj> > const&, mongo::UUID const&, mongo::RepairData)",
                    "s+": "8C"
                },
                {
                    "a": "557947886858",
                    "b": "557944827000",
                    "o": "305F858",
                    "s": "_ZN5mongo26rebuildIndexesOnCollectionEPNS_16OperationContextERKNS_13CollectionPtrERKSt6vectorINS_7BSONObjESaIS6_EENS_10RepairDataE",
                    "C": "mongo::rebuildIndexesOnCollection(mongo::OperationContext*, mongo::CollectionPtr const&, std::vector<mongo::BSONObj, std::allocator<mongo::BSONObj> > const&, mongo::RepairData)",
                    "s+": "88"
                },
                {
                    "a": "5579469A86D3",
                    "b": "557944827000",
                    "o": "21816D3",
                    "s": "_ZN5mongo26rebuildIndexesForNamespaceEPNS_16OperationContextERKNS_15NamespaceStringEPNS_13StorageEngineE",
                    "C": "mongo::rebuildIndexesForNamespace(mongo::OperationContext*, mongo::NamespaceString const&, mongo::StorageEngine*)",
                    "s+": "2F3"
                },
                {
                    "a": "5579469A9796",
                    "b": "557944827000",
                    "o": "2182796",
                    "s": "_ZN5mongo6repair16repairCollectionEPNS_16OperationContextEPNS_13StorageEngineERKNS_15NamespaceStringE",
                    "C": "mongo::repair::repairCollection(mongo::OperationContext*, mongo::StorageEngine*, mongo::NamespaceString const&)",
                    "s+": "F26"
                },
                {
                    "a": "5579469AA686",
                    "b": "557944827000",
                    "o": "2183686",
                    "s": "_ZN5mongo6repair14repairDatabaseEPNS_16OperationContextEPNS_13StorageEngineERKNS_18TenantDatabaseNameE",
                    "C": "mongo::repair::repairDatabase(mongo::OperationContext*, mongo::StorageEngine*, mongo::TenantDatabaseName const&)",
                    "s+": "306"
                },
                {
                    "a": "5579469A36F5",
                    "b": "557944827000",
                    "o": "217C6F5",
                    "s": "_ZN5mongo16startup_recovery25repairAndRecoverDatabasesEPNS_16OperationContextENS_13StorageEngine17LastShutdownStateE",
                    "C": "mongo::startup_recovery::repairAndRecoverDatabases(mongo::OperationContext*, mongo::StorageEngine::LastShutdownState)",
                    "s+": "C95"
                },
                {
                    "a": "557946967431",
                    "b": "557944827000",
                    "o": "2140431",
                    "s": "_ZN5mongo12_GLOBAL__N_114_initAndListenEPNS_14ServiceContextEi.isra.2072",
                    "C": "mongo::(anonymous namespace)::_initAndListen(mongo::ServiceContext*, int) [clone .isra.2072]",
                    "s+": "A81"
                },
                {
                    "a": "55794696A641",
                    "b": "557944827000",
                    "o": "2143641",
                    "s": "_ZN5mongo11mongod_mainEiPPc",
                    "C": "mongo::mongod_main(int, char**)",
                    "s+": "B61"
                },
                {
                    "a": "55794674BE7E",
                    "b": "557944827000",
                    "o": "1F24E7E",
                    "s": "main",
                    "s+": "E"
                },
                {
                    "a": "7F54C6850D90",
                    "b": "7F54C6827000",
                    "o": "29D90",
                    "s": "__libc_init_first",
                    "s+": "90"
                },
                {
                    "a": "7F54C6850E40",
                    "b": "7F54C6827000",
                    "o": "29E40",
                    "s": "__libc_start_main",
                    "s+": "80"
                },
                {
                    "a": "557946964EE5",
                    "b": "557944827000",
                    "o": "213DEE5",
                    "s": "_start",
                    "s+": "25"
                }
            ],
            "processInfo": {
                "mongodbVersion": "6.0.10",
                "gitVersion": "8e4b5670df9b9fe814e57cb5f3f8ee9407237b5a",
                "compiledModules": [],
                "uname": {
                    "sysname": "Linux",
                    "release": "6.1.29-0-virt",
                    "version": "#1-Alpine SMP PREEMPT_DYNAMIC Wed, 17 May 2023 14:22:15 +0000",
                    "machine": "x86_64"
                },
                "somap": [
                    {
                        "b": "557944827000",
                        "elfType": 3,
                        "buildId": "EEC9F2A4DFEBC9960E47B604D74B1B0D58A0DC49"
                    },
                    {
                        "b": "7F54C6827000",
                        "path": "/lib/x86_64-linux-gnu/libc.so.6",
                        "elfType": 3,
                        "buildId": "229B7DC509053FE4DF5E29E8629911F0C3BC66DD"
                    }
                ]
            }
        }
    },
    "tags": []
}

Because of the failure I couldn’t restart the mongodb
I’d like to know how to fix it. Thank you.

1 Like

Hi Leon,

Did you ever found a solution to this issue? I got one almost similar while trying to repair a DB.

Thank You,
Laurent.

Any fix about this issue ? We meet the same problem to repair the database. ( Our version is 6.0.15)

To reproduce the issue, just remove a big index file (about 380000 records), and then try repair the database.

The only way I found to work around is to remove the corresponding collection file as well. But in this way, the data of this collection will be totally lost.

I have reported an issue [SERVER-94183] Failed to rebuild index in repair mode due to the yielding lock issue - MongoDB Jira

And I found another workaround is to set the internal parameter internalIndexBuildBulkLoadYieldIterations to a large number, e.g. 2147483647

mongod --dbpath=… --repair --setParameter “internalIndexBuildBulkLoadYieldIterations=2147483647”