#3367 smf: fix issue when remove node inv_id between threads

[Nguyen Huynh Tai]

The core dump was raised when executing procedures in parallel and invoking the callback

[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/usr/local/lib/opensaf/osafsmfd --tracemask=0xffffffff'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  SmfCallback::send_callback_msg (this=0x7f240400acf0, phase=<optimized out>, step_dn=...) at src/smf/smfd/SmfCallback.cc:331
[Current thread is 1 (Thread 0x7f242056ab00 (LWP 1217))]

Thread 1 (Thread 0x7f242056ab00 (LWP 1217)):
#0  SmfCallback::send_callback_msg (this=0x7f240400acf0, phase=<optimized out>, step_dn=...) at src/smf/smfd/SmfCallback.cc:331
        dn = "safSmfStep=0001,safSmfProc=RollingSUfromNoExistingSG_ParallellExecution2,safSmfCampaign=rollingNodes,safApp=safSmfService"
        cbk_mbx = 0x7f241400c7bc
        smfsv_evt = {next = 0x0, type = SMFSV_EVT_TYPE_SMFND, cb_hdl = 0, mds_ctxt = {length = 0 '\000', data = '\000' <repeats 11 times>}, fr_dest = 15, fr_svc = 30, fr_node_id = 0, rcvd_prio = 0, info = {smfd = {type = SMFD_EVT_CBK_RSP, event = {mds_info = {change = NCSMDS_NONE, dest = 1, svc_id = 0, node_id = 1634926713, rem_svc_pvt_ver = 102 'f', role = 1885697107}, cmd_rsp = {result = 0}, cbk_rsp = {evt_type = SMF_CLBK_EVT, evt = {cbk_evt = {inv_id = 1, scope_id = 0, object_name = {_opaque = {121, 24947, 21350, 26221, 29779, 28773, 12349, 12336, 11313, 24947, 21350, 26221, 29264, 25455, 21053, 27759, 26988, 26478, 21843, 29286, 28015, 28494, 30789, 29545, 26996, 26478, 18259, 20575, 29281, 27745, 25964, 27756, 30789, 25445, 29813, 28521, 12910, 29484, 26209, 27987, 17254, 28001, 24944, 26473, 15726, 28530, 27756, 28265, 20071, 25711, 29541, 29484, 26209, 28737, 15728, 24947, 21350, 26221, 25939, 30322, 25449, 101, 0 <repeats 67 times>}}, camp_phase = SA_SMF_UPGRADE, cbk_label = {labelSize = 25, label = 0x7f23fc0052f0 "OsafSmfCbkUtil-UpgradeCmd"}, params_len = 31, params = 0x7f23fc002dd0 "logger NTAI-firstStepBeforeLock"}, resp_evt = {inv_id = 1, err = 0}}, next = 0x0}}}, smfnd = {type = SMFND_EVT_CBK_RSP, event = {mds_info = {change = NCSMDS_NONE, dest = 1, svc_id = 0, node_id = 1634926713, rem_svc_pvt_ver = 102 'f', role = 1885697107}, cmd_req = {cmd_len = 0, cmd = 0x1 <error: Cannot access memory at address 0x1>}, cbk_req_rsp = {evt_type = SMF_CLBK_EVT, evt = {cbk_evt = {inv_id = 1, scope_id = 0, object_name = {_opaque = {121, 24947, 21350, 26221, 29779, 28773, 12349, 12336, 11313, 24947, 21350, 26221, 29264, 25455, 21053, 27759, 26988, 26478, 21843, 29286, 28015, 28494, 30789, 29545, 26996, 26478, 18259, 20575, 29281, 27745, 25964, 27756, 30789, 25445, 29813, 28521, 12910, 29484, 26209, 27987, 17254, 28001, 24944, 26473, 15726, 28530, 27756, 28265, 20071, 25711, 29541, 29484, 26209, 28737, 15728, 24947, 21350, 26221, 25939, 30322, 25449, 101, 0 <repeats 67 times>}}, camp_phase = SA_SMF_UPGRADE, cbk_label = {labelSize = 25, label = 0x7f23fc0052f0 "OsafSmfCbkUtil-UpgradeCmd"}, params_len = 31, params = 0x7f23fc002dd0 "logger NTAI-firstStepBeforeLock"}, resp_evt = {inv_id = 1, err = 0}}, next = 0x0}, cmd_req_asynch = {timeout = 0, cmd_len = 0, cmd = 0x1 <error: Cannot access memory at address 0x1>}}}, smfa = {type = SMFA_EVT_CBK, event = {mds_info = {change = NCSMDS_NONE, dest = 1, svc_id = 0, node_id = 1634926713, rem_svc_pvt_ver = 102 'f', role = 1885697107}, cbk_req_rsp = {evt_type = SMF_CLBK_EVT, evt = {cbk_evt = {inv_id = 1, scope_id = 0, object_name = {_opaque = {121, 24947, 21350, 26221, 29779, 28773, 12349, 12336, 11313, 24947, 21350, 26221, 29264, 25455, 21053, 27759, 26988, 26478, 21843, 29286, 28015, 28494, 30789, 29545, 26996, 26478, 18259, 20575, 29281, 27745, 25964, 27756, 30789, 25445, 29813, 28521, 12910, 29484, 26209, 27987, 17254, 28001, 24944, 26473, 15726, 28530, 27756, 28265, 20071, 25711, 29541, 29484, 26209, 28737, 15728, 24947, 21350, 26221, 25939, 30322, 25449, 101, 0 <repeats 67 times>}}, camp_phase = SA_SMF_UPGRADE, cbk_label = {labelSize = 25, label = 0x7f23fc0052f0 "OsafSmfCbkUtil-UpgradeCmd"}, params_len = 31, params = 0x7f23fc002dd0 "logger NTAI-firstStepBeforeLock"}, resp_evt = {inv_id = 1, err = 0}}, next = 0x0}}}}}
        evt = <optimized out>
        mds_info = {i_mds_hdl = 65551, i_svc_id = 30, i_op = MDS_SEND, info = {svc_install = {i_yr_svc_hdl = 139793138097216, i_install_scope = 31, i_svc_cb = 0xa, o_dest = 4, o_anc = 0, i_mds_q_ownership = false, o_sel_obj = {raise_obj = 0, rmv_obj = 0}, i_mds_svc_pvt_ver = 0 '\000', i_fail_no_active_sends = false, i_msg_loss_indication = false}, svc_uninstall = {i_msg_free_cb = 0x7f2420569c40}, svc_subscribe = {i_scope = 542547008, i_num_svcs = 36 '$', i_svc_ids = 0x20000001f}, red_subscribe = {i_scope = 542547008, i_num_svcs = 36 '$', i_svc_ids = 0x20000001f}, svc_cancel = {i_num_svcs = 64 '@', i_svc_ids = 0x20000001f}, svc_sys_subscribe = {i_evt_map = 542547008}, svc_send = {i_msg = 0x7f2420569c40, i_to_svc = 31, i_priority = MDS_SEND_PRIORITY_MEDIUM, i_sendtype = MDS_SENDTYPE_BCAST, info = {snd = {i_to_dest = 4}, sndrsp = {i_to_dest = 4, i_time_to_wait = 0, o_rsp = 0x0, buff = 0x0, len = 0, o_msg_fmt_ver = 0}, sndrack = {i_sender_dest = 4, i_time_to_wait = 0, i_msg_ctxt = {length = 0 '\000', data = '\000' <repeats 11 times>}}, sndack = {i_to_dest = 4, i_time_to_wait = 0}, rsp = {i_sender_dest = 4, i_msg_ctxt = {length = 0 '\000', data = '\000' <repeats 11 times>}}, red = {i_to_vdest = 4, i_to_anc = 0}, redrsp = {i_to_vdest = 4, i_to_anc = 0, i_time_to_wait = 0, o_rsp = 0x0, buff = 0x0, len = 0, o_msg_fmt_ver = 0}, redrack = {i_to_vdest = 4, i_to_anc = 0, i_time_to_wait = 0, i_msg_ctxt = {length = 0 '\000', data = '\000' <repeats 11 times>}}, redack = {i_to_vdest = 4, i_to_anc = 0, i_time_to_wait = 0}, rrsp = {i_to_dest = 4, i_to_anc = 0, i_msg_ctxt = {length = 0 '\000', data = '\000' <repeats 11 times>}}, bcast = {i_bcast_scope = NCSMDS_SCOPE_NONE}, rbcast = {i_bcast_scope = NCSMDS_SCOPE_NONE}}}, svc_direct_send = {i_direct_buff = 0x7f2420569c40 "", i_direct_buff_len = 31, i_to_svc = 2, i_priority = 10, i_sendtype = MDS_SENDTYPE_SND, i_msg_fmt_ver = 4, info = {snd = {i_to_dest = 0}, sndrsp = {i_to_dest = 0, i_time_to_wait = 0, o_rsp = 0x0, buff = 0x0, len = 0, o_msg_fmt_ver = 0}, sndrack = {i_sender_dest = 0, i_time_to_wait = 0, i_msg_ctxt = {length = 0 '\000', data = '\000' <repeats 11 times>}}, sndack = {i_to_dest = 0, i_time_to_wait = 0}, rsp = {i_sender_dest = 0, i_msg_ctxt = {length = 0 '\000', data = '\000' <repeats 11 times>}}, red = {i_to_vdest = 0, i_to_anc = 0}, redrsp = {i_to_vdest = 0, i_to_anc = 0, i_time_to_wait = 0, o_rsp = 0x0, buff = 0x0, len = 0, o_msg_fmt_ver = 0}, redrack = {i_to_vdest = 0, i_to_anc = 0, i_time_to_wait = 0, i_msg_ctxt = {length = 0 '\000', data = '\000' <repeats 11 times>}}, redack = {i_to_vdest = 0, i_to_anc = 0, i_time_to_wait = 0}, rrsp = {i_to_dest = 0, i_to_anc = 0, i_msg_ctxt = {length = 0 '\000', data = '\000' <repeats 11 times>}}, bcast = {i_bcast_scope = 0}, rbcast = {i_bcast_scope = 0}}}, retrieve_msg = {i_dispatchFlags = 542547008}, chg_role = {new_role = 542547008}, query_dest = {i_dest = 139793138097216, i_svc_id = 31, i_query_for_role = 2, info = {query_for_anc = {i_vdest_rl = 10, o_anc = 4}, query_for_role = {i_anc = 10, o_vdest_rl = 4}}, o_local = false, o_node_id = 0, o_adest = 0}, query_pwe = {o_pwe_id = 40000, o_absolute = 86, info = {abs_info = {o_adest = 8589934623}, virt_info = {o_vdest = 8589934623, o_anc = 10, o_role = 4}}}, subscribe_node = {i_dummy = 542547008}, unsubscribe_node = {i_dummy = 542547008}}}
        fds = {{fd = 53, events = 1, revents = 1}}
        temp = 0x7f240800a210
        new_inv_id = 0x7f23fc00f190
        rsp_evt = <optimized out>
        rc = <optimized out>
        ais_err = SA_AIS_OK
        inv_id_sent = 1
        t_ = {trace_leave_called = false, file_ = 0x55ec3be7fb8c "src/smf/smfd/SmfCallback.cc", function_ = 0x55ec3be7fba8 "send_callback_msg"}
        __FUNCTION__ = "send_callback_msg"
        mbx_fd = <optimized out>
#1  0x000055ec3bde36a4 in SmfCallback::execute (this=this@entry=0x7f240400acf0, step_dn="safSmfStep=0001,safSmfProc=RollingSUfromNoExistingSG_ParallellExecution2,safSmfCampaign=rollingNodes,safApp=safSmfService") at src/smf/smfd/SmfCallback.cc:78
        t_ = {trace_leave_called = false, file_ = 0x55ec3be7fb8c "src/smf/smfd/SmfCallback.cc", function_ = 0x55ec3be7fc51 "execute"}
        __FUNCTION__ = "execute"
        rc = <optimized out>
#2  0x000055ec3be6494b in SmfUpgradeStep::checkAndInvokeCallback (this=<optimized out>, callbackList=std::__cxx11::list = {...}, camp_phase=camp_phase@entry=1) at src/smf/smfd/SmfUpgradeStep.cc:2513
        stepCount = <optimized out>
        cbkElem = @0x7f2414017aa0: 0x7f240400acf0
        __for_range = std::__cxx11::list = {[0] = 0x7f240400acf0}
        __for_begin = <optimized out>
        __for_end = <optimized out>
        stepDn = "safSmfStep=0001,safSmfProc=RollingSUfromNoExistingSG_ParallellExecution2,safSmfCampaign=rollingNodes,safApp=safSmfService"
        rc = <optimized out>
        iter = <optimized out>
        t_ = {trace_leave_called = false, file_ = 0x55ec3be94948 "src/smf/smfd/SmfUpgradeStep.cc", function_ = 0x55ec3be9645e "checkAndInvokeCallback"}
        __FUNCTION__ = "checkAndInvokeCallback"
        procSteps = std::vector of length 5, capacity 8 = {0x7f23fc0039c0, 0x7f23fc00d020, 0x7f23fc00d560, 0x7f23fc00d730, 0x7f23fc00da40}
#3  0x000055ec3be2f373 in SmfStepTypeAuLock::execute (this=0x7f241400cab0) at src/smf/smfd/SmfStepTypes.cc:323
        cbkList = std::__cxx11::list = {[0] = 0x7f240400acf0}
        t_ = {trace_leave_called = false, file_ = 0x55ec3be8d580 "src/smf/smfd/SmfStepTypes.cc", function_ = 0x55ec3be7fc51 "execute"}
        __FUNCTION__ = "execute"
#4  0x000055ec3be2d238 in SmfStepStateExecuting::execute (this=0x7f23fc004d70, i_step=0x7f23fc0039c0) at src/smf/smfd/SmfStepState.cc:221
        t_ = {trace_leave_called = false, file_ = 0x55ec3be8ce0e "src/smf/smfd/SmfStepState.cc", function_ = 0x55ec3be7fc51 "execute"}
        __FUNCTION__ = "execute"
        stepType = <optimized out>
#5  0x000055ec3be63eff in SmfUpgradeStep::execute (this=0x7f23fc0039c0) at src/smf/smfd/SmfUpgradeStep.cc:2440
        stepResult = <optimized out>
        t_ = {trace_leave_called = false, file_ = 0x55ec3be94948 "src/smf/smfd/SmfUpgradeStep.cc", function_ = 0x55ec3be7fc51 "execute"}
        __FUNCTION__ = "execute"
#6  0x000055ec3be24db6 in SmfProcStateExecuting::executeStep (this=0x7f2408002c50, i_proc=0x7f240400b120) at src/smf/smfd/SmfProcState.cc:357
        stepResult = <optimized out>
        elem = @0x7f23fc002c00: 0x7f23fc0039c0
        __for_range = std::vector of length 5, capacity 8 = {0x7f23fc0039c0, 0x7f23fc00d020, 0x7f23fc00d560, 0x7f23fc00d730, 0x7f23fc00da40}
        __for_begin = <optimized out>
        __for_end = <optimized out>
        t_ = {trace_leave_called = false, file_ = 0x55ec3be8baf5 "src/smf/smfd/SmfProcState.cc", function_ = 0x55ec3be8bb12 "executeStep"}
        __FUNCTION__ = "executeStep"
        procSteps = std::vector of length 5, capacity 8 = {0x7f23fc0039c0, 0x7f23fc00d020, 0x7f23fc00d560, 0x7f23fc00d730, 0x7f23fc00da40}
#7  0x000055ec3be47844 in SmfUpgradeProcedure::execute (this=0x7f240400b120) at src/smf/smfd/SmfUpgradeProcedure.cc:4378
        t_ = {trace_leave_called = false, file_ = 0x55ec3be90de0 "src/smf/smfd/SmfUpgradeProcedure.cc", function_ = 0x55ec3be7fc51 "execute"}
        __FUNCTION__ = "execute"
        procResult = <optimized out>
#8  0x000055ec3be278d4 in SmfProcedureThread::processEvt (this=0x7f241400c7b0) at src/smf/smfd/SmfProcedureThread.cc:617
        evt = 0x7f2414005240
        procResult = SMF_PROC_DONE
#9  0x000055ec3be27a98 in SmfProcedureThread::handleEvents (this=this@entry=0x7f241400c7b0) at src/smf/smfd/SmfProcedureThread.cc:697
        ret = <optimized out>
        mbx_fd = <optimized out>
        fds = {{fd = 51, events = 1, revents = 1}}
#10 0x000055ec3bdd4e42 in SmfProcedureThread::main (this=0x7f241400c7b0) at src/smf/smfd/SmfProcedureThread.cc:717
        t_ = {trace_leave_called = false, file_ = 0x55ec3be8bd80 "src/smf/smfd/SmfProcedureThread.cc", function_ = 0x55ec3be8374c "main"}
        __FUNCTION__ = "main"
#11 0x000055ec3bdd4efd in SmfProcedureThread::main (info=0x7f241400c7b0) at src/smf/smfd/SmfProcedureThread.cc:57
        self = 0x7f241400c7b0
#12 0x00007f2422ee9609 in start_thread (arg=<optimized out>) at pthread_create.c:477
        ret = <optimized out>
        pd = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139793138100992, -5628119038026148695, 139793138366654, 139793138366655, 139793138366816, 139793138099008, 5741147383385199785, 5741142093694538921}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
        not_first_call = 0
#13 0x00007f2422e0e353 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

[Nguyen Huynh Tai]

commit bfced2a4c511c327f6c97a0d6a057d956ebfdccd (HEAD -> develop, origin/develop)
Author: tai.h.nguyen tai.h.nguyen@endava.com
Date: Fri Mar 21 13:07:40 2025 +0700

smf: Fix osafsmfd coredump [#3367]

The node 'inv_id' will be removed in the main thread when the callbacks
are executed successfully. It will then continue to be checked and removed
in the procedure thread if it has not been removed yet. A core dump may occur
if procedures executing in parallel access an invalid pointer to 'inv_id'
after it has already been removed by the main thread.

To avoid this, smfd must avoid removing the node in the sender thread until
the callback has successfully completed and the node has been deleted from
the main thread. The sender thread is only removed when transmitting fails.