[[ Bug 23240 ]] Fix engine lockup after error in modal stack

livecodeian · livecodeian · commit 330a959ded27 · 2021-06-30T09:19:16.000+01:00
This patch fixes an infinite loop that occurs when the "traceError"
message is sent and not handled. By returning false from MCB_error when
the message is not handled, the while loops in
MCExecContext::TryToEvaluate* methods will exit
diff --git a/engine/src/debug.cpp b/engine/src/debug.cpp
@@ -192,8 +192,10 @@ void MCB_setmsg(MCExecContext &ctxt, MCStringRef p_string)
     
 }
 
-void MCB_message(MCExecContext &ctxt, MCNameRef mess, MCParameter *p)
+Exec_stat MCB_message(MCExecContext &ctxt, MCNameRef mess, MCParameter *p)
 {
+	Exec_stat t_stat = ES_NOT_HANDLED;
+
 	Boolean exitall = MCexitall;
 	MCSaveprops sp;
 	MCU_saveprops(sp);
@@ -208,8 +210,8 @@ void MCB_message(MCExecContext &ctxt, MCNameRef mess, MCParameter *p)
 	/* UNCHECKED */ MCVariable::createwithname(MCNAME("MCdebugresult"), MCresult);
 	MCtracereturn = False;
 	MCtraceabort = False;
-    
-    MCExecResultMode t_oldresultmode = MCresultmode;
+
+	MCExecResultMode t_oldresultmode = MCresultmode;
 
 	Boolean oldcheck;
 	oldcheck = MCcheckstack;
@@ -220,19 +222,21 @@ void MCB_message(MCExecContext &ctxt, MCNameRef mess, MCParameter *p)
 
 	// OK-2008-11-28: [[Bug 7491]] - It seems that using the "send" parameter causes problems with the MetaCard debugger
 	// So instead of doing that, I've added a new optional parameter to MCObject::send, called p_force, and used this instead.
-	if (ctxt.GetObject() -> message(mess, p, True, False, True) == ES_NORMAL)
+
+	t_stat = ctxt.GetObject() -> message(mess, p, True, False, True);
+	if (t_stat == ES_NORMAL)
 	{
 		MCcheckstack = oldcheck;
 		
-        while (!MCtracereturn)
+		while (!MCtracereturn)
 		{
 			MCU_resetprops(True);
 			MCscreen->wait(REFRESH_INTERVAL, True, True);
 		}
 		
-        if (!MCtracedobject)
+		if (!MCtracedobject)
 			MCtracedobject = ctxt.GetObject();
-        
+
 		if (MCtraceabort)
 		{
 			MCtraceabort = False;
@@ -241,19 +245,22 @@ void MCB_message(MCExecContext &ctxt, MCNameRef mess, MCParameter *p)
 		else
 			if (MCtracestackptr)
 				MCtrace = True;
-	 }
-	 MCcheckstack = oldcheck;
-	 MCtracewindow = NULL;
-	 delete MCresult;
-	 MCresult = oldresult;
-	 MCU_restoreprops(sp);
-	 MCexitall = exitall;
-     MCresultmode = t_oldresultmode;
-
+	}
+	MCcheckstack = oldcheck;
+	MCtracewindow = NULL;
+	delete MCresult;
+	MCresult = oldresult;
+	MCU_restoreprops(sp);
+	MCexitall = exitall;
+	MCresultmode = t_oldresultmode;
+
+	return t_stat;
 }
 
-void MCB_prepmessage(MCExecContext &ctxt, MCNameRef mess, uint2 line, uint2 pos, uint2 id, MCStringRef p_info)
+Exec_stat MCB_prepmessage(MCExecContext &ctxt, MCNameRef mess, uint2 line, uint2 pos, uint2 id, MCStringRef p_info)
 {
+	Exec_stat t_stat = ES_NOT_HANDLED;
+
 	Boolean added = False;
 	if (MCnexecutioncontexts < MAX_CONTEXTS)
 	{
@@ -276,22 +283,26 @@ void MCB_prepmessage(MCExecContext &ctxt, MCNameRef mess, uint2 line, uint2 pos,
 		ctxt.GetObject()->names(P_LONG_ID, &t_val);
 		MCeerror->add(EE_OBJECT_NAME, 0, 0, *t_val);
 
-        MCAutoStringRef t_error;
-        MCeerror -> copyasstringref(&t_error);
-        p4.setvalueref_argument(*t_error);
+		MCAutoStringRef t_error;
+		MCeerror -> copyasstringref(&t_error);
+		p4.setvalueref_argument(*t_error);
 	}
 	else if (!MCStringIsEmpty(p_info))
 	{
 		p3.setnext(&p4);
 		p4.setvalueref_argument(p_info);
 	}
-    MCDeletedObjectsFreezePool();
-    MCB_message(ctxt, mess, &p1);
-    MCDeletedObjectsThawPool();
+
+	MCDeletedObjectsFreezePool();
+	t_stat = MCB_message(ctxt, mess, &p1);
+	MCDeletedObjectsThawPool();
+
 	if (id != 0)
 		MCeerror->clear();
 	if (added)
 		MCnexecutioncontexts--;
+
+	return t_stat;
 }
 
 void MCB_trace(MCExecContext &ctxt, uint2 line, uint2 pos)
@@ -337,19 +348,23 @@ bool s_in_trace_error = false;
 
 bool MCB_error(MCExecContext &ctxt, uint2 line, uint2 pos, uint2 id)
 {
-    // An unhandled error has been thrown - end all modal loops
-    MCscreen->breakModalLoops();
-    
-    // OK-2009-03-25: [[Bug 7517]] - The crash described in this bug report is probably caused by a stack overflow. This overflow is due to
+	// An unhandled error has been thrown - end all modal loops
+	MCscreen->breakModalLoops();
+
+	// OK-2009-03-25: [[Bug 7517]] - The crash described in this bug report is probably caused by a stack overflow. This overflow is due to
 	// errors being thrown in the IDE (or in this case GLX2) component of the debugger. This should prevent traceError from recursing.
 	if (s_in_trace_error)
 		return false;
-	
+
+
+	// MCB_error should not be called again if 'traceError' was not handled
+	bool t_handled;
 	s_in_trace_error = true;
-	MCB_prepmessage(ctxt, MCM_trace_error, line, pos, id);
-	MCerrorlock++; // suppress errors as stack unwinds
+	t_handled = MCB_prepmessage(ctxt, MCM_trace_error, line, pos, id) == ES_NORMAL;
 	s_in_trace_error = false;
-    return true;
+	MCerrorlock++; // suppress errors as stack unwinds
+
+	return t_handled;
 }
 
 void MCB_done(MCExecContext &ctxt)
diff --git a/engine/src/debug.h b/engine/src/debug.h
@@ -102,8 +102,8 @@ extern MCNameRef MClogmessage;
 struct MCExecValue;
 
 extern void MCB_setmsg(MCExecContext &ctxt, MCStringRef p_string);
-extern void MCB_message(MCExecContext &ctxt, MCNameRef message, MCParameter *p);
-extern void MCB_prepmessage(MCExecContext &ctxt, MCNameRef message, uint2 line, uint2 pos, uint2 id, MCStringRef p_info = kMCEmptyString);
+extern Exec_stat MCB_message(MCExecContext &ctxt, MCNameRef message, MCParameter *p);
+extern Exec_stat MCB_prepmessage(MCExecContext &ctxt, MCNameRef message, uint2 line, uint2 pos, uint2 id, MCStringRef p_info = kMCEmptyString);
 extern void MCB_break(MCExecContext &ctxt, uint2 line, uint2 pos);
 extern void MCB_trace(MCExecContext &ctxt, uint2 line, uint2 pos);
 extern bool MCB_error(MCExecContext &ctxt, uint2 line, uint2 pos, uint2 id);
