GP-3649 - DemangledObjects - separate lref/rref from pointerLeverls; fix MDMang function pointers indirection

Ghidra/Features/Base/src/main/java/ghidra/app/util/demangler/AbstractDemangledFunctionDefinitionDataType.java

@@ -227,13 +227,21 @@ public abstract class AbstractDemangledFunctionDefinitionDataType extends Demang
 
 		StringBuilder typeBuffer = new StringBuilder();
 		int pointerLevels = getPointerLevels();
-		if (pointerLevels > 0) {
+		if (pointerLevels > 0 || isReference() || isRValueReference()) {
 
 			addParentName(typeBuffer);
 
 			for (int i = 0; i < pointerLevels; ++i) {
 				typeBuffer.append(getTypeString());
 			}
+			// kludge for now... current type-emitting mechanism in DemangledObject hierarchy
+			//  lacks a lot... needs revamped.
+			if (isLValueReference()) {
+				typeBuffer.append(" &");
+			}
+			else if (isRValueReference()) {
+				typeBuffer.append(" &&");
+			}
 		}
 
 		if (!StringUtils.isBlank(typeBuffer)) {
@@ -331,7 +339,25 @@ public abstract class AbstractDemangledFunctionDefinitionDataType extends Demang
 			dt = fddt;
 		}
 
-		return new PointerDataType(dt, dataTypeManager);
+		// This was also totally wonked in terms of type-emitting.  Whole mangled type needs
+		//  gone through and revamped;  not sure it is good to have pointer levels or reference
+		//  information in a FunctionPointer... need to investigate more.
+		int numPointers = getPointerLevels();
+
+		for (int i = 0; i < numPointers; ++i) {
+			dt = PointerDataType.getPointer(dt, dataTypeManager);
+		}
+
+		if (isLValueReference()) {
+			// Placeholder in prep for more lref work
+			dt = PointerDataType.getPointer(dt, dataTypeManager);
+		}
+		else if (isRValueReference()) {
+			// Placeholder in prep for more rref work
+			dt = PointerDataType.getPointer(dt, dataTypeManager);
+		}
+
+		return dt;
 	}
 
 	private void setParameters(FunctionDefinitionDataType fddt, DataTypeManager dataTypeManager) {

Ghidra/Features/Base/src/main/java/ghidra/app/util/demangler/DemangledDataType.java

@@ -47,6 +47,7 @@ public class DemangledDataType extends DemangledType {
 
 	public static final String ARR_NOTATION = "[]";
 	public static final String REF_NOTATION = "&";
+	public static final String RIGHT_REF_NOTATION = "&&";
 	public static final String PTR_NOTATION = "*";
 
 	public static final String VOLATILE = "volatile";
@@ -100,7 +101,10 @@ public class DemangledDataType extends DemangledType {
 	private boolean isComplex;
 	private boolean isEnum;
 	private boolean isPointer64;
-	private boolean isReference;
+	// Cannot be both lref and rref.  Prior to C++11, we only had reference (& operator).
+	// This is now distinguished as left-value reference (l-value reference or lref), and there
+	// is now an additional right-value reference (r-value reference or rref) with the && operator.
+	private boolean isLValueReference;
 	private boolean isRValueReference;
 	private boolean isSigned;
 	private boolean isStruct;
@@ -208,13 +212,20 @@ public class DemangledDataType extends DemangledType {
 		}
 
 		int numPointers = getPointerLevels();
-		if (isReference()) {
-			numPointers++;
-		}
 
 		for (int i = 0; i < numPointers; ++i) {
 			dt = PointerDataType.getPointer(dt, dataTypeManager);
 		}
+
+		if (isLValueReference()) {
+			// Placeholder in prep for more lref work
+			dt = PointerDataType.getPointer(dt, dataTypeManager);
+		}
+		else if (isRValueReference()) {
+			// Placeholder in prep for more rref work
+			dt = PointerDataType.getPointer(dt, dataTypeManager);
+		}
+
 		return dt;
 	}
 
@@ -463,7 +474,13 @@ public class DemangledDataType extends DemangledType {
 	}
 
 	public void setReference() {
-		isReference = true;
+		setLValueReference();
+	}
+
+	public void setLValueReference() {
+		isLValueReference = true;
+		// Cannot be both
+		isRValueReference = false;
 	}
 
 	/**
@@ -471,6 +488,8 @@ public class DemangledDataType extends DemangledType {
 	 */
 	public void setRValueReference() {
 		isRValueReference = true;
+		// Cannot be both
+		isLValueReference = false;
 	}
 
 	public void setSigned() {
@@ -550,7 +569,15 @@ public class DemangledDataType extends DemangledType {
 	}
 
 	public boolean isReference() {
-		return isReference;
+		return isLValueReference() || isRValueReference();
+	}
+
+	public boolean isLValueReference() {
+		return isLValueReference;
+	}
+
+	public boolean isRValueReference() {
+		return isRValueReference;
 	}
 
 	public boolean isSigned() {
@@ -697,11 +724,11 @@ public class DemangledDataType extends DemangledType {
 			buffer.append(SPACE + PTR_NOTATION);
 		}
 
-		if (isReference) {
+		if (isLValueReference) {
 			buffer.append(SPACE + REF_NOTATION);
-			if (isRValueReference) {
+		}
-				buffer.append(REF_NOTATION); // &&
+		else if (isRValueReference) {
-			}
+			buffer.append(SPACE + RIGHT_REF_NOTATION);
 		}
 
 		// the order of __ptr64 and __restrict can vary--with fuzzing...

Ghidra/Features/GnuDemangler/src/main/java/ghidra/app/util/demangler/gnu/GnuDemanglerParser.java

@@ -870,7 +870,7 @@ public class GnuDemanglerParser {
 			}
 			else if (ch == '&') {
 				if (!dt.isReference()) {
-					dt.setReference();
+					dt.setLValueReference();
 				}
 				else {
 					dt.setRValueReference();
@@ -1286,7 +1286,7 @@ public class GnuDemanglerParser {
 			dt.incrementPointerLevels();
 		}
 		else if (type.equals("&")) {
-			dt.setReference();
+			dt.setLValueReference();
 		}
 		else {
 			throw new DemanglerParseException("Unexpected charater inside of parens: " + type);

Ghidra/Features/GnuDemangler/src/test/java/ghidra/app/util/demangler/GnuDemanglerParserTest.java

@@ -1927,8 +1927,18 @@ public class GnuDemanglerParserTest extends AbstractGenericTest {
 		assertName(object, name, "WebCore", "TextCodecICU");
 
 		String signature = object.getSignature(false);
+
+		// 20230719: Note that argument in the following function is not correctly parsed.
+		//  The argument is a non-pointer/ref reference to a function that returns void and
+		//  which takes the argument list shown.  Parser looks for double closing parenthesis,
+		//  but that is found on the last argument of the template parameter.  It actually needed
+		//  to find the last closing parethesis and know that there could be nested pairs.
+		// Modification on 20230719 has "(* &&)" emitted in place of "(*)" because of lref/rref
+		//  work in GP-3649, but the "&&" was found due to the closing parenthesis issue, which
+		//  already existed.
+		//  TODO: needs fixed
 		assertEquals(
-			"undefined WebCore::TextCodecICU::registerCodecs(void (*)(char const *,WTF::Function<std::__1::unique_ptr<WebCore::TextCodec,std::__1::default_delete<WebCore::TextCodec>> ()> &&))",
+			"undefined WebCore::TextCodecICU::registerCodecs(void (* &&)(char const *,WTF::Function<std::__1::unique_ptr<WebCore::TextCodec,std::__1::default_delete<WebCore::TextCodec>> ()> &&))",
 			signature);
 	}
 

Ghidra/Features/MicrosoftDemangler/src/test/java/ghidra/app/plugin/core/analysis/MicrosoftDemanglerAnalyzerTest.java

@@ -26,6 +26,7 @@ import ghidra.framework.options.Options;
 import ghidra.program.database.ProgramBuilder;
 import ghidra.program.database.ProgramDB;
 import ghidra.program.model.address.Address;
+import ghidra.program.model.data.ParameterDefinition;
 import ghidra.program.model.listing.*;
 import ghidra.program.model.symbol.SourceType;
 import ghidra.test.AbstractGhidraHeadedIntegrationTest;
@@ -97,6 +98,29 @@ public class MicrosoftDemanglerAnalyzerTest extends AbstractGhidraHeadedIntegrat
 		assertEquals("undefined InvokeHelperV(void)", function.getSignature().toString());
 	}
 
+	@Test
+	public void testApplyComplicatedFunctionSignatureHavingReference() throws Exception {
+
+		String mangled = "?f2@@YAP6AP6AHH@ZP6ADD@Z@ZAAP6AP6AHH@Z0@Z@Z";
+
+		Address addr = addr("0x110");
+		createSymbol(addr, mangled);
+
+		analyze();
+
+		FunctionManager fm = program.getFunctionManager();
+		Function function = fm.getFunctionAt(addr);
+		assertNotNull(function);
+		ParameterDefinition[] params = function.getSignature().getArguments();
+		assertEquals(1, params.length);
+		ParameterDefinition param = params[0];
+		assertEquals("_func__func_int_int_ptr__func_char_char_ptr * * param_1",
+			param.toString());
+		assertEquals(
+			"_func__func_int_int_ptr__func_char_char_ptr * f2(_func__func_int_int_ptr__func_char_char_ptr * * param_1)",
+			function.getSignature().toString());
+	}
+
 //==================================================================================================
 // Private Methods
 //==================================================================================================

Ghidra/Features/MicrosoftDmang/src/main/java/mdemangler/MDMangGhidra.java

@@ -452,7 +452,8 @@ public class MDMangGhidra extends MDMang {
 	}
 
 	private DemangledFunctionReference processDemangledFunctionReference(MDModifierType refType) {
-		if (!((refType instanceof MDReferenceType) || (refType instanceof MDDataRightReferenceType))) {
+		if (!((refType instanceof MDReferenceType) ||
+			(refType instanceof MDDataRightReferenceType))) {
 			return null; // Not planning on anything else yet.
 		}
 		DemangledFunctionReference functionReference =
@@ -610,18 +611,19 @@ public class MDMangGhidra extends MDMang {
 				// modifierType.getArrayString();
 				// resultDataType.setArray();
 				//Processing the referenced type (for Ghidra, and then setting attributes on it)
-				processDataType(resultDataType, (MDDataType) modifierType.getReferencedType());
+				DemangledDataType newResult =
-				resultDataType.incrementPointerLevels();
+					processDataType(resultDataType, (MDDataType) modifierType.getReferencedType());
+				newResult.incrementPointerLevels();
 				if (modifierType.getCVMod().isConst()) {
-					resultDataType.setConst();
+					newResult.setConst();
 				}
 				if (modifierType.getCVMod().isVolatile()) {
-					resultDataType.setVolatile();
+					newResult.setVolatile();
 				}
 				if (modifierType.getCVMod().isPointer64()) {
-					resultDataType.setPointer64();
+					newResult.setPointer64();
 				}
-				return resultDataType;
+				return newResult;
 			}
 			// TODO: fix. Following is a kludge because DemangledObject has no
 			// DemangledReference
@@ -648,18 +650,19 @@ public class MDMangGhidra extends MDMang {
 					return fr;
 				}
 				//Processing the referenced type (for Ghidra, and then setting attributes on it)
-				processDataType(resultDataType, (MDDataType) modifierType.getReferencedType());
+				DemangledDataType newResult =
-				resultDataType.setReference(); // Not sure if we should do/use this.
+					processDataType(resultDataType, (MDDataType) modifierType.getReferencedType());
+				newResult.setLValueReference();
 				if (modifierType.getCVMod().isConst()) {
-					resultDataType.setConst();
+					newResult.setConst();
 				}
 				if (modifierType.getCVMod().isVolatile()) {
-					resultDataType.setVolatile();
+					newResult.setVolatile();
 				}
 				if (modifierType.getCVMod().isPointer64()) {
-					resultDataType.setPointer64();
+					newResult.setPointer64();
 				}
-				return resultDataType;
+				return newResult;
 			}
 			// TODO: fix. Following is a kludge because DemangledObject has no DemangledReference
 			// with corresponding referencedType.
@@ -725,7 +728,7 @@ public class MDMangGhidra extends MDMang {
 				}
 				//Processing the referenced type (for Ghidra, and then setting attributes on it)
 				processDataType(resultDataType, (MDDataType) modifierType.getReferencedType());
-				resultDataType.setReference(); // Not sure if we should do/use this.
+				resultDataType.setRValueReference();
 				if (modifierType.getCVMod().isConst()) {
 					resultDataType.setConst();
 				}
@@ -808,7 +811,7 @@ public class MDMangGhidra extends MDMang {
 			}
 		}
 		else if (datatype instanceof MDReferenceType) {
-			resultDataType.setReference();
+			resultDataType.setLValueReference();
 		}
 		else if (datatype instanceof MDArrayBasicType) {
 			resultDataType.setArray(1);