Add new calling convention callbacks to Python API

Author: D0ntPanic

binaryninjaapi.h

@@ -17903,17 +17903,19 @@ namespace BinaryNinja {
 		    void* ctxt, const BNVariable* var, BNFunction* func, BNVariable* result);
 
 		static bool IsReturnTypeRegisterCompatibleCallback(void* ctxt, BNType* type);
-		static BNVariable GetIndirectReturnValueLocationCallback(void* ctxt);
+		static void GetIndirectReturnValueLocationCallback(void* ctxt, BNVariable* outVar);
 		static bool GetReturnedIndirectReturnValuePointerCallback(void* ctxt, BNVariable* outVar);
 
 		static bool IsArgumentTypeRegisterCompatibleCallback(void* ctxt, BNType* type);
 		static bool AreNonRegisterArgumentsIndirectCallback(void* ctxt);
 		static bool AreStackArgumentsNaturallyAlignedCallback(void* ctxt);
 
-		static BNCallLayout GetCallLayoutCallback(void* ctxt, BNReturnValue* returnValue, BNFunctionParameter* params,
-			size_t paramCount, bool hasPermittedRegs, uint32_t* permittedRegs, size_t permittedRegCount);
+		static void GetCallLayoutCallback(void* ctxt, BNReturnValue* returnValue, BNFunctionParameter* params,
+			size_t paramCount, bool hasPermittedRegs, uint32_t* permittedRegs, size_t permittedRegCount,
+			BNCallLayout* result);
 		static void FreeCallLayoutCallback(void* ctxt, BNCallLayout* layout);
-		static BNValueLocation GetReturnValueLocationCallback(void* ctxt, BNReturnValue* returnValue);
+		static void GetReturnValueLocationCallback(
+			void* ctxt, BNReturnValue* returnValue, BNValueLocation* outLocation);
 		static void FreeValueLocationCallback(void* ctxt, BNValueLocation* location);
 		static BNValueLocation* GetParameterLocationsCallback(void* ctxt, BNValueLocation* returnValue,
 			BNFunctionParameter* params, size_t paramCount, bool hasPermittedRegs, uint32_t* permittedRegs,

binaryninjacore.h

@@ -2918,7 +2918,7 @@ extern "C"
 		void (*getIncomingFlagValue)(void* ctxt, uint32_t flag, BNFunction* func, BNRegisterValue* result);
 
 		bool (*isReturnTypeRegisterCompatible)(void* ctxt, BNType* type);
-		BNVariable (*getIndirectReturnValueLocation)(void* ctxt);
+		void (*getIndirectReturnValueLocation)(void* ctxt, BNVariable* outVar);
 		bool (*getReturnedIndirectReturnValuePointer)(void* ctxt, BNVariable* outVar);
 
 		bool (*isArgumentTypeRegisterCompatible)(void* ctxt, BNType* type);
@@ -2932,10 +2932,11 @@ extern "C"
 
 		bool (*areArgumentRegistersUsedForVarArgs)(void* ctxt);
 
-		BNCallLayout (*getCallLayout)(void* ctxt, BNReturnValue* returnValue, BNFunctionParameter* params,
-			size_t paramCount, bool hasPermittedRegs, uint32_t* permittedRegs, size_t permittedRegCount);
+		void (*getCallLayout)(void* ctxt, BNReturnValue* returnValue, BNFunctionParameter* params,
+			size_t paramCount, bool hasPermittedRegs, uint32_t* permittedRegs, size_t permittedRegCount,
+			BNCallLayout* result);
 		void (*freeCallLayout)(void* ctxt, BNCallLayout* layout);
-		BNValueLocation (*getReturnValueLocation)(void* ctxt, BNReturnValue* returnValue);
+		void (*getReturnValueLocation)(void* ctxt, BNReturnValue* returnValue, BNValueLocation* outLocation);
 		void (*freeValueLocation)(void* ctxt, BNValueLocation* location);
 		BNValueLocation* (*getParameterLocations)(void* ctxt, BNValueLocation* returnValue, BNFunctionParameter* params,
 			size_t paramCount, bool hasPermittedRegs, uint32_t* permittedRegs, size_t permittedRegCount,

callingconvention.cpp

@@ -353,10 +353,10 @@ bool CallingConvention::IsReturnTypeRegisterCompatibleCallback(void* ctxt, BNTyp
 }
 
 
-BNVariable CallingConvention::GetIndirectReturnValueLocationCallback(void* ctxt)
+void CallingConvention::GetIndirectReturnValueLocationCallback(void* ctxt, BNVariable* outVar)
 {
 	CallbackRef<CallingConvention> cc(ctxt);
-	return cc->GetIndirectReturnValueLocation();
+	*outVar = cc->GetIndirectReturnValueLocation();
 }
 
 
@@ -394,9 +394,9 @@ bool CallingConvention::AreStackArgumentsNaturallyAlignedCallback(void* ctxt)
 }
 
 
-BNCallLayout CallingConvention::GetCallLayoutCallback(void* ctxt, BNReturnValue* returnValue,
+void CallingConvention::GetCallLayoutCallback(void* ctxt, BNReturnValue* returnValue,
 	BNFunctionParameter* params, size_t paramCount, bool hasPermittedRegs, uint32_t* permittedRegs,
-	size_t permittedRegCount)
+	size_t permittedRegCount, BNCallLayout* result)
 {
 	CallbackRef<CallingConvention> cc(ctxt);
 	auto ret = ReturnValue::FromAPIObject(returnValue);
@@ -414,7 +414,7 @@ BNCallLayout CallingConvention::GetCallLayoutCallback(void* ctxt, BNReturnValue*
 	}
 
 	auto layout = cc->GetCallLayout(ret, paramObjs, regOpt);
-	return layout.ToAPIObject();
+	*result = layout.ToAPIObject();
 }
 
 
@@ -424,12 +424,13 @@ void CallingConvention::FreeCallLayoutCallback(void*, BNCallLayout* layout)
 }
 
 
-BNValueLocation CallingConvention::GetReturnValueLocationCallback(void* ctxt, BNReturnValue* returnValue)
+void CallingConvention::GetReturnValueLocationCallback(
+	void* ctxt, BNReturnValue* returnValue, BNValueLocation* outLocation)
 {
 	CallbackRef<CallingConvention> cc(ctxt);
 	ReturnValue ret = ReturnValue::FromAPIObject(returnValue);
 	ValueLocation location = cc->GetReturnValueLocation(ret);
-	return location.ToAPIObject();
+	*outLocation = location.ToAPIObject();
 }
 
 

python/architecture.py

@@ -1051,7 +1051,7 @@ def calling_conventions(self) -> Mapping[str, 'callingconvention.CallingConventi
 		result = {}
 		try:
 			for i in range(0, count.value):
-				obj = callingconvention.CallingConvention(handle=core.BNNewCallingConventionReference(cc[i]))
+				obj = callingconvention.CoreCallingConvention(handle=core.BNNewCallingConventionReference(cc[i]))
 				result[obj.name] = obj
 		finally:
 			core.BNFreeCallingConventionList(cc, count.value)
@@ -2613,7 +2613,7 @@ def default_calling_convention(self):
 		cc_handle = core.BNGetArchitectureDefaultCallingConvention(self.handle)
 		if cc_handle is None:
 			return None
-		return callingconvention.CallingConvention(handle=cc_handle)
+		return callingconvention.CoreCallingConvention(handle=cc_handle)
 
 	@default_calling_convention.setter
 	def default_calling_convention(self, cc: 'callingconvention.CallingConvention'):
@@ -2633,7 +2633,7 @@ def cdecl_calling_convention(self):
 		cc_handle = core.BNGetArchitectureCdeclCallingConvention(self.handle)
 		if cc_handle is None:
 			return None
-		return callingconvention.CallingConvention(handle=cc_handle)
+		return callingconvention.CoreCallingConvention(handle=cc_handle)
 
 	@cdecl_calling_convention.setter
 	def cdecl_calling_convention(self, cc: 'callingconvention.CallingConvention'):
@@ -2653,7 +2653,7 @@ def stdcall_calling_convention(self):
 		cc_handle = core.BNGetArchitectureStdcallCallingConvention(self.handle)
 		if cc_handle is None:
 			return None
-		return callingconvention.CallingConvention(handle=cc_handle)
+		return callingconvention.CoreCallingConvention(handle=cc_handle)
 
 	@stdcall_calling_convention.setter
 	def stdcall_calling_convention(self, cc: 'callingconvention.CallingConvention'):
@@ -2673,7 +2673,7 @@ def fastcall_calling_convention(self):
 		cc_handle = core.BNGetArchitectureFastcallCallingConvention(self.handle)
 		if cc_handle is None:
 			return None
-		return callingconvention.CallingConvention(handle=cc_handle)
+		return callingconvention.CoreCallingConvention(handle=cc_handle)
 
 	@fastcall_calling_convention.setter
 	def fastcall_calling_convention(self, cc: 'callingconvention.CallingConvention'):