typestub bug fixes

Ghidra/Features/PyGhidra/src/main/py/pyproject.toml

@@ -30,6 +30,7 @@ classifiers = [
 ]
 dependencies = [
     "Jpype1>=1.5.0",
+    "packaging"
 ]
 
 [project.optional-dependencies]

Ghidra/Features/PyGhidra/src/main/py/src/pyghidra/launcher.py

@@ -28,12 +28,13 @@ import subprocess
 import sys
 import tempfile
 import threading
+from importlib.machinery import ModuleSpec
 from pathlib import Path
 from typing import List, NoReturn, Tuple, Union
 
 import jpype
 from jpype import imports, _jpype
-from importlib.machinery import ModuleSpec
+from packaging.version import Version
 
 from .javac import java_compile
 from .script import PyGhidraScript
@@ -349,7 +350,7 @@ class PyGhidraLauncher:
         Checks if the currently installed Ghidra version is supported.
         The launcher will report the problem and terminate if it is not supported.
         """
-        if self.app_info.version < MINIMUM_GHIDRA_VERSION:
+        if Version(self.app_info.version) < Version(MINIMUM_GHIDRA_VERSION):
             msg = f"Ghidra version {self.app_info.version} is not supported" + os.linesep + \
                   f"The minimum required version is {MINIMUM_GHIDRA_VERSION}"
             self._report_fatal_error("Unsupported Version", msg, ValueError(msg))

GhidraBuild/BuildFiles/Doclets/src/main/java/ghidra/doclets/typestubs/GhidraBuiltinsBuilder.java

@@ -188,6 +188,7 @@ class GhidraBuiltinsBuilder {
 	 * @param printer the printer
 	 */
 	private void printScriptImports(PrintWriter printer) {
+		printer.println("from __future__ import annotations");
 		printer.println("import collections.abc");
 		printer.println("import typing");
 		printer.println("from warnings import deprecated # type: ignore");

GhidraBuild/BuildFiles/Doclets/src/main/java/ghidra/doclets/typestubs/JavadocConverter.java

@@ -553,7 +553,7 @@ public class JavadocConverter extends DocConverter {
 	private static String sanitizeQualifiedName(ExecutableElement el, TypeMirror type) {
 		Element self = el.getEnclosingElement();
 		PackageElement pkg = PythonTypeStubElement.getPackage(self);
-		return PythonTypeStubElement.sanitizeQualifiedName(self, type, pkg);
+		return PythonTypeStubElement.sanitizeQualifiedName(type, pkg);
 	}
 
 	/**

GhidraBuild/BuildFiles/Doclets/src/main/java/ghidra/doclets/typestubs/PythonTypeStubElement.java

@@ -23,7 +23,6 @@ import java.util.List;
 import java.util.Set;
 
 import javax.lang.model.element.Element;
-import javax.lang.model.element.ElementKind;
 import javax.lang.model.element.Modifier;
 import javax.lang.model.element.Name;
 import javax.lang.model.element.PackageElement;
@@ -106,24 +105,6 @@ abstract class PythonTypeStubElement<T extends Element> {
 		return false;
 	}
 
-	/**
-	 * Gets the type string for the provided type and quotes if necessary<p/>
-	 *
-	 * This string value is safe to be used as a parameter or return type
-	 * as well as for use in a generic type.
-	 *
-	 * @param self the type to become typing.Self if encountered
-	 * @param type the type to get the string for
-	 * @return the type string
-	 */
-	String getTypeString(Element self, TypeMirror type) {
-		String typeName = sanitizeQualifiedName(self, type);
-		if (isSamePackage(type) && !typeName.equals("typing.Self")) {
-			typeName = '"' + typeName + '"';
-		}
-		return typeName;
-	}
-
 	/**
 	 * Gets the Python safe name for this element
 	 *
@@ -248,18 +229,17 @@ abstract class PythonTypeStubElement<T extends Element> {
 	/**
 	 * Makes the provided type Python safe if necessary
 	 *
-	 * @param self the type to become typing.Self if encountered
 	 * @param type the type to make Python safe
 	 * @param pkg the current package
 	 * @return the Python safe type name
 	 */
-	static String sanitize(Element self, TypeMirror type, PackageElement pkg) {
+	static String sanitize(TypeMirror type, PackageElement pkg) {
 		return switch (type.getKind()) {
 			case DECLARED -> throw new RuntimeException(
 				"declared types should use the qualified name");
 			case ARRAY -> {
 				TypeMirror component = ((ArrayType) type).getComponentType();
-				yield "jpype.JArray[" + sanitizeQualifiedName(self, component, pkg) + "]";
+				yield "jpype.JArray[" + sanitizeQualifiedName(component, pkg) + "]";
 			}
 			case BOOLEAN -> "jpype.JBoolean";
 			case BYTE -> "jpype.JByte";
@@ -270,55 +250,31 @@ abstract class PythonTypeStubElement<T extends Element> {
 			case LONG -> "jpype.JLong";
 			case SHORT -> "jpype.JShort";
 			case TYPEVAR -> type.toString();
-			case WILDCARD -> getWildcardVarName(self, (WildcardType) type, pkg);
+			case WILDCARD -> getWildcardVarName((WildcardType) type, pkg);
 			default -> throw new RuntimeException("unexpected TypeKind " + type.getKind());
 		};
 	}
 
-	/**
-	 * Checks if the provided type is the same as the provided element
-	 *
-	 * @param self the element of the type to become typing.Self
-	 * @param type the type to check
-	 * @return true if the inputs represent the same type
-	 */
-	static final boolean isSelfType(Element self, TypeMirror type) {
-		if (self.getKind() == ElementKind.ENUM) {
-			// typing.Self is usually invalid here
-			return false;
-		}
-		if (type instanceof DeclaredType dt) {
-			return self.equals(dt.asElement());
-		}
-		return false;
-	}
-
 	/**
 	 * Makes the qualified name for the provided type Python safe if necessary
 	 *
-	 * @param self the type to become typing.Self if encountered
 	 * @param type the type to make Python safe
 	 * @return the Python safe qualified type name
 	 */
-	final String sanitizeQualifiedName(Element self, TypeMirror type) {
+	final String sanitizeQualifiedName(TypeMirror type) {
-		return sanitizeQualifiedName(self, type, pkg);
+		return sanitizeQualifiedName(type, pkg);
 	}
 
 	/**
 	 * Makes the qualified name for the provided type Python safe if necessary<p/>
 	 *
 	 * The provided package is used to check each type and generic components.
-	 * If they require a "forward declaration", it is handled accordingly.
 	 *
-	 * @param self the type to become typing.Self if encountered
 	 * @param type the type to make Python safe
 	 * @param pkg the current package
 	 * @return the Python safe qualified type name
 	 */
-	static final String sanitizeQualifiedName(Element self, TypeMirror type, PackageElement pkg) {
+	static final String sanitizeQualifiedName(TypeMirror type, PackageElement pkg) {
-		if (isSelfType(self, type)) {
-			return "typing.Self";
-		}
 		if (type instanceof DeclaredType dt) {
 			TypeElement el = (TypeElement) dt.asElement();
 			PackageElement typePkg = getPackage(el);
@@ -341,11 +297,41 @@ abstract class PythonTypeStubElement<T extends Element> {
 				return name;
 			}
 			Iterable<String> it = () -> args.stream()
-					.map(paramType -> sanitizeQualifiedName(self, paramType, pkg))
+					.map(paramType -> sanitizeQualifiedName(paramType, pkg))
 					.iterator();
 			return name + "[" + String.join(", ", it) + "]";
 		}
-		return sanitize(self, type, pkg);
+		return sanitize(type, pkg);
+	}
+
+	/**
+	 * Recursively adds the type and it's generic parameters to the provided imports set.
+	 *
+	 * @param imports the set of imported types
+	 * @param type the type to add to the imports
+	 */
+	static void addNeededTypes(Set<TypeElement> imports, TypeMirror type) {
+		switch (type.getKind()) {
+			case DECLARED:
+				DeclaredType dt = (DeclaredType) type;;
+				imports.add((TypeElement) dt.asElement());
+				for (TypeMirror genericType : dt.getTypeArguments()) {
+					addNeededTypes(imports, genericType);
+				}
+				break;
+			case WILDCARD:
+				WildcardType wt = (WildcardType) type;
+				TypeMirror base = wt.getExtendsBound();
+				if (base == null) {
+					base = wt.getSuperBound();
+				}
+				if (base != null) {
+					addNeededTypes(imports, base);
+				}
+				break;
+			default:
+				break;
+		}
 	}
 
 	/**
@@ -411,18 +397,17 @@ abstract class PythonTypeStubElement<T extends Element> {
 	/**
 	 * Gets the name for a wildcard type if possible
 	 *
-	 * @param self the type to become typing.Self if encountered
 	 * @param type the wildcard type
 	 * @param pkg the current package
 	 * @return the determined type name if possible otherwise typing.Any
 	 */
-	private static String getWildcardVarName(Element self, WildcardType type, PackageElement pkg) {
+	private static String getWildcardVarName(WildcardType type, PackageElement pkg) {
 		TypeMirror base = type.getExtendsBound();
 		if (base == null) {
 			base = type.getSuperBound();
 		}
 		if (base != null) {
-			return sanitizeQualifiedName(self, base, pkg);
+			return sanitizeQualifiedName(base, pkg);
 		}
 		return "typing.Any";
 	}

GhidraBuild/BuildFiles/Doclets/src/main/java/ghidra/doclets/typestubs/PythonTypeStubMethod.java

@@ -87,7 +87,6 @@ final class PythonTypeStubMethod extends PythonTypeStubElement<ExecutableElement
 			"java.lang.Short", "int",
 			"java.lang.String", "str"));
 
-	private final PythonTypeStubType parent;
 	private final boolean filterSelf;
 	List<String> typevars;
 	Set<TypeElement> imports;
@@ -111,7 +110,6 @@ final class PythonTypeStubMethod extends PythonTypeStubElement<ExecutableElement
 	 */
 	PythonTypeStubMethod(PythonTypeStubType parent, ExecutableElement el, boolean filterSelf) {
 		super(parent.doclet, el);
-		this.parent = parent;
 		this.filterSelf = filterSelf;
 	}
 
@@ -193,20 +191,14 @@ final class PythonTypeStubMethod extends PythonTypeStubElement<ExecutableElement
 			return imports;
 		}
 
-		List<? extends VariableElement> parameters = el.getParameters();
+		List<? extends TypeMirror> parameters = getParameterTypes();
-		TypeMirror resType = el.getReturnType();
 
 		// make the set big enough for all paramters and the return type
 		imports = new HashSet<>(parameters.size() + 1);
 
-		if (resType instanceof DeclaredType dt) {
+		addNeededTypes(imports, getReturnType());
-			imports.add((TypeElement) dt.asElement());
+		for (TypeMirror param : parameters) {
-		}
+			addNeededTypes(imports, param);
-
-		for (VariableElement param : parameters) {
-			if (param.asType() instanceof DeclaredType dt) {
-				imports.add((TypeElement) dt.asElement());
-			}
 		}
 
 		return imports;
@@ -329,7 +321,7 @@ final class PythonTypeStubMethod extends PythonTypeStubElement<ExecutableElement
 				printer.print(convertedType);
 			}
 			else {
-				printer.print(getTypeString(parent.el, res));
+				printer.print(sanitizeQualifiedName(res));
 			}
 		}
 	}
@@ -398,7 +390,7 @@ final class PythonTypeStubMethod extends PythonTypeStubElement<ExecutableElement
 		if (convertedType != null) {
 			return name + ": " + convertedType;
 		}
-		return name + ": " + getTypeString(parent.el, type);
+		return name + ": " + sanitizeQualifiedName(type);
 	}
 
 	/**

GhidraBuild/BuildFiles/Doclets/src/main/java/ghidra/doclets/typestubs/PythonTypeStubPackage.java

@@ -147,6 +147,7 @@ final class PythonTypeStubPackage extends PythonTypeStubElement<PackageElement>
 	 */
 	private void process(PrintWriter printer, String indent) {
 		writeJavaDoc(printer, indent, "");
+		printer.println("from __future__ import annotations");
 		printer.println("import collections.abc");
 		printer.println("import datetime");
 		printer.println("import typing");

GhidraBuild/BuildFiles/Doclets/src/main/java/ghidra/doclets/typestubs/PythonTypeStubType.java

@@ -125,18 +125,13 @@ class PythonTypeStubType extends PythonTypeStubElement<TypeElement> {
 			imports.add((TypeElement) dt.asElement());
 		}
 		for (TypeMirror iface : el.getInterfaces()) {
-			if (iface instanceof DeclaredType dt) {
+			addNeededTypes(imports, iface);
-				imports.add((TypeElement) dt.asElement());
-			}
 		}
 		for (PythonTypeStubNestedType nested : getNestedTypes()) {
 			imports.addAll(nested.getImportedTypes());
 		}
 		for (VariableElement field : getFields()) {
-			TypeMirror fieldType = field.asType();
+			addNeededTypes(imports, field.asType());
-			if (fieldType instanceof DeclaredType dt) {
-				imports.add((TypeElement) dt.asElement());
-			}
 		}
 		for (PythonTypeStubMethod method : getMethods()) {
 			imports.addAll(method.getImportedTypes());
@@ -327,7 +322,7 @@ class PythonTypeStubType extends PythonTypeStubElement<TypeElement> {
 		else {
 			TypeMirror type = field.asType();
 			printer.print(": ");
-			String sanitizedType = getTypeString(el, type);
+			String sanitizedType = sanitizeQualifiedName(type);
 
 			// only one of these may be applied
 			// prefer Final over ClassVar
@@ -648,11 +643,7 @@ class PythonTypeStubType extends PythonTypeStubElement<TypeElement> {
 			}
 			return OBJECT_NAME;
 		}
-		return sanitizeQualifiedName(el, base);
+		return sanitizeQualifiedName(base);
-	}
-
-	private String sanitizeQualifiedName(TypeMirror type) {
-		return sanitizeQualifiedName(el, type);
 	}
 
 	/**