GP-5171: Generating pypredef files for PyDev. Fixing PyDev

errors/warnings in PyGhidra.
This commit is contained in:
Ryan Kurtz 2024-11-27 09:38:51 -05:00
parent e66bbc5231
commit 56d6af4531
16 changed files with 108 additions and 92 deletions

View file

@ -1 +1,2 @@
/.pytest_cache/ /.pytest_cache/
!.pydevproject

View file

@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?eclipse-pydev version="1.0"?><pydev_project>
<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python interpreter</pydev_property>
<pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
<path>/${PROJECT_DIR_NAME}/ghidra_scripts</path>
<path>/${PROJECT_DIR_NAME}/src/main/py/src/pyghidra</path>
</pydev_pathproperty>
</pydev_project>

View file

@ -54,10 +54,10 @@ debug_callback = _debug_callback
# Expose API # Expose API
from .core import run_script, start, started, open_program from pyghidra.core import run_script, start, started, open_program
from .launcher import DeferredPyGhidraLauncher, GuiPyGhidraLauncher, HeadlessPyGhidraLauncher from pyghidra.launcher import DeferredPyGhidraLauncher, GuiPyGhidraLauncher, HeadlessPyGhidraLauncher
from .script import get_current_interpreter from pyghidra.script import get_current_interpreter
from .version import ApplicationInfo, ExtensionDetails from pyghidra.version import ApplicationInfo, ExtensionDetails
__all__ = [ __all__ = [

View file

@ -20,7 +20,6 @@ import logging
import sys import sys
from pathlib import Path from pathlib import Path
import pyghidra
import pyghidra.core import pyghidra.core
import pyghidra.gui import pyghidra.gui

View file

@ -14,7 +14,6 @@
# limitations under the License. # limitations under the License.
## ##
import contextlib import contextlib
from pathlib import Path
from typing import Union, TYPE_CHECKING, Tuple, ContextManager, List, Optional from typing import Union, TYPE_CHECKING, Tuple, ContextManager, List, Optional
from pyghidra.converters import * # pylint: disable=wildcard-import, unused-wildcard-import from pyghidra.converters import * # pylint: disable=wildcard-import, unused-wildcard-import
@ -51,29 +50,29 @@ def started() -> bool:
return PyGhidraLauncher.has_launched() return PyGhidraLauncher.has_launched()
def _get_language(id: str) -> "Language": def _get_language(lang_id: str) -> "Language":
from ghidra.program.util import DefaultLanguageService from ghidra.program.util import DefaultLanguageService
from ghidra.program.model.lang import LanguageID, LanguageNotFoundException from ghidra.program.model.lang import LanguageID, LanguageNotFoundException
try: try:
service: "LanguageService" = DefaultLanguageService.getLanguageService() service: "LanguageService" = DefaultLanguageService.getLanguageService()
return service.getLanguage(LanguageID(id)) return service.getLanguage(LanguageID(lang_id))
except LanguageNotFoundException: except LanguageNotFoundException:
# suppress the java exception # suppress the java exception
pass pass
raise ValueError("Invalid Language ID: "+id) raise ValueError("Invalid Language ID: " + lang_id)
def _get_compiler_spec(lang: "Language", id: str = None) -> "CompilerSpec": def _get_compiler_spec(lang: "Language", compiler: str = None) -> "CompilerSpec":
if id is None: if compiler is None:
return lang.getDefaultCompilerSpec() return lang.getDefaultCompilerSpec()
from ghidra.program.model.lang import CompilerSpecID, CompilerSpecNotFoundException from ghidra.program.model.lang import CompilerSpecID, CompilerSpecNotFoundException
try: try:
return lang.getCompilerSpecByID(CompilerSpecID(id)) return lang.getCompilerSpecByID(CompilerSpecID(compiler))
except CompilerSpecNotFoundException: except CompilerSpecNotFoundException:
# suppress the java exception # suppress the java exception
pass pass
lang_id = lang.getLanguageID() lang_id = lang.getLanguageID()
raise ValueError(f"Invalid CompilerSpecID: {id} for Language: {lang_id.toString()}") raise ValueError(f"Invalid CompilerSpecID: {compiler} for Language: {lang_id.toString()}")
def _setup_project( def _setup_project(
@ -85,8 +84,8 @@ def _setup_project(
loader: Union[str, JClass] = None loader: Union[str, JClass] = None
) -> Tuple["GhidraProject", "Program"]: ) -> Tuple["GhidraProject", "Program"]:
from ghidra.base.project import GhidraProject from ghidra.base.project import GhidraProject
from java.lang import ClassLoader from java.lang import ClassLoader # type:ignore @UnresolvedImport
from java.io import IOException from java.io import IOException # type:ignore @UnresolvedImport
if binary_path is not None: if binary_path is not None:
binary_path = Path(binary_path) binary_path = Path(binary_path)
if project_location: if project_location:
@ -99,7 +98,7 @@ def _setup_project(
project_location.mkdir(exist_ok=True, parents=True) project_location.mkdir(exist_ok=True, parents=True)
if isinstance(loader, str): if isinstance(loader, str):
from java.lang import ClassNotFoundException from java.lang import ClassNotFoundException # type:ignore @UnresolvedImport
try: try:
gcl = ClassLoader.getSystemClassLoader() gcl = ClassLoader.getSystemClassLoader()
loader = JClass(loader, gcl) loader = JClass(loader, gcl)
@ -108,7 +107,8 @@ def _setup_project(
if isinstance(loader, JClass): if isinstance(loader, JClass):
from ghidra.app.util.opinion import Loader from ghidra.app.util.opinion import Loader
if not Loader.class_.isAssignableFrom(loader): loader_cls = Loader.class_
if not loader_cls.isAssignableFrom(loader):
raise TypeError(f"{loader} does not implement ghidra.app.util.opinion.Loader") raise TypeError(f"{loader} does not implement ghidra.app.util.opinion.Loader")
# Open/Create project # Open/Create project
@ -157,8 +157,8 @@ def _setup_script(project: "GhidraProject", program: "Program"):
from ghidra.program.util import ProgramLocation from ghidra.program.util import ProgramLocation
from ghidra.util.task import TaskMonitor from ghidra.util.task import TaskMonitor
from java.io import PrintWriter from java.io import PrintWriter # type:ignore @UnresolvedImport
from java.lang import System from java.lang import System # type:ignore @UnresolvedImport
if project is not None: if project is not None:
project = project.getProject() project = project.getProject()
@ -185,7 +185,7 @@ def _analyze_program(flat_api, program):
if hasattr(GhidraProgramUtilities, "markProgramAnalyzed"): if hasattr(GhidraProgramUtilities, "markProgramAnalyzed"):
GhidraProgramUtilities.markProgramAnalyzed(program) GhidraProgramUtilities.markProgramAnalyzed(program)
else: else:
GhidraProgramUtilities.setAnalyzedFlag(program, True) GhidraProgramUtilities.setAnalyzedFlag(program, True) # @UndefinedVariable
finally: finally:
GhidraScriptUtil.releaseBundleHostReference() GhidraScriptUtil.releaseBundleHostReference()

View file

@ -19,7 +19,7 @@ from pathlib import Path
import sys import sys
import threading import threading
from .launcher import PyGhidraLauncher, _run_mac_app from pyghidra.launcher import PyGhidraLauncher, _run_mac_app
class GhidraLauncher(PyGhidraLauncher): class GhidraLauncher(PyGhidraLauncher):
@ -31,12 +31,12 @@ class GhidraLauncher(PyGhidraLauncher):
def _launch(self): def _launch(self):
from ghidra import Ghidra from ghidra import Ghidra
from java.lang import Runtime, Thread from java.lang import Runtime, Thread # type:ignore @UnresolvedImport
if self._gui: if self._gui:
if sys.platform == "win32": if sys.platform == "win32":
appid = ctypes.c_wchar_p(self.app_info.name) appid = ctypes.c_wchar_p(self.app_info.name)
ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID(appid) ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID(appid) # @UndefinedVariable
Thread(lambda: Ghidra.main([self._class_name, *self.args])).start() Thread(lambda: Ghidra.main([self._class_name, *self.args])).start()
is_exiting = threading.Event() is_exiting = threading.Event()
Runtime.getRuntime().addShutdownHook(Thread(is_exiting.set)) Runtime.getRuntime().addShutdownHook(Thread(is_exiting.set))

View file

@ -21,8 +21,8 @@ import json
from pathlib import Path from pathlib import Path
import zipfile import zipfile
from java.lang import Class from java.lang import Class # type:ignore @UnresolvedImport
from java.io import PrintWriter from java.io import PrintWriter # type:ignore @UnresolvedImport
from jpype import JMethod, JObject, JClass from jpype import JMethod, JObject, JClass
from ghidra.framework import Application from ghidra.framework import Application
@ -85,7 +85,7 @@ class _Helper:
with zipfile.ZipFile(javadoc_zip, "r") as docs: with zipfile.ZipFile(javadoc_zip, "r") as docs:
with docs.open(json_path) as f: with docs.open(json_path) as f:
jsondoc = json.load(f) jsondoc = json.load(f)
except (IOError, KeyError) as e: except (IOError, KeyError):
pass pass
return jsondoc return jsondoc

View file

@ -112,12 +112,12 @@ def _gui():
# where tkinter can't be imported. Since there may not be an attached # where tkinter can't be imported. Since there may not be an attached
# terminal, the problem still needs to be reported somehow. # terminal, the problem still needs to be reported somehow.
try: try:
import tkinter.messagebox as _ import tkinter.messagebox as _ # @UnusedImport
except ImportError as e: except ImportError as e:
if platform.system() == "Windows": if platform.system() == "Windows":
# there is no console/terminal to report the error # there is no console/terminal to report the error
import ctypes import ctypes
MessageBox = ctypes.windll.user32.MessageBoxW MessageBox = ctypes.windll.user32.MessageBoxW # @UndefinedVariable
MessageBox(None, str(e), "Import Error", 0) MessageBox(None, str(e), "Import Error", 0)
sys.exit(1) sys.exit(1)
# report this before detaching from the console or no # report this before detaching from the console or no

View file

@ -22,7 +22,7 @@ from types import CodeType, FunctionType, MappingProxyType, MethodType, ModuleTy
from docking.widgets.label import GLabel from docking.widgets.label import GLabel
from generic.theme import GColor from generic.theme import GColor
from ghidra.app.plugin.core.console import CodeCompletion from ghidra.app.plugin.core.console import CodeCompletion
from java.util import Arrays, Collections from java.util import Arrays, Collections # type:ignore @UnresolvedImport
from jpype import JPackage from jpype import JPackage
from jpype.types import JDouble, JFloat, JInt, JLong, JShort from jpype.types import JDouble, JFloat, JInt, JLong, JShort
@ -80,9 +80,10 @@ class PythonCodeCompleter(Completer):
if attr is PythonCodeCompleter._BUILTIN_ATTRIBUTE: if attr is PythonCodeCompleter._BUILTIN_ATTRIBUTE:
if iskeyword(match.rstrip()): if iskeyword(match.rstrip()):
return label return label
attr = builtins.__dict__.get(match, PythonCodeCompleter._BUILTIN_ATTRIBUTE) builtins_dict = builtins.__dict__
attr = builtins_dict.get(match, PythonCodeCompleter._BUILTIN_ATTRIBUTE)
if attr is not PythonCodeCompleter._BUILTIN_ATTRIBUTE and not match.startswith("__"): if attr is not PythonCodeCompleter._BUILTIN_ATTRIBUTE and not match.startswith("__"):
attr = builtins.__dict__[match] attr = builtins_dict[match]
else: else:
return label return label
color = _TYPE_COLORS.get(type(attr), PythonCodeCompleter._BUILTIN_ATTRIBUTE) color = _TYPE_COLORS.get(type(attr), PythonCodeCompleter._BUILTIN_ATTRIBUTE)

View file

@ -26,11 +26,11 @@ from code import InteractiveConsole
from ghidra.framework import Application from ghidra.framework import Application
from ghidra.pyghidra import PyGhidraScriptProvider, PyGhidraPlugin from ghidra.pyghidra import PyGhidraScriptProvider, PyGhidraPlugin
from ghidra.pyghidra.interpreter import PyGhidraConsole from ghidra.pyghidra.interpreter import PyGhidraConsole
from java.io import BufferedReader, InputStreamReader from java.io import BufferedReader, InputStreamReader # type:ignore @UnresolvedImport
from java.lang import String from java.lang import String # type:ignore @UnresolvedImport
from java.lang import Thread as JThread from java.lang import Thread as JThread # type:ignore @UnresolvedImport
from java.util import Collections from java.util import Collections # type:ignore @UnresolvedImport
from java.util.function import Consumer from java.util.function import Consumer # type:ignore @UnresolvedImport
from jpype import JClass, JImplements, JOverride from jpype import JClass, JImplements, JOverride
from pyghidra.internal.plugin.completions import PythonCodeCompleter from pyghidra.internal.plugin.completions import PythonCodeCompleter

View file

@ -29,8 +29,8 @@ COMPILER_OPTIONS = ["-target", "21", "-source", "21"]
def _to_jar_(jar_path: Path, root: Path): def _to_jar_(jar_path: Path, root: Path):
from java.io import ByteArrayOutputStream from java.io import ByteArrayOutputStream # type:ignore @UnresolvedImport
from java.util.jar import JarEntry, JarOutputStream from java.util.jar import JarEntry, JarOutputStream # type:ignore @UnresolvedImport
out = ByteArrayOutputStream() out = ByteArrayOutputStream()
with JarOutputStream(out) as jar: with JarOutputStream(out) as jar:
@ -46,12 +46,12 @@ def _to_jar_(jar_path: Path, root: Path):
class _CompilerDiagnosticListener: class _CompilerDiagnosticListener:
def __init__(self): def __init__(self):
from javax.tools import Diagnostic from javax.tools import Diagnostic # type:ignore @UnresolvedImport
self.errors: List[Diagnostic] = [] self.errors: List[Diagnostic] = []
@JOverride @JOverride
def report(self, diagnostic): def report(self, diagnostic):
from javax.tools import Diagnostic from javax.tools import Diagnostic # type:ignore @UnresolvedImport
diagnostic: Diagnostic = diagnostic diagnostic: Diagnostic = diagnostic
kind = diagnostic.getKind() kind = diagnostic.getKind()
@ -71,10 +71,10 @@ def java_compile(src_path: Path, jar_path: Path):
:raises ValueError: If an error occurs when compiling the Java source :raises ValueError: If an error occurs when compiling the Java source
""" """
from java.lang import System from java.lang import System # type:ignore @UnresolvedImport
from java.io import Writer from java.io import Writer # type:ignore @UnresolvedImport
from java.nio.file import Path as JPath from java.nio.file import Path as JPath # type:ignore @UnresolvedImport
from javax.tools import StandardLocation, ToolProvider from javax.tools import StandardLocation, ToolProvider # type:ignore @UnresolvedImport
with tempfile.TemporaryDirectory() as out: with tempfile.TemporaryDirectory() as out:
outdir = Path(out).resolve() outdir = Path(out).resolve()

View file

@ -14,7 +14,6 @@
# limitations under the License. # limitations under the License.
## ##
import contextlib import contextlib
import ctypes
import ctypes.util import ctypes.util
import html import html
import importlib.metadata import importlib.metadata
@ -36,17 +35,17 @@ import jpype
from jpype import imports, _jpype from jpype import imports, _jpype
from packaging.version import Version from packaging.version import Version
from .javac import java_compile from pyghidra.javac import java_compile
from .script import PyGhidraScript from pyghidra.script import PyGhidraScript
from .version import ApplicationInfo, ExtensionDetails, MINIMUM_GHIDRA_VERSION from pyghidra.version import ApplicationInfo, ExtensionDetails, MINIMUM_GHIDRA_VERSION
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@contextlib.contextmanager @contextlib.contextmanager
def _silence_java_output(stdout=True, stderr=True): def _silence_java_output(stdout=True, stderr=True):
from java.io import OutputStream, PrintStream from java.io import OutputStream, PrintStream # type:ignore @UnresolvedImport
from java.lang import System from java.lang import System # type:ignore @UnresolvedImport
out = System.out out = System.out
err = System.err err = System.err
null = PrintStream(OutputStream.nullOutputStream()) null = PrintStream(OutputStream.nullOutputStream())
@ -116,7 +115,7 @@ def _plugin_lock():
""" """
File lock for processing plugins File lock for processing plugins
""" """
from java.io import RandomAccessFile from java.io import RandomAccessFile # type:ignore @UnresolvedImport
path = Path(tempfile.gettempdir()) / "pyghidra_plugin_lock" path = Path(tempfile.gettempdir()) / "pyghidra_plugin_lock"
try: try:
# Python doesn't have a file lock except for unix systems # Python doesn't have a file lock except for unix systems
@ -431,7 +430,7 @@ class PyGhidraLauncher:
# Add extra class paths # Add extra class paths
# Do this before installing plugins incase dependencies are needed # Do this before installing plugins incase dependencies are needed
if self.class_files: if self.class_files:
from java.lang import ClassLoader from java.lang import ClassLoader # type:ignore @UnresolvedImport
gcl = ClassLoader.getSystemClassLoader() gcl = ClassLoader.getSystemClassLoader()
for path in self.class_files: for path in self.class_files:
gcl.addPath(path) gcl.addPath(path)
@ -451,7 +450,7 @@ class PyGhidraLauncher:
self._layout = GhidraLauncher.initializeGhidraEnvironment() self._layout = GhidraLauncher.initializeGhidraEnvironment()
# import properties to register the property customizer # import properties to register the property customizer
from . import properties as _ from pyghidra import properties as _ # @UnusedImport
_load_entry_points("pyghidra.pre_launch") _load_entry_points("pyghidra.pre_launch")
@ -654,7 +653,7 @@ class GuiPyGhidraLauncher(PyGhidraLauncher):
@staticmethod @staticmethod
def _get_thread(name: str): def _get_thread(name: str):
from java.lang import Thread from java.lang import Thread # type:ignore @UnresolvedImport
for t in Thread.getAllStackTraces().keySet(): for t in Thread.getAllStackTraces().keySet():
if t.getName() == name: if t.getName() == name:
return t return t
@ -662,11 +661,11 @@ class GuiPyGhidraLauncher(PyGhidraLauncher):
def _launch(self): def _launch(self):
from ghidra import Ghidra from ghidra import Ghidra
from java.lang import Runtime, Thread from java.lang import Runtime, Thread # type:ignore @UnresolvedImport
if sys.platform == "win32": if sys.platform == "win32":
appid = ctypes.c_wchar_p(self.app_info.name) appid = ctypes.c_wchar_p(self.app_info.name)
ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID(appid) ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID(appid) # @UndefinedVariable
stdout = _PyGhidraStdOut(sys.stdout) stdout = _PyGhidraStdOut(sys.stdout)
stderr = _PyGhidraStdOut(sys.stderr) stderr = _PyGhidraStdOut(sys.stderr)

View file

@ -14,7 +14,6 @@
# limitations under the License. # limitations under the License.
## ##
import functools import functools
import importlib
import importlib.util import importlib.util
import inspect import inspect
import logging import logging
@ -157,7 +156,7 @@ class PyGhidraScript(dict):
global _headless_interpreter global _headless_interpreter
from ghidra.util import SystemUtilities from ghidra.util import SystemUtilities
from .ghidradoc import _Helper from pyghidra.ghidradoc import _Helper
if SystemUtilities.isInHeadlessMode() and _headless_interpreter is None: if SystemUtilities.isInHeadlessMode() and _headless_interpreter is None:
_headless_interpreter = jobj _headless_interpreter = jobj

View file

@ -56,6 +56,16 @@ class GhidraBuiltinsBuilder {
catch (IOException e) { catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
File pypredefDir = new File(doclet.getDestDir().getParentFile(), "pypredef");
File pypredefFile = new File(pypredefDir, "ghidra.ghidra_builtins.pypredef");
pypredefDir.mkdirs();
try (PrintWriter printer = new PrintWriter(new FileWriter(pypredefFile))) {
process(printer);
}
catch (IOException e) {
e.printStackTrace();
}
} }
/** /**

View file

@ -15,20 +15,10 @@
*/ */
package ghidra.doclets.typestubs; package ghidra.doclets.typestubs;
import java.io.File; import java.io.*;
import java.io.FileWriter; import java.util.*;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import javax.lang.model.element.Element; import javax.lang.model.element.*;
import javax.lang.model.element.PackageElement;
import javax.lang.model.element.TypeElement;
/** /**
* {@link PythonTypeStubElement} for a package<p/> * {@link PythonTypeStubElement} for a package<p/>
@ -105,6 +95,16 @@ final class PythonTypeStubPackage extends PythonTypeStubElement<PackageElement>
catch (IOException e) { catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
File pypredefDir = new File(doclet.getDestDir().getParentFile(), "pypredef");
File pypredefFile = new File(pypredefDir, packageName + ".pypredef");
pypredefDir.mkdirs();
try (PrintWriter printer = new PrintWriter(new FileWriter(pypredefFile))) {
process(printer, "");
}
catch (IOException e) {
e.printStackTrace();
}
} }
/** /**

View file

@ -16,27 +16,12 @@
package ghidra.doclets.typestubs; package ghidra.doclets.typestubs;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.util.ArrayList; import java.util.*;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import javax.lang.model.element.Element; import javax.lang.model.element.*;
import javax.lang.model.element.ExecutableElement; import javax.lang.model.type.*;
import javax.lang.model.element.TypeElement;
import javax.lang.model.element.TypeParameterElement;
import javax.lang.model.element.VariableElement;
import javax.lang.model.type.DeclaredType;
import javax.lang.model.type.TypeKind;
import javax.lang.model.type.TypeMirror;
import com.sun.source.doctree.DocTree; import com.sun.source.doctree.DocTree;
@ -278,12 +263,11 @@ class PythonTypeStubType extends PythonTypeStubElement<TypeElement> {
for (PythonTypeStubNestedType nested : getNestedTypes()) { for (PythonTypeStubNestedType nested : getNestedTypes()) {
nested.process(printer, indent); nested.process(printer, indent);
} }
printClassLiteralField(printer, indent);
for (VariableElement field : getFields()) { for (VariableElement field : getFields()) {
printField(field, printer, indent, isStatic(field)); printField(field, printer, indent, isStatic(field));
} }
if (!getFields().isEmpty()) {
printer.println(); printer.println();
}
ListIterator<PythonTypeStubMethod> methodIterator = getMethods().listIterator(); ListIterator<PythonTypeStubMethod> methodIterator = getMethods().listIterator();
while (methodIterator.hasNext()) { while (methodIterator.hasNext()) {
PythonTypeStubMethod method = methodIterator.next(); PythonTypeStubMethod method = methodIterator.next();
@ -342,6 +326,17 @@ class PythonTypeStubType extends PythonTypeStubElement<TypeElement> {
} }
} }
/**
* Prints the class literal field to the provided printer
*
* @param printer the printer
* @param indent the indentation
*/
void printClassLiteralField(PrintWriter printer, String indent) {
printer.print(indent);
printer.println("class_: " + applyClassVar(Class.class.getName()));
}
/** /**
* Wraps the provided type in typing.ClassVar * Wraps the provided type in typing.ClassVar
* *