GT-3328 corrected sleighArgs.txt use with module dependency paths

Ghidra/Framework/SoftwareModeling/Sleigh.launch

@@ -1,27 +1,27 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <launchConfiguration type="org.eclipse.jdt.launching.localJavaApplication">
-<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
+    <listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
-<listEntry value="/Framework Utility/src/main/java/ghidra/GhidraLauncher.java"/>
+        <listEntry value="/Framework Utility/src/main/java/ghidra/GhidraLauncher.java"/>
-</listAttribute>
+    </listAttribute>
-<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
+    <listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
-<listEntry value="1"/>
+        <listEntry value="1"/>
-</listAttribute>
+    </listAttribute>
-<listAttribute key="org.eclipse.debug.ui.favoriteGroups">
+    <listAttribute key="org.eclipse.debug.ui.favoriteGroups">
-<listEntry value="org.eclipse.debug.ui.launchGroup.debug"/>
+        <listEntry value="org.eclipse.debug.ui.launchGroup.debug"/>
-<listEntry value="org.eclipse.debug.ui.launchGroup.run"/>
+        <listEntry value="org.eclipse.debug.ui.launchGroup.run"/>
-</listAttribute>
+    </listAttribute>
-<booleanAttribute key="org.eclipse.jdt.launching.ATTR_USE_CLASSPATH_ONLY_JAR" value="false"/>
+    <booleanAttribute key="org.eclipse.jdt.launching.ATTR_USE_CLASSPATH_ONLY_JAR" value="false"/>
-<booleanAttribute key="org.eclipse.jdt.launching.ATTR_USE_START_ON_FIRST_THREAD" value="true"/>
+    <booleanAttribute key="org.eclipse.jdt.launching.ATTR_USE_START_ON_FIRST_THREAD" value="true"/>
-<listAttribute key="org.eclipse.jdt.launching.CLASSPATH">
+    <listAttribute key="org.eclipse.jdt.launching.CLASSPATH">
-<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot; standalone=&quot;no&quot;?&gt;&#10;&lt;runtimeClasspathEntry path=&quot;5&quot; projectName=&quot;Framework Utility&quot; type=&quot;1&quot;/&gt;&#10;"/>
+        <listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot; standalone=&quot;no&quot;?&gt;&#10;&lt;runtimeClasspathEntry path=&quot;5&quot; projectName=&quot;Framework Utility&quot; type=&quot;1&quot;/&gt;&#10;"/>
-</listAttribute>
+    </listAttribute>
-<booleanAttribute key="org.eclipse.jdt.launching.DEFAULT_CLASSPATH" value="false"/>
+    <booleanAttribute key="org.eclipse.jdt.launching.DEFAULT_CLASSPATH" value="false"/>
-<stringAttribute key="org.eclipse.jdt.launching.JRE_CONTAINER" value="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-11"/>
+    <stringAttribute key="org.eclipse.jdt.launching.JRE_CONTAINER" value="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-11"/>
-<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="ghidra.GhidraLauncher"/>
+    <stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="ghidra.GhidraLauncher"/>
-<listAttribute key="org.eclipse.jdt.launching.MODULEPATH">
+    <listAttribute key="org.eclipse.jdt.launching.MODULEPATH">
-<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot; standalone=&quot;no&quot;?&gt;&#10;&lt;runtimeClasspathEntry containerPath=&quot;org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-11&quot; path=&quot;4&quot; type=&quot;4&quot;/&gt;&#10;"/>
+        <listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot; standalone=&quot;no&quot;?&gt;&#10;&lt;runtimeClasspathEntry containerPath=&quot;org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-11&quot; path=&quot;4&quot; type=&quot;4&quot;/&gt;&#10;"/>
-</listAttribute>
+    </listAttribute>
-<stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="ghidra.pcodeCPort.slgh_compile.SleighCompileLauncher -i &quot;${project_loc}/build/data/sleighArgs.txt&quot; -a &quot;${project_loc}/data/languages&quot;"/>
+    <stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="ghidra.pcodeCPort.slgh_compile.SleighCompileLauncher -DBaseDir=&quot;${project_loc}/../../../../&quot; -i &quot;${project_loc}/build/tmp/sleighArgs.txt&quot; -a &quot;${project_loc}/data/languages&quot;"/>
-<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="Framework Utility"/>
+    <stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="Framework Utility"/>
-<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Djava.awt.headless=true -Xmx1048M -XX:+IgnoreUnrecognizedVMOptions -Djava.system.class.loader=ghidra.GhidraClassLoader -Dfile.encoding=UTF-8 -Duser.country=US -Duser.language=en -Duser.variant -Xdock:name=&quot;Sleigh&quot; -Dvisualvm.display.name=Sleigh"/>
+    <stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Djava.awt.headless=true -Xmx1048M -XX:+IgnoreUnrecognizedVMOptions -Djava.system.class.loader=ghidra.GhidraClassLoader -Dfile.encoding=UTF-8 -Duser.country=US -Duser.language=en -Duser.variant -Xdock:name=&quot;Sleigh&quot; -Dvisualvm.display.name=Sleigh"/>
 </launchConfiguration>

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguage.java

@@ -422,7 +422,7 @@ public class SleighLanguage implements Language {
 		String languageName = specName + ".slaspec";
 		ResourceFile languageFile = new ResourceFile(slaFile.getParentFile(), languageName);
 
-		// see gradleScripts/processorUtils.gradle for sleighArgs.txt generation
+		// see gradle/processorUtils.gradle for sleighArgs.txt generation
 		ResourceFile sleighArgsFile = null;
 		ResourceFile languageModule = Application.getModuleContainingResourceFile(languageFile);
 		if (languageModule != null) {
@@ -430,21 +430,23 @@ public class SleighLanguage implements Language {
 				sleighArgsFile = new ResourceFile(languageModule, "data/sleighArgs.txt");
 			}
 			else {
-				sleighArgsFile = new ResourceFile(languageModule, "build/data/sleighArgs.txt");
+				sleighArgsFile = new ResourceFile(languageModule, "build/tmp/sleighArgs.txt");
 			}
 		}
 
-		Map<String, String> defineMap;
 		String[] args;
 		if (sleighArgsFile != null && sleighArgsFile.isFile()) {
-			args = new String[] { "-i", sleighArgsFile.getAbsolutePath(),
+			String baseDir = Application.getInstallationDirectory().getAbsolutePath().replace(
+				File.separatorChar, '/');
+			if (!baseDir.endsWith("/")) {
+				baseDir += "/";
+			}
+			args = new String[] { "-DBaseDir=" + baseDir, "-i", sleighArgsFile.getAbsolutePath(),
 				languageFile.getAbsolutePath(), description.getSlaFile().getAbsolutePath() };
-			defineMap = new HashMap<>();
 		}
 		else {
 			args = new String[] { languageFile.getAbsolutePath(),
 				description.getSlaFile().getAbsolutePath() };
-			defineMap = ModuleDefinitionsMap.getModuleMap();
 		}
 
 		try {
@@ -454,7 +456,7 @@ public class SleighLanguage implements Language {
 				buf.append(" ");
 			}
 			Msg.debug(this, "Sleigh compile: " + buf);
-			int returnCode = SleighCompileLauncher.runMain(args, defineMap);
+			int returnCode = SleighCompileLauncher.runMain(args);
 			if (returnCode != 0) {
 				throw new SleighException("Errors compiling " + languageFile.getAbsolutePath() +
 					" -- please check log messages for details");

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompile.java

@@ -1688,6 +1688,6 @@ public class SleighCompile extends SleighBase {
 	 * @throws RecognitionException
 	 */
 	public static void main(String[] args) throws JDOMException, IOException, RecognitionException {
-		System.exit(SleighCompileLauncher.runMain(args, new HashMap<String, String>()));
+		System.exit(SleighCompileLauncher.runMain(args));
 	}
 }

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompileLauncher.java

@@ -71,24 +71,24 @@ public class SleighCompileLauncher implements GhidraLaunchable {
 		ApplicationConfiguration configuration = new ApplicationConfiguration();
 		Application.initializeApplication(layout, configuration);
 
-		System.exit(runMain(args, new HashMap<String, String>()));
+		System.exit(runMain(args));
 	}
 
 	/**
 	 * Execute the Sleigh compiler process
 	 * 
 	 * @param args sleigh compiler command line arguments
-	 * @param preprocs additional preprocessor macro
 	 * @return exit code (TODO: exit codes are not well defined)
 	 * @throws JDOMException
 	 * @throws IOException
 	 * @throws RecognitionException
 	 */
-	public static int runMain(String[] args, Map<String, String> preprocs)
+	public static int runMain(String[] args)
 			throws JDOMException, IOException, RecognitionException {
 		int retval;
 		String filein = null;
 		String fileout = null;
+		Map<String, String> preprocs = new HashMap<>();
 
 		SleighCompile.yydebug = false;
 		boolean allMode = false;
@@ -218,8 +218,8 @@ public class SleighCompileLauncher implements GhidraLaunchable {
 				System.out.println("Compiling " + input + ":");
 				SleighCompile compiler = new SleighCompile();
 				initCompiler(compiler, preprocs, unnecessaryPcodeWarning, lenientConflict,
-					allCollisionWarning,
+					allCollisionWarning, allNopWarning, deadTempWarning, unusedFieldWarning,
-					allNopWarning, deadTempWarning, unusedFieldWarning, enforceLocalKeyWord);
+					enforceLocalKeyWord);
 
 				String outname = input.getName().replace(".slaspec", ".sla");
 				File output = new File(input.getParent(), outname);
@@ -247,8 +247,8 @@ public class SleighCompileLauncher implements GhidraLaunchable {
 		// single file compile
 		SleighCompile compiler = new SleighCompile();
 		initCompiler(compiler, preprocs, unnecessaryPcodeWarning, lenientConflict,
-			allCollisionWarning, allNopWarning,
+			allCollisionWarning, allNopWarning, deadTempWarning, unusedFieldWarning,
-			deadTempWarning, unusedFieldWarning, enforceLocalKeyWord);
+			enforceLocalKeyWord);
 		if (i == args.length) {
 			Msg.error(SleighCompile.class, "Missing input file name");
 			return 1;
@@ -418,6 +418,7 @@ public class SleighCompileLauncher implements GhidraLaunchable {
 			return 4;
 		}
 		catch (PreprocessorException e) {
+			Msg.error(SleighCompile.class, e.getMessage());
 			Msg.error(SleighCompile.class, "Errors during preprocessing, halting compilation");
 			return 5;
 		}

Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/pcodeCPort/slgh_compile/regression/SleighCompileRegressionTest.java

@@ -16,7 +16,7 @@
  */
 package ghidra.pcodeCPort.slgh_compile.regression;
 
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
 
 import java.io.*;
 import java.util.*;
@@ -167,11 +167,9 @@ public class SleighCompileRegressionTest extends AbstractGenericTest {
 
 	private int runActualCompiler(File inputFile, File actualFile)
 			throws JDOMException, IOException, RecognitionException {
-		return SleighCompileLauncher.runMain(
+		return SleighCompileLauncher.runMain(new String[] { "-DBaseDir=../../../../../../",
-			new String[] { "-DMIPS=../../../../../../ghidra/Ghidra/Processors/MIPS",
+			"-DMIPS=ghidra/Ghidra/Processors/MIPS", "-D8051=ghidra/Ghidra/Processors/8051",
-				"-D8051=../../../../../../ghidra/Ghidra/Processors/8051",
+			inputFile.getAbsolutePath(), actualFile.getAbsolutePath() });
-				inputFile.getAbsolutePath(), actualFile.getAbsolutePath() },
-			new HashMap<String, String>());
 	}
 
 	private static final Pattern SPACEMATCH = Pattern.compile("^\\s*<print piece=\" \"/>\\s*$");

Ghidra/Processors/AARCH64/build.gradle

@@ -7,9 +7,9 @@ apply plugin: 'eclipse'
 
 eclipse.project.name = 'Processors AARCH64'
 
-sleighCompile {
+sleighCompileOptions = [
-	args '-l'
+	'-l'
-}
+]
 
 dependencies {
 	compile project(':Base')

Ghidra/Processors/Atmel/build.gradle

@@ -10,6 +10,6 @@ dependencies {
 	compile project(':Base')
 }
 
-sleighCompile {
+sleighCompileOptions = [
-	// args '-l'
+	// '-l'
-}
+]

Ghidra/Processors/Dalvik/build.gradle

@@ -7,9 +7,9 @@ apply plugin: 'eclipse'
 eclipse.project.name = 'Processors Dalvik'
 
 
-sleighCompile {
+sleighCompileOptions = [
-	args '-l'
+	'-l'
-}
+]
 
 dependencies {
 	compile project(':Base')

Ghidra/Processors/HCS08/build.gradle

@@ -3,6 +3,6 @@ apply from: "$rootProject.projectDir/gradle/processorProject.gradle"
 apply plugin: 'eclipse'
 eclipse.project.name = 'Processors HCS08'
 
-sleighCompile {
+sleighCompileOptions = [
-	args '-l'
+	'-l'
-}
+]

Ghidra/Processors/HCS12/build.gradle

@@ -6,9 +6,9 @@ apply plugin: 'eclipse'
 
 eclipse.project.name = 'Processors HCS12'
 
-sleighCompile {
+sleighCompileOptions = [
-	args '-l'
+	'-l'
-}
+]
 
 dependencies {
 	compile project(':Base')

Ghidra/Processors/MCS96/build.gradle

@@ -3,7 +3,6 @@ apply from: "$rootProject.projectDir/gradle/processorProject.gradle"
 apply plugin: 'eclipse'
 eclipse.project.name = 'Processors MCS96'
 
-
+sleighCompileOptions = [
-sleighCompile {
+	'-l'
-	args '-l'
+]
-}

Ghidra/Processors/MIPS/build.gradle

@@ -10,6 +10,6 @@ dependencies {
 	compile project(':Base')
 }
 
-sleighCompile {
+sleighCompileOptions = [
-	args '-l'
+	'-l'
-}
+]

GhidraBuild/BuildFiles/sleighDevBuild.template

@@ -46,8 +46,9 @@
 			fork="true"
 			failonerror="true">
 			<jvmarg value="-Xmx2048M"/>
+			<arg value="-DBaseDir=../../../../../../" />  <!-- repos root directory -->
 			<arg value="-i"/>
-			<arg value="../build/data/sleighArgs.txt"/>
+			<arg value="../build/tmp/sleighArgs.txt"/>
 			<arg value="-a"/>
 			<arg value="./languages"/>
 		</java>

GhidraBuild/BuildFiles/sleighDistBuild.template

@@ -39,6 +39,7 @@
 			fork="true"
 			failonerror="true">
 			<jvmarg value="-Xmx2048M"/>
+			<arg value="-DBaseDir=../../../../../" />  <!-- Ghidra install directory -->
 			<arg value="-i"/>
 			<arg value="sleighArgs.txt"/>
 			<arg value="-a"/>

gradle/processorProject.gradle

@@ -7,8 +7,6 @@
 		apply from: "$rootProject.projectDir/gradle/nativeProject.gradle"
  *****************************************************************************************/
 
-
-
 /*****************************************************************************************
  *
  *  Create a configuration so the a dependency can be declared on the the software modeling
@@ -26,22 +24,71 @@ dependencies {
 
 /*****************************************************************************************
  *
- *  Task to write sleigh compiler args to build/data/sleighArgs.txt for use with sleigh 
+ *  Sleigh compile options to be written to sleighArgs.txt in support of the following
- *  external sleigh compiler.
+ *  use cases:
+ *    - Ant build using data/build.xml (development and distribution)
+ *    - Eclipse Sleigh launcher (development only)
+ *    - Ghidra runtime language rebuild (SleighLanguage.reloadLanguage; development and distribution)
+ *    - Distribution build (sleighCompile task; development layout)
+ *
+ *  This list may be added to or replaced by a specific processor project/module.
+ *
+ *  Example: MIPS processor module dependency within a slaspec specified as:
+ *
+ *    @include "$(BaseDir)$(MIPS)/data/language/maips.sinc
+ *
+ *  with the corresponding MIPS definition specified within the sleighCompileOptions
+ *  list specified within the module's build.gradle file:
+ *
+ *      sleighCompileOptions.add "-DMIPS=%%MIPS%%"
+ *  -or-
+ *      sleighCompileOptions = [
+ *          "-l",
+ *          "-DMIPS=%%MIPS%%"
+ *      ]
+ *
+ *****************************************************************************************/
+ ext.sleighCompileOptions = [ ]
+ 
+ /*****************************************************************************************
+ *
+ *  Check for invalid sleighCompileOptions
+ *
+ *****************************************************************************************/
+ def checkSleighCompileOptions() {
+ 	sleighCompileOptions.each { a ->
+ 		assert !(a.startsWith("-a") || a.startsWith("-i")) : "Invalid sleighCompileOption: ${a}"
+ 	}
+ }
+ 
+/*****************************************************************************************
+ *
+ * Task to write sleigh compiler args for use with sleigh compiler.
+ * Due to the possible presence of module dependency paths two different sleighArgs.txt
+ * files are produced: one for development layout (build/tmp/sleighArgs.txt) and
+ * one for distribution layout ([build/]data/sleighArgs.txt).  When invoking the 
+ * Sleigh compiler and using a sleighArgs.txt file the appropriate 'BaseDir' property
+ * must be specified.  Withing a distribution install 'BaseDir' must specifiy the 
+ * path to the install directory while in a development layout 'BaseDir' must specify
+ * the repos root directory which contains the 'ghidra' repo directory.
  *
  *****************************************************************************************/
 task saveSleighArgs {
 	def sleighArgsFile = file("build/data/sleighArgs.txt")
-	outputs.files sleighArgsFile
+	def sleighArgsDevFile = file("build/tmp/sleighArgs.txt")
+	outputs.files sleighArgsFile, sleighArgsDevFile
 	outputs.upToDateWhen { false }
 	doLast {
+		checkSleighCompileOptions()
 		sleighArgsFile.withWriter { out->
-			project.sleighCompile.args.each { a-> 
+			sleighCompileOptions.each { a-> 
-				// don't save -a option
+				out.println resolveSleighArg(a, false) 
-				if (!"-a".equals(a)) {
-					out.println a 
 			}
 		}
+		sleighArgsDevFile.withWriter { out->
+			sleighCompileOptions.each { a-> 
+				out.println resolveSleighArg(a, true) 
+			}
 		}
 	}
 }
@@ -50,6 +97,7 @@ rootProject.prepDev.dependsOn(saveSleighArgs)
 apply plugin: 'base'
 clean {
 	delete file("build/data/sleighArgs.txt")
+	delete file("build/tmp/sleighArgs.txt")
 }
 
 /*****************************************************************************************
@@ -94,6 +142,7 @@ rootProject.assembleDistribution {
  *
  *****************************************************************************************/
 task sleighCompile (type: JavaExec) {
+	dependsOn saveSleighArgs
 	group = rootProject.GHIDRA_GROUP
 	description "  Compiles all the sleigh languages.    [processorUtils.gradle]\n"
 	 
@@ -104,7 +153,12 @@ task sleighCompile (type: JavaExec) {
 	// Delay adding the directory argument until the first part of the execution phase, so
 	// that any extra args added by a project override will be added to the arg list before
 	// these arguments.
+	// NOTE: projects should no longer add arguments to this task and should instead
+	// add such args to the sleighCompileOptions list.
 	doFirst {
+		args "-i"
+		args "./build/tmp/sleighArgs.txt"
+		args "-DBaseDir=${getProjectReposRootPath()}"
 		args '-a'
 		args './data/languages'
 	}
@@ -115,11 +169,9 @@ task sleighCompile (type: JavaExec) {
 // The task that copies the common files to the distribution folder must depend on
 // the sleigh tasks before executing.
 rootProject.assembleDistribution.dependsOn sleighCompile
-rootProject.assembleDistribution.dependsOn saveSleighArgs
 
 // Add in this projects sleighCompile to the allSleighCompile task
 rootProject.allSleighCompile.dependsOn sleighCompile
-rootProject.allSleighCompile.dependsOn saveSleighArgs
 
 /*****************************************************************************************
  *
@@ -154,3 +206,57 @@ sleighCompile.outputs.files (taskOutputs)
 
 // define the sleigh compile inputs to saveSleighArgs to limit task creation to language modules
 saveSleighArgs.inputs.files (taskInputs)
+
+/*****************************************************************************************
+ *
+ * Gets the absolute repos root directory path with a trailing File separator.
+ * This path may be used for specifying 'BaseDir' to the sleigh compiler within a 
+ * development layout.
+ *
+ *****************************************************************************************/
+def getProjectReposRootPath() {
+	def rootPath = rootProject.projectDir.absolutePath
+	int index = rootPath.lastIndexOf(File.separator)
+	rootPath = rootPath.substring(0, index+1)
+	return rootPath
+}
+
+/*****************************************************************************************
+ *
+ * Filter a sleigh compiler argument replacing any project/module reference of the form
+ * %%MODULE%% witha that MODULE's relative path.  If useDevPath is true the path will
+ * include the containing repo directory (e.g., ghidra/Ghidra/...), otherwise the
+ * path should start at the application root 'Ghidra/'.  Only a single replacement per
+ * arg is supported.
+ *
+ * This mechanism relies on the relative depth of a language module project within a
+ * repository directory hierarchy.  In general language module projects must reside 
+ * within the directory Ghidra/Processors.
+ *
+ *****************************************************************************************/
+def resolveSleighArg(String arg, boolean useDevPath) {
+	int index = arg.indexOf("%%")
+	if (index < 0) {
+		return arg
+	}
+	String newArg = arg.substring(0, index)
+	String tail = arg.substring(index+2)
+	index = tail.indexOf("%%")
+	assert index > 0 : "Badly formed sleigh path-replacment option: ${arg}"
+	String moduleName = tail.substring(0, index)
+	tail = tail.substring(index+2)
+	def moduleProject = project(":${moduleName}")
+	def modulePath
+	if (useDevPath) {
+		// first path element is the containing repo directory
+		modulePath = moduleProject.projectDir.absolutePath
+		modulePath = modulePath.substring(getProjectReposRootPath().length())
+	}
+	else {
+		// first path element is the Ghidra directory
+		modulePath =  getZipPath(moduleProject)
+	}
+	newArg += modulePath
+	newArg += tail
+	return newArg
+}