GP-467 simplifed build for Decompiler making installation of bison/flex

optional for most end-users

Ghidra/Features/Decompiler/build.gradle

@@ -8,8 +8,6 @@ apply plugin: 'eclipse'
 
 eclipse.project.name = 'Features Decompiler'
 
-
-
 dependencies {
 	compile project(':Base')
 	compile project(':SoftwareModeling')
@@ -20,48 +18,64 @@ dependencies {
     helpPath project(path: ":Base", configuration: 'helpPath')
 }
 
+ext.cppSourceDir = "src/decompile/cpp"
 
 /**
- * This task calls bison to process the xml.y and grammar.y file into cpp files that will then
- * be compiled with the rest of the decompiler code.
+ * The bison and flex generated files are maintained in source control and 
+ * do not need regeneration unless changes have been made to the yacc (*.y),
+ * lex (*.l) or any critical header file.  This stand-alone task should be 
+ * executed as needed prior to building the decompiler or sleigh native
+ * executables.
+ */
+task generateParsers {
+	dependsOn "lexSleigh", "yaccDecompiler"
+}
+
+ /**
+ * This task calls bison to process the grammar.y and xml.y files (yacc) to
+ * generate the corresponding *.cc files that will then be compiled with the rest 
+ * of the decompiler code.
  */
 task yaccDecompiler {
-	Task t1 = createBisonTask("xml", "decompile", true);
-	Task t2 = createBisonTask("grammar", "decompile", true);
-	
-	dependsOn t1, t2
 
-	// these variables are used by the cpp sources sets as an input to the decompiler native build	
-	ext.sourceDir = file("build/generated/decompile/cpp")
-	ext.headerDir = file("build/generated/decompile/headers") 
+	Task t1 = createBisonTask("xml", "decompile", false, true);
+	Task t2 = createBisonTask("grammar", "decompile", false, true);
+	
+	if (t1 != null) {
+		dependsOn t1, t2
+	}
 }
 
 /**
- * This task calls bison to process the xml.y file (yacc) into an cpp file that will then
- * be compiled with the rest of the decompiler code.
+ * This task calls bison to process the slghparse.y, pcodeparse.y and xml.y 
+ * files (yacc) to generate the corresponding *.cc files that will then be compiled 
+ * with the rest of the sleigh code.
  */
 task yaccSleigh {
-	Task t1 = createBisonTask("slghparse", "sleigh", false);
-	Task t2 = createBisonTask("pcodeparse", "sleigh", true);
-	Task t3 = createBisonTask("xml", "sleigh", true);
 	
-	dependsOn t1,t2,t3
-
-	ext.sourceDir = file("build/generated/sleigh/cpp")
-	ext.headerDir = file("build/generated/sleigh/headers")  
+	Task t1 = createBisonTask("slghparse", "sleigh", true, false); // also produces slghparse.hh header file
+	Task t2 = createBisonTask("pcodeparse", "sleigh", false, true);
+	Task t3 = createBisonTask("xml", "sleigh", false, true);
+	
+	if (t1 != null) {
+		dependsOn t1,t2,t3
+	}
 }
 
 /**
- * This task calls flex to process the slghscan. file (yacc) into an cpp file that will then
- * be compiled with the rest of the decompiler code.
+ * This task calls flex to process the slghscan.l file (lex) into a cpp file that will then
+ * be compiled with the rest of the sleigh code.
  */
  task lexSleigh {
+ 
+ 	dependsOn "yaccSleigh"
+ 	
 	Task t1 = createLexTask("slghscan", "sleighlex")
 	
-	dependsOn t1
-		
-	ext.sourceDir = file("build/generated/sleighlex")
-	ext.headerDir = file("build/generated/sleighlex")  
+	if (t1 != null) {
+		// TODO: add missing task input file dependencies
+		dependsOn t1
+	}
 }
 
 def buildDir = "../../../build" // Relative to the 'workingDir' Exec task property.
@@ -362,9 +376,12 @@ model {
 			targetPlatform "osx64"
 			sources {
 				cpp {
+					// NOTE: The bison/flex generated files are assumed to be up-to-date.
+					//       The task `generateParsers` should be executed if needed.
+					// builtBy yaccDecompiler
 		            source {
 		                srcDir "src/decompile/cpp"
-		               // include "xml.cc"				// generated by yacc task
+		                
 		                include "space.cc"
 		                include "float.cc"
 		                include "address.cc"
@@ -387,7 +404,6 @@ model {
 		                include "fspec.cc"
 		                include "action.cc"
 		                include "loadimage.cc"
-		                //include "grammar.cc"	 // doesn't seem to be used
 		                include "varnode.cc"
 		                include "op.cc"
 		                include "type.cc"
@@ -436,25 +452,21 @@ model {
 		                include "comment_ghidra.cc"
 						include "string_ghidra.cc"
 		         //       include "callgraph.cc"			// uncomment for debug
-		         //       include "ifacedecomp.cc"		// uncomment for debug
+		         //       include "ifacedecomp.cc"			// uncomment for debug
 		         //       include "ifaceterm.cc"			// uncomment for debug
 		         //       include "interface.cc"			// uncomment for debug
+		         
+		         		// generated source files
+		         		
+		         	 	include "xml.cc"
+		         // 	  include "grammar.cc"				// used by diagnostic console mode
 		            }
 					exportedHeaders {
 						srcDir "src/decompile/cpp"
 					}
-				} // end cpp
-				
-				// this creates a source set that is basically the output of bison yacc task
-				// defined above that compiles the xml.y file to a cpp file.
-				yacc(CppSourceSet) {
-	 				generatedBy yaccDecompiler
-					exportedHeaders {
-						srcDir "src/decompile/cpp"
-					}
-				}				
+				} // end cpp				
 			} // end sources
-		}// end decompile
+		} // end decompile
 		
 		sleigh(NativeExecutableSpec) {
 			targetPlatform "win64"
@@ -462,9 +474,12 @@ model {
 			targetPlatform "osx64"
 			sources {
 				cpp {
+					// NOTE: The bison/flex generated files are assumed to be up-to-date.
+					//       The task `generateParsers` should be executed if needed.
+					// builtBy lexSleigh
 					source {
 						srcDir "src/decompile/cpp"
-						//include "xml.cc"
+						
 						include "space.cc"
 						include "float.cc"
 						include "address.cc"
@@ -473,7 +488,6 @@ model {
 						include "opcodes.cc"
 						include "globalcontext.cc"
 						include "sleigh.cc"
-						//include "pcodeparse.cc"
 						include "pcodecompile.cc"
 						include "sleighbase.cc"
 						include "slghsymbol.cc"
@@ -483,31 +497,21 @@ model {
 						include "context.cc"
 						include "filemanage.cc"
 						include "slgh_compile.cc"
-						//include "slghparse.cc"
-						//include "slghscan.cc"
+						
+						// generated source files
+						
+						include "xml.cc"
+						include "pcodeparse.cc"
+						include "slghparse.cc"
+						include "slghscan.cc"
 					}
 					exportedHeaders {
 						srcDir "src/decompile/cpp"
 					}
-				}
-				yacc(CppSourceSet) {
-					generatedBy yaccSleigh
-					exportedHeaders {
-						srcDir "src/decompile/cpp"
-					}
-				}
-				lex(CppSourceSet) {
-					generatedBy lexSleigh
-					builtBy yaccSleigh // Requires headers produced by bison
-					exportedHeaders {
-						srcDir "src/decompile/cpp"
-						srcDir yaccSleigh.headerDir
-						srcDir lexSleigh.headerDir
-					}
-				}
-				
+				} // end cpp
 			} // end sources (sleigh)
-		}
+		} // end sleigh
+		
 	}  // end components
 	
 	binaries {
@@ -555,32 +559,53 @@ model {
 	}
 } // end model
 
-
+// Perform simple dependency change detection for generated files 
+// produced by bison or flex.  A 5-second tolerance is used
+// to avoid arbitrary last-modified times which may be produced 
+// by a git checkout.
+boolean isUpToDate(File srcFile, File resultFile) {
+	long resultLm = resultFile.lastModified()
+	if (resultLm == 0) {
+		return true
+	}
+	long srcLm = srcFile.lastModified()
+	long elapsedTime = srcLm - resultLm
+	if (elapsedTime < 5000) {
+	  println "Is UpToDate: ${resultFile}"
+	  return true;
+	}
+	return false;
+}
 
 /**
- * create a bison task to compile a yacc file for the decompiler
+ * Create a bison task to compile a yacc file (*.y) for the sleigh/decompiler
  */
-Task createBisonTask(String filename, String binaryName, boolean qualifyVariables) {
+Task createBisonTask(String filename, String binaryName, boolean generateHeader,  boolean qualifyVariables) {
+
+	def outputCppDir = "${cppSourceDir}"
+	def outputHeadersDir = "${cppSourceDir}"
+	
+	def yaccFile = "${cppSourceDir}/${filename}.y"
+	def ccFile = "${outputCppDir}/${filename}.cc"
+	def headerFile = "${outputCppDir}/${filename}.hh"
 
 	return task("bison_${binaryName}_$filename", type: Exec) {
-		String inputFile = "src/decompile/cpp/${filename}.y"
-		String outputCppDir = "build/generated/$binaryName/cpp"
-		String outputHeadersDir = "build/generated/$binaryName/headers"
 
-		inputs.file inputFile
-		outputs.dir outputCppDir
-
-		// make sure output directories exist. Must be done in execution phase. Otherwise,
-		// if a clean task is executed it will wipe them out
-		doFirst {
-			file(outputCppDir).mkdirs()
-			file(outputHeadersDir)mkdirs()
-		}		
+		inputs.file "${yaccFile}"
+		outputs.file "${ccFile}"
+		if (generateHeader) {
+			outputs.file "${headerFile}"
+		}
+		
+		// doFirst {
+		//	file(outputCppDir).mkdirs()
+		//	file(outputHeadersDir)mkdirs()
+		// }		
 		
 		executable 'bison'			// use bison program to process yacc files
 		
 		// specify the bison's output file
-		args "-o", "$outputCppDir/${filename}.tab.cc"
+		args "-o", "${ccFile}"
 		
 		// most of the yacc files should be compiled with a variable qualifyer to avoid dupes.
 		// Unfortunately there is one (slghparse) that can't use a qualifyer because it 
@@ -591,41 +616,42 @@ Task createBisonTask(String filename, String binaryName, boolean qualifyVariable
 		}
 		
 		// tell bison where to put the hh file.
-		args "--defines=${outputHeadersDir}/${filename}.tab.hh"
+		if (generateHeader) {
+			args "--defines=${headerFile}"
+		}
 		
 		// tell bison the file to compile
-		args inputFile
+		args "${yaccFile}"
 	}
 }
 
 /**
- * create a lex task to compile a yacc file for the decompiler
+ * Create a flex task to compile a lex file (*.l) for sleigh.
  */
 Task createLexTask(String filename, String binaryName) {
-	return task("lex_${binaryName}_$filename", type: Exec) {
-		String outputDir = "build/generated/${binaryName}/cpp"
-		String inputFile = "src/decompile/cpp/${filename}.l"
 
+	def outputCppDir = "${cppSourceDir}"
+	
+	def lexFile = "${cppSourceDir}/${filename}.l"
+	def ccFile = "${outputCppDir}/${filename}.cc"
+	
+	return task("lex_${binaryName}_$filename", type: Exec) {
+		
+		// set up inputs and outputs so that gradle knows when this needs to be rebuilt
+		inputs.file "${lexFile}"
+		outputs.files "${ccFile}"
+
+		// doFirst {
+		//	file(outputCppDir).mkdirs();
+		// }
+		
 		executable 'flex'			// the program to execute
 
-		// set up inputs and outputs so that gradle knows when this needs to be rebuilt
-	
-		inputs.file inputFile
-		outputs.dir outputDir	
-	
-		// 	make sure the output dirs are created, but do it in the execution phase.
-		// 	Can't do it in the configure phase, otherwise
-		// 	a clean will remove it during the execution phase
-	
-		doFirst {
-			file(outputDir).mkdirs();
-		}	
-
 		// tell flex where to put the output
-		args "-o", "$outputDir/${filename}.yy.cc"
+		args "-o", "${ccFile}"
 		
 		// tell flex the input file
-		args inputFile
+		args "${lexFile}"
 	}
 }
 

Ghidra/Features/Decompiler/src/decompile/cpp/.gitignore

@@ -1,9 +1,3 @@
-grammar.cc
-pcodeparse.cc
-slghparse.cc
-slghparse.tab.hh
-slghscan.cc
-xml.cc
 coreext_*
 ghidraext_*
 consoleext_*

Ghidra/Features/Decompiler/src/decompile/cpp/Makefile

@@ -140,7 +140,7 @@ SLEIGH_OPT_OBJS=$(SLEIGH_NAMES:%=sla_opt/%.o)
 LIBSLA_DBG_OBJS=$(LIBSLA_NAMES:%=com_dbg/%.o)
 LIBSLA_OPT_OBJS=$(LIBSLA_NAMES:%=com_opt/%.o)
 LIBSLA_SOURCE=$(LIBSLA_NAMES:%=%.cc) $(LIBSLA_NAMES:%=%.hh) \
-	$(SLACOMP:%=%.cc) slgh_compile.hh slghparse.tab.hh types.h \
+	$(SLACOMP:%=%.cc) slgh_compile.hh slghparse.hh types.h \
 	partmap.hh error.hh slghparse.y pcodeparse.y xml.y slghscan.l loadimage_bfd.hh loadimage_bfd.cc
 LIBDECOMP_DBG_OBJS=$(LIBDECOMP_NAMES:%=com_dbg/%.o)
 LIBDECOMP_OPT_OBJS=$(LIBDECOMP_NAMES:%=com_opt/%.o)
@@ -223,16 +223,14 @@ pcodeparse.cc:	pcodeparse.y
 	$(YACC) -p pcode -o $@ $<
 slghparse.cc:	slghparse.y
 	$(YACC) -d -o $@ $<
-	mv slghparse.hh slghparse.tab.hh
 slghscan.cc:	slghscan.l
 	$(LEX) -o$@ $<
 ruleparse.cc:	ruleparse.y
 	$(YACC) -p ruleparse -d -o $@ $<
-	mv ruleparse.hh ruleparse.tab.hh
 
-slghparse.tab.hh:	slghparse.y slghparse.cc
-slghscan.cc:	slghparse.tab.hh
-ruleparse.tab.hh:	ruleparse.y ruleparse.cc
+slghparse.hh:	slghparse.y slghparse.cc
+slghscan.cc:	slghparse.hh slgh_compile.hh
+ruleparse.hh:	ruleparse.y ruleparse.cc
 
 decomp_dbg:	$(COMMANDLINE_DBG_OBJS)
 	$(CXX) $(DBG_CXXFLAGS) $(ARCH_TYPE) -o decomp_dbg $(COMMANDLINE_DBG_OBJS) $(BFDLIB) $(LNK)
@@ -382,7 +380,6 @@ resetgcov:
 
 reallyclean:	clean	
 	rm -rf coreext_*.cc coreext_*.hh ghidraext_*.cc ghidraext_*.hh consoleext_*.cc consoleext_*.hh
-	rm -rf grammar.cc xml.cc pcodeparse.cc slghparse.cc slghparse.tab.hh slghscan.cc ruleparse.cc ruleparse.tab.hh
 	rm -rf com_dbg com_opt ghi_dbg ghi_opt sla_dbg sla_opt
 	rm -f $(EXECS) TAGS *~
 

Ghidra/Features/Decompiler/src/decompile/cpp/slghparse.hh

@@ -0,0 +1,239 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* A Bison parser, made by GNU Bison 3.0.4.  */
+
+/* Bison interface for Yacc-like parsers in C
+
+   Copyright (C) 1984, 1989-1990, 2000-2015 Free Software Foundation, Inc.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+/* As a special exception, you may create a larger work that contains
+   part or all of the Bison parser skeleton and distribute that work
+   under terms of your choice, so long as that work isn't itself a
+   parser generator using the skeleton or a modified version thereof
+   as a parser skeleton.  Alternatively, if you modify or redistribute
+   the parser skeleton itself, you may (at your option) remove this
+   special exception, which will cause the skeleton and the resulting
+   Bison output files to be licensed under the GNU General Public
+   License without this special exception.
+
+   This special exception was added by the Free Software Foundation in
+   version 2.2 of Bison.  */
+
+#ifndef YY_YY_SRC_DECOMPILE_CPP_SLGHPARSE_HH_INCLUDED
+# define YY_YY_SRC_DECOMPILE_CPP_SLGHPARSE_HH_INCLUDED
+/* Debug traces.  */
+#ifndef YYDEBUG
+# define YYDEBUG 0
+#endif
+#if YYDEBUG
+extern int yydebug;
+#endif
+
+/* Token type.  */
+#ifndef YYTOKENTYPE
+# define YYTOKENTYPE
+  enum yytokentype
+  {
+    OP_BOOL_OR = 258,
+    OP_BOOL_AND = 259,
+    OP_BOOL_XOR = 260,
+    OP_OR = 261,
+    OP_XOR = 262,
+    OP_AND = 263,
+    OP_EQUAL = 264,
+    OP_NOTEQUAL = 265,
+    OP_FEQUAL = 266,
+    OP_FNOTEQUAL = 267,
+    OP_GREATEQUAL = 268,
+    OP_LESSEQUAL = 269,
+    OP_SLESS = 270,
+    OP_SGREATEQUAL = 271,
+    OP_SLESSEQUAL = 272,
+    OP_SGREAT = 273,
+    OP_FLESS = 274,
+    OP_FGREAT = 275,
+    OP_FLESSEQUAL = 276,
+    OP_FGREATEQUAL = 277,
+    OP_LEFT = 278,
+    OP_RIGHT = 279,
+    OP_SRIGHT = 280,
+    OP_FADD = 281,
+    OP_FSUB = 282,
+    OP_SDIV = 283,
+    OP_SREM = 284,
+    OP_FMULT = 285,
+    OP_FDIV = 286,
+    OP_ZEXT = 287,
+    OP_CARRY = 288,
+    OP_BORROW = 289,
+    OP_SEXT = 290,
+    OP_SCARRY = 291,
+    OP_SBORROW = 292,
+    OP_NAN = 293,
+    OP_ABS = 294,
+    OP_SQRT = 295,
+    OP_CEIL = 296,
+    OP_FLOOR = 297,
+    OP_ROUND = 298,
+    OP_INT2FLOAT = 299,
+    OP_FLOAT2FLOAT = 300,
+    OP_TRUNC = 301,
+    OP_CPOOLREF = 302,
+    OP_NEW = 303,
+    OP_POPCOUNT = 304,
+    BADINTEGER = 305,
+    GOTO_KEY = 306,
+    CALL_KEY = 307,
+    RETURN_KEY = 308,
+    IF_KEY = 309,
+    DEFINE_KEY = 310,
+    ATTACH_KEY = 311,
+    MACRO_KEY = 312,
+    SPACE_KEY = 313,
+    TYPE_KEY = 314,
+    RAM_KEY = 315,
+    DEFAULT_KEY = 316,
+    REGISTER_KEY = 317,
+    ENDIAN_KEY = 318,
+    WITH_KEY = 319,
+    ALIGN_KEY = 320,
+    OP_UNIMPL = 321,
+    TOKEN_KEY = 322,
+    SIGNED_KEY = 323,
+    NOFLOW_KEY = 324,
+    HEX_KEY = 325,
+    DEC_KEY = 326,
+    BIG_KEY = 327,
+    LITTLE_KEY = 328,
+    SIZE_KEY = 329,
+    WORDSIZE_KEY = 330,
+    OFFSET_KEY = 331,
+    NAMES_KEY = 332,
+    VALUES_KEY = 333,
+    VARIABLES_KEY = 334,
+    PCODEOP_KEY = 335,
+    IS_KEY = 336,
+    LOCAL_KEY = 337,
+    DELAYSLOT_KEY = 338,
+    CROSSBUILD_KEY = 339,
+    EXPORT_KEY = 340,
+    BUILD_KEY = 341,
+    CONTEXT_KEY = 342,
+    ELLIPSIS_KEY = 343,
+    GLOBALSET_KEY = 344,
+    BITRANGE_KEY = 345,
+    CHAR = 346,
+    INTEGER = 347,
+    INTB = 348,
+    STRING = 349,
+    SYMBOLSTRING = 350,
+    SPACESYM = 351,
+    SECTIONSYM = 352,
+    TOKENSYM = 353,
+    USEROPSYM = 354,
+    VALUESYM = 355,
+    VALUEMAPSYM = 356,
+    CONTEXTSYM = 357,
+    NAMESYM = 358,
+    VARSYM = 359,
+    BITSYM = 360,
+    SPECSYM = 361,
+    VARLISTSYM = 362,
+    OPERANDSYM = 363,
+    STARTSYM = 364,
+    ENDSYM = 365,
+    MACROSYM = 366,
+    LABELSYM = 367,
+    SUBTABLESYM = 368
+  };
+#endif
+
+/* Value type.  */
+#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
+
+union YYSTYPE
+{
+#line 29 "src/decompile/cpp/slghparse.y" /* yacc.c:1909  */
+
+  char ch;
+  uintb *i;
+  intb *big;
+  string *str;
+  vector<string> *strlist;
+  vector<intb> *biglist;
+  vector<ExprTree *> *param;
+  SpaceQuality *spacequal;
+  FieldQuality *fieldqual;
+  StarQuality *starqual;
+  VarnodeTpl *varnode;
+  ExprTree *tree;
+  vector<OpTpl *> *stmt;
+  ConstructTpl *sem;
+  SectionVector *sectionstart;
+  Constructor *construct;
+  PatternEquation *pateq;
+  PatternExpression *patexp;
+
+  vector<SleighSymbol *> *symlist;
+  vector<ContextChange *> *contop;
+  SleighSymbol *anysym;
+  SpaceSymbol *spacesym;
+  SectionSymbol *sectionsym;
+  TokenSymbol *tokensym;
+  UserOpSymbol *useropsym;
+  MacroSymbol *macrosym;
+  LabelSymbol *labelsym;
+  SubtableSymbol *subtablesym;
+  StartSymbol *startsym;
+  EndSymbol *endsym;
+  OperandSymbol *operandsym;
+  VarnodeListSymbol *varlistsym;
+  VarnodeSymbol *varsym;
+  BitrangeSymbol *bitsym;
+  NameSymbol *namesym;
+  ValueSymbol *valuesym;
+  ValueMapSymbol *valuemapsym;
+  ContextSymbol *contextsym;
+  FamilySymbol *famsym;
+  SpecificSymbol *specsym;
+
+#line 212 "src/decompile/cpp/slghparse.hh" /* yacc.c:1909  */
+};
+
+typedef union YYSTYPE YYSTYPE;
+# define YYSTYPE_IS_TRIVIAL 1
+# define YYSTYPE_IS_DECLARED 1
+#endif
+
+
+extern YYSTYPE yylval;
+
+int yyparse (void);
+
+#endif /* !YY_YY_SRC_DECOMPILE_CPP_SLGHPARSE_HH_INCLUDED  */

Ghidra/Features/Decompiler/src/decompile/cpp/slghscan.l

@@ -16,7 +16,7 @@
  */
 %{
 #include "slgh_compile.hh"
-#include "slghparse.tab.hh"
+#include "slghparse.hh"
 #define yywrap() 1
 #define YY_SKIP_YYWRAP