/* ### * IP: GHIDRA * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /********************************************************************************* * test.gradle * * Contains tasks for running unit/integration tests and generating associated * reports. * * Testing tasks: unitTestReport * integrationTestReport * * *********************************************************************************/ /********************************************************************************* * Define some vars. Note that the first few vars must be defined. If they * aren't set by a calling script, we default them. * *********************************************************************************/ apply from: "gradle/support/testUtils.gradle" if (!project.hasProperty('rootTestDir')) { project.ext.rootTestDir = "build" } if (!project.hasProperty('machineName')) { try { project.ext.machineName = InetAddress.localHost.hostName } catch (UnknownHostException e) { project.ext.machineName = "STANDALONE" } } if (!project.hasProperty('xms')) { project.ext.xms = "512M" } if (!project.hasProperty('xmx')) { project.ext.xmx = "2048M" } loadApplicationProperties() loadMachineSpecificProperties() if (!project.hasProperty('srcTreeName')) { project.ext.srcTreeName = System.properties.get("application.version") } project.ext.testRootDirName = "JunitTest_" + srcTreeName project.ext.pcodeTestRootDirName = "PCodeTest_" + srcTreeName project.ext.shareDir = System.properties.get('share.dir') if (project.ext.shareDir != null) { // add src tree name to machine specified share path project.ext.testShareDir = shareDir + "/" + testRootDirName project.ext.pcodeTestShareDir = shareDir + "/" + pcodeTestRootDirName } else { project.ext.testShareDir = "$rootTestDir" + "/JUnit" project.ext.pcodeTestShareDir = "$rootTestDir" + "/PCodeTest" } project.ext.upgradeProgramErrorMessage = "WARNING! Attempting data upgrade for " project.ext.upgradeTimeErrorMessage = "Upgrade Time (ms): " project.ext.logPropertiesUrl = getLogFileUrl() // Specify final report directory via cmd line using "-PreportDir=". This is useful for // the 'parallelCombinedTestReport' task. project.ext.reportDir = project.hasProperty('reportDir') ? project.getProperty('reportDir') + "/reports" : testShareDir + "/reports" project.ext.reportArchivesDir = testShareDir + "/reportsArchive" project.ext.archivedReportDir = reportArchivesDir + "/reports" project.ext.testOutputDir = file(rootTestDir).getAbsolutePath() + "/output" project.ext.userHome = System.properties.get("user.home") // 'parallelMode' will change configs to allow concurrent test execution. // Enable this mode if 'parallelCombinedTestReport' invoked in the command line. project.ext.parallelMode = project.gradle.startParameter.taskNames.contains('parallelCombinedTestReport') project.ext.targetMode = project.gradle.startParameter.taskNames.contains('targetedTestReport') project.ext.targetFiles = targetMode ? project.getProperty('targetFiles') : null project.ext.targetProjects = targetMode ? getTestProjects(targetFiles) : null // 'parallelCombinedTestReport' task will parse a JUnit test report for test duration. // Specify the location of the report via cmd line using "-PtestTimeParserInputDir=". // If the location is not specified via cmd line, it will look for the latest test report // in /../reportsArchive/reports_ // If a reportsArchive path does not exist, it will assume all tests have the same duration and // continue with test task creation. project.ext.testTimeParserInputDir = project.hasProperty('testTimeParserInputDir') ? project.getProperty('testTimeParserInputDir') + "/reports/classes" : getLastArchivedReport("$reportArchivesDir") + "/classes" // A port of 0 will allow the kernel to give a free port number in the set of "User" // or "Registered" Ports (usually 1024 - 49151). This will prevent port collisions among // concurrent JUnit tests. project.ext.debugPort = parallelMode ? 0 : generateRandomDebugPort() /********************************************************************************* * Create specified absolute directory if it does not exist *********************************************************************************/ def createDir(dirPath) { println "Creating directory: " + dirPath File dir = new File(dirPath); dir.mkdirs(); if (!dir.exists()) { throw new RuntimeException("Failed to create required directory: " + dirPath); } if (!dir.canWrite()) { throw new RuntimeException("Process does not have write permission for directory: " + dirPath); } } /********************************************************************************* * Retrieves log4j properties file and returns its path. *********************************************************************************/ def getLogFileUrl() { String logFilePath = "$rootDir/Ghidra/Framework/Generic/src/main/resources/generic.log4jtest.xml" File logFile = new File(logFilePath) if (logFile.exists()) { return logFile.toURI().toString() } throw new GradleException("Cannot find log4j properties file using path '$logFilePath'") } /** * Given a set of files (passed in via the -PtargetFiles command line param), this * will search the project tree for each files' package to see if there is a test * class defined that can be run. * * The param is a list of file paths created by 'git diff --names-only' */ def List getTestProjects(String files) { List projects = new ArrayList<>(); List paths = files.tokenize('%') for (String path : paths) { File file = new File(path) // might blow up? // Find the project for this file def ps while (file.parentFile != null) { ps = rootProject.allprojects.findAll { it.projectDir == file } if (ps) { ps.each { projects.add(it) } break; } file = file.parentFile } } return projects } /********************************************************************************* * Creates a new debug port randomly. *********************************************************************************/ def generateRandomDebugPort() { // keep port within narrow range (NOTE: must be greater than 1024 and less than 65535) // Generate random byte value Random random = new Random() def ran = random.nextInt(); for (int i = 1; i < 4; i++) { ran ^= (ran >> 8); } ran &= 0xff; // Generate random port between 18300 and 18555 def port = 18300 + ran return port } /********************************************************************************* * Loads application specific property file that contains info we need. * Properties will be immediately added to the global System.properties file so we * can readily access them from just one place. *********************************************************************************/ def loadApplicationProperties() { Properties props = new Properties() File appProperties = new File("Ghidra/application.properties"); if (!appProperties.exists()) { return; } props.load(new FileInputStream(appProperties)); props.each {k, v -> System.setProperty(k, v) } } /********************************************************************************* * Loads any machine specific property file that contains info we need. * Only those properties with our machine name prefix will be immediately * added to the global System.properties file so we can readily access them from * just one place (machine name prefix will be omitted). *********************************************************************************/ def loadMachineSpecificProperties() { if (project.hasProperty('testPropertiesPath')) { Properties props = new Properties() def testPropertiesPath = project.getProperty('testPropertiesPath') File testProperties = new File(testPropertiesPath); if (!testProperties.exists()) { return; } println "loadMachineSpecificProperties: Using machine specific properties file '$testProperties'" props.load(new FileInputStream(testProperties)); // Note: Only load those properties that contain our machine name (set above). This means // that local test runs will not use these values. Also, if the test machine name // changes, then so too will have to change the properties file. props.each { k, v -> if (k.startsWith(machineName)) { // strip off _ prefix from property key def key = k.substring(machineName.length()+1) println "loadMachineSpecificProperties: Setting system property $key:$v" System.setProperty(key, v) } } } } /********************************************************************************* * Archive previously generated test report *********************************************************************************/ task archiveTestReport { File reports = file(reportDir) File archivedReports = file(archivedReportDir) File reportArchives = file(reportArchivesDir) onlyIf { containsIndexFile(reports) } doLast { reportArchives.mkdirs() delete archivedReports reports.renameTo(archivedReports) renameReportArchive(archivedReports) // renames archived reports directory generateArchiveIndex(reportArchives) } } /********************************************************************************* * Remove remnants of previous tests. *********************************************************************************/ task deleteTestTempAndReportDirs() { doFirst { delete file(rootTestDir).getAbsolutePath() delete file(reportDir).getAbsolutePath() } } /********************************************************************************* * Initialize test task *********************************************************************************/ def initTestJVM(Task task, String rootDirName) { def testTempDir = file(rootTestDir).getAbsolutePath() def testReportDir = file(reportDir).getAbsolutePath() task.doFirst { println "Test Machine Name: " + machineName println "Root Test Dir: " + rootTestDir println "Test Output Dir: " + testOutputDir println "Test Temp Dir: " + testTempDir println "Test Report Dir: " + testReportDir println "Java Debug Port: " + debugPort createDir(testTempDir) createDir(testOutputDir) } // If false, testing will halt when an error is found. task.ignoreFailures true // If false, then tests are re-run every time, even if no code has changed. task.outputs.upToDateWhen {false} // Must set this to see System.out print statements. task.testLogging.showStandardStreams = true // Min/Max heap size. These are passed in. task.minHeapSize xms task.maxHeapSize xmx task.doFirst { task.jvmArgs '-DupgradeProgramErrorMessage=' + upgradeProgramErrorMessage, '-DupgradeTimeErrorMessage=' + upgradeTimeErrorMessage, '-Dlog4j.configurationFile=' + logPropertiesUrl, '-Dghidra.test.property.batch.mode=true', '-Dghidra.test.property.parallel.mode=' + parallelMode, '-Dghidra.test.property.output.dir=' + testOutputDir, '-Dghidra.test.property.report.dir=' + testReportDir, '-DSystemUtilities.isTesting=true', '-Dmachine.name=' + machineName, '-Djava.io.tmpdir=' + testTempDir, '-Duser.data.dir=' + userHome + '/.ghidra/.ghidra-' + srcTreeName + '-Test', '-Dcpu.core.override=8', '-XX:ParallelGCThreads=8', '-XX:+UseParallelGC', '-Djava.awt.headless=false', '-DDecompilerFunctionAnalyzer.enabled=false', // prevent long-winded analysis when testing (see DecompilerFunctionAnalyzer) '-Dfile.encoding=UTF8', '-Duser.country=US', '-Duser.language=en', '-Djdk.attach.allowAttachSelf', '-XX:TieredStopAtLevel=1', '-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=' + debugPort, // Allow illegal reflective accesses '--add-opens=java.base/java.util.concurrent=ALL-UNNAMED', '--add-opens=java.desktop/sun.awt=ALL-UNNAMED', '--add-opens=java.desktop/sun.swing=ALL-UNNAMED', '--add-opens=java.desktop/java.awt=ALL-UNNAMED', '--add-opens=java.desktop/javax.swing=ALL-UNNAMED', '--add-opens=java.desktop/javax.swing.text=ALL-UNNAMED' // Note: this args are used to speed-up the tests, but are not safe for production code // -noverify and -XX:TieredStopAtLevel=1 // Note: modern remote debug invocation; // -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 // // We removed these lines, which should not be needed in modern JVMs // -Xdebug // -Xnoagent // -Djava.compiler=NONE // -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8000 } } /********************************************************************************* * Create this TestReport task so that individual test tasks may later assign themselves * (using 'reportOn'). These tasks run tests concurrently. * * Invoking this task via command line enables 'parallelMode'. * * Example: * ./gradlew parallelCombinedTestReport -PtestTimeParserInputDir= -PreportDir= * *********************************************************************************/ task parallelCombinedTestReport(type: TestReport) { t -> group = "test" destinationDirectory = file("$reportDir") logger.debug("parallelCombinedTestReport: Using destinationDir = $reportDir") dependsOn ":deleteTestTempAndReportDirs" mustRunAfter ":deleteTestTempAndReportDirs" } /********************************************************************************* * UNIT TEST REPORT * * Summary: Runs all unit tests and generates a single report. * *********************************************************************************/ task unitTestReport(type: TestReport) { t -> group = "test" description = "Run unit tests and save HTML report." destinationDirectory = file("$reportDir/unitTests") outputs.upToDateWhen {false} } /********************************************************************************* * INTEGRATION TEST REPORT * * Summary: Runs all integration tests and generates a single report. * *********************************************************************************/ task integrationTestReport(type: TestReport) { t -> group = "test" description = "Run integration tests and save HTML report." destinationDirectory = file("$reportDir/integrationTests") outputs.upToDateWhen {false} } /********************************************************************************* * TARGETED TEST REPORT * * Summary: Runs unit tests against a set of projects * * The projects to be run against are stored in the project.ext.targetProjects * var, which is populated at the top of this file. This is currently only * used by the runAllTests.sh script which executes this task on each commit, * so we can run all the tests for the files contained in that commit. * *********************************************************************************/ task targetedTestReport(type: TestReport) { t -> group = "test" description = "Run unit tests against a specific set of projects." destinationDirectory = file("$reportDir/unitTests") outputs.upToDateWhen {false} dependsOn ":deleteTestTempAndReportDirs" // This is a bit tricky but the tasks for each project are not yet defined // at this point, so we have to put this in an afterEvaluate block before // we can set up the reportOn dependency project.ext.targetProjects.each { it.afterEvaluate { p -> if (p.tasks.findByName('test')) { reportOn p.test } } } } /********************************************************************************* * Create a task of type: Test for a subproject. * Each task will run a maximum of 'numMaxParallelForks' tests at a time. * The task will include tests from classesList[classesListPosition, classesListPosition+numMaxParallelForks] * and inherit excludes from its non-parallel counterpart. * * @param subproject * @param testType - Either 'test' or 'integrationTest'. The sourceSets defined in setupJava.gradle * @param bucketName - the test category (appConfig, dockingConfig, integrationConfig, etc...) * @param taskNameCounter - Number to append to task name. * @param classesList - List of classes for this subproject.testType. Sorted by duration. * @param classesListPosition - Start position in 'classesList' * @param numMaxParallelForks - Number of concurrent tests to run. *********************************************************************************/ def createTestTask(Project subproject, String testType, String bucketName, int taskNameCounter, ArrayList classesList, int classesListPosition, int numMaxParallelForks) { subproject.task(testType+"_$taskNameCounter"+"_$bucketName", type: Test) { t -> group = "test" testClassesDirs = files subproject.sourceSets["$testType"].output.classesDirs classpath = subproject.sourceSets["$testType"].runtimeClasspath maxParallelForks = numMaxParallelForks initTestJVM(t, testRootDirName) rootProject.tasks['parallelCombinedTestReport'].reportOn t // include classes in task up to the maxParallelForks value for (int i = 0; (i < maxParallelForks && classesListPosition < classesList.size); i++) { // Convert my.package.MyClass to my/package/MyClass.class include classesList[classesListPosition].replace(".","/") + ".class" classesListPosition++ } // Inherit excludes from subproject build.gradle. // These excludes will override any includes. excludes = subproject.tasks["$testType"].excludes // Display granularity of 0 will log method-level events // as "Task > Worker > org.SomeClass > org.someMethod". testLogging { displayGranularity 0 events "started", "passed", "skipped", "failed" } // Current task mustRunAfter previous task (test_1 before test_2,etc.) if (taskNameCounter > 1) { def prevTaskName = "$testType" + "_" + (taskNameCounter -1) + "_$bucketName" mustRunAfter prevTaskName } logger.info("createTestTask: Created $subproject.name" + ":$testType" + "_$taskNameCounter" + "_$bucketName" + "\n\tincludes = \n" + t.getIncludes() + "\n\tinheriting excludes (overrides any 'includes') = \n" + t.excludes) } // end task } /********************************************************************************* * Split each subproject's integrationTest ("src/test.slow") and test ("src/test") * task into multiple tasks, based on duration parsed from a previous JUnit report. * Each task will run tests concurrently. * * Each task will run a maximum of the 'maxParallelForks' value at a time. *********************************************************************************/ configure(subprojects.findAll {parallelMode == true}) { subproject -> afterEvaluate { if (!shouldSkipTestTaskCreation(subproject)) { logger.info("parallelCombinedTestReport: Creating 'test' tasks for " + subproject.name + " subproject.") Map testMap = getTestsForSubProject(subproject.sourceSets.test.java) for (Map.Entry classMap : testMap.entrySet()) { String bucketName = classMap.getKey(); int classesListPosition = 0 // current position in classesList int taskNameCounter = 1 // task suffix int numMaxParallelForks = 40 // unit tests are fast; 40 seems to be a reasonable number List tests = classMap.getValue(); while (classesListPosition < tests.size()) { createTestTask(subproject, "test", bucketName, taskNameCounter, tests, classesListPosition, numMaxParallelForks) classesListPosition+=numMaxParallelForks taskNameCounter+=1; // "test_1_appConfig", "test_2_appConfig, etc. } } } if (!shouldSkipIntegrationTestTaskCreation(subproject)) { logger.info("parallelCombinedTestReport: Creating 'integrationTest' tasks for " + subproject.name + " subproject.") Map testMap = getTestsForSubProject(subproject.sourceSets.integrationTest.java) for (Map.Entry classMap : testMap.entrySet()) { String bucketName = classMap.getKey(); int classesListPosition = 0 // current position in classesList int taskNameCounter = 1 // task suffix // Through trial-and-error we found that 40 is too many // concurrent integration tests (ghidratest server has 40 CPUs). // 20 seems like a good balance of throughput vs resource usage for ghidratest server. int numMaxParallelForks = 20 List tests = classMap.getValue(); while (classesListPosition < tests.size()) { createTestTask(subproject, "integrationTest", bucketName, taskNameCounter, tests, classesListPosition, numMaxParallelForks) classesListPosition+=numMaxParallelForks taskNameCounter+=1; // "integrationTest_1_appConfig", "integrationTest_2_appConfig, etc. } } } } // end afterEvaluate }// end subprojects /********************************************************************************* * COMBINED TEST REPORT * * Summary: Runs all integration and unit tests, and creates a single report. * *********************************************************************************/ task combinedTestReport(type: TestReport) { t -> group = "test" destinationDirectory = file("$reportDir") dependsOn ":deleteTestTempAndReportDirs" mustRunAfter ":deleteTestTempAndReportDirs" } /********************************************************************************* * P-CODE TEST REPORT * * Summary: Runs all P-Code Tests * *********************************************************************************/ task pcodeTestReport(type: TestReport) { t -> group = "pcodeTest" destinationDirectory = file("$pcodeTestShareDir" + "/reports") } /********************************************************************************* * P-CODE TEST REPORT w/ MATRIX REPORT * * Summary: Runs all P-Code Tests and consolidates JUnit and Matrix report in * results directory. * *********************************************************************************/ task pcodeConsolidatedTestReport(type: Copy) { group = "pcodeTest" dependsOn ':pcodeTestReport' into (pcodeTestShareDir + "/reports") from (testOutputDir + "/test-output") { exclude '**/*.xml', 'cache/**' } } /********************************************************************************* * Rename archived report directory to reflect lastModified date *********************************************************************************/ def renameReportArchive(dir) { long modifiedTime = dir.lastModified() Calendar calendar = new GregorianCalendar() calendar.setTimeInMillis( modifiedTime ) java.text.SimpleDateFormat dateFormat = new java.text.SimpleDateFormat("_MMM_d_yyyy") String newFilename = dir.getName() + dateFormat.format(calendar.getTime()) File newFile = new File( dir.getParent(), newFilename ) newFile = getUniqueName( newFile ) dir.renameTo( newFile ) println "Archived reports: " + newFile.getCanonicalPath() } /********************************************************************************* * Attempt to generate unique directory/file name to avoid duplication *********************************************************************************/ def getUniqueName(file) { if (!file.exists()) { return file } for (int i = 2; i < 22; i++ ) { String newName = file.getName() + "_" + i File newFile = new File( file.getParentFile(), newName ) if (!newFile.exists()) { return newFile } } return file } /********************************************************************************* * Generate report archive index.html file within the archivesDir directory *********************************************************************************/ def generateArchiveIndex(archivesDir) { File file = new File( archivesDir, "index.html" ); println "Generating Archived Reports Index: " + file.getCanonicalPath() PrintWriter p = new PrintWriter(file); try { p.println(""); p.println(""); printStyleSheet(p); p.println(""); p.println(""); p.println("
"); p.println("

Past Test Reports

"); p.println(" "); // get a dir listing of all the files from the report dir and // create a hyperlink for the filename File[] reportDirFiles = archivesDir.listFiles(); Arrays.sort(reportDirFiles, new Comparator() { public int compare(File file1, File file2) { if ( file1 == file2 ) { return 0; } long timestamp1 = file1.lastModified(); long timestamp2 = file2.lastModified(); if (timestamp1 == timestamp2) { return file1.getName().compareTo(file2.getName()); } return (timestamp1 < timestamp2) ? 1 : -1; } }); // archived reports older than one month will be removed long now = System.currentTimeMillis(); println "Current time: " + now long oldestTime = now - (1000L * 60 * 60 * 24 * 30); println "Delete date: " + oldestTime for (File f : reportDirFiles) { if (!f.isDirectory()) { continue; } println "File: " + f.toString() println "\tlast modified: " + f.lastModified() if (f.lastModified() < oldestTime) { println "\t\tFile older than the oldest time--deleting!..." f.deleteDir(); // delete old archive results } else { println "\t\tFile is a spring chicken--creating link..." createLinkForDir(p, f); } } p.println("
"); p.println("
"); p.println(""); p.println(""); } finally { p.close(); } } /********************************************************************************* * Add HTML code to PrintWriter p which provides link to specified file which * is assumed to reside within the same directory as the index.html file being written *********************************************************************************/ def createLinkForDir(p, file) { // skip files - dirs only if ( !file.isDirectory() ) { return; } String columnPadding = " " p.println( columnPadding + "" ) String dirName = file.getName() if (containsIndexFile(file)) { String linkText = "" + dirName + "" p.println( columnPadding + linkText); } else { p.println( columnPadding + dirName); } p.println( columnPadding + "" ) } /********************************************************************************* * Returns true if the specified directory File contains an index.html file *********************************************************************************/ def containsIndexFile(dir) { File indexFile = new File(dir, "index.html"); return indexFile.exists(); } /********************************************************************************* * Add HTML code to PrintWriter p which provides the STYLE tag *********************************************************************************/ def printStyleSheet(p) { p.println( "" ); }