/* ### * IP: GHIDRA * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /******************************************************************************* * fetchDependencies.gradle * * * * Fetches/downloads required dependencies that aren't available in the * * standard online repositories (eg: maven) and configures a flat * * directory-style respository that points to them. This should be run * * immediately after cloning the Ghidra repository before any other gradle * * tasks are run. * * * * usage: from the command line in the main ghidra repository directory, run * * the following: * * * * gradle -I gradle/support/fetchDependencies.gradle init * * * * Note: When running the script, files will only be downloaded if * * necessary (eg: they are not already in the dependencies/downloads/ * * directory). * * * *******************************************************************************/ import java.util.zip.*; import java.nio.file.*; import java.security.MessageDigest; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.WildcardFileFilter; initscript { repositories { mavenCentral() } dependencies { classpath 'commons-io:commons-io:2.11.0' } } ext.NUM_RETRIES = 3 // # of times to try to download a file before failing ext.REPO_DIR = ((Script)this).buildscript.getSourceFile().getParentFile().getParentFile().getParentFile() ext.DEPS_DIR = file("${REPO_DIR}/dependencies") ext.DOWNLOADS_DIR = file("${DEPS_DIR}/downloads") ext.FID_DIR = file("${DEPS_DIR}/fidb") ext.FLAT_REPO_DIR = file("${DEPS_DIR}/flatRepo") ext.OFFLINE = System.properties["offline"] != null ext.HIDE_DOWNLOAD_PROGRESS = System.properties["hideDownloadProgress"] != null ext.createdDirs = [] as Set file("${REPO_DIR}/Ghidra/application.properties").withReader { reader -> def ghidraProps = new Properties() ghidraProps.load(reader) ext.RELEASE_VERSION = ghidraProps.getProperty('application.version') } ext.deps = [ [ name: "dex2jar-2.1.zip", url: "https://github.com/pxb1988/dex2jar/releases/download/v2.1/dex2jar-2.1.zip", sha256: "7a9bdf843d43de4d1e94ec2e7b6f55825017b0c4a7ee39ff82660e2493a46f08", destination: { unzip(DOWNLOADS_DIR, DOWNLOADS_DIR, "dex2jar-2.1.zip") copyDirectory(new File(DOWNLOADS_DIR, "dex-tools-2.1/lib/"), FLAT_REPO_DIR, new WildcardFileFilter("dex-*")); } ], [ name: "java-sarif-2.1-modified.jar", url: "https://github.com/NationalSecurityAgency/ghidra-data/raw/Ghidra_${RELEASE_VERSION}/lib/java-sarif-2.1-modified.jar", sha256: "7f736566494756d271aa5e4b1af6c89dc50d074ab1c6374a47df822264226b01", destination: FLAT_REPO_DIR ], [ name: "AXMLPrinter2.jar", url: "https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/android4me/AXMLPrinter2.jar", sha256: "00ed038eb6abaf6ddec8d202a3ed7a81b521458f4cd459948115cfd02ff59d6d", destination: FLAT_REPO_DIR ], [ name: "yajsw-stable-13.12.zip", url: "https://sourceforge.net/projects/yajsw/files/yajsw/yajsw-stable-13.12/yajsw-stable-13.12.zip", sha256: "c6fc59815d3800d14ec977926a8afd3f606a0ebd74d2cfd60601677466edeaa2", destination: file("${DEPS_DIR}/GhidraServer") ], [ name: "postgresql-15.3.tar.gz", url: "https://ftp.postgresql.org/pub/source/v15.3/postgresql-15.3.tar.gz", sha256: "086d38533e28747966a4d5f1e78ea432e33a78f21dcb9133010ecb5189fad98c", destination: file("${DEPS_DIR}/BSim") ], [ name: "PyDev 6.3.1.zip", url: "https://sourceforge.net/projects/pydev/files/pydev/PyDev%206.3.1/PyDev%206.3.1.zip", sha256: "4d81fe9d8afe7665b8ea20844d3f5107f446742927c59973eade4f29809b0699", destination: file("${DEPS_DIR}/GhidraDev") ], [ name: "cdt-8.6.0.zip", url: "https://archive.eclipse.org/tools/cdt/releases/8.6/cdt-8.6.0.zip", sha256: "81b7d19d57c4a3009f4761699a72e8d642b5e1d9251d2bb98df438b1e28f8ba9", destination: file("${DEPS_DIR}/GhidraDev") ], [ name: "vs2012_x64.fidb", url: "https://github.com/NationalSecurityAgency/ghidra-data/raw/Ghidra_${RELEASE_VERSION}/FunctionID/vs2012_x64.fidb", sha256: "d4e98ab3f790b831793218430bba0d8b24a5fbf4da65b0c1ffa8cb0cfbeb0cdc", destination: FID_DIR ], [ name: "vs2012_x86.fidb", url: "https://github.com/NationalSecurityAgency/ghidra-data/raw/Ghidra_${RELEASE_VERSION}/FunctionID/vs2012_x86.fidb", sha256: "a490ed7e2ed21e587459feaeace7036b7ede4ce84e72e10dfd8c57434a6918b6", destination: FID_DIR ], [ name: "vs2015_x64.fidb", url: "https://github.com/NationalSecurityAgency/ghidra-data/raw/Ghidra_${RELEASE_VERSION}/FunctionID/vs2015_x64.fidb", sha256: "e04e9e40f9ecb601c85f4d84ed9bf66b45363be1d1e82c162e4c9902b8cb508f", destination: FID_DIR ], [ name: "vs2015_x86.fidb", url: "https://github.com/NationalSecurityAgency/ghidra-data/raw/Ghidra_${RELEASE_VERSION}/FunctionID/vs2015_x86.fidb", sha256: "b66ee696653e2ed365919deaaef885103120c792e22e79af70d1209d7e1d8644", destination: FID_DIR ], [ name: "vs2017_x64.fidb", url: "https://github.com/NationalSecurityAgency/ghidra-data/raw/Ghidra_${RELEASE_VERSION}/FunctionID/vs2017_x64.fidb", sha256: "d5fa5f697298174fa53d247d3599e6a12884605ad181c7b954e2380ec1f0bd89", destination: FID_DIR ], [ name: "vs2017_x86.fidb", url: "https://github.com/NationalSecurityAgency/ghidra-data/raw/Ghidra_${RELEASE_VERSION}/FunctionID/vs2017_x86.fidb", sha256: "d389cb8d76ff4a59ca35f891b8521c72ad5f0df96e253973a2d21a8614a4cc81", destination: FID_DIR ], [ name: "vs2019_x64.fidb", url: "https://github.com/NationalSecurityAgency/ghidra-data/raw/Ghidra_${RELEASE_VERSION}/FunctionID/vs2019_x64.fidb", sha256: "150007796fc36a4069660ad62449aadaaf3dd11b3864a5ef21e79831c9ce9118", destination: FID_DIR ], [ name: "vs2019_x86.fidb", url: "https://github.com/NationalSecurityAgency/ghidra-data/raw/Ghidra_${RELEASE_VERSION}/FunctionID/vs2019_x86.fidb", sha256: "eb630a36faa586a371eb734dc0bbd8d13ccaef697f3db5872596358f3dd2432a", destination: FID_DIR ], [ name: "vsOlder_x64.fidb", url: "https://github.com/NationalSecurityAgency/ghidra-data/raw/Ghidra_${RELEASE_VERSION}/FunctionID/vsOlder_x64.fidb", sha256: "8c3b51f4660cd27e1a0d610a9f3f2d5fbc833a66ac9ee4393ee2f2481e855866", destination: FID_DIR ], [ name: "vsOlder_x86.fidb", url: "https://github.com/NationalSecurityAgency/ghidra-data/raw/Ghidra_${RELEASE_VERSION}/FunctionID/vsOlder_x86.fidb", sha256: "98605c6b6b9214a945d844e41c85860d54649a45bca7873ef6991c0e93720787", destination: FID_DIR ], [ name: "protobuf-3.20.3-py2.py3-none-any.whl", url: "https://files.pythonhosted.org/packages/8d/14/619e24a4c70df2901e1f4dbc50a6291eb63a759172558df326347dce1f0d/protobuf-3.20.3-py2.py3-none-any.whl", sha256: "a7ca6d488aa8ff7f329d4c545b2dbad8ac31464f1d8b1c87ad1346717731e4db", destination: file("${DEPS_DIR}/Debugger-rmi-trace/") ], [ name: "psutil-5.9.8.tar.gz", url: "https://files.pythonhosted.org/packages/90/c7/6dc0a455d111f68ee43f27793971cf03fe29b6ef972042549db29eec39a2/psutil-5.9.8.tar.gz", sha256: "6be126e3225486dff286a8fb9a06246a5253f4c7c53b475ea5f5ac934e64194c", destination: file("${DEPS_DIR}/Debugger-rmi-trace/") ], [ name: "setuptools-68.0.0-py3-none-any.whl", url: "https://files.pythonhosted.org/packages/c7/42/be1c7bbdd83e1bfb160c94b9cafd8e25efc7400346cf7ccdbdb452c467fa/setuptools-68.0.0-py3-none-any.whl", sha256: "11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f", destination: [file("${DEPS_DIR}/Debugger-rmi-trace/"), file("${DEPS_DIR}/PyGhidra/")] ], [ name: "wheel-0.37.1-py2.py3-none-any.whl", url: "https://files.pythonhosted.org/packages/27/d6/003e593296a85fd6ed616ed962795b2f87709c3eee2bca4f6d0fe55c6d00/wheel-0.37.1-py2.py3-none-any.whl", sha256: "4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a", destination: file("${DEPS_DIR}/Debugger-rmi-trace/") ], [ name: "Pybag-2.2.10-py3-none-any.whl", url: "https://files.pythonhosted.org/packages/2d/e0/f877c91e036fcaed2a827f80d6cbdf1d26cffc3333c9ebda31c55c45f050/Pybag-2.2.10-py3-none-any.whl", sha256: "81cf1e33dd667dd217dc56a123326796e6799b8569f2c6efb78c16375caf9b2b", destination: file("${DEPS_DIR}/Debugger-agent-dbgeng/") ], [ name: "capstone-5.0.1-py3-none-win_amd64.whl", url: "https://files.pythonhosted.org/packages/d0/dd/b28df50316ca193dd1275a4c47115a720796d9e1501c1888c4bfa5dc2260/capstone-5.0.1-py3-none-win_amd64.whl", sha256: "1bfa5c81e6880caf41a31946cd6d2d069c048bcc22edf121254b501a048de675", destination: file("${DEPS_DIR}/Debugger-agent-dbgeng/") ], [ name: "comtypes-1.4.1-py3-none-any.whl", url: "https://files.pythonhosted.org/packages/50/8f/518a37381e55a8857a638afa86143efa5508434613541402d20611a1b322/comtypes-1.4.1-py3-none-any.whl", sha256: "a208a0e3ca1c0a5362735da0ff661822801dce87312b894d7d752add010a21b0", destination: file("${DEPS_DIR}/Debugger-agent-dbgeng/") ], [ name: "pywin32-306-cp312-cp312-win_amd64.whl", url: "https://files.pythonhosted.org/packages/83/1c/25b79fc3ec99b19b0a0730cc47356f7e2959863bf9f3cd314332bddb4f68/pywin32-306-cp312-cp312-win_amd64.whl", sha256: "37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e", destination: file("${DEPS_DIR}/Debugger-agent-dbgeng/") ], [ name: "JPype1-1.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", url: "https://files.pythonhosted.org/packages/5d/cf/7b89469bcede4b2fd69c2db7d1d61e8759393cfeec46f7b0c84f5006a691/JPype1-1.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", sha256: "f7aa1469d75f9b310f709b61bb2faa4cef4cbd4d670531ad1d1bb53e29cfda05", destination: file("${DEPS_DIR}/PyGhidra/") ], [ name: "JPype1-1.5.0-cp39-cp39-win_amd64.whl", url: "https://files.pythonhosted.org/packages/b9/fd/d38a8e401b089adce04c48021ddcb366891d1932db2f7653054feb470ae6/JPype1-1.5.0-cp39-cp39-win_amd64.whl", sha256: "6bfdc101c56cab0b6b16e974fd8cbb0b3f7f14178286b8b55413c5d82d5f2bea", destination: file("${DEPS_DIR}/PyGhidra/") ], [ name: "JPype1-1.5.0-cp310-cp310-macosx_10_9_universal2.whl", url: "https://files.pythonhosted.org/packages/84/9c/80d5edf6d610f82d0658b6402cdf3f8cdd6a7d4f36afb2149da90e0cad47/JPype1-1.5.0-cp310-cp310-macosx_10_9_universal2.whl", sha256: "7b6b1af3f9e0033080e3532c2686a224cd14706f36c14ef36160a2a1db751a17", destination: file("${DEPS_DIR}/PyGhidra/") ], [ name: "JPype1-1.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", url: "https://files.pythonhosted.org/packages/74/98/d6517002355b0585d0e66f7b0283c7f6e2271c898a886e1ebac09836b100/JPype1-1.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", sha256: "a02b2f05621c119d35f4acc501b4261eeb48a4af7cc13d9afc2e9eb316c4bd29", destination: file("${DEPS_DIR}/PyGhidra/") ], [ name: "JPype1-1.5.0-cp310-cp310-win_amd64.whl", url: "https://files.pythonhosted.org/packages/da/5f/253c1c1dba6f7f457b6c3aa2ea9c517287d49764e0ee1042d5818c36e781/JPype1-1.5.0-cp310-cp310-win_amd64.whl", sha256: "0b40c76e075d4fed2c83340bb30b7b95bbc396fd370c564c6b608faab00ea4ef", destination: file("${DEPS_DIR}/PyGhidra/") ], [ name: "JPype1-1.5.0-cp311-cp311-macosx_10_9_universal2.whl", url: "https://files.pythonhosted.org/packages/98/37/0049866cbfecb879b46d8e9f9b70944624ab17152a282ad5cf60909054ec/JPype1-1.5.0-cp311-cp311-macosx_10_9_universal2.whl", sha256: "85a31b30b482eaf788b21af421e0750aa0be7758307314178143a76632b0ad04", destination: file("${DEPS_DIR}/PyGhidra/") ], [ name: "JPype1-1.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", url: "https://files.pythonhosted.org/packages/17/1e/7728ae8fb41e8fbf3a7309f8936d07b0b1622f2860733df0e7ec30b1ce76/JPype1-1.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", sha256: "5ef976e0f3b2e9604469f449f30bb2031941a159a0637f4c16adb2c5076f3e81", destination: file("${DEPS_DIR}/PyGhidra/") ], [ name: "JPype1-1.5.0-cp311-cp311-win_amd64.whl", url: "https://files.pythonhosted.org/packages/1f/19/144f3a767b563ba5c6d4aa534ea1f3fad9a5067c3917df4458a6e1afe0ef/JPype1-1.5.0-cp311-cp311-win_amd64.whl", sha256: "2bc987205ff8d2d8e36dfbef05430e0638e85d4fee1166ba58ebfa6f7a67cdf8", destination: file("${DEPS_DIR}/PyGhidra/") ], [ name: "JPype1-1.5.0-cp312-cp312-macosx_10_9_universal2.whl", url: "https://files.pythonhosted.org/packages/30/0d/9ac6f0e59427fc5ebf4547c2fdbb38e347b46c2dc20b430490236d037ed8/JPype1-1.5.0-cp312-cp312-macosx_10_9_universal2.whl", sha256: "8714bfaf09d6877160bc7ac97812016ccb09f6d7ba5ea2a9f519178aefcca93f", destination: file("${DEPS_DIR}/PyGhidra/") ], [ name: "JPype1-1.5.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", url: "https://files.pythonhosted.org/packages/7d/ed/549766039d17550da6e3fa59ed776a021b400324d7766358d3b6e33d8b28/JPype1-1.5.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", sha256: "8649b526eccb4047881ad60bdb1974eb71a09cdb7f8bda17c96fdc0f9a3f2d1e", destination: file("${DEPS_DIR}/PyGhidra/") ], [ name: "JPype1-1.5.0-cp312-cp312-win_amd64.whl", url: "https://files.pythonhosted.org/packages/20/47/9606af72e21703e5fca5e29e5bd5e345506977b6ba492c549648adef47ef/JPype1-1.5.0-cp312-cp312-win_amd64.whl", sha256: "9aafc00b00bf8c1b624081e5d4ab87f7752e6c7ee6a141cfc332250b05c6d42f", destination: file("${DEPS_DIR}/PyGhidra/") ], [ name: "packaging-23.2-py3-none-any.whl", url: "https://files.pythonhosted.org/packages/ec/1a/610693ac4ee14fcdf2d9bf3c493370e4f2ef7ae2e19217d7a237ff42367d/packaging-23.2-py3-none-any.whl", sha256: "8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7", destination: file("${DEPS_DIR}/PyGhidra/") ] ] // Download dependencies (if necessary) and verify their hashes mkdirs(DOWNLOADS_DIR) deps.each { File file = new File(DOWNLOADS_DIR, it.name) if (OFFLINE || !it.sha256.equals(generateHash(file))) { download(it.url, file) if (!OFFLINE) { assert(it.sha256.equals(generateHash(file))); } } } // Copies the downloaded dependencies to their required destination. // Some downloads require pre-processing before their relevant pieces can be copied. deps.each { def copier = { File fp -> if (!OFFLINE) { println "Copying " + it.name + " to " + fp } mkdirs(fp) copyFile(new File(DOWNLOADS_DIR, it.name), new File(fp, it.name)); } if (it.destination instanceof File) { copier(it.destination) } else if (it.destination instanceof List) { it.destination.each { fp -> copier(fp) } } else if (it.destination instanceof Closure) { if (!OFFLINE) { println "Processing " + it.name } it.destination() } else { throw new GradleException("Unexpected destination type: " + it.destination) } } //-------------------------------------Helper methods---------------------------------------------- /** * Downloads a file from a URL. The download attempt will be tried NUM_RETRIES times before failing. * * Progress is shown on the command line in the form of the number of bytes downloaded and a * percentage of the total. * * Note: We do not validate that the number of bytes downloaded matches the expected total here; any * discrepencies will be caught when checking the SHA-256s later on. * * @param url the file to download * @param file the local file to create for the download */ def download(url, file) { if (OFFLINE) { println "curl -L -o " + relative(file) + " '" + url + "'" return } println "URL: " + url def(InputStream istream, size) = establishConnection(url, NUM_RETRIES); assert istream != null : " ***CONNECTION FAILURE***\n max attempts exceeded; exiting\n" FileOutputStream ostream = new FileOutputStream(file); def dataBuffer = new byte[1024]; int bytesRead; int totalRead; if (HIDE_DOWNLOAD_PROGRESS) { print " Downloading..." } while ((bytesRead = istream.read(dataBuffer, 0, 1024)) != -1) { ostream.write(dataBuffer, 0, bytesRead); totalRead += bytesRead if (!HIDE_DOWNLOAD_PROGRESS) { print "\r" print " Downloading: " + totalRead + " of " + size if (!size.equals("???")) { int pctComplete = (totalRead / size) * 100 print " (" + pctComplete + "%)" } print " " // overwrite gradle timer output System.out.flush() } } println() istream.close(); ostream.close(); } /** * Attempts to establish a connection to the given URL * * @param url the URL to connect to * @param retries the number of times to attempt to connect if there are failures * @return the InputStream for the URL, and the size of the download in bytes as a string */ def establishConnection(url, retries) { for (int i = 0; i < retries; i++) { try { if (i == 0) { println " Connecting..." } else { println " Connecting (" + (i+1) + "/" + retries + ")..." } URLConnection conn = new URL(url).openConnection(); conn.setRequestMethod("HEAD"); def size = conn.getContentLengthLong(); if (size == -1) { size = "???" } return [new BufferedInputStream(new URL(url).openStream()), size]; } catch (Exception e) { println " Connection error! " + e } } } /** * Unzips a file to a directory * * @param sourceDir the directory where the zip file resides * @param targetDir the directory where the unzipped files should be placed * @param zipFileName the name of the file to unpack */ def unzip(sourceDir, targetDir, zipFileName) { def zipFile = new File(sourceDir, zipFileName) if (OFFLINE) { println "unzip " + relative(zipFile) + " -d " + relative(targetDir) return } def zip = new ZipFile(zipFile) zip.entries().findAll { !it.directory }.each { e -> (e.name as File).with { f -> if (f.parentFile != null) { File destPath = new File(targetDir.path, f.parentFile.path) mkdirs(destPath) File targetFile = new File(destPath.path, f.name) targetFile.withOutputStream { w -> w << zip.getInputStream(e) } } } } zip.close() } /** * Creates the given directory, including any necessary but nonexistent parent directories * * @return true if and only if the directory was created, along with all necessary parent * directories; false otherwise */ def mkdirs(dir) { if (OFFLINE) { if (!createdDirs.contains(dir)) { println "mkdir -p " + relative(dir) createdDirs << dir } return } return dir.mkdirs() } /** * Copies a file to a new location * * @param sourceFile the file to copy * @param targetFile the new file */ def copyFile(sourceFile, targetFile) { if (OFFLINE) { println "cp " + relative(sourceFile) + " " + relative(targetFile) return } FileUtils.copyFile(sourceFile, targetFile) } /** * Copies a filtered directory to a new location * * @param sourceDir the directory to copy * @param targetDir the new directory * @param filter the filter to apply; null to copy everything */ def copyDirectory(sourceDir, targetDir, filter) { if (OFFLINE) { println "cp -r " + relative(sourceDir) + " " + relative(targetDir) return } FileUtils.copyDirectory(sourceDir, targetDir, filter) } /** * Returns the path of the file relative to the repository * * @return The path of the file relative to the repository */ def relative(file) { return "\"" + file.absolutePath.substring(REPO_DIR.absolutePath.length() + 1).replaceAll("\\\\", "/") + "\"" } /** * Generates the SHA-256 hash for the given file * * @param file the file to generate the SHA-256 hash for * @return the generated SHA-256 hash, or null if the file does not exist */ def generateHash(file) { if (!file.exists()) { return null } MessageDigest md = MessageDigest.getInstance("SHA-256"); md.update(Files.readAllBytes(Paths.get(file.path))); byte[] digest = md.digest(); StringBuilder sb = new StringBuilder(); for (byte b : digest) { sb.append(String.format("%02x", b)); } return sb.toString(); }