mirror of
https://github.com/NationalSecurityAgency/ghidra.git
synced 2025-10-03 17:59:46 +02:00
GP-1830 BSim migration to gson use. Corrected various bugs with BSim
elasticsearch use.
This commit is contained in:
ghidra1
parent
237d0445b5
commit
593fd98e0d
18 changed files with 769 additions and 1275 deletions
|
|
@ -1,7 +1,6 @@
|
|||
##MODULE IP: Oxygen Icons - LGPL 3.0
|
||||
MODULE FILE LICENSE: postgresql-15.3.tar.gz Postgresql License
|
||||
MODULE FILE LICENSE: lib/postgresql-42.6.2.jar PostgresqlJDBC License
|
||||
MODULE FILE LICENSE: lib/json-simple-1.1.1.jar Apache License 2.0
|
||||
MODULE FILE LICENSE: lib/commons-dbcp2-2.9.0.jar Apache License 2.0
|
||||
MODULE FILE LICENSE: lib/commons-pool2-2.11.1.jar Apache License 2.0
|
||||
MODULE FILE LICENSE: lib/commons-logging-1.2.jar Apache License 2.0
|
||||
|
|
|
|||
|
|
@ -33,7 +33,6 @@ dependencies {
|
|||
api project(":CodeCompare")
|
||||
|
||||
api "org.postgresql:postgresql:42.6.2"
|
||||
api "com.googlecode.json-simple:json-simple:1.1.1"
|
||||
api "org.apache.commons:commons-dbcp2:2.9.0"
|
||||
api "org.apache.commons:commons-pool2:2.11.1"
|
||||
api "commons-logging:commons-logging:1.2"
|
||||
|
|
|
|||
|
|
@ -17,8 +17,6 @@ package ghidra.features.bsim.gui.filters;
|
|||
|
||||
import java.sql.SQLException;
|
||||
|
||||
import org.json.simple.JSONObject;
|
||||
|
||||
import ghidra.features.bsim.query.client.IDSQLResolution;
|
||||
import ghidra.features.bsim.query.client.SQLEffects;
|
||||
import ghidra.features.bsim.query.description.ExecutableRecord;
|
||||
|
|
@ -50,7 +48,7 @@ public class ExecutableNameBSimFilterType extends BSimFilterType {
|
|||
IDElasticResolution resolution) throws ElasticException {
|
||||
StringBuilder buffer = new StringBuilder();
|
||||
buffer.append("\"filter\": { \"term\": { \"name_exec\": \"");
|
||||
buffer.append(JSONObject.escape(atom.value));
|
||||
buffer.append(ElasticDatabase.escape(atom.value));
|
||||
buffer.append("\" } } ");
|
||||
effect.addStandalone(this, buffer.toString());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,8 +17,6 @@ package ghidra.features.bsim.gui.filters;
|
|||
|
||||
import java.sql.SQLException;
|
||||
|
||||
import org.json.simple.JSONObject;
|
||||
|
||||
import ghidra.features.bsim.query.client.IDSQLResolution;
|
||||
import ghidra.features.bsim.query.client.SQLEffects;
|
||||
import ghidra.features.bsim.query.description.ExecutableRecord;
|
||||
|
|
@ -49,7 +47,7 @@ public class NotExecutableNameBSimFilterType extends BSimFilterType {
|
|||
IDElasticResolution resolution) throws ElasticException {
|
||||
StringBuilder buffer = new StringBuilder();
|
||||
buffer.append("\"must_not\": { \"term\": { \"name_exec\": \"");
|
||||
buffer.append(JSONObject.escape(atom.value));
|
||||
buffer.append(ElasticDatabase.escape(atom.value));
|
||||
buffer.append("\" } } ");
|
||||
effect.addStandalone(this, buffer.toString());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,10 +48,9 @@ public class PathStartsBSimFilterType extends BSimFilterType {
|
|||
@Override
|
||||
public void gatherElasticEffect(ElasticEffects effect, FilterAtom atom,
|
||||
IDElasticResolution resolution) throws ElasticException {
|
||||
effect.addDocValue("String path = doc['path'].value; ");
|
||||
effect.addDocValue("String path = doc['path'].size() == 0 ? null : doc['path'].value; ");
|
||||
String argName = effect.assignArgument();
|
||||
effect.addScriptElement(this,
|
||||
"(path != null) && path.startsWith(params." + argName + ')');
|
||||
effect.addScriptElement(this, "(path != null) && path.startsWith(params." + argName + ')');
|
||||
effect.addParam(argName, atom.value);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -284,7 +284,9 @@ public class BSimPostgresDBConnectionManager {
|
|||
|
||||
String loginError = null;
|
||||
|
||||
if (bds.getPassword() == null) {
|
||||
serverInfo.setUserInfo(bds);
|
||||
}
|
||||
|
||||
connectionType = serverInfo.hasPassword() ? ConnectionType.SSL_Password_Authentication
|
||||
: ConnectionType.SSL_No_Authentication;
|
||||
|
|
|
|||
|
|
@ -186,7 +186,7 @@ public abstract class AbstractSQLFunctionDatabase<VF extends LSHVectorFactory>
|
|||
* @throws SQLException if error occurs obtaining connection
|
||||
*/
|
||||
protected Connection initConnection() throws SQLException {
|
||||
if (db == null) {
|
||||
if (db == null || db.isClosed()) {
|
||||
db = ds.getConnection();
|
||||
}
|
||||
return db;
|
||||
|
|
@ -435,6 +435,12 @@ public abstract class AbstractSQLFunctionDatabase<VF extends LSHVectorFactory>
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop this database
|
||||
* @throws SQLException if a database error occured
|
||||
*/
|
||||
abstract protected void dropDatabase() throws SQLException;
|
||||
|
||||
protected void setConnectionOnTables(Connection db) {
|
||||
|
||||
weightTable.setConnection(db);
|
||||
|
|
@ -1201,8 +1207,8 @@ public abstract class AbstractSQLFunctionDatabase<VF extends LSHVectorFactory>
|
|||
}
|
||||
else if (msg.contains("authentication failed") ||
|
||||
msg.contains("requires a valid client certificate")) {
|
||||
lasterror =
|
||||
new BSimError(ErrorCategory.Authentication, "Could not authenticate with database");
|
||||
lasterror = new BSimError(ErrorCategory.Authentication,
|
||||
"Could not authenticate with database");
|
||||
}
|
||||
else if (msg.contains("does not exist") && !msg.contains(" role ")) {
|
||||
lasterror = new BSimError(ErrorCategory.Nodatabase, cause.getMessage());
|
||||
|
|
@ -1572,6 +1578,9 @@ public abstract class AbstractSQLFunctionDatabase<VF extends LSHVectorFactory>
|
|||
else if (query instanceof QueryExeCount q) {
|
||||
fdbQueryExeCount(q);
|
||||
}
|
||||
else if (query instanceof DropDatabase q) {
|
||||
fdbDatabaseDrop(q);
|
||||
}
|
||||
else if (query instanceof CreateDatabase q) {
|
||||
fdbDatabaseCreate(q);
|
||||
}
|
||||
|
|
@ -1622,7 +1631,8 @@ public abstract class AbstractSQLFunctionDatabase<VF extends LSHVectorFactory>
|
|||
|
||||
lasterror = null;
|
||||
try {
|
||||
if (!(query instanceof CreateDatabase) && !initialize()) {
|
||||
if (!(query instanceof CreateDatabase) && !(query instanceof DropDatabase) &&
|
||||
!initialize()) {
|
||||
lasterror = new BSimError(ErrorCategory.Nodatabase, "The database does not exist");
|
||||
return null;
|
||||
}
|
||||
|
|
@ -2087,6 +2097,29 @@ public abstract class AbstractSQLFunctionDatabase<VF extends LSHVectorFactory>
|
|||
}
|
||||
}
|
||||
|
||||
private void fdbDatabaseDrop(DropDatabase query) throws LSHException {
|
||||
ResponseDropDatabase response = query.getResponse();
|
||||
if (query.databaseName == null) {
|
||||
throw new LSHException("Missing databaseName for drop database");
|
||||
}
|
||||
if (!query.databaseName.equals(ds.getServerInfo().getDBName())) {
|
||||
throw new UnsupportedOperationException("drop database name must match");
|
||||
}
|
||||
response.dropSuccessful = true; // Response parameters assuming success
|
||||
response.errorMessage = null;
|
||||
try {
|
||||
dropDatabase();
|
||||
}
|
||||
catch (SQLException e) {
|
||||
String msg = e.getMessage();
|
||||
if (msg.indexOf("database \"" + query.databaseName + "\" does not exist") > 0) {
|
||||
return; // missing database
|
||||
}
|
||||
response.dropSuccessful = false;
|
||||
response.errorMessage = e.getMessage();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Entry point for the CreateDatabase command
|
||||
* @param query the query to execute
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ import ghidra.features.bsim.query.client.tables.CachedStatement;
|
|||
import ghidra.features.bsim.query.client.tables.SQLStringTable;
|
||||
import ghidra.features.bsim.query.description.*;
|
||||
import ghidra.features.bsim.query.protocol.*;
|
||||
import ghidra.util.Msg;
|
||||
|
||||
/**
|
||||
* Defines the BSim {@link FunctionDatabase} backed by a PostgreSQL database.
|
||||
|
|
@ -200,9 +201,6 @@ public final class PostgresFunctionDatabase
|
|||
st.executeUpdate(createdbstring);
|
||||
postgresDs.initializeFrom(defaultDs);
|
||||
}
|
||||
finally {
|
||||
defaultDs.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -244,6 +242,64 @@ public final class PostgresFunctionDatabase
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void dropDatabase() throws SQLException {
|
||||
|
||||
if (getStatus() == Status.Busy || postgresDs.getActiveConnections() != 0) {
|
||||
throw new SQLException("database in use");
|
||||
}
|
||||
|
||||
BSimServerInfo serverInfo = postgresDs.getServerInfo();
|
||||
BSimServerInfo defaultServerInfo =
|
||||
new BSimServerInfo(DBType.postgres, serverInfo.getUserInfo(),
|
||||
serverInfo.getServerName(), serverInfo.getPort(), DEFAULT_DATABASE_NAME);
|
||||
|
||||
BSimPostgresDataSource defaultDs =
|
||||
BSimPostgresDBConnectionManager.getDataSource(defaultServerInfo);
|
||||
if (getStatus() == Status.Ready) {
|
||||
defaultDs.initializeFrom(postgresDs);
|
||||
}
|
||||
|
||||
close(); // close this instance
|
||||
|
||||
try (Connection defaultDb = defaultDs.getConnection();
|
||||
Statement defaultSt = defaultDb.createStatement()) {
|
||||
try (ResultSet rs = defaultSt.executeQuery(
|
||||
"SELECT 1 FROM pg_database WHERE datname='" + serverInfo.getDBName() + "'")) {
|
||||
if (!rs.next()) {
|
||||
return; // database does not exist
|
||||
}
|
||||
}
|
||||
|
||||
// Connect to database and examine schema
|
||||
HashSet<String> tableNames = new HashSet<>();
|
||||
postgresDs.initializeFrom(defaultDs);
|
||||
try (Connection c = initConnection(); Statement st = c.createStatement()) {
|
||||
try (ResultSet rs = st.executeQuery(
|
||||
"SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' ORDER BY table_name")) {
|
||||
while (rs.next()) {
|
||||
tableNames.add(rs.getString(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Spot check for a few BSim table names that always exist
|
||||
if (!tableNames.contains("keyvaluetable") || !tableNames.contains("desctable") ||
|
||||
!tableNames.contains("weighttable")) {
|
||||
throw new SQLException("attempted to drop non-BSim database");
|
||||
}
|
||||
|
||||
postgresDs.dispose(); // disconnect before dropping database
|
||||
|
||||
Msg.info(this, "Dropping BSim postgresql database: " + serverInfo);
|
||||
defaultSt.executeUpdate("DROP DATABASE \"" + serverInfo.getDBName() + '"');
|
||||
}
|
||||
finally {
|
||||
// ensure
|
||||
postgresDs.initializeFrom(defaultDs);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @throws SQLException if there is a problem creating or executing the query
|
||||
|
|
|
|||
|
|
@ -19,9 +19,9 @@ import java.io.*;
|
|||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
|
||||
import org.json.simple.JSONObject;
|
||||
import org.json.simple.parser.JSONParser;
|
||||
import org.json.simple.parser.ParseException;
|
||||
import com.google.gson.*;
|
||||
|
||||
import ghidra.util.Msg;
|
||||
|
||||
public class ElasticConnection {
|
||||
public static final String POST = "POST";
|
||||
|
|
@ -38,24 +38,51 @@ public class ElasticConnection {
|
|||
httpURLbase = url + '/' + repo + '_';
|
||||
}
|
||||
|
||||
public void close() {
|
||||
// nothing to do - http connections do not persist
|
||||
}
|
||||
|
||||
public boolean lastRequestSuccessful() {
|
||||
return (lastResponseCode >= 200) && (lastResponseCode < 300);
|
||||
}
|
||||
|
||||
/**
|
||||
* Assuming the writer has been closed and connection.getResponseCode() is called
|
||||
* placing the value in lastResponseCode, read the response and parse into a JSONObject
|
||||
* @return the JSONObject
|
||||
* @throws IOException for problems with the socket
|
||||
* @throws ParseException for JSON parse errors
|
||||
* Get String held by a JsonElement, allowing for a null object.
|
||||
* @param element is the JsonElement or null
|
||||
* @return the underlying String or null
|
||||
*/
|
||||
private JSONObject grabResponse(HttpURLConnection connection)
|
||||
throws IOException, ParseException {
|
||||
JSONParser parser = new JSONParser();
|
||||
static String convertToString(JsonElement element) {
|
||||
if (isNull(element)) {
|
||||
return null;
|
||||
}
|
||||
return element.getAsString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get String held by a JsonElement, allowing for a null object.
|
||||
* @param element is the JsonElement or null
|
||||
* @param defaultStr default string to be returned if element or string is null
|
||||
* @return the underlying String or defaultStr if null
|
||||
*/
|
||||
static String convertToString(JsonElement element, String defaultStr) {
|
||||
String str = convertToString(element);
|
||||
return str != null ? str : defaultStr;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check element for null value
|
||||
* @param element json element
|
||||
* @return true if null else false
|
||||
*/
|
||||
static boolean isNull(JsonElement element) {
|
||||
return (element == null || element instanceof JsonNull);
|
||||
}
|
||||
|
||||
/**
|
||||
* Assuming the writer has been closed and connection.getResponseCode() is called
|
||||
* placing the value in lastResponseCode, read the response and parse into a JsonObject
|
||||
* @return the JsonObject
|
||||
* @throws IOException for problems with the socket
|
||||
* @throws JsonParseException for JSON parse errors
|
||||
*/
|
||||
private JsonObject grabResponse(HttpURLConnection connection)
|
||||
throws IOException, JsonParseException {
|
||||
InputStream in;
|
||||
if (lastRequestSuccessful()) {
|
||||
in = connection.getInputStream();
|
||||
|
|
@ -68,7 +95,7 @@ public class ElasticConnection {
|
|||
throw new IOException(connection.getResponseMessage());
|
||||
}
|
||||
Reader reader = new InputStreamReader(in);
|
||||
JSONObject jsonObject = (JSONObject) parser.parse(reader);
|
||||
JsonObject jsonObject = JsonParser.parseReader(reader).getAsJsonObject();
|
||||
return jsonObject;
|
||||
}
|
||||
|
||||
|
|
@ -78,39 +105,86 @@ public class ElasticConnection {
|
|||
* @param resp is the parsed error document
|
||||
* @return the exception String
|
||||
*/
|
||||
private String parseErrorJSON(JSONObject resp) {
|
||||
private static String parseErrorJSON(JsonObject resp) {
|
||||
Object errorObj = resp.get("error");
|
||||
if (errorObj == null) {
|
||||
return "Unknown error format";
|
||||
}
|
||||
if (errorObj instanceof String) {
|
||||
return (String) errorObj;
|
||||
}
|
||||
if (!(errorObj instanceof JSONObject)) {
|
||||
if (!(errorObj instanceof JsonObject err)) {
|
||||
return "Unknown error format";
|
||||
}
|
||||
JSONObject jsonObj = (JSONObject) errorObj;
|
||||
String typeString = (String) jsonObj.get("type");
|
||||
String reasonString = (String) jsonObj.get("reason");
|
||||
if (typeString == null) {
|
||||
typeString = "Unknown Error";
|
||||
|
||||
String typeString = convertToString(err.get("type"), "Unknown Error");
|
||||
if (typeString.endsWith("_exception")) {
|
||||
// Log elastic exception root cause to assist debug
|
||||
String errorDetail = parseErrorCause(err);
|
||||
if (errorDetail.length() != 0) {
|
||||
Msg.error(ElasticConnection.class, "Elasticsearch exception: " + errorDetail);
|
||||
}
|
||||
if (reasonString == null) {
|
||||
reasonString = "Unknown reason";
|
||||
}
|
||||
|
||||
String reasonString = convertToString(err.get("reason"), "Unknown Reason");
|
||||
return typeString + " : " + reasonString;
|
||||
}
|
||||
|
||||
private static StringBuilder conditionalNewLine(StringBuilder buf) {
|
||||
if (!buf.isEmpty()) {
|
||||
buf.append("\n");
|
||||
}
|
||||
return buf;
|
||||
}
|
||||
|
||||
private static String parseErrorCause(JsonObject error) {
|
||||
|
||||
StringBuilder buf = new StringBuilder();
|
||||
|
||||
JsonElement reason = error.get("reason");
|
||||
|
||||
String typeString = convertToString(error.get("type"));
|
||||
if (typeString != null) {
|
||||
String reasonString = convertToString(reason); // "reason" is string when "type" is present
|
||||
String errorStr = typeString + " : " + reasonString;
|
||||
conditionalNewLine(buf).append(errorStr);
|
||||
}
|
||||
|
||||
JsonElement scriptStack = error.get("script_stack");
|
||||
if (scriptStack instanceof JsonArray scriptStackArray) {
|
||||
scriptStackArray
|
||||
.forEach(e -> conditionalNewLine(buf).append(" ").append(convertToString(e)));
|
||||
}
|
||||
|
||||
JsonElement causedBy = error.get("caused_by");
|
||||
if (causedBy instanceof JsonObject causedByObject) {
|
||||
conditionalNewLine(buf).append(" ").append(parseErrorCause(causedByObject));
|
||||
}
|
||||
|
||||
JsonElement failedShards = error.get("failed_shards");
|
||||
if (failedShards instanceof JsonArray failedShardsArray) {
|
||||
for (JsonElement failedShardElement : failedShardsArray) {
|
||||
JsonObject failedShard = (JsonObject) failedShardElement;
|
||||
String indexStr = convertToString(failedShard.get("index"));
|
||||
conditionalNewLine(buf).append(" Failed shard index: ").append(indexStr);
|
||||
conditionalNewLine(buf).append(" ").append(parseErrorCause(failedShard));
|
||||
}
|
||||
}
|
||||
|
||||
if (reason instanceof JsonObject reasonObject) {
|
||||
conditionalNewLine(buf).append(parseErrorCause(reasonObject));
|
||||
}
|
||||
|
||||
return buf.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a raw request to the server that is not specific to the repository.
|
||||
* Intended for general configuration or security commands
|
||||
* @param command is the type of command
|
||||
* @param path is the specific URL path receiving the command
|
||||
* @param body is JSON document describing the command
|
||||
* @return the response as parsed JSONObject
|
||||
* @return the response as parsed JsonObject
|
||||
* @throws ElasticException for any problems with the connection
|
||||
*/
|
||||
public JSONObject executeRawStatement(String command, String path, String body)
|
||||
public JsonObject executeRawStatement(String command, String path, String body)
|
||||
throws ElasticException {
|
||||
HttpURLConnection connection = null;
|
||||
try {
|
||||
|
|
@ -123,7 +197,7 @@ public class ElasticConnection {
|
|||
writer.write(body);
|
||||
}
|
||||
lastResponseCode = connection.getResponseCode();
|
||||
JSONObject resp = grabResponse(connection);
|
||||
JsonObject resp = grabResponse(connection);
|
||||
if (!lastRequestSuccessful()) {
|
||||
throw new ElasticException(parseErrorJSON(resp));
|
||||
}
|
||||
|
|
@ -132,7 +206,7 @@ public class ElasticConnection {
|
|||
catch (IOException e) {
|
||||
throw new ElasticException("Error sending request: " + e.getMessage());
|
||||
}
|
||||
catch (ParseException e) {
|
||||
catch (JsonParseException e) {
|
||||
throw new ElasticException("Error parsing response: " + e.getMessage());
|
||||
}
|
||||
finally {
|
||||
|
|
@ -163,7 +237,7 @@ public class ElasticConnection {
|
|||
writer.write(body);
|
||||
}
|
||||
lastResponseCode = connection.getResponseCode();
|
||||
JSONObject resp = grabResponse(connection);
|
||||
JsonObject resp = grabResponse(connection);
|
||||
if (!lastRequestSuccessful()) {
|
||||
throw new ElasticException(parseErrorJSON(resp));
|
||||
}
|
||||
|
|
@ -171,7 +245,7 @@ public class ElasticConnection {
|
|||
catch (IOException e) {
|
||||
throw new ElasticException("Error sending request: " + e.getMessage());
|
||||
}
|
||||
catch (ParseException e) {
|
||||
catch (JsonParseException e) {
|
||||
throw new ElasticException("Error parsing response: " + e.getMessage());
|
||||
}
|
||||
finally {
|
||||
|
|
@ -186,10 +260,10 @@ public class ElasticConnection {
|
|||
* @param command is the type of command
|
||||
* @param path is the overarching index/type/<command>
|
||||
* @param body is JSON document describing the request
|
||||
* @return the parsed response as a JSONObject
|
||||
* @return the parsed response as a JsonObject
|
||||
* @throws ElasticException for any problems with the connection
|
||||
*/
|
||||
public JSONObject executeStatement(String command, String path, String body)
|
||||
public JsonObject executeStatement(String command, String path, String body)
|
||||
throws ElasticException {
|
||||
HttpURLConnection connection = null;
|
||||
try {
|
||||
|
|
@ -202,7 +276,7 @@ public class ElasticConnection {
|
|||
writer.write(body);
|
||||
}
|
||||
lastResponseCode = connection.getResponseCode();
|
||||
JSONObject resp = grabResponse(connection);
|
||||
JsonObject resp = grabResponse(connection);
|
||||
if (!lastRequestSuccessful()) {
|
||||
throw new ElasticException(parseErrorJSON(resp));
|
||||
}
|
||||
|
|
@ -211,7 +285,7 @@ public class ElasticConnection {
|
|||
catch (IOException e) {
|
||||
throw new ElasticException("Error sending request: " + e.getMessage());
|
||||
}
|
||||
catch (ParseException e) {
|
||||
catch (JsonParseException e) {
|
||||
throw new ElasticException("Error parsing response: " + e.getMessage());
|
||||
}
|
||||
finally {
|
||||
|
|
@ -227,10 +301,10 @@ public class ElasticConnection {
|
|||
* @param command is the type of command
|
||||
* @param path is the overarching index/type/<command>
|
||||
* @param body is JSON document describing the request
|
||||
* @return the parsed response as a JSONObject
|
||||
* @return the parsed response as a JsonObject
|
||||
* @throws ElasticException for any problems with the connection
|
||||
*/
|
||||
public JSONObject executeStatementExpectFailure(String command, String path, String body)
|
||||
public JsonObject executeStatementExpectFailure(String command, String path, String body)
|
||||
throws ElasticException {
|
||||
HttpURLConnection connection = null;
|
||||
try {
|
||||
|
|
@ -243,13 +317,13 @@ public class ElasticConnection {
|
|||
writer.write(body);
|
||||
}
|
||||
lastResponseCode = connection.getResponseCode();
|
||||
JSONObject resp = grabResponse(connection);
|
||||
JsonObject resp = grabResponse(connection);
|
||||
return resp;
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw new ElasticException("Error sending request: " + e.getMessage());
|
||||
}
|
||||
catch (ParseException e) {
|
||||
catch (JsonParseException e) {
|
||||
throw new ElasticException("Error parsing response: " + e.getMessage());
|
||||
}
|
||||
finally {
|
||||
|
|
@ -264,10 +338,10 @@ public class ElasticConnection {
|
|||
* and is structured slightly differently from other commands.
|
||||
* @param path is the specific URL path receiving the bulk command
|
||||
* @param body is structured list of JSON commands and source
|
||||
* @return the response as parsed JSONObject
|
||||
* @return the response as parsed JsonObject
|
||||
* @throws ElasticException for any problems with the connection
|
||||
*/
|
||||
public JSONObject executeBulk(String path, String body) throws ElasticException {
|
||||
public JsonObject executeBulk(String path, String body) throws ElasticException {
|
||||
HttpURLConnection connection = null;
|
||||
try {
|
||||
URL httpURL = new URL(hostURL + path);
|
||||
|
|
@ -279,7 +353,7 @@ public class ElasticConnection {
|
|||
writer.write(body);
|
||||
}
|
||||
lastResponseCode = connection.getResponseCode();
|
||||
JSONObject resp = grabResponse(connection);
|
||||
JsonObject resp = grabResponse(connection);
|
||||
if (!lastRequestSuccessful()) {
|
||||
throw new ElasticException(parseErrorJSON(resp));
|
||||
}
|
||||
|
|
@ -288,7 +362,7 @@ public class ElasticConnection {
|
|||
catch (IOException e) {
|
||||
throw new ElasticException("Error sending request: " + e.getMessage());
|
||||
}
|
||||
catch (ParseException e) {
|
||||
catch (JsonParseException e) {
|
||||
throw new ElasticException("Error parsing response: " + e.getMessage());
|
||||
}
|
||||
finally {
|
||||
|
|
@ -298,7 +372,7 @@ public class ElasticConnection {
|
|||
}
|
||||
}
|
||||
|
||||
public JSONObject executeURIOnly(String command, String path) throws ElasticException {
|
||||
public JsonObject executeURIOnly(String command, String path) throws ElasticException {
|
||||
HttpURLConnection connection = null;
|
||||
try {
|
||||
URL httpURL = new URL(httpURLbase + path);
|
||||
|
|
@ -306,7 +380,7 @@ public class ElasticConnection {
|
|||
connection.setRequestMethod(command);
|
||||
connection.setDoOutput(true);
|
||||
lastResponseCode = connection.getResponseCode();
|
||||
JSONObject resp = grabResponse(connection);
|
||||
JsonObject resp = grabResponse(connection);
|
||||
if (!lastRequestSuccessful()) {
|
||||
throw new ElasticException(parseErrorJSON(resp));
|
||||
}
|
||||
|
|
@ -315,7 +389,7 @@ public class ElasticConnection {
|
|||
catch (IOException e) {
|
||||
throw new ElasticException("Error sending request: " + e.getMessage());
|
||||
}
|
||||
catch (ParseException e) {
|
||||
catch (JsonParseException e) {
|
||||
throw new ElasticException("Error parsing response: " + e.getMessage());
|
||||
}
|
||||
finally {
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -172,7 +172,7 @@ public class BSimH2FileDBConnectionManager {
|
|||
|
||||
dispose();
|
||||
|
||||
if (dbf.isFile()) {
|
||||
if (!dbf.isFile()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -286,6 +286,9 @@ public class BSimH2FileDBConnectionManager {
|
|||
public synchronized Connection getConnection() throws SQLException {
|
||||
|
||||
if (successfulConnection) {
|
||||
if (bds.isClosed()) {
|
||||
bds.restart();
|
||||
}
|
||||
return bds.getConnection();
|
||||
}
|
||||
|
||||
|
|
@ -317,6 +320,9 @@ public class BSimH2FileDBConnectionManager {
|
|||
* @throws SQLException if connection or authentication error occurs
|
||||
*/
|
||||
private Connection connect() throws SQLException {
|
||||
if (bds.isClosed()) {
|
||||
bds.restart();
|
||||
}
|
||||
Connection c = bds.getConnection();
|
||||
successfulConnection = true;
|
||||
return c;
|
||||
|
|
|
|||
|
|
@ -22,13 +22,16 @@ import java.util.*;
|
|||
import generic.concurrent.*;
|
||||
import generic.lsh.vector.LSHVector;
|
||||
import generic.lsh.vector.VectorCompare;
|
||||
import ghidra.features.bsim.query.BSimServerInfo;
|
||||
import ghidra.features.bsim.query.LSHException;
|
||||
import ghidra.features.bsim.query.*;
|
||||
import ghidra.features.bsim.query.BSimPostgresDBConnectionManager.BSimPostgresDataSource;
|
||||
import ghidra.features.bsim.query.BSimServerInfo.DBType;
|
||||
import ghidra.features.bsim.query.FunctionDatabase.Status;
|
||||
import ghidra.features.bsim.query.client.*;
|
||||
import ghidra.features.bsim.query.description.*;
|
||||
import ghidra.features.bsim.query.elastic.Base64VectorFactory;
|
||||
import ghidra.features.bsim.query.file.BSimH2FileDBConnectionManager.BSimH2FileDataSource;
|
||||
import ghidra.features.bsim.query.protocol.*;
|
||||
import ghidra.util.Msg;
|
||||
import ghidra.util.task.TaskMonitor;
|
||||
|
||||
public class H2FileFunctionDatabase extends AbstractSQLFunctionDatabase<Base64VectorFactory> {
|
||||
|
|
@ -120,6 +123,48 @@ public class H2FileFunctionDatabase extends AbstractSQLFunctionDatabase<Base64Ve
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void dropDatabase() throws SQLException {
|
||||
|
||||
if (getStatus() == Status.Busy || fileDs.getActiveConnections() != 0) {
|
||||
throw new SQLException("database in use");
|
||||
}
|
||||
|
||||
close(); // close this instance
|
||||
|
||||
if (!fileDs.exists()) {
|
||||
// ignore request and return
|
||||
return;
|
||||
}
|
||||
|
||||
// Connect to database and examine schema
|
||||
HashSet<String> tableNames = new HashSet<>();
|
||||
try (Connection c = initConnection(); Statement st = c.createStatement()) {
|
||||
try (ResultSet rs = st.executeQuery(
|
||||
"SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' ORDER BY table_name")) {
|
||||
while (rs.next()) {
|
||||
tableNames.add(rs.getString(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Spot check for a few BSim table names that always exist
|
||||
if (!tableNames.contains("keyvaluetable") || !tableNames.contains("desctable") ||
|
||||
!tableNames.contains("weighttable")) {
|
||||
throw new SQLException("attempted to drop non-BSim database");
|
||||
}
|
||||
|
||||
fileDs.dispose(); // disconnect before deleting database
|
||||
|
||||
BSimServerInfo serverInfo = fileDs.getServerInfo();
|
||||
if (!fileDs.delete()) {
|
||||
throw new SQLException("failed to delete H2-file database: " + serverInfo);
|
||||
}
|
||||
|
||||
Msg.info(this, "Deleted BSim H2-file database: " + serverInfo);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Create vector map which maps vector ID to {@link VectorStoreEntry}
|
||||
* @return vector map
|
||||
|
|
@ -174,8 +219,8 @@ public class H2FileFunctionDatabase extends AbstractSQLFunctionDatabase<Base64Ve
|
|||
}
|
||||
|
||||
@Override
|
||||
protected int queryNearestVector(List<VectorResult> resultset, LSHVector vec,
|
||||
double simthresh, double sigthresh, int max) throws SQLException {
|
||||
protected int queryNearestVector(List<VectorResult> resultset, LSHVector vec, double simthresh,
|
||||
double sigthresh, int max) throws SQLException {
|
||||
VectorCompare comp;
|
||||
List<VectorResult> resultsToSort = new ArrayList<>();
|
||||
for (VectorStoreEntry entry : vectorStore) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,53 @@
|
|||
/* ###
|
||||
* IP: GHIDRA
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package ghidra.features.bsim.query.protocol;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Writer;
|
||||
|
||||
import generic.lsh.vector.LSHVectorFactory;
|
||||
import ghidra.util.xml.XmlUtilities;
|
||||
import ghidra.xml.XmlElement;
|
||||
import ghidra.xml.XmlPullParser;
|
||||
|
||||
public class DropDatabase extends BSimQuery<ResponseDropDatabase> {
|
||||
public String databaseName;
|
||||
public ResponseDropDatabase dropResponse;
|
||||
|
||||
public DropDatabase() {
|
||||
super("dropdatabase");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void buildResponseTemplate() {
|
||||
if (response == null)
|
||||
response = dropResponse = new ResponseDropDatabase();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveXml(Writer fwrite) throws IOException {
|
||||
fwrite.append('<').append(XmlUtilities.escapeElementEntities(name));
|
||||
fwrite.append(" dbname=\"").append(databaseName).append("\" />\n");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void restoreXml(XmlPullParser parser, LSHVectorFactory vectorFactory) {
|
||||
XmlElement el = parser.start(name);
|
||||
databaseName = XmlUtilities.unEscapeElementEntities(el.getAttribute("dbname"));
|
||||
parser.end();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
/* ###
|
||||
* IP: GHIDRA
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package ghidra.features.bsim.query.protocol;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Writer;
|
||||
|
||||
import generic.lsh.vector.LSHVectorFactory;
|
||||
import ghidra.features.bsim.query.LSHException;
|
||||
import ghidra.util.xml.SpecXmlUtils;
|
||||
import ghidra.xml.XmlElement;
|
||||
import ghidra.xml.XmlPullParser;
|
||||
|
||||
/**
|
||||
* Response of server indicating whether a password change request ({@link PasswordChange}) succeeded
|
||||
*/
|
||||
public class ResponseDropDatabase extends QueryResponseRecord {
|
||||
|
||||
public boolean operationSupported; // true if the back-end supports this operation
|
||||
public boolean dropSuccessful; // true if drop was successful
|
||||
public String errorMessage; // Error message if change was not successful
|
||||
|
||||
public ResponseDropDatabase() {
|
||||
super("responsedropdatabase");
|
||||
operationSupported = true;
|
||||
dropSuccessful = false;
|
||||
errorMessage = null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveXml(Writer fwrite) throws IOException {
|
||||
fwrite.append('<').append(name);
|
||||
fwrite.append(" success=\"");
|
||||
SpecXmlUtils.encodeBoolean(dropSuccessful);
|
||||
fwrite.append("\">");
|
||||
if (errorMessage != null) {
|
||||
SpecXmlUtils.xmlEscapeWriter(fwrite, errorMessage);
|
||||
}
|
||||
fwrite.append("</").append(name).append(">\n");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void restoreXml(XmlPullParser parser, LSHVectorFactory vectorFactory)
|
||||
throws LSHException {
|
||||
XmlElement el = parser.start(name);
|
||||
dropSuccessful = SpecXmlUtils.decodeBoolean(el.getAttribute("success"));
|
||||
errorMessage = parser.end().getText();
|
||||
if (errorMessage != null && errorMessage.length() == 0) {
|
||||
errorMessage = null;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -23,7 +23,6 @@ import java.util.*;
|
|||
import org.junit.*;
|
||||
|
||||
import ghidra.features.bsim.query.*;
|
||||
import ghidra.features.bsim.query.BSimServerInfo.DBType;
|
||||
import ghidra.features.bsim.query.FunctionDatabase.BSimError;
|
||||
import ghidra.features.bsim.query.description.DatabaseInformation;
|
||||
import ghidra.features.bsim.query.file.BSimH2FileDBConnectionManager.BSimH2FileDataSource;
|
||||
|
|
|
|||
|
|
@ -1,732 +0,0 @@
|
|||
/* ###
|
||||
* IP: GHIDRA
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package ghidra.features.bsim.query.test;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import java.io.*;
|
||||
import java.net.URL;
|
||||
import java.sql.Date;
|
||||
import java.time.Instant;
|
||||
import java.util.*;
|
||||
|
||||
import org.junit.*;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
import generic.jar.ResourceFile;
|
||||
import generic.lsh.vector.*;
|
||||
import generic.util.Path;
|
||||
import ghidra.GhidraTestApplicationLayout;
|
||||
import ghidra.app.util.headless.HeadlessAnalyzer;
|
||||
import ghidra.app.util.headless.HeadlessOptions;
|
||||
import ghidra.features.bsim.gui.filters.ExecutableCategoryBSimFilterType;
|
||||
import ghidra.features.bsim.gui.filters.HasNamedChildBSimFilterType;
|
||||
import ghidra.features.bsim.query.*;
|
||||
import ghidra.features.bsim.query.FunctionDatabase.BSimError;
|
||||
import ghidra.features.bsim.query.client.tables.ExeTable.ExeTableOrderColumn;
|
||||
import ghidra.features.bsim.query.description.*;
|
||||
import ghidra.features.bsim.query.ingest.BSimLaunchable;
|
||||
import ghidra.features.bsim.query.protocol.*;
|
||||
import ghidra.framework.*;
|
||||
import ghidra.framework.client.HeadlessClientAuthenticator;
|
||||
import ghidra.util.xml.SpecXmlUtils;
|
||||
import ghidra.xml.NonThreadedXmlPullParserImpl;
|
||||
import ghidra.xml.XmlPullParser;
|
||||
|
||||
// These tests require specific data files and paths to be set up. See BSimServerTestUtil.
|
||||
// The "ignore" directive is to prevent these from running as part of the automated nightly tests.
|
||||
@Ignore
|
||||
public class BSimServerTest {
|
||||
private static final String PROPERTIES_FILE = "RegressionSignatures.properties";
|
||||
|
||||
private static BSimServerTestUtil util;
|
||||
private static LSHVectorFactory vectorFactory;
|
||||
private static FunctionDatabase client;
|
||||
private static BSimLaunchable bulk;
|
||||
private static ResourceFile dumpFile;
|
||||
private static DescriptionManager originalBash;
|
||||
|
||||
private static XmlPullParser getParser(ResourceFile file) {
|
||||
XmlPullParser parser;
|
||||
try {
|
||||
InputStream input = file.getInputStream();
|
||||
parser = new NonThreadedXmlPullParserImpl(input, "BSim test parser",
|
||||
SpecXmlUtils.getXmlHandler(), false);
|
||||
}
|
||||
catch (SAXException e) {
|
||||
return null;
|
||||
}
|
||||
catch (IOException e) {
|
||||
return null;
|
||||
}
|
||||
return parser;
|
||||
}
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
util = new BSimServerTestUtil();
|
||||
util.verifyDirectories();
|
||||
GhidraTestApplicationLayout layout =
|
||||
new GhidraTestApplicationLayout(new File(util.ghidraDir));
|
||||
ApplicationConfiguration config = new HeadlessGhidraApplicationConfiguration();
|
||||
Application.initializeApplication(layout, config);
|
||||
ghidra.framework.protocol.ghidra.Handler.registerHandler(); /// Register ghidra: protocol
|
||||
ghidra.features.bsim.query.postgresql.Handler.registerHandler(); // Register postgresql: protocol
|
||||
HeadlessClientAuthenticator.installHeadlessClientAuthenticator(null, null, true);
|
||||
bulk = new BSimLaunchable();
|
||||
|
||||
util.verifyRaw();
|
||||
File propFile = new File(util.xmlDir, PROPERTIES_FILE);
|
||||
if (!propFile.isFile()) {
|
||||
createPropertiesFile();
|
||||
runHeadless();
|
||||
}
|
||||
|
||||
util.startServer();
|
||||
doIngest();
|
||||
BSimServerInfo bsimServerInfo;
|
||||
try {
|
||||
bsimServerInfo = new BSimServerInfo(new URL(util.bsimURLString));
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new AssertionError(e);
|
||||
}
|
||||
client = BSimClientFactory.buildClient(bsimServerInfo, false);
|
||||
if (!client.initialize()) {
|
||||
throw new IOException("Unable to connect to server");
|
||||
}
|
||||
vectorFactory = client.getLSHVectorFactory();
|
||||
|
||||
ResourceFile xmlFile =
|
||||
new ResourceFile(new ResourceFile(util.xmlDir), "sigs_" + BSimServerTestUtil.BASH_MD5);
|
||||
if (!xmlFile.isFile()) {
|
||||
throw new IOException("Basic signature generation did not happen");
|
||||
}
|
||||
XmlPullParser parser = getParser(xmlFile);
|
||||
originalBash = new DescriptionManager();
|
||||
originalBash.restoreXml(parser, vectorFactory);
|
||||
parser.dispose();
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void shutdown() throws Exception {
|
||||
if (client != null) {
|
||||
client.close();
|
||||
}
|
||||
util.shutdownServer();
|
||||
if (dumpFile != null && dumpFile.exists()) {
|
||||
dumpFile.delete();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testLibHistoryXml() {
|
||||
ResourceFile xmlFile = new ResourceFile(new ResourceFile(util.xmlDir),
|
||||
"sigs_" + BSimServerTestUtil.LIBHISTORY_MD5);
|
||||
assertTrue(xmlFile.isFile());
|
||||
XmlPullParser parser = getParser(xmlFile);
|
||||
DescriptionManager manager = new DescriptionManager();
|
||||
try {
|
||||
manager.restoreXml(parser, vectorFactory);
|
||||
assertTrue(manager.getExecutableRecordSet().size() == 2);
|
||||
ExecutableRecord eRec = manager.findExecutable(BSimServerTestUtil.LIBHISTORY_MD5);
|
||||
// make sure basic meta-data comes through
|
||||
assertTrue(eRec.getNameExec().equals("libhistory.so.7.0"));
|
||||
assertTrue(eRec.getNameCompiler().equals("gcc"));
|
||||
assertTrue(eRec.getPath().equals("raw"));
|
||||
assertTrue(eRec.getRepository().equals("ghidra://localhost/repo"));
|
||||
assertTrue(eRec.hasCategory("Test Category", "shared"));
|
||||
ExecutableRecord libRec = manager.findExecutable("unknown", "x86:LE:64:default", "");
|
||||
assertTrue(libRec.isLibrary());
|
||||
FunctionDescription fDesc = manager.findFunctionByName("close", libRec);
|
||||
assertNotNull(fDesc);
|
||||
assertEquals(fDesc.getAddress(), -1);
|
||||
fDesc = manager.findFunctionByName("read_history_range", eRec);
|
||||
assertNotNull(fDesc);
|
||||
assertEquals(fDesc.getAddress(), 0x105f60);
|
||||
FunctionDescription addHistory = null;
|
||||
FunctionDescription malloc = null;
|
||||
for (CallgraphEntry entry : fDesc.getCallgraphRecord()) {
|
||||
String name = entry.getFunctionDescription().getFunctionName();
|
||||
if (name.equals("add_history")) {
|
||||
addHistory = entry.getFunctionDescription();
|
||||
}
|
||||
else if (name.equals("malloc")) {
|
||||
malloc = entry.getFunctionDescription();
|
||||
}
|
||||
}
|
||||
assertNotNull(addHistory);
|
||||
assertNotNull(malloc);
|
||||
assertEquals(addHistory.getAddress(), 0x102770);
|
||||
assertEquals(malloc.getAddress(), -1);
|
||||
assertTrue(addHistory.getExecutableRecord() == eRec); // Should be same object
|
||||
assertTrue(malloc.getExecutableRecord() == libRec);
|
||||
}
|
||||
catch (LSHException e) {
|
||||
Assert.fail("Failure processing libhistory");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBashLibReadline() {
|
||||
try {
|
||||
ResourceFile xmlFile = new ResourceFile(new ResourceFile(util.xmlDir),
|
||||
"sigs_" + BSimServerTestUtil.LIBREADLINE_MD5);
|
||||
XmlPullParser parser = getParser(xmlFile);
|
||||
DescriptionManager manager = new DescriptionManager();
|
||||
manager.restoreXml(parser, vectorFactory);
|
||||
parser.dispose();
|
||||
assertEquals(manager.getExecutableRecordSet().size(), 2);
|
||||
ExecutableRecord bashRec = originalBash.findExecutable(BSimServerTestUtil.BASH_MD5);
|
||||
assertTrue(bashRec.hasCategory("Test Category", "static"));
|
||||
ExecutableRecord readRec = manager.findExecutable(BSimServerTestUtil.LIBREADLINE_MD5);
|
||||
assertTrue(readRec.hasCategory("Test Category", "shared"));
|
||||
// Comparing function "history_filename"
|
||||
FunctionDescription bashFunc =
|
||||
originalBash.findFunction("FUN_001cc840", 0x1cc840, bashRec);
|
||||
FunctionDescription readFunc = manager.findFunction("FUN_00134a70", 0x134a70, readRec);
|
||||
VectorCompare compareData = new VectorCompare();
|
||||
double sig = bashFunc.getSignatureRecord()
|
||||
.getLSHVector()
|
||||
.compare(readFunc.getSignatureRecord().getLSHVector(), compareData);
|
||||
assertTrue(sig > 0.99999);
|
||||
}
|
||||
catch (LSHException e) {
|
||||
Assert.fail("Failure processing bash and libreadline");
|
||||
}
|
||||
}
|
||||
|
||||
private static void compareExe(DescriptionManager manager1, DescriptionManager manager2,
|
||||
String md5) throws Exception {
|
||||
ExecutableRecord eRec1 = manager1.findExecutable(md5);
|
||||
ExecutableRecord eRec2 = manager2.findExecutable(md5);
|
||||
Iterator<FunctionDescription> iter = manager1.listFunctions(eRec1);
|
||||
while (iter.hasNext()) {
|
||||
FunctionDescription func1 = iter.next();
|
||||
FunctionDescription func2 =
|
||||
manager2.findFunction(func1.getFunctionName(), func1.getAddress(), eRec2);
|
||||
assertEquals(func1.getFlags(), func2.getFlags());
|
||||
if (func1.getSignatureRecord() == null) {
|
||||
assertTrue(func2.getSignatureRecord() == null);
|
||||
}
|
||||
else {
|
||||
assertNotNull(func2.getSignatureRecord());
|
||||
assertTrue(func1.getSignatureRecord()
|
||||
.getLSHVector()
|
||||
.equals(func2.getSignatureRecord().getLSHVector()));
|
||||
}
|
||||
if (func1.getCallgraphRecord() == null) {
|
||||
assertTrue(func2.getCallgraphRecord() == null);
|
||||
continue;
|
||||
}
|
||||
assertNotNull(func2.getCallgraphRecord());
|
||||
func1.sortCallgraph();
|
||||
func2.sortCallgraph();
|
||||
Iterator<CallgraphEntry> iter1 = func1.getCallgraphRecord().iterator();
|
||||
Iterator<CallgraphEntry> iter2 = func2.getCallgraphRecord().iterator();
|
||||
while (iter1.hasNext()) {
|
||||
assertTrue(iter2.hasNext());
|
||||
FunctionDescription call1 = iter1.next().getFunctionDescription();
|
||||
FunctionDescription call2 = iter2.next().getFunctionDescription();
|
||||
assertTrue(call1.equals(call2));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDumpFile() {
|
||||
try {
|
||||
assertNotNull(dumpFile);
|
||||
assertTrue(dumpFile.exists());
|
||||
XmlPullParser parser = getParser(dumpFile);
|
||||
DescriptionManager manager1 = new DescriptionManager();
|
||||
manager1.restoreXml(parser, vectorFactory);
|
||||
parser.dispose();
|
||||
compareExe(manager1, originalBash, BSimServerTestUtil.BASH_MD5);
|
||||
}
|
||||
catch (Exception e) {
|
||||
Assert.fail("Failed to perform dumpexexml: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static void testForError(QueryResponseRecord response) throws LSHException {
|
||||
if (response == null) {
|
||||
BSimError lastError = client.getLastError();
|
||||
if (lastError == null) {
|
||||
throw new LSHException("Unknown error");
|
||||
}
|
||||
throw new LSHException(lastError.message);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testQueryInfo() throws LSHException {
|
||||
QueryInfo queryInfo = new QueryInfo();
|
||||
ResponseInfo response = queryInfo.execute(client);
|
||||
testForError(response);
|
||||
DatabaseInformation info = response.info;
|
||||
assertTrue(info.databasename.equals("TestName"));
|
||||
assertTrue(info.owner.equals("TestOwner"));
|
||||
assertTrue(info.description.equals("TestDescription"));
|
||||
assertEquals(info.settings, 0x49);
|
||||
assertTrue(info.trackcallgraph);
|
||||
assertNotNull(info.execats);
|
||||
assertEquals(info.execats.size(), 1);
|
||||
assertTrue(info.execats.get(0).equals("Test Category"));
|
||||
}
|
||||
|
||||
private void compareExecutableRecords(ExecutableRecord exe1, ExecutableRecord exe2) {
|
||||
assertEquals(exe1.getMd5(), exe2.getMd5());
|
||||
assertEquals(exe1.getNameExec(), exe2.getNameExec());
|
||||
assertEquals(exe1.getArchitecture(), exe2.getArchitecture());
|
||||
assertEquals(exe1.getNameCompiler(), exe2.getNameCompiler());
|
||||
assertEquals(exe1.getPath(), exe2.getPath());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testQueryExeInfo() throws LSHException {
|
||||
ExecutableRecord libHistory = new ExecutableRecord(BSimServerTestUtil.LIBHISTORY_MD5,
|
||||
"libhistory.so.7.0", "gcc", "x86:LE:64:default", null, null, null, "raw");
|
||||
ExecutableRecord libReadline = new ExecutableRecord(BSimServerTestUtil.LIBREADLINE_MD5,
|
||||
"libreadline.so.7.0", "gcc", "x86:LE:64:default", null, null, null, "raw");
|
||||
QueryExeInfo queryExeInfo = new QueryExeInfo();
|
||||
queryExeInfo.includeFakes = true;
|
||||
ResponseExe responseExe = queryExeInfo.execute(client);
|
||||
testForError(responseExe);
|
||||
assertEquals(responseExe.recordCount, 3);
|
||||
ExecutableRecord exe1 = responseExe.records.get(0);
|
||||
compareExecutableRecords(exe1, libHistory);
|
||||
assertEquals(exe1.getExeCategoryAlphabetic("Test Category"), "shared");
|
||||
ExecutableRecord exe2 = responseExe.records.get(1);
|
||||
compareExecutableRecords(exe2, libReadline);
|
||||
assertEquals(exe2.getExeCategoryAlphabetic("Test Category"), "shared");
|
||||
ExecutableRecord exe3 = responseExe.records.get(2);
|
||||
assertEquals(exe3.getMd5(), "bbbbbbbbaaaaaaaa4b13cd7905584d9f");
|
||||
assertEquals(exe3.getNameExec(), "unknown");
|
||||
|
||||
queryExeInfo = new QueryExeInfo();
|
||||
queryExeInfo.filterMd5 = BSimServerTestUtil.LIBREADLINE_MD5;
|
||||
responseExe = queryExeInfo.execute(client);
|
||||
testForError(responseExe);
|
||||
assertEquals(responseExe.recordCount, 1);
|
||||
exe1 = responseExe.records.get(0);
|
||||
compareExecutableRecords(exe1, libReadline);
|
||||
|
||||
queryExeInfo = new QueryExeInfo();
|
||||
queryExeInfo.filterMd5 = "0a860"; // Partial md5
|
||||
responseExe = queryExeInfo.execute(client);
|
||||
testForError(responseExe);
|
||||
assertEquals(responseExe.recordCount, 1);
|
||||
exe1 = responseExe.getDescriptionManager().findExecutable("libhistory.so.7.0", null, null);
|
||||
compareExecutableRecords(exe1, libHistory);
|
||||
|
||||
queryExeInfo = new QueryExeInfo();
|
||||
queryExeInfo.filterExeName = "lib";
|
||||
queryExeInfo.sortColumn = ExeTableOrderColumn.NAME;
|
||||
responseExe = queryExeInfo.execute(client);
|
||||
testForError(responseExe);
|
||||
assertEquals(responseExe.records.size(), 2);
|
||||
exe1 = responseExe.records.get(0);
|
||||
exe2 = responseExe.records.get(1);
|
||||
compareExecutableRecords(exe1, libHistory);
|
||||
compareExecutableRecords(exe2, libReadline);
|
||||
|
||||
QueryExeCount queryExeCount = new QueryExeCount();
|
||||
responseExe = queryExeCount.execute(client);
|
||||
testForError(responseExe);
|
||||
assertEquals(responseExe.recordCount, 2);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testQueryName() throws LSHException {
|
||||
QueryName queryName = new QueryName();
|
||||
queryName.spec.exename = "libhistory.so.7.0";
|
||||
queryName.funcname = "history_arg_extract";
|
||||
ResponseName responseName = queryName.execute(client);
|
||||
testForError(responseName);
|
||||
assertTrue(responseName.uniqueexecutable);
|
||||
ExecutableRecord eRec =
|
||||
responseName.manage.findExecutable(BSimServerTestUtil.LIBHISTORY_MD5);
|
||||
assertTrue(eRec.getNameExec().equals("libhistory.so.7.0"));
|
||||
assertTrue(eRec.getMd5().equals(BSimServerTestUtil.LIBHISTORY_MD5));
|
||||
assertTrue(eRec.getNameCompiler().equals("gcc"));
|
||||
assertTrue(eRec.getPath().equals("raw"));
|
||||
Iterator<FunctionDescription> iter = responseName.manage.listAllFunctions();
|
||||
assertTrue(iter.hasNext());
|
||||
FunctionDescription func = iter.next();
|
||||
assertFalse(iter.hasNext()); // Should be exactly one function
|
||||
assertTrue(func.getFunctionName().equals("history_arg_extract"));
|
||||
assertEquals(func.getAddress(), 0x103d40);
|
||||
SignatureRecord sigRec = func.getSignatureRecord();
|
||||
assertNotNull(sigRec);
|
||||
assertEquals(sigRec.getCount(), 2);
|
||||
ExecutableRecord bashRec = originalBash.findExecutable(BSimServerTestUtil.BASH_MD5);
|
||||
FunctionDescription bashFunc =
|
||||
originalBash.findFunctionByName("history_arg_extract", bashRec);
|
||||
assertNotNull(bashFunc);
|
||||
VectorCompare vectorCompare = new VectorCompare();
|
||||
double sim = sigRec.getLSHVector()
|
||||
.compare(bashFunc.getSignatureRecord().getLSHVector(), vectorCompare);
|
||||
assertTrue(sim > 0.8);
|
||||
assertTrue(sim < 0.999);
|
||||
}
|
||||
|
||||
private static QueryResponseRecord doStagedQuery(BSimQuery<?> query,
|
||||
StagingManager stagingManager) throws LSHException {
|
||||
|
||||
boolean haveMore = stagingManager.initialize(query);
|
||||
query.buildResponseTemplate();
|
||||
|
||||
QueryResponseRecord globalResponse = query.getResponse();
|
||||
|
||||
while (haveMore) {
|
||||
// Get the current staged form of the query
|
||||
BSimQuery<?> stagedQuery = stagingManager.getQuery();
|
||||
QueryResponseRecord response = stagedQuery.execute(client);
|
||||
if (response != null) {
|
||||
if (globalResponse != response) {
|
||||
globalResponse.mergeResults(response); // Merge the staged response with the global response
|
||||
}
|
||||
|
||||
haveMore = stagingManager.nextStage();
|
||||
if (haveMore) {
|
||||
stagedQuery.clearResponse(); // Make space for next stage
|
||||
}
|
||||
}
|
||||
else {
|
||||
throw new LSHException(client.getLastError().message);
|
||||
}
|
||||
}
|
||||
|
||||
return globalResponse;
|
||||
}
|
||||
|
||||
private static void testSimilarityResult(SimilarityResult simRes) {
|
||||
assertEquals(simRes.getTotalCount(), 2);
|
||||
Iterator<SimilarityNote> iter = simRes.iterator();
|
||||
SimilarityNote note1 = iter.next();
|
||||
SimilarityNote note2 = iter.next();
|
||||
FunctionDescription func1 = note1.getFunctionDescription();
|
||||
FunctionDescription func2 = note2.getFunctionDescription();
|
||||
assertTrue(func1.getExecutableRecord().getNameExec().equals("libhistory.so.7.0"));
|
||||
assertTrue(func2.getExecutableRecord().getNameExec().equals("libreadline.so.7.0"));
|
||||
assertNotNull(func1.getSignatureRecord());
|
||||
assertNotNull(func2.getSignatureRecord());
|
||||
assertNotNull(simRes.getBase().getSignatureRecord());
|
||||
LSHVector baseVector = simRes.getBase().getSignatureRecord().getLSHVector();
|
||||
VectorCompare vectorCompare = new VectorCompare();
|
||||
double sim1 = func1.getSignatureRecord().getLSHVector().compare(baseVector, vectorCompare);
|
||||
assertEquals(note1.getSimilarity(), sim1, 0.0001);
|
||||
double sim2 = func2.getSignatureRecord().getLSHVector().compare(baseVector, vectorCompare);
|
||||
assertEquals(note2.getSimilarity(), sim2, 0.0001);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testQueryNearest() throws LSHException {
|
||||
QueryNearest queryNearest = new QueryNearest();
|
||||
ExecutableRecord bashRec = originalBash.findExecutable(BSimServerTestUtil.BASH_MD5);
|
||||
FunctionDescription func1 = originalBash.findFunctionByName("_rl_adjust_point", bashRec);
|
||||
FunctionDescription func2 = originalBash.findFunctionByName("_rl_compare_chars", bashRec);
|
||||
FunctionDescription func3 = originalBash.findFunctionByName("add_history", bashRec);
|
||||
FunctionDescription func4 = originalBash.findFunctionByName("get_history_event", bashRec);
|
||||
queryNearest.manage.transferSettings(originalBash);
|
||||
queryNearest.manage.transferFunction(func1, true);
|
||||
queryNearest.manage.transferFunction(func2, true);
|
||||
queryNearest.manage.transferFunction(func3, true);
|
||||
queryNearest.manage.transferFunction(func4, true);
|
||||
StagingManager functionStage = new FunctionStaging(2);
|
||||
QueryResponseRecord response = doStagedQuery(queryNearest, functionStage);
|
||||
testForError(response);
|
||||
ResponseNearest respNearest = (ResponseNearest) response;
|
||||
respNearest.sort();
|
||||
int matchCount = 0;
|
||||
for (SimilarityResult simRes : respNearest.result) {
|
||||
if (simRes.getBase().getAddress() == func1.getAddress()) {
|
||||
assertTrue(func1.equals(simRes.getBase()));
|
||||
matchCount += 1;
|
||||
}
|
||||
else if (simRes.getBase().getAddress() == func2.getAddress()) {
|
||||
assertTrue(func2.equals(simRes.getBase()));
|
||||
matchCount += 1;
|
||||
}
|
||||
else if (simRes.getBase().getAddress() == func3.getAddress()) {
|
||||
assertTrue(func3.equals(simRes.getBase()));
|
||||
matchCount += 1;
|
||||
}
|
||||
else if (simRes.getBase().getAddress() == func4.getAddress()) {
|
||||
assertTrue(func4.equals(simRes.getBase()));
|
||||
matchCount += 1;
|
||||
}
|
||||
testSimilarityResult(simRes);
|
||||
}
|
||||
assertEquals(matchCount, 4); // Make sure we hit all functions
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testQueryVector() throws LSHException {
|
||||
QueryNearestVector queryVector = new QueryNearestVector();
|
||||
ExecutableRecord bashRec = originalBash.findExecutable(BSimServerTestUtil.BASH_MD5);
|
||||
FunctionDescription func = originalBash.findFunctionByName("_rl_kill_kbd_macro", bashRec);
|
||||
queryVector.manage.transferSettings(originalBash);
|
||||
queryVector.manage.transferFunction(func, true);
|
||||
ResponseNearestVector respVector = queryVector.execute(client);
|
||||
testForError(respVector);
|
||||
respVector.sort();
|
||||
assertEquals(respVector.totalvec, 1);
|
||||
assertEquals(respVector.totalmatch, 2);
|
||||
assertEquals(respVector.uniquematch, 0); // 1 vector matches 2 functions
|
||||
Iterator<SimilarityVectorResult> iter = respVector.result.iterator();
|
||||
SimilarityVectorResult simVecRes = iter.next();
|
||||
assertFalse(iter.hasNext());
|
||||
assertTrue(simVecRes.getBase().equals(func));
|
||||
assertEquals(simVecRes.getTotalCount(), 2);
|
||||
Iterator<VectorResult> iter2 = simVecRes.iterator();
|
||||
VectorResult vec1 = iter2.next();
|
||||
VectorResult vec2 = iter2.next();
|
||||
assertFalse(iter2.hasNext());
|
||||
if (vec1.sim > vec2.sim) {
|
||||
VectorResult tmp = vec1;
|
||||
vec1 = vec2;
|
||||
vec2 = tmp;
|
||||
}
|
||||
VectorCompare vectorCompare = new VectorCompare();
|
||||
LSHVector baseVector = func.getSignatureRecord().getLSHVector();
|
||||
double sim1 = baseVector.compare(vec1.vec, vectorCompare);
|
||||
assertEquals(sim1, vec1.sim, 0.0001);
|
||||
assertEquals(vec1.hitcount, 1);
|
||||
double sim2 = baseVector.compare(vec2.vec, vectorCompare);
|
||||
assertEquals(sim2, vec2.sim, 0.0001);
|
||||
assertEquals(vec1.hitcount, 1);
|
||||
assertTrue(vec2.sim > 0.999); // Second vector should be 1.0 match
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testChildFilter() throws LSHException {
|
||||
QueryNearest query = new QueryNearest();
|
||||
query.manage.transferSettings(originalBash);
|
||||
ExecutableRecord bashRec = originalBash.findExecutable(BSimServerTestUtil.BASH_MD5);
|
||||
FunctionDescription funcDesc = originalBash.findFunctionByName("_rl_errmsg", bashRec);
|
||||
query.manage.transferFunction(funcDesc, true);
|
||||
query.bsimFilter = new BSimFilter();
|
||||
query.bsimFilter.addAtom(new HasNamedChildBSimFilterType(), "[unknown]__fprintf_chk");
|
||||
ResponseNearest respNearest = query.execute(client);
|
||||
testForError(respNearest);
|
||||
assertEquals(respNearest.totalfunc, 1);
|
||||
assertEquals(respNearest.totalmatch, 1); // Filtered all but one response
|
||||
assertEquals(respNearest.uniquematch, 1);
|
||||
ExecutableRecord eRec = respNearest.manage.getExecutableRecordSet().first();
|
||||
assertTrue(eRec.getNameExec().equals("libreadline.so.7.0"));
|
||||
assertTrue(eRec.getMd5().equals(BSimServerTestUtil.LIBREADLINE_MD5));
|
||||
assertEquals(respNearest.result.size(), 1);
|
||||
SimilarityResult simRes = respNearest.result.get(0);
|
||||
assertEquals(simRes.size(), 1); // Only one function returned
|
||||
assertEquals(simRes.getTotalCount(), 3); // 3 functions similar to vector
|
||||
SimilarityNote note = simRes.iterator().next();
|
||||
assertTrue(note.getSimilarity() > 0.800);
|
||||
FunctionDescription resFunc = note.getFunctionDescription();
|
||||
assertTrue(resFunc.getFunctionName().equals("FUN_0011ece0"));
|
||||
assertEquals(resFunc.getAddress(), 0x11ece0);
|
||||
assertEquals(resFunc.getSignatureRecord().getCount(), 1); // only function with this exact vector
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUpdate() throws Exception {
|
||||
QueryUpdate update = new QueryUpdate();
|
||||
|
||||
Date dt = new Date(Instant.parse("2010-12-25T10:15:30.00Z").toEpochMilli());
|
||||
ExecutableRecord exerec = update.manage.newExecutableRecord(
|
||||
BSimServerTestUtil.LIBREADLINE_MD5, "libreadline.so.7.0", "gcc", "x86:LE:64:default",
|
||||
dt, "ghidra://localhost/repo", "/raw", null);
|
||||
List<CategoryRecord> catrec = new ArrayList<>();
|
||||
catrec.add(new CategoryRecord("Test Category", "SHARED!"));
|
||||
update.manage.setExeCategories(exerec, catrec);
|
||||
update.manage.newFunctionDescription("my_remove_history", 0x131c00, exerec);
|
||||
ResponseUpdate respUpdate = update.execute(client);
|
||||
testForError(respUpdate);
|
||||
assertEquals(respUpdate.exeupdate, 1);
|
||||
assertEquals(respUpdate.funcupdate, 1);
|
||||
assertTrue(respUpdate.badexe.isEmpty());
|
||||
assertTrue(respUpdate.badfunc.isEmpty());
|
||||
if (util.isElasticSearch) {
|
||||
Thread.sleep(2000); // Give chance for refresh timer to expire
|
||||
}
|
||||
QueryNearest nearest = new QueryNearest();
|
||||
nearest.manage.transferSettings(originalBash);
|
||||
|
||||
ExecutableRecord bash = originalBash.findExecutable(BSimServerTestUtil.BASH_MD5);
|
||||
FunctionDescription desc = originalBash.findFunctionByName("remove_history", bash);
|
||||
nearest.manage.transferFunction(desc, true);
|
||||
nearest.bsimFilter = new BSimFilter();
|
||||
nearest.bsimFilter.addAtom(new ExecutableCategoryBSimFilterType("Test Category"),
|
||||
"SHARED!");
|
||||
ResponseNearest respNearest = nearest.execute(client);
|
||||
testForError(respNearest);
|
||||
assertEquals(respNearest.totalfunc, 1);
|
||||
assertEquals(respNearest.totalmatch, 1);
|
||||
assertEquals(respNearest.uniquematch, 1);
|
||||
SimilarityResult simRes = respNearest.result.get(0);
|
||||
assertTrue(simRes.getBase().equals(desc)); // base should match original function
|
||||
assertEquals(simRes.size(), 1);
|
||||
assertEquals(simRes.getTotalCount(), 2); // The filtered libhistory version is also counted here
|
||||
SimilarityNote note = simRes.iterator().next();
|
||||
FunctionDescription resFunc = note.getFunctionDescription();
|
||||
assertTrue(resFunc.getFunctionName().equals("my_remove_history"));
|
||||
ExecutableRecord resRec = resFunc.getExecutableRecord();
|
||||
assertTrue(resRec.getDate().equals(dt));
|
||||
|
||||
// Restore the original records
|
||||
update = new QueryUpdate();
|
||||
exerec = update.manage.newExecutableRecord(BSimServerTestUtil.LIBREADLINE_MD5,
|
||||
"libreadline.so.7.0", "gcc", "x86:LE:64:default", bash.getDate(),
|
||||
"ghidra://localhost/repo", "/raw", null);
|
||||
catrec = new ArrayList<>();
|
||||
catrec.add(new CategoryRecord("Test Category", "shared"));
|
||||
update.manage.setExeCategories(exerec, catrec);
|
||||
update.manage.newFunctionDescription("remove_history", 0x131c00, exerec);
|
||||
testForError(update.execute(client));
|
||||
if (util.isElasticSearch) {
|
||||
Thread.sleep(2000); // Give chance for refresh timer to expire
|
||||
}
|
||||
}
|
||||
|
||||
private static void doIngest() throws Exception {
|
||||
if (dumpFile != null && dumpFile.exists()) {
|
||||
return;
|
||||
}
|
||||
runCreateDatabase();
|
||||
if (util.isElasticSearch) {
|
||||
Thread.sleep(2000); // Give chance for refresh timer to expire
|
||||
}
|
||||
runSetMetaData();
|
||||
runSetExeCategory();
|
||||
runDropIndex();
|
||||
if (util.isElasticSearch) {
|
||||
Thread.sleep(2000); // Give chance for refresh timer to expire
|
||||
}
|
||||
runCommitSigs();
|
||||
runRebuildIndex();
|
||||
if (util.isElasticSearch) {
|
||||
Thread.sleep(2000); // Give chance for refresh timer to expire
|
||||
}
|
||||
runDumpFile(BSimServerTestUtil.BASH_MD5);
|
||||
dumpFile =
|
||||
new ResourceFile(new ResourceFile(util.testDir), "sigs_" + BSimServerTestUtil.BASH_MD5);
|
||||
runDelete();
|
||||
if (util.isElasticSearch) {
|
||||
Thread.sleep(2000); // Give chance for refresh timer to expire
|
||||
}
|
||||
}
|
||||
|
||||
private static void createPropertiesFile() throws IOException {
|
||||
File props = new File(util.xmlDir, PROPERTIES_FILE);
|
||||
FileWriter writer = new FileWriter(props);
|
||||
writer.write("RegressionSignatures: Working directory = " + util.xmlDir + '\n');
|
||||
writer.close();
|
||||
}
|
||||
|
||||
private static void runHeadless() throws IOException {
|
||||
HeadlessAnalyzer analyzer = HeadlessAnalyzer.getInstance();
|
||||
HeadlessOptions options = analyzer.getOptions();
|
||||
List<String> preScripts = new ArrayList<>();
|
||||
List<String> postScripts = new ArrayList<>();
|
||||
List<File> inputFiles = new ArrayList<>();
|
||||
|
||||
options.setPropertiesFileDirectories(util.xmlDir);
|
||||
Path scriptPath = new Path(Path.GHIDRA_HOME + "/Features/BSim/other/testscripts");
|
||||
options.setScriptDirectories(scriptPath.getPath().getAbsolutePath());
|
||||
|
||||
inputFiles.add(new File(util.rawDir));
|
||||
preScripts.add("TailoredAnalysis.java");
|
||||
preScripts.add("InstallMetadataTest.java");
|
||||
postScripts.add("RegressionSignatures.java");
|
||||
|
||||
options.setPreScripts(preScripts);
|
||||
options.setPostScripts(postScripts);
|
||||
|
||||
analyzer.processLocal(util.projectDir, util.repoName, "/", inputFiles);
|
||||
}
|
||||
|
||||
private static void runCreateDatabase() throws Exception {
|
||||
String params[] = new String[3];
|
||||
params[0] = "createdatabase";
|
||||
params[1] = util.bsimURLString;
|
||||
params[2] = "medium_64";
|
||||
bulk.run(params);
|
||||
}
|
||||
|
||||
private static void runSetMetaData() throws Exception {
|
||||
String params[] = new String[5];
|
||||
params[0] = "setmetadata";
|
||||
params[1] = util.bsimURLString;
|
||||
params[2] = "name=TestName";
|
||||
params[3] = "owner=TestOwner";
|
||||
params[4] = "description=TestDescription";
|
||||
bulk.run(params);
|
||||
}
|
||||
|
||||
private static void runSetExeCategory() throws Exception {
|
||||
String params[] = new String[3];
|
||||
params[0] = "addexecategory";
|
||||
params[1] = util.bsimURLString;
|
||||
params[2] = "Test Category";
|
||||
bulk.run(params);
|
||||
}
|
||||
|
||||
private static void runDropIndex() throws Exception {
|
||||
if (util.isH2Database) {
|
||||
return;
|
||||
}
|
||||
String params[] = new String[2];
|
||||
params[0] = "dropindex";
|
||||
params[1] = util.bsimURLString;
|
||||
bulk.run(params);
|
||||
}
|
||||
|
||||
private static void runCommitSigs() throws Exception {
|
||||
String params[] = new String[3];
|
||||
params[0] = "commitsigs";
|
||||
params[1] = util.bsimURLString;
|
||||
params[2] = util.xmlDir;
|
||||
bulk.run(params);
|
||||
}
|
||||
|
||||
private static void runRebuildIndex() throws Exception {
|
||||
if (util.isH2Database) {
|
||||
return;
|
||||
}
|
||||
String params[] = new String[2];
|
||||
params[0] = "rebuildindex";
|
||||
params[1] = util.bsimURLString;
|
||||
bulk.run(params);
|
||||
}
|
||||
|
||||
private static void runDelete() throws Exception {
|
||||
String params[] = new String[3];
|
||||
params[0] = "delete";
|
||||
params[1] = util.bsimURLString;
|
||||
params[2] = "md5=" + BSimServerTestUtil.BASH_MD5;
|
||||
bulk.run(params);
|
||||
}
|
||||
|
||||
private static void runDumpFile(String md5) throws Exception {
|
||||
String params[] = new String[4];
|
||||
params[0] = "dumpsigs";
|
||||
params[1] = util.bsimURLString;
|
||||
params[2] = util.testDir;
|
||||
params[3] = "md5=" + BSimServerTestUtil.BASH_MD5;
|
||||
bulk.run(params);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,192 +0,0 @@
|
|||
/* ###
|
||||
* IP: GHIDRA
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package ghidra.features.bsim.query.test;
|
||||
|
||||
import java.io.*;
|
||||
|
||||
import ghidra.features.bsim.query.BSimControlLaunchable;
|
||||
import ghidra.util.MD5Utilities;
|
||||
import utilities.util.FileUtilities;
|
||||
|
||||
/**
|
||||
* The TEST_DIRECTORY String should be changed to point to a directory that will
|
||||
* hold data for the server and for the tests. To start, this directory should contain
|
||||
* a subdirectory "raw", and within this subdirectory should be the following 3 specific binary
|
||||
* executables:
|
||||
* libreadline.so.7.0
|
||||
* libhistory.so.7.0
|
||||
* bash
|
||||
*
|
||||
* all pulled from Ubuntu 18.04.5.
|
||||
*/
|
||||
public class BSimServerTestUtil {
|
||||
private static final String HOST_URL = "postgresql://localhost";
|
||||
// private static final String HOST_URL = "https://localhost:9200";
|
||||
// private static final String HOST_URL = "file:///tmp/bsimtest/db";
|
||||
private static final String TEST_DIRECTORY = "/tmp/bsimtest";
|
||||
public static final String REPO_NAME = "repo";
|
||||
public static final String LIBHISTORY_MD5 = "0a860a716d5bec97c64db652549b72fd";
|
||||
public static final String LIBREADLINE_MD5 = "71b5761b43b840eb88d053790deaf77c";
|
||||
public static final String BASH_MD5 = "557c0271e30cf474e0f46f93721fd1ba";
|
||||
public String repoName;
|
||||
public String bsimURLString = HOST_URL + '/' + REPO_NAME;
|
||||
public String testDir;
|
||||
public String ghidraDir;
|
||||
public String projectDir;
|
||||
public String rawDir;
|
||||
public String xmlDir;
|
||||
public String serverDir;
|
||||
public String serverTouchDir;
|
||||
public boolean isElasticSearch;
|
||||
public boolean isH2Database;
|
||||
|
||||
public BSimServerTestUtil() {
|
||||
testDir = TEST_DIRECTORY;
|
||||
ghidraDir = TEST_DIRECTORY + "/ghidra";
|
||||
repoName = REPO_NAME;
|
||||
projectDir = testDir + "/project";
|
||||
rawDir = testDir + "/raw";
|
||||
xmlDir = testDir + "/xml";
|
||||
serverDir = testDir + "/db";
|
||||
serverTouchDir = testDir + "/servertouch";
|
||||
isElasticSearch = HOST_URL.startsWith("http") || HOST_URL.startsWith("elastic");
|
||||
isH2Database = HOST_URL.startsWith("file");
|
||||
}
|
||||
|
||||
public void verifyDirectories() throws FileNotFoundException {
|
||||
File dir0 = new File(testDir);
|
||||
if (!dir0.exists()) {
|
||||
throw new FileNotFoundException("Could not find test directory");
|
||||
}
|
||||
File dir1 = new File(projectDir);
|
||||
if (!dir1.exists()) {
|
||||
if (!dir1.mkdir()) {
|
||||
throw new FileNotFoundException("Could not create project directory");
|
||||
}
|
||||
}
|
||||
File dir2 = new File(xmlDir);
|
||||
if (!dir2.exists()) {
|
||||
if (!dir2.mkdir()) {
|
||||
throw new FileNotFoundException("Could not create xml directory");
|
||||
}
|
||||
}
|
||||
File dir3 = new File(ghidraDir);
|
||||
if (!dir3.exists()) {
|
||||
if (!dir3.mkdir()) {
|
||||
throw new FileNotFoundException("Could not create ghidra directory");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void verifyRaw() throws IOException {
|
||||
File rawDirectory = new File(rawDir);
|
||||
if (!rawDirectory.exists()) {
|
||||
throw new FileNotFoundException(rawDir);
|
||||
}
|
||||
if (!rawDirectory.isDirectory()) {
|
||||
throw new FileNotFoundException("/raw is not a directory");
|
||||
}
|
||||
String[] list = rawDirectory.list();
|
||||
boolean readlinePresent = false;
|
||||
boolean historyPresent = false;
|
||||
boolean bashPresent = false;
|
||||
for (String element : list) {
|
||||
File lib = new File(rawDirectory, element);
|
||||
if (element.equals("libreadline.so.7.0")) {
|
||||
String md5 = MD5Utilities.getMD5Hash(lib);
|
||||
if (md5.equals(LIBREADLINE_MD5)) {
|
||||
readlinePresent = true;
|
||||
}
|
||||
else {
|
||||
throw new FileNotFoundException("libreadline.so.7.0 md5 does not match");
|
||||
}
|
||||
}
|
||||
else if (element.equals("libhistory.so.7.0")) {
|
||||
String md5 = MD5Utilities.getMD5Hash(lib);
|
||||
if (md5.equals(LIBHISTORY_MD5)) {
|
||||
historyPresent = true;
|
||||
}
|
||||
else {
|
||||
throw new FileNotFoundException("libhistory.so.7.0 md5 does not match");
|
||||
}
|
||||
}
|
||||
else if (element.equals("bash")) {
|
||||
String md5 = MD5Utilities.getMD5Hash(lib);
|
||||
if (md5.equals(BASH_MD5)) {
|
||||
bashPresent = true;
|
||||
}
|
||||
else {
|
||||
throw new FileNotFoundException("bash md5 does not match");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!readlinePresent) {
|
||||
throw new FileNotFoundException("Missing libreadline.so.7.0");
|
||||
}
|
||||
if (!historyPresent) {
|
||||
throw new FileNotFoundException("Missing libhistory.so.7.0");
|
||||
}
|
||||
if (!bashPresent) {
|
||||
throw new FileNotFoundException("Missing bash");
|
||||
}
|
||||
}
|
||||
|
||||
public void startServer() throws Exception {
|
||||
if (isElasticSearch || isH2Database) {
|
||||
return; // Don't try to start elasticsearch server
|
||||
}
|
||||
File touch = new File(serverTouchDir);
|
||||
if (touch.exists()) {
|
||||
return;
|
||||
}
|
||||
File dir = new File(serverDir);
|
||||
if (dir.isDirectory()) {
|
||||
FileUtilities.deleteDir(dir);
|
||||
}
|
||||
String[] params = new String[2];
|
||||
|
||||
params[0] = "start";
|
||||
params[1] = serverDir;
|
||||
dir.mkdir(); // Create the data directory
|
||||
|
||||
new BSimControlLaunchable().run(params);
|
||||
|
||||
byte[] touchBytes = new byte[2];
|
||||
touchBytes[0] = 'a';
|
||||
touchBytes[1] = 'b';
|
||||
FileUtilities.writeBytes(touch, touchBytes);
|
||||
}
|
||||
|
||||
public void shutdownServer() throws Exception {
|
||||
if (isElasticSearch || isH2Database) {
|
||||
return;
|
||||
}
|
||||
File touch = new File(serverTouchDir);
|
||||
if (!touch.exists()) {
|
||||
return;
|
||||
}
|
||||
String[] params = new String[2];
|
||||
|
||||
params[0] = "stop";
|
||||
params[1] = serverDir;
|
||||
new BSimControlLaunchable().run(params);
|
||||
touch.delete(); // Remove the touch file
|
||||
File dir = new File(serverDir);
|
||||
if (dir.isDirectory()) {
|
||||
FileUtilities.deleteDir(dir); // Clean up database files
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1 +0,0 @@
|
|||
MODULE FILE LICENSE: lib/json-simple-1.1.1.jar Apache License 2.0
|
||||
Loading…
Add table
Add a link
Reference in a new issue