GP-1830 BSim migration to gson use. Corrected various bugs with BSim

elasticsearch use.

Ghidra/Features/BSim/Module.manifest

@@ -1,7 +1,6 @@
 ##MODULE IP: Oxygen Icons - LGPL 3.0
 MODULE FILE LICENSE: postgresql-15.3.tar.gz Postgresql License
 MODULE FILE LICENSE: lib/postgresql-42.6.2.jar PostgresqlJDBC License
-MODULE FILE LICENSE: lib/json-simple-1.1.1.jar Apache License 2.0
 MODULE FILE LICENSE: lib/commons-dbcp2-2.9.0.jar Apache License 2.0
 MODULE FILE LICENSE: lib/commons-pool2-2.11.1.jar Apache License 2.0
 MODULE FILE LICENSE: lib/commons-logging-1.2.jar Apache License 2.0

Ghidra/Features/BSim/build.gradle

@@ -33,7 +33,6 @@ dependencies {
 	api project(":CodeCompare")
 
 	api "org.postgresql:postgresql:42.6.2"
-    api "com.googlecode.json-simple:json-simple:1.1.1"
     api "org.apache.commons:commons-dbcp2:2.9.0"
     api "org.apache.commons:commons-pool2:2.11.1"
     api "commons-logging:commons-logging:1.2"

Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/filters/ExecutableNameBSimFilterType.java

@@ -17,8 +17,6 @@ package ghidra.features.bsim.gui.filters;
 
 import java.sql.SQLException;
 
-import org.json.simple.JSONObject;
-
 import ghidra.features.bsim.query.client.IDSQLResolution;
 import ghidra.features.bsim.query.client.SQLEffects;
 import ghidra.features.bsim.query.description.ExecutableRecord;
@@ -50,7 +48,7 @@ public class ExecutableNameBSimFilterType extends BSimFilterType {
 			IDElasticResolution resolution) throws ElasticException {
 		StringBuilder buffer = new StringBuilder();
 		buffer.append("\"filter\": { \"term\": { \"name_exec\": \"");
-		buffer.append(JSONObject.escape(atom.value));
+		buffer.append(ElasticDatabase.escape(atom.value));
 		buffer.append("\" } } ");
 		effect.addStandalone(this, buffer.toString());
 	}

Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/filters/NotExecutableNameBSimFilterType.java

@@ -17,8 +17,6 @@ package ghidra.features.bsim.gui.filters;
 
 import java.sql.SQLException;
 
-import org.json.simple.JSONObject;
-
 import ghidra.features.bsim.query.client.IDSQLResolution;
 import ghidra.features.bsim.query.client.SQLEffects;
 import ghidra.features.bsim.query.description.ExecutableRecord;
@@ -49,7 +47,7 @@ public class NotExecutableNameBSimFilterType extends BSimFilterType {
 			IDElasticResolution resolution) throws ElasticException {
 		StringBuilder buffer = new StringBuilder();
 		buffer.append("\"must_not\": { \"term\": { \"name_exec\": \"");
-		buffer.append(JSONObject.escape(atom.value));
+		buffer.append(ElasticDatabase.escape(atom.value));
 		buffer.append("\" } } ");
 		effect.addStandalone(this, buffer.toString());
 	}

Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/filters/PathStartsBSimFilterType.java

@@ -48,10 +48,9 @@ public class PathStartsBSimFilterType extends BSimFilterType {
 	@Override
 	public void gatherElasticEffect(ElasticEffects effect, FilterAtom atom,
 			IDElasticResolution resolution) throws ElasticException {
-		effect.addDocValue("String path = doc['path'].value; ");
+		effect.addDocValue("String path = doc['path'].size() == 0 ? null : doc['path'].value; ");
 		String argName = effect.assignArgument();
-		effect.addScriptElement(this,
+		effect.addScriptElement(this, "(path != null) && path.startsWith(params." + argName + ')');
-			"(path != null) && path.startsWith(params." + argName + ')');
 		effect.addParam(argName, atom.value);
 	}
 

Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/BSimPostgresDBConnectionManager.java

@@ -284,7 +284,9 @@ public class BSimPostgresDBConnectionManager {
 
 			String loginError = null;
 
+			if (bds.getPassword() == null) {
 				serverInfo.setUserInfo(bds);
+			}
 
 			connectionType = serverInfo.hasPassword() ? ConnectionType.SSL_Password_Authentication
 					: ConnectionType.SSL_No_Authentication;

Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/client/AbstractSQLFunctionDatabase.java

@@ -186,7 +186,7 @@ public abstract class AbstractSQLFunctionDatabase<VF extends LSHVectorFactory>
 	 * @throws SQLException if error occurs obtaining connection
 	 */
 	protected Connection initConnection() throws SQLException {
-		if (db == null) {
+		if (db == null || db.isClosed()) {
 			db = ds.getConnection();
 		}
 		return db;
@@ -435,6 +435,12 @@ public abstract class AbstractSQLFunctionDatabase<VF extends LSHVectorFactory>
 		}
 	}
 
+	/**
+	 * Drop this database
+	 * @throws SQLException if a database error occured
+	 */
+	abstract protected void dropDatabase() throws SQLException;
+
 	protected void setConnectionOnTables(Connection db) {
 
 		weightTable.setConnection(db);
@@ -1201,8 +1207,8 @@ public abstract class AbstractSQLFunctionDatabase<VF extends LSHVectorFactory>
 			}
 			else if (msg.contains("authentication failed") ||
 				msg.contains("requires a valid client certificate")) {
-				lasterror =
+				lasterror = new BSimError(ErrorCategory.Authentication,
-					new BSimError(ErrorCategory.Authentication, "Could not authenticate with database");
+					"Could not authenticate with database");
 			}
 			else if (msg.contains("does not exist") && !msg.contains(" role ")) {
 				lasterror = new BSimError(ErrorCategory.Nodatabase, cause.getMessage());
@@ -1572,6 +1578,9 @@ public abstract class AbstractSQLFunctionDatabase<VF extends LSHVectorFactory>
 		else if (query instanceof QueryExeCount q) {
 			fdbQueryExeCount(q);
 		}
+		else if (query instanceof DropDatabase q) {
+			fdbDatabaseDrop(q);
+		}
 		else if (query instanceof CreateDatabase q) {
 			fdbDatabaseCreate(q);
 		}
@@ -1622,7 +1631,8 @@ public abstract class AbstractSQLFunctionDatabase<VF extends LSHVectorFactory>
 
 		lasterror = null;
 		try {
-			if (!(query instanceof CreateDatabase) && !initialize()) {
+			if (!(query instanceof CreateDatabase) && !(query instanceof DropDatabase) &&
+				!initialize()) {
 				lasterror = new BSimError(ErrorCategory.Nodatabase, "The database does not exist");
 				return null;
 			}
@@ -2087,6 +2097,29 @@ public abstract class AbstractSQLFunctionDatabase<VF extends LSHVectorFactory>
 		}
 	}
 
+	private void fdbDatabaseDrop(DropDatabase query) throws LSHException {
+		ResponseDropDatabase response = query.getResponse();
+		if (query.databaseName == null) {
+			throw new LSHException("Missing databaseName for drop database");
+		}
+		if (!query.databaseName.equals(ds.getServerInfo().getDBName())) {
+			throw new UnsupportedOperationException("drop database name must match");
+		}
+		response.dropSuccessful = true;		// Response parameters assuming success
+		response.errorMessage = null;
+		try {
+			dropDatabase();
+		}
+		catch (SQLException e) {
+			String msg = e.getMessage();
+			if (msg.indexOf("database \"" + query.databaseName + "\" does not exist") > 0) {
+				return; // missing database
+			}
+			response.dropSuccessful = false;
+			response.errorMessage = e.getMessage();
+		}
+	}
+
 	/**
 	 * Entry point for the CreateDatabase command
 	 * @param query the query to execute

Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/client/PostgresFunctionDatabase.java

@@ -31,6 +31,7 @@ import ghidra.features.bsim.query.client.tables.CachedStatement;
 import ghidra.features.bsim.query.client.tables.SQLStringTable;
 import ghidra.features.bsim.query.description.*;
 import ghidra.features.bsim.query.protocol.*;
+import ghidra.util.Msg;
 
 /**
  * Defines the BSim {@link FunctionDatabase} backed by a PostgreSQL database.
@@ -200,9 +201,6 @@ public final class PostgresFunctionDatabase
 			st.executeUpdate(createdbstring);
 			postgresDs.initializeFrom(defaultDs);
 		}
-		finally {
-			defaultDs.dispose();
-		}
 	}
 
 	@Override
@@ -244,6 +242,64 @@ public final class PostgresFunctionDatabase
 		}
 	}
 
+	@Override
+	protected void dropDatabase() throws SQLException {
+
+		if (getStatus() == Status.Busy || postgresDs.getActiveConnections() != 0) {
+			throw new SQLException("database in use");
+		}
+
+		BSimServerInfo serverInfo = postgresDs.getServerInfo();
+		BSimServerInfo defaultServerInfo =
+			new BSimServerInfo(DBType.postgres, serverInfo.getUserInfo(),
+				serverInfo.getServerName(), serverInfo.getPort(), DEFAULT_DATABASE_NAME);
+
+		BSimPostgresDataSource defaultDs =
+			BSimPostgresDBConnectionManager.getDataSource(defaultServerInfo);
+		if (getStatus() == Status.Ready) {
+			defaultDs.initializeFrom(postgresDs);
+		}
+
+		close(); // close this instance
+
+		try (Connection defaultDb = defaultDs.getConnection();
+				Statement defaultSt = defaultDb.createStatement()) {
+			try (ResultSet rs = defaultSt.executeQuery(
+				"SELECT 1 FROM pg_database WHERE datname='" + serverInfo.getDBName() + "'")) {
+				if (!rs.next()) {
+					return; // database does not exist
+				}
+			}
+
+			// Connect to database and examine schema
+			HashSet<String> tableNames = new HashSet<>();
+			postgresDs.initializeFrom(defaultDs);
+			try (Connection c = initConnection(); Statement st = c.createStatement()) {
+				try (ResultSet rs = st.executeQuery(
+					"SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' ORDER BY table_name")) {
+					while (rs.next()) {
+						tableNames.add(rs.getString(1));
+					}
+				}
+			}
+
+			// Spot check for a few BSim table names that always exist
+			if (!tableNames.contains("keyvaluetable") || !tableNames.contains("desctable") ||
+				!tableNames.contains("weighttable")) {
+				throw new SQLException("attempted to drop non-BSim database");
+			}
+
+			postgresDs.dispose(); // disconnect before dropping database
+
+			Msg.info(this, "Dropping BSim postgresql database: " + serverInfo);
+			defaultSt.executeUpdate("DROP DATABASE \"" + serverInfo.getDBName() + '"');
+		}
+		finally {
+			// ensure 
+			postgresDs.initializeFrom(defaultDs);
+		}
+	}
+
 	/**
 	 * 
 	 * @throws SQLException if there is a problem creating or executing the query

Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/elastic/ElasticConnection.java

@@ -19,9 +19,9 @@ import java.io.*;
 import java.net.HttpURLConnection;
 import java.net.URL;
 
-import org.json.simple.JSONObject;
+import com.google.gson.*;
-import org.json.simple.parser.JSONParser;
+
-import org.json.simple.parser.ParseException;
+import ghidra.util.Msg;
 
 public class ElasticConnection {
 	public static final String POST = "POST";
@@ -38,24 +38,51 @@ public class ElasticConnection {
 		httpURLbase = url + '/' + repo + '_';
 	}
 
-	public void close() {
-		// nothing to do - http connections do not persist
-	}
-
 	public boolean lastRequestSuccessful() {
 		return (lastResponseCode >= 200) && (lastResponseCode < 300);
 	}
 
 	/**
-	 * Assuming the writer has been closed and connection.getResponseCode() is called
+	 * Get String held by a JsonElement, allowing for a null object.
-	 * placing the value in lastResponseCode, read the response and parse into a JSONObject
+	 * @param element is the JsonElement or null
-	 * @return the JSONObject
+	 * @return the underlying String or null
-	 * @throws IOException for problems with the socket
-	 * @throws ParseException for JSON parse errors
 	 */
-	private JSONObject grabResponse(HttpURLConnection connection)
+	static String convertToString(JsonElement element) {
-			throws IOException, ParseException {
+		if (isNull(element)) {
-		JSONParser parser = new JSONParser();
+			return null;
+		}
+		return element.getAsString();
+	}
+
+	/**
+	 * Get String held by a JsonElement, allowing for a null object.
+	 * @param element is the JsonElement or null
+	 * @param defaultStr default string to be returned if element or string is null
+	 * @return the underlying String or defaultStr if null
+	 */
+	static String convertToString(JsonElement element, String defaultStr) {
+		String str = convertToString(element);
+		return str != null ? str : defaultStr;
+	}
+
+	/**
+	 * Check element for null value
+	 * @param element json element
+	 * @return true if null else false
+	 */
+	static boolean isNull(JsonElement element) {
+		return (element == null || element instanceof JsonNull);
+	}
+
+	/**
+	 * Assuming the writer has been closed and connection.getResponseCode() is called
+	 * placing the value in lastResponseCode, read the response and parse into a JsonObject
+	 * @return the JsonObject
+	 * @throws IOException for problems with the socket
+	 * @throws JsonParseException for JSON parse errors
+	 */
+	private JsonObject grabResponse(HttpURLConnection connection)
+			throws IOException, JsonParseException {
 		InputStream in;
 		if (lastRequestSuccessful()) {
 			in = connection.getInputStream();
@@ -68,7 +95,7 @@ public class ElasticConnection {
 			throw new IOException(connection.getResponseMessage());
 		}
 		Reader reader = new InputStreamReader(in);
-		JSONObject jsonObject = (JSONObject) parser.parse(reader);
+		JsonObject jsonObject = JsonParser.parseReader(reader).getAsJsonObject();
 		return jsonObject;
 	}
 
@@ -78,39 +105,86 @@ public class ElasticConnection {
 	 * @param resp is the parsed error document
 	 * @return the exception String
 	 */
-	private String parseErrorJSON(JSONObject resp) {
+	private static String parseErrorJSON(JsonObject resp) {
 		Object errorObj = resp.get("error");
-		if (errorObj == null) {
-			return "Unknown error format";
-		}
 		if (errorObj instanceof String) {
 			return (String) errorObj;
 		}
-		if (!(errorObj instanceof JSONObject)) {
+		if (!(errorObj instanceof JsonObject err)) {
 			return "Unknown error format";
 		}
-		JSONObject jsonObj = (JSONObject) errorObj;
+
-		String typeString = (String) jsonObj.get("type");
+		String typeString = convertToString(err.get("type"), "Unknown Error");
-		String reasonString = (String) jsonObj.get("reason");
+		if (typeString.endsWith("_exception")) {
-		if (typeString == null) {
+			// Log elastic exception root cause to assist debug
-			typeString = "Unknown Error";
+			String errorDetail = parseErrorCause(err);
+			if (errorDetail.length() != 0) {
+				Msg.error(ElasticConnection.class, "Elasticsearch exception: " + errorDetail);
 			}
-		if (reasonString == null) {
-			reasonString = "Unknown reason";
 		}
+
+		String reasonString = convertToString(err.get("reason"), "Unknown Reason");
 		return typeString + " : " + reasonString;
 	}
 
+	private static StringBuilder conditionalNewLine(StringBuilder buf) {
+		if (!buf.isEmpty()) {
+			buf.append("\n");
+		}
+		return buf;
+	}
+
+	private static String parseErrorCause(JsonObject error) {
+
+		StringBuilder buf = new StringBuilder();
+
+		JsonElement reason = error.get("reason");
+
+		String typeString = convertToString(error.get("type"));
+		if (typeString != null) {
+			String reasonString = convertToString(reason); // "reason" is string when "type" is present
+			String errorStr = typeString + " : " + reasonString;
+			conditionalNewLine(buf).append(errorStr);
+		}
+
+		JsonElement scriptStack = error.get("script_stack");
+		if (scriptStack instanceof JsonArray scriptStackArray) {
+			scriptStackArray
+					.forEach(e -> conditionalNewLine(buf).append("   ").append(convertToString(e)));
+		}
+
+		JsonElement causedBy = error.get("caused_by");
+		if (causedBy instanceof JsonObject causedByObject) {
+			conditionalNewLine(buf).append("   ").append(parseErrorCause(causedByObject));
+		}
+
+		JsonElement failedShards = error.get("failed_shards");
+		if (failedShards instanceof JsonArray failedShardsArray) {
+			for (JsonElement failedShardElement : failedShardsArray) {
+				JsonObject failedShard = (JsonObject) failedShardElement;
+				String indexStr = convertToString(failedShard.get("index"));
+				conditionalNewLine(buf).append("   Failed shard index: ").append(indexStr);
+				conditionalNewLine(buf).append("   ").append(parseErrorCause(failedShard));
+			}
+		}
+
+		if (reason instanceof JsonObject reasonObject) {
+			conditionalNewLine(buf).append(parseErrorCause(reasonObject));
+		}
+
+		return buf.toString();
+	}
+
 	/**
 	 * Send a raw request to the server that is not specific to the repository.
 	 * Intended for general configuration or security commands
 	 * @param command is the type of command
 	 * @param path is the specific URL path receiving the command
 	 * @param body is JSON document describing the command
-	 * @return the response as parsed JSONObject
+	 * @return the response as parsed JsonObject
 	 * @throws ElasticException for any problems with the connection
 	 */
-	public JSONObject executeRawStatement(String command, String path, String body)
+	public JsonObject executeRawStatement(String command, String path, String body)
 			throws ElasticException {
 		HttpURLConnection connection = null;
 		try {
@@ -123,7 +197,7 @@ public class ElasticConnection {
 				writer.write(body);
 			}
 			lastResponseCode = connection.getResponseCode();
-			JSONObject resp = grabResponse(connection);
+			JsonObject resp = grabResponse(connection);
 			if (!lastRequestSuccessful()) {
 				throw new ElasticException(parseErrorJSON(resp));
 			}
@@ -132,7 +206,7 @@ public class ElasticConnection {
 		catch (IOException e) {
 			throw new ElasticException("Error sending request: " + e.getMessage());
 		}
-		catch (ParseException e) {
+		catch (JsonParseException e) {
 			throw new ElasticException("Error parsing response: " + e.getMessage());
 		}
 		finally {
@@ -163,7 +237,7 @@ public class ElasticConnection {
 				writer.write(body);
 			}
 			lastResponseCode = connection.getResponseCode();
-			JSONObject resp = grabResponse(connection);
+			JsonObject resp = grabResponse(connection);
 			if (!lastRequestSuccessful()) {
 				throw new ElasticException(parseErrorJSON(resp));
 			}
@@ -171,7 +245,7 @@ public class ElasticConnection {
 		catch (IOException e) {
 			throw new ElasticException("Error sending request: " + e.getMessage());
 		}
-		catch (ParseException e) {
+		catch (JsonParseException e) {
 			throw new ElasticException("Error parsing response: " + e.getMessage());
 		}
 		finally {
@@ -186,10 +260,10 @@ public class ElasticConnection {
 	 * @param command is the type of command
 	 * @param path is the overarching index/type/<command>
 	 * @param body is JSON document describing the request
-	 * @return the parsed response as a JSONObject
+	 * @return the parsed response as a JsonObject
 	 * @throws ElasticException for any problems with the connection
 	 */
-	public JSONObject executeStatement(String command, String path, String body)
+	public JsonObject executeStatement(String command, String path, String body)
 			throws ElasticException {
 		HttpURLConnection connection = null;
 		try {
@@ -202,7 +276,7 @@ public class ElasticConnection {
 				writer.write(body);
 			}
 			lastResponseCode = connection.getResponseCode();
-			JSONObject resp = grabResponse(connection);
+			JsonObject resp = grabResponse(connection);
 			if (!lastRequestSuccessful()) {
 				throw new ElasticException(parseErrorJSON(resp));
 			}
@@ -211,7 +285,7 @@ public class ElasticConnection {
 		catch (IOException e) {
 			throw new ElasticException("Error sending request: " + e.getMessage());
 		}
-		catch (ParseException e) {
+		catch (JsonParseException e) {
 			throw new ElasticException("Error parsing response: " + e.getMessage());
 		}
 		finally {
@@ -227,10 +301,10 @@ public class ElasticConnection {
 	 * @param command is the type of command
 	 * @param path is the overarching index/type/<command>
 	 * @param body is JSON document describing the request
-	 * @return the parsed response as a JSONObject
+	 * @return the parsed response as a JsonObject
 	 * @throws ElasticException for any problems with the connection
 	 */
-	public JSONObject executeStatementExpectFailure(String command, String path, String body)
+	public JsonObject executeStatementExpectFailure(String command, String path, String body)
 			throws ElasticException {
 		HttpURLConnection connection = null;
 		try {
@@ -243,13 +317,13 @@ public class ElasticConnection {
 				writer.write(body);
 			}
 			lastResponseCode = connection.getResponseCode();
-			JSONObject resp = grabResponse(connection);
+			JsonObject resp = grabResponse(connection);
 			return resp;
 		}
 		catch (IOException e) {
 			throw new ElasticException("Error sending request: " + e.getMessage());
 		}
-		catch (ParseException e) {
+		catch (JsonParseException e) {
 			throw new ElasticException("Error parsing response: " + e.getMessage());
 		}
 		finally {
@@ -264,10 +338,10 @@ public class ElasticConnection {
 	 * and is structured slightly differently from other commands.
 	 * @param path is the specific URL path receiving the bulk command
 	 * @param body is structured list of JSON commands and source
-	 * @return the response as parsed JSONObject
+	 * @return the response as parsed JsonObject
 	 * @throws ElasticException for any problems with the connection
 	 */
-	public JSONObject executeBulk(String path, String body) throws ElasticException {
+	public JsonObject executeBulk(String path, String body) throws ElasticException {
 		HttpURLConnection connection = null;
 		try {
 			URL httpURL = new URL(hostURL + path);
@@ -279,7 +353,7 @@ public class ElasticConnection {
 				writer.write(body);
 			}
 			lastResponseCode = connection.getResponseCode();
-			JSONObject resp = grabResponse(connection);
+			JsonObject resp = grabResponse(connection);
 			if (!lastRequestSuccessful()) {
 				throw new ElasticException(parseErrorJSON(resp));
 			}
@@ -288,7 +362,7 @@ public class ElasticConnection {
 		catch (IOException e) {
 			throw new ElasticException("Error sending request: " + e.getMessage());
 		}
-		catch (ParseException e) {
+		catch (JsonParseException e) {
 			throw new ElasticException("Error parsing response: " + e.getMessage());
 		}
 		finally {
@@ -298,7 +372,7 @@ public class ElasticConnection {
 		}
 	}
 
-	public JSONObject executeURIOnly(String command, String path) throws ElasticException {
+	public JsonObject executeURIOnly(String command, String path) throws ElasticException {
 		HttpURLConnection connection = null;
 		try {
 			URL httpURL = new URL(httpURLbase + path);
@@ -306,7 +380,7 @@ public class ElasticConnection {
 			connection.setRequestMethod(command);
 			connection.setDoOutput(true);
 			lastResponseCode = connection.getResponseCode();
-			JSONObject resp = grabResponse(connection);
+			JsonObject resp = grabResponse(connection);
 			if (!lastRequestSuccessful()) {
 				throw new ElasticException(parseErrorJSON(resp));
 			}
@@ -315,7 +389,7 @@ public class ElasticConnection {
 		catch (IOException e) {
 			throw new ElasticException("Error sending request: " + e.getMessage());
 		}
-		catch (ParseException e) {
+		catch (JsonParseException e) {
 			throw new ElasticException("Error parsing response: " + e.getMessage());
 		}
 		finally {

Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/file/BSimH2FileDBConnectionManager.java

@@ -172,7 +172,7 @@ public class BSimH2FileDBConnectionManager {
 
 			dispose();
 
-			if (dbf.isFile()) {
+			if (!dbf.isFile()) {
 				return true;
 			}
 
@@ -286,6 +286,9 @@ public class BSimH2FileDBConnectionManager {
 		public synchronized Connection getConnection() throws SQLException {
 
 			if (successfulConnection) {
+				if (bds.isClosed()) {
+					bds.restart();
+				}
 				return bds.getConnection();
 			}
 
@@ -317,6 +320,9 @@ public class BSimH2FileDBConnectionManager {
 		 * @throws SQLException if connection or authentication error occurs 
 		 */
 		private Connection connect() throws SQLException {
+			if (bds.isClosed()) {
+				bds.restart();
+			}
 			Connection c = bds.getConnection();
 			successfulConnection = true;
 			return c;

Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/file/H2FileFunctionDatabase.java

@@ -22,13 +22,16 @@ import java.util.*;
 import generic.concurrent.*;
 import generic.lsh.vector.LSHVector;
 import generic.lsh.vector.VectorCompare;
-import ghidra.features.bsim.query.BSimServerInfo;
+import ghidra.features.bsim.query.*;
-import ghidra.features.bsim.query.LSHException;
+import ghidra.features.bsim.query.BSimPostgresDBConnectionManager.BSimPostgresDataSource;
+import ghidra.features.bsim.query.BSimServerInfo.DBType;
+import ghidra.features.bsim.query.FunctionDatabase.Status;
 import ghidra.features.bsim.query.client.*;
 import ghidra.features.bsim.query.description.*;
 import ghidra.features.bsim.query.elastic.Base64VectorFactory;
 import ghidra.features.bsim.query.file.BSimH2FileDBConnectionManager.BSimH2FileDataSource;
 import ghidra.features.bsim.query.protocol.*;
+import ghidra.util.Msg;
 import ghidra.util.task.TaskMonitor;
 
 public class H2FileFunctionDatabase extends AbstractSQLFunctionDatabase<Base64VectorFactory> {
@@ -120,6 +123,48 @@ public class H2FileFunctionDatabase extends AbstractSQLFunctionDatabase<Base64Ve
 		}
 	}
 
+	@Override
+	protected void dropDatabase() throws SQLException {
+
+		if (getStatus() == Status.Busy || fileDs.getActiveConnections() != 0) {
+			throw new SQLException("database in use");
+		}
+
+		close(); // close this instance
+
+		if (!fileDs.exists()) {
+			// ignore request and return
+			return;
+		}
+
+		// Connect to database and examine schema
+		HashSet<String> tableNames = new HashSet<>();
+		try (Connection c = initConnection(); Statement st = c.createStatement()) {
+			try (ResultSet rs = st.executeQuery(
+				"SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' ORDER BY table_name")) {
+				while (rs.next()) {
+					tableNames.add(rs.getString(1));
+				}
+			}
+		}
+
+		// Spot check for a few BSim table names that always exist
+		if (!tableNames.contains("keyvaluetable") || !tableNames.contains("desctable") ||
+			!tableNames.contains("weighttable")) {
+			throw new SQLException("attempted to drop non-BSim database");
+		}
+
+		fileDs.dispose(); // disconnect before deleting database
+
+		BSimServerInfo serverInfo = fileDs.getServerInfo();
+		if (!fileDs.delete()) {
+			throw new SQLException("failed to delete H2-file database: " + serverInfo);
+		}
+
+		Msg.info(this, "Deleted BSim H2-file database: " + serverInfo);
+
+	}
+
 	/**
 	 * Create vector map which maps vector ID to {@link VectorStoreEntry}
 	 * @return vector map
@@ -174,8 +219,8 @@ public class H2FileFunctionDatabase extends AbstractSQLFunctionDatabase<Base64Ve
 	}
 
 	@Override
-	protected int queryNearestVector(List<VectorResult> resultset, LSHVector vec,
+	protected int queryNearestVector(List<VectorResult> resultset, LSHVector vec, double simthresh,
-		double simthresh, double sigthresh, int max) throws SQLException {
+			double sigthresh, int max) throws SQLException {
 		VectorCompare comp;
 		List<VectorResult> resultsToSort = new ArrayList<>();
 		for (VectorStoreEntry entry : vectorStore) {

Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/protocol/DropDatabase.java

@@ -0,0 +1,53 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.features.bsim.query.protocol;
+
+import java.io.IOException;
+import java.io.Writer;
+
+import generic.lsh.vector.LSHVectorFactory;
+import ghidra.util.xml.XmlUtilities;
+import ghidra.xml.XmlElement;
+import ghidra.xml.XmlPullParser;
+
+public class DropDatabase extends BSimQuery<ResponseDropDatabase> {
+	public String databaseName;
+	public ResponseDropDatabase dropResponse;
+
+	public DropDatabase() {
+		super("dropdatabase");
+	}
+
+	@Override
+	public void buildResponseTemplate() {
+		if (response == null)
+			response = dropResponse = new ResponseDropDatabase();
+	}
+
+	@Override
+	public void saveXml(Writer fwrite) throws IOException {
+		fwrite.append('<').append(XmlUtilities.escapeElementEntities(name));
+		fwrite.append(" dbname=\"").append(databaseName).append("\" />\n");
+	}
+
+	@Override
+	public void restoreXml(XmlPullParser parser, LSHVectorFactory vectorFactory) {
+		XmlElement el = parser.start(name);
+		databaseName = XmlUtilities.unEscapeElementEntities(el.getAttribute("dbname"));
+		parser.end();
+	}
+
+}

Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/protocol/ResponseDropDatabase.java

@@ -0,0 +1,66 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.features.bsim.query.protocol;
+
+import java.io.IOException;
+import java.io.Writer;
+
+import generic.lsh.vector.LSHVectorFactory;
+import ghidra.features.bsim.query.LSHException;
+import ghidra.util.xml.SpecXmlUtils;
+import ghidra.xml.XmlElement;
+import ghidra.xml.XmlPullParser;
+
+/**
+ *  Response of server indicating whether a password change request ({@link PasswordChange}) succeeded
+ */
+public class ResponseDropDatabase extends QueryResponseRecord {
+
+	public boolean operationSupported;	// true if the back-end supports this operation
+	public boolean dropSuccessful;		// true if drop was successful
+	public String errorMessage;			// Error message if change was not successful
+
+	public ResponseDropDatabase() {
+		super("responsedropdatabase");
+		operationSupported = true;
+		dropSuccessful = false;
+		errorMessage = null;
+	}
+
+	@Override
+	public void saveXml(Writer fwrite) throws IOException {
+		fwrite.append('<').append(name);
+		fwrite.append(" success=\"");
+		SpecXmlUtils.encodeBoolean(dropSuccessful);
+		fwrite.append("\">");
+		if (errorMessage != null) {
+			SpecXmlUtils.xmlEscapeWriter(fwrite, errorMessage);
+		}
+		fwrite.append("</").append(name).append(">\n");
+	}
+
+	@Override
+	public void restoreXml(XmlPullParser parser, LSHVectorFactory vectorFactory)
+			throws LSHException {
+		XmlElement el = parser.start(name);
+		dropSuccessful = SpecXmlUtils.decodeBoolean(el.getAttribute("success"));
+		errorMessage = parser.end().getText();
+		if (errorMessage != null && errorMessage.length() == 0) {
+			errorMessage = null;
+		}
+	}
+
+}

Ghidra/Features/BSim/src/test.slow/java/ghidra/features/bsim/query/file/BSimH2DatabaseManagerTest.java

@@ -23,7 +23,6 @@ import java.util.*;
 import org.junit.*;
 
 import ghidra.features.bsim.query.*;
-import ghidra.features.bsim.query.BSimServerInfo.DBType;
 import ghidra.features.bsim.query.FunctionDatabase.BSimError;
 import ghidra.features.bsim.query.description.DatabaseInformation;
 import ghidra.features.bsim.query.file.BSimH2FileDBConnectionManager.BSimH2FileDataSource;

Ghidra/Features/BSim/src/test.slow/java/ghidra/features/bsim/query/test/BSimServerTest.java

@@ -1,732 +0,0 @@
-/* ###
- * IP: GHIDRA
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package ghidra.features.bsim.query.test;
-
-import static org.junit.Assert.*;
-
-import java.io.*;
-import java.net.URL;
-import java.sql.Date;
-import java.time.Instant;
-import java.util.*;
-
-import org.junit.*;
-import org.xml.sax.SAXException;
-
-import generic.jar.ResourceFile;
-import generic.lsh.vector.*;
-import generic.util.Path;
-import ghidra.GhidraTestApplicationLayout;
-import ghidra.app.util.headless.HeadlessAnalyzer;
-import ghidra.app.util.headless.HeadlessOptions;
-import ghidra.features.bsim.gui.filters.ExecutableCategoryBSimFilterType;
-import ghidra.features.bsim.gui.filters.HasNamedChildBSimFilterType;
-import ghidra.features.bsim.query.*;
-import ghidra.features.bsim.query.FunctionDatabase.BSimError;
-import ghidra.features.bsim.query.client.tables.ExeTable.ExeTableOrderColumn;
-import ghidra.features.bsim.query.description.*;
-import ghidra.features.bsim.query.ingest.BSimLaunchable;
-import ghidra.features.bsim.query.protocol.*;
-import ghidra.framework.*;
-import ghidra.framework.client.HeadlessClientAuthenticator;
-import ghidra.util.xml.SpecXmlUtils;
-import ghidra.xml.NonThreadedXmlPullParserImpl;
-import ghidra.xml.XmlPullParser;
-
-// These tests require specific data files and paths to be set up.  See BSimServerTestUtil.
-// The "ignore" directive is to prevent these from running as part of the automated nightly tests.
-@Ignore
-public class BSimServerTest {
-	private static final String PROPERTIES_FILE = "RegressionSignatures.properties";
-
-	private static BSimServerTestUtil util;
-	private static LSHVectorFactory vectorFactory;
-	private static FunctionDatabase client;
-	private static BSimLaunchable bulk;
-	private static ResourceFile dumpFile;
-	private static DescriptionManager originalBash;
-
-	private static XmlPullParser getParser(ResourceFile file) {
-		XmlPullParser parser;
-		try {
-			InputStream input = file.getInputStream();
-			parser = new NonThreadedXmlPullParserImpl(input, "BSim test parser",
-				SpecXmlUtils.getXmlHandler(), false);
-		}
-		catch (SAXException e) {
-			return null;
-		}
-		catch (IOException e) {
-			return null;
-		}
-		return parser;
-	}
-
-	@BeforeClass
-	public static void setUp() throws Exception {
-		util = new BSimServerTestUtil();
-		util.verifyDirectories();
-		GhidraTestApplicationLayout layout =
-			new GhidraTestApplicationLayout(new File(util.ghidraDir));
-		ApplicationConfiguration config = new HeadlessGhidraApplicationConfiguration();
-		Application.initializeApplication(layout, config);
-		ghidra.framework.protocol.ghidra.Handler.registerHandler();	/// Register ghidra: protocol
-		ghidra.features.bsim.query.postgresql.Handler.registerHandler();			// Register postgresql: protocol
-		HeadlessClientAuthenticator.installHeadlessClientAuthenticator(null, null, true);
-		bulk = new BSimLaunchable();
-
-		util.verifyRaw();
-		File propFile = new File(util.xmlDir, PROPERTIES_FILE);
-		if (!propFile.isFile()) {
-			createPropertiesFile();
-			runHeadless();
-		}
-
-		util.startServer();
-		doIngest();
-		BSimServerInfo bsimServerInfo;
-		try {
-			bsimServerInfo = new BSimServerInfo(new URL(util.bsimURLString));
-		}
-		catch (Exception e) {
-			throw new AssertionError(e);
-		}
-		client = BSimClientFactory.buildClient(bsimServerInfo, false);
-		if (!client.initialize()) {
-			throw new IOException("Unable to connect to server");
-		}
-		vectorFactory = client.getLSHVectorFactory();
-
-		ResourceFile xmlFile =
-			new ResourceFile(new ResourceFile(util.xmlDir), "sigs_" + BSimServerTestUtil.BASH_MD5);
-		if (!xmlFile.isFile()) {
-			throw new IOException("Basic signature generation did not happen");
-		}
-		XmlPullParser parser = getParser(xmlFile);
-		originalBash = new DescriptionManager();
-		originalBash.restoreXml(parser, vectorFactory);
-		parser.dispose();
-	}
-
-	@AfterClass
-	public static void shutdown() throws Exception {
-		if (client != null) {
-			client.close();
-		}
-		util.shutdownServer();
-		if (dumpFile != null && dumpFile.exists()) {
-			dumpFile.delete();
-		}
-	}
-
-	@Test
-	public void testLibHistoryXml() {
-		ResourceFile xmlFile = new ResourceFile(new ResourceFile(util.xmlDir),
-			"sigs_" + BSimServerTestUtil.LIBHISTORY_MD5);
-		assertTrue(xmlFile.isFile());
-		XmlPullParser parser = getParser(xmlFile);
-		DescriptionManager manager = new DescriptionManager();
-		try {
-			manager.restoreXml(parser, vectorFactory);
-			assertTrue(manager.getExecutableRecordSet().size() == 2);
-			ExecutableRecord eRec = manager.findExecutable(BSimServerTestUtil.LIBHISTORY_MD5);
-			// make sure basic meta-data comes through
-			assertTrue(eRec.getNameExec().equals("libhistory.so.7.0"));
-			assertTrue(eRec.getNameCompiler().equals("gcc"));
-			assertTrue(eRec.getPath().equals("raw"));
-			assertTrue(eRec.getRepository().equals("ghidra://localhost/repo"));
-			assertTrue(eRec.hasCategory("Test Category", "shared"));
-			ExecutableRecord libRec = manager.findExecutable("unknown", "x86:LE:64:default", "");
-			assertTrue(libRec.isLibrary());
-			FunctionDescription fDesc = manager.findFunctionByName("close", libRec);
-			assertNotNull(fDesc);
-			assertEquals(fDesc.getAddress(), -1);
-			fDesc = manager.findFunctionByName("read_history_range", eRec);
-			assertNotNull(fDesc);
-			assertEquals(fDesc.getAddress(), 0x105f60);
-			FunctionDescription addHistory = null;
-			FunctionDescription malloc = null;
-			for (CallgraphEntry entry : fDesc.getCallgraphRecord()) {
-				String name = entry.getFunctionDescription().getFunctionName();
-				if (name.equals("add_history")) {
-					addHistory = entry.getFunctionDescription();
-				}
-				else if (name.equals("malloc")) {
-					malloc = entry.getFunctionDescription();
-				}
-			}
-			assertNotNull(addHistory);
-			assertNotNull(malloc);
-			assertEquals(addHistory.getAddress(), 0x102770);
-			assertEquals(malloc.getAddress(), -1);
-			assertTrue(addHistory.getExecutableRecord() == eRec);		// Should be same object
-			assertTrue(malloc.getExecutableRecord() == libRec);
-		}
-		catch (LSHException e) {
-			Assert.fail("Failure processing libhistory");
-		}
-	}
-
-	@Test
-	public void testBashLibReadline() {
-		try {
-			ResourceFile xmlFile = new ResourceFile(new ResourceFile(util.xmlDir),
-				"sigs_" + BSimServerTestUtil.LIBREADLINE_MD5);
-			XmlPullParser parser = getParser(xmlFile);
-			DescriptionManager manager = new DescriptionManager();
-			manager.restoreXml(parser, vectorFactory);
-			parser.dispose();
-			assertEquals(manager.getExecutableRecordSet().size(), 2);
-			ExecutableRecord bashRec = originalBash.findExecutable(BSimServerTestUtil.BASH_MD5);
-			assertTrue(bashRec.hasCategory("Test Category", "static"));
-			ExecutableRecord readRec = manager.findExecutable(BSimServerTestUtil.LIBREADLINE_MD5);
-			assertTrue(readRec.hasCategory("Test Category", "shared"));
-			// Comparing function "history_filename"
-			FunctionDescription bashFunc =
-				originalBash.findFunction("FUN_001cc840", 0x1cc840, bashRec);
-			FunctionDescription readFunc = manager.findFunction("FUN_00134a70", 0x134a70, readRec);
-			VectorCompare compareData = new VectorCompare();
-			double sig = bashFunc.getSignatureRecord()
-					.getLSHVector()
-					.compare(readFunc.getSignatureRecord().getLSHVector(), compareData);
-			assertTrue(sig > 0.99999);
-		}
-		catch (LSHException e) {
-			Assert.fail("Failure processing bash and libreadline");
-		}
-	}
-
-	private static void compareExe(DescriptionManager manager1, DescriptionManager manager2,
-			String md5) throws Exception {
-		ExecutableRecord eRec1 = manager1.findExecutable(md5);
-		ExecutableRecord eRec2 = manager2.findExecutable(md5);
-		Iterator<FunctionDescription> iter = manager1.listFunctions(eRec1);
-		while (iter.hasNext()) {
-			FunctionDescription func1 = iter.next();
-			FunctionDescription func2 =
-				manager2.findFunction(func1.getFunctionName(), func1.getAddress(), eRec2);
-			assertEquals(func1.getFlags(), func2.getFlags());
-			if (func1.getSignatureRecord() == null) {
-				assertTrue(func2.getSignatureRecord() == null);
-			}
-			else {
-				assertNotNull(func2.getSignatureRecord());
-				assertTrue(func1.getSignatureRecord()
-						.getLSHVector()
-						.equals(func2.getSignatureRecord().getLSHVector()));
-			}
-			if (func1.getCallgraphRecord() == null) {
-				assertTrue(func2.getCallgraphRecord() == null);
-				continue;
-			}
-			assertNotNull(func2.getCallgraphRecord());
-			func1.sortCallgraph();
-			func2.sortCallgraph();
-			Iterator<CallgraphEntry> iter1 = func1.getCallgraphRecord().iterator();
-			Iterator<CallgraphEntry> iter2 = func2.getCallgraphRecord().iterator();
-			while (iter1.hasNext()) {
-				assertTrue(iter2.hasNext());
-				FunctionDescription call1 = iter1.next().getFunctionDescription();
-				FunctionDescription call2 = iter2.next().getFunctionDescription();
-				assertTrue(call1.equals(call2));
-			}
-		}
-	}
-
-	@Test
-	public void testDumpFile() {
-		try {
-			assertNotNull(dumpFile);
-			assertTrue(dumpFile.exists());
-			XmlPullParser parser = getParser(dumpFile);
-			DescriptionManager manager1 = new DescriptionManager();
-			manager1.restoreXml(parser, vectorFactory);
-			parser.dispose();
-			compareExe(manager1, originalBash, BSimServerTestUtil.BASH_MD5);
-		}
-		catch (Exception e) {
-			Assert.fail("Failed to perform dumpexexml: " + e.getMessage());
-		}
-	}
-
-	private static void testForError(QueryResponseRecord response) throws LSHException {
-		if (response == null) {
-			BSimError lastError = client.getLastError();
-			if (lastError == null) {
-				throw new LSHException("Unknown error");
-			}
-			throw new LSHException(lastError.message);
-		}
-	}
-
-	@Test
-	public void testQueryInfo() throws LSHException {
-		QueryInfo queryInfo = new QueryInfo();
-		ResponseInfo response = queryInfo.execute(client);
-		testForError(response);
-		DatabaseInformation info = response.info;
-		assertTrue(info.databasename.equals("TestName"));
-		assertTrue(info.owner.equals("TestOwner"));
-		assertTrue(info.description.equals("TestDescription"));
-		assertEquals(info.settings, 0x49);
-		assertTrue(info.trackcallgraph);
-		assertNotNull(info.execats);
-		assertEquals(info.execats.size(), 1);
-		assertTrue(info.execats.get(0).equals("Test Category"));
-	}
-
-	private void compareExecutableRecords(ExecutableRecord exe1, ExecutableRecord exe2) {
-		assertEquals(exe1.getMd5(), exe2.getMd5());
-		assertEquals(exe1.getNameExec(), exe2.getNameExec());
-		assertEquals(exe1.getArchitecture(), exe2.getArchitecture());
-		assertEquals(exe1.getNameCompiler(), exe2.getNameCompiler());
-		assertEquals(exe1.getPath(), exe2.getPath());
-	}
-
-	@Test
-	public void testQueryExeInfo() throws LSHException {
-		ExecutableRecord libHistory = new ExecutableRecord(BSimServerTestUtil.LIBHISTORY_MD5,
-			"libhistory.so.7.0", "gcc", "x86:LE:64:default", null, null, null, "raw");
-		ExecutableRecord libReadline = new ExecutableRecord(BSimServerTestUtil.LIBREADLINE_MD5,
-			"libreadline.so.7.0", "gcc", "x86:LE:64:default", null, null, null, "raw");
-		QueryExeInfo queryExeInfo = new QueryExeInfo();
-		queryExeInfo.includeFakes = true;
-		ResponseExe responseExe = queryExeInfo.execute(client);
-		testForError(responseExe);
-		assertEquals(responseExe.recordCount, 3);
-		ExecutableRecord exe1 = responseExe.records.get(0);
-		compareExecutableRecords(exe1, libHistory);
-		assertEquals(exe1.getExeCategoryAlphabetic("Test Category"), "shared");
-		ExecutableRecord exe2 = responseExe.records.get(1);
-		compareExecutableRecords(exe2, libReadline);
-		assertEquals(exe2.getExeCategoryAlphabetic("Test Category"), "shared");
-		ExecutableRecord exe3 = responseExe.records.get(2);
-		assertEquals(exe3.getMd5(), "bbbbbbbbaaaaaaaa4b13cd7905584d9f");
-		assertEquals(exe3.getNameExec(), "unknown");
-
-		queryExeInfo = new QueryExeInfo();
-		queryExeInfo.filterMd5 = BSimServerTestUtil.LIBREADLINE_MD5;
-		responseExe = queryExeInfo.execute(client);
-		testForError(responseExe);
-		assertEquals(responseExe.recordCount, 1);
-		exe1 = responseExe.records.get(0);
-		compareExecutableRecords(exe1, libReadline);
-
-		queryExeInfo = new QueryExeInfo();
-		queryExeInfo.filterMd5 = "0a860";			// Partial md5
-		responseExe = queryExeInfo.execute(client);
-		testForError(responseExe);
-		assertEquals(responseExe.recordCount, 1);
-		exe1 = responseExe.getDescriptionManager().findExecutable("libhistory.so.7.0", null, null);
-		compareExecutableRecords(exe1, libHistory);
-
-		queryExeInfo = new QueryExeInfo();
-		queryExeInfo.filterExeName = "lib";
-		queryExeInfo.sortColumn = ExeTableOrderColumn.NAME;
-		responseExe = queryExeInfo.execute(client);
-		testForError(responseExe);
-		assertEquals(responseExe.records.size(), 2);
-		exe1 = responseExe.records.get(0);
-		exe2 = responseExe.records.get(1);
-		compareExecutableRecords(exe1, libHistory);
-		compareExecutableRecords(exe2, libReadline);
-
-		QueryExeCount queryExeCount = new QueryExeCount();
-		responseExe = queryExeCount.execute(client);
-		testForError(responseExe);
-		assertEquals(responseExe.recordCount, 2);
-	}
-
-	@Test
-	public void testQueryName() throws LSHException {
-		QueryName queryName = new QueryName();
-		queryName.spec.exename = "libhistory.so.7.0";
-		queryName.funcname = "history_arg_extract";
-		ResponseName responseName = queryName.execute(client);
-		testForError(responseName);
-		assertTrue(responseName.uniqueexecutable);
-		ExecutableRecord eRec =
-			responseName.manage.findExecutable(BSimServerTestUtil.LIBHISTORY_MD5);
-		assertTrue(eRec.getNameExec().equals("libhistory.so.7.0"));
-		assertTrue(eRec.getMd5().equals(BSimServerTestUtil.LIBHISTORY_MD5));
-		assertTrue(eRec.getNameCompiler().equals("gcc"));
-		assertTrue(eRec.getPath().equals("raw"));
-		Iterator<FunctionDescription> iter = responseName.manage.listAllFunctions();
-		assertTrue(iter.hasNext());
-		FunctionDescription func = iter.next();
-		assertFalse(iter.hasNext());				// Should be exactly one function
-		assertTrue(func.getFunctionName().equals("history_arg_extract"));
-		assertEquals(func.getAddress(), 0x103d40);
-		SignatureRecord sigRec = func.getSignatureRecord();
-		assertNotNull(sigRec);
-		assertEquals(sigRec.getCount(), 2);
-		ExecutableRecord bashRec = originalBash.findExecutable(BSimServerTestUtil.BASH_MD5);
-		FunctionDescription bashFunc =
-			originalBash.findFunctionByName("history_arg_extract", bashRec);
-		assertNotNull(bashFunc);
-		VectorCompare vectorCompare = new VectorCompare();
-		double sim = sigRec.getLSHVector()
-				.compare(bashFunc.getSignatureRecord().getLSHVector(), vectorCompare);
-		assertTrue(sim > 0.8);
-		assertTrue(sim < 0.999);
-	}
-
-	private static QueryResponseRecord doStagedQuery(BSimQuery<?> query,
-			StagingManager stagingManager) throws LSHException {
-
-		boolean haveMore = stagingManager.initialize(query);
-		query.buildResponseTemplate();
-
-		QueryResponseRecord globalResponse = query.getResponse();
-
-		while (haveMore) {
-			// Get the current staged form of the query
-			BSimQuery<?> stagedQuery = stagingManager.getQuery();
-			QueryResponseRecord response = stagedQuery.execute(client);
-			if (response != null) {
-				if (globalResponse != response) {
-					globalResponse.mergeResults(response);	// Merge the staged response with the global response
-				}
-
-				haveMore = stagingManager.nextStage();
-				if (haveMore) {
-					stagedQuery.clearResponse(); // Make space for next stage
-				}
-			}
-			else {
-				throw new LSHException(client.getLastError().message);
-			}
-		}
-
-		return globalResponse;
-	}
-
-	private static void testSimilarityResult(SimilarityResult simRes) {
-		assertEquals(simRes.getTotalCount(), 2);
-		Iterator<SimilarityNote> iter = simRes.iterator();
-		SimilarityNote note1 = iter.next();
-		SimilarityNote note2 = iter.next();
-		FunctionDescription func1 = note1.getFunctionDescription();
-		FunctionDescription func2 = note2.getFunctionDescription();
-		assertTrue(func1.getExecutableRecord().getNameExec().equals("libhistory.so.7.0"));
-		assertTrue(func2.getExecutableRecord().getNameExec().equals("libreadline.so.7.0"));
-		assertNotNull(func1.getSignatureRecord());
-		assertNotNull(func2.getSignatureRecord());
-		assertNotNull(simRes.getBase().getSignatureRecord());
-		LSHVector baseVector = simRes.getBase().getSignatureRecord().getLSHVector();
-		VectorCompare vectorCompare = new VectorCompare();
-		double sim1 = func1.getSignatureRecord().getLSHVector().compare(baseVector, vectorCompare);
-		assertEquals(note1.getSimilarity(), sim1, 0.0001);
-		double sim2 = func2.getSignatureRecord().getLSHVector().compare(baseVector, vectorCompare);
-		assertEquals(note2.getSimilarity(), sim2, 0.0001);
-	}
-
-	@Test
-	public void testQueryNearest() throws LSHException {
-		QueryNearest queryNearest = new QueryNearest();
-		ExecutableRecord bashRec = originalBash.findExecutable(BSimServerTestUtil.BASH_MD5);
-		FunctionDescription func1 = originalBash.findFunctionByName("_rl_adjust_point", bashRec);
-		FunctionDescription func2 = originalBash.findFunctionByName("_rl_compare_chars", bashRec);
-		FunctionDescription func3 = originalBash.findFunctionByName("add_history", bashRec);
-		FunctionDescription func4 = originalBash.findFunctionByName("get_history_event", bashRec);
-		queryNearest.manage.transferSettings(originalBash);
-		queryNearest.manage.transferFunction(func1, true);
-		queryNearest.manage.transferFunction(func2, true);
-		queryNearest.manage.transferFunction(func3, true);
-		queryNearest.manage.transferFunction(func4, true);
-		StagingManager functionStage = new FunctionStaging(2);
-		QueryResponseRecord response = doStagedQuery(queryNearest, functionStage);
-		testForError(response);
-		ResponseNearest respNearest = (ResponseNearest) response;
-		respNearest.sort();
-		int matchCount = 0;
-		for (SimilarityResult simRes : respNearest.result) {
-			if (simRes.getBase().getAddress() == func1.getAddress()) {
-				assertTrue(func1.equals(simRes.getBase()));
-				matchCount += 1;
-			}
-			else if (simRes.getBase().getAddress() == func2.getAddress()) {
-				assertTrue(func2.equals(simRes.getBase()));
-				matchCount += 1;
-			}
-			else if (simRes.getBase().getAddress() == func3.getAddress()) {
-				assertTrue(func3.equals(simRes.getBase()));
-				matchCount += 1;
-			}
-			else if (simRes.getBase().getAddress() == func4.getAddress()) {
-				assertTrue(func4.equals(simRes.getBase()));
-				matchCount += 1;
-			}
-			testSimilarityResult(simRes);
-		}
-		assertEquals(matchCount, 4);		// Make sure we hit all functions
-	}
-
-	@Test
-	public void testQueryVector() throws LSHException {
-		QueryNearestVector queryVector = new QueryNearestVector();
-		ExecutableRecord bashRec = originalBash.findExecutable(BSimServerTestUtil.BASH_MD5);
-		FunctionDescription func = originalBash.findFunctionByName("_rl_kill_kbd_macro", bashRec);
-		queryVector.manage.transferSettings(originalBash);
-		queryVector.manage.transferFunction(func, true);
-		ResponseNearestVector respVector = queryVector.execute(client);
-		testForError(respVector);
-		respVector.sort();
-		assertEquals(respVector.totalvec, 1);
-		assertEquals(respVector.totalmatch, 2);
-		assertEquals(respVector.uniquematch, 0);		// 1 vector matches 2 functions
-		Iterator<SimilarityVectorResult> iter = respVector.result.iterator();
-		SimilarityVectorResult simVecRes = iter.next();
-		assertFalse(iter.hasNext());
-		assertTrue(simVecRes.getBase().equals(func));
-		assertEquals(simVecRes.getTotalCount(), 2);
-		Iterator<VectorResult> iter2 = simVecRes.iterator();
-		VectorResult vec1 = iter2.next();
-		VectorResult vec2 = iter2.next();
-		assertFalse(iter2.hasNext());
-		if (vec1.sim > vec2.sim) {
-			VectorResult tmp = vec1;
-			vec1 = vec2;
-			vec2 = tmp;
-		}
-		VectorCompare vectorCompare = new VectorCompare();
-		LSHVector baseVector = func.getSignatureRecord().getLSHVector();
-		double sim1 = baseVector.compare(vec1.vec, vectorCompare);
-		assertEquals(sim1, vec1.sim, 0.0001);
-		assertEquals(vec1.hitcount, 1);
-		double sim2 = baseVector.compare(vec2.vec, vectorCompare);
-		assertEquals(sim2, vec2.sim, 0.0001);
-		assertEquals(vec1.hitcount, 1);
-		assertTrue(vec2.sim > 0.999);		// Second vector should be 1.0 match
-	}
-
-	@Test
-	public void testChildFilter() throws LSHException {
-		QueryNearest query = new QueryNearest();
-		query.manage.transferSettings(originalBash);
-		ExecutableRecord bashRec = originalBash.findExecutable(BSimServerTestUtil.BASH_MD5);
-		FunctionDescription funcDesc = originalBash.findFunctionByName("_rl_errmsg", bashRec);
-		query.manage.transferFunction(funcDesc, true);
-		query.bsimFilter = new BSimFilter();
-		query.bsimFilter.addAtom(new HasNamedChildBSimFilterType(), "[unknown]__fprintf_chk");
-		ResponseNearest respNearest = query.execute(client);
-		testForError(respNearest);
-		assertEquals(respNearest.totalfunc, 1);
-		assertEquals(respNearest.totalmatch, 1);		// Filtered all but one response
-		assertEquals(respNearest.uniquematch, 1);
-		ExecutableRecord eRec = respNearest.manage.getExecutableRecordSet().first();
-		assertTrue(eRec.getNameExec().equals("libreadline.so.7.0"));
-		assertTrue(eRec.getMd5().equals(BSimServerTestUtil.LIBREADLINE_MD5));
-		assertEquals(respNearest.result.size(), 1);
-		SimilarityResult simRes = respNearest.result.get(0);
-		assertEquals(simRes.size(), 1);			// Only one function returned
-		assertEquals(simRes.getTotalCount(), 3);	// 3 functions similar to vector
-		SimilarityNote note = simRes.iterator().next();
-		assertTrue(note.getSimilarity() > 0.800);
-		FunctionDescription resFunc = note.getFunctionDescription();
-		assertTrue(resFunc.getFunctionName().equals("FUN_0011ece0"));
-		assertEquals(resFunc.getAddress(), 0x11ece0);
-		assertEquals(resFunc.getSignatureRecord().getCount(), 1);		// only function with this exact vector
-	}
-
-	@Test
-	public void testUpdate() throws Exception {
-		QueryUpdate update = new QueryUpdate();
-
-		Date dt = new Date(Instant.parse("2010-12-25T10:15:30.00Z").toEpochMilli());
-		ExecutableRecord exerec = update.manage.newExecutableRecord(
-			BSimServerTestUtil.LIBREADLINE_MD5, "libreadline.so.7.0", "gcc", "x86:LE:64:default",
-			dt, "ghidra://localhost/repo", "/raw", null);
-		List<CategoryRecord> catrec = new ArrayList<>();
-		catrec.add(new CategoryRecord("Test Category", "SHARED!"));
-		update.manage.setExeCategories(exerec, catrec);
-		update.manage.newFunctionDescription("my_remove_history", 0x131c00, exerec);
-		ResponseUpdate respUpdate = update.execute(client);
-		testForError(respUpdate);
-		assertEquals(respUpdate.exeupdate, 1);
-		assertEquals(respUpdate.funcupdate, 1);
-		assertTrue(respUpdate.badexe.isEmpty());
-		assertTrue(respUpdate.badfunc.isEmpty());
-		if (util.isElasticSearch) {
-			Thread.sleep(2000);		// Give chance for refresh timer to expire
-		}
-		QueryNearest nearest = new QueryNearest();
-		nearest.manage.transferSettings(originalBash);
-
-		ExecutableRecord bash = originalBash.findExecutable(BSimServerTestUtil.BASH_MD5);
-		FunctionDescription desc = originalBash.findFunctionByName("remove_history", bash);
-		nearest.manage.transferFunction(desc, true);
-		nearest.bsimFilter = new BSimFilter();
-		nearest.bsimFilter.addAtom(new ExecutableCategoryBSimFilterType("Test Category"),
-			"SHARED!");
-		ResponseNearest respNearest = nearest.execute(client);
-		testForError(respNearest);
-		assertEquals(respNearest.totalfunc, 1);
-		assertEquals(respNearest.totalmatch, 1);
-		assertEquals(respNearest.uniquematch, 1);
-		SimilarityResult simRes = respNearest.result.get(0);
-		assertTrue(simRes.getBase().equals(desc));		// base should match original function
-		assertEquals(simRes.size(), 1);
-		assertEquals(simRes.getTotalCount(), 2);		// The filtered libhistory version is also counted here
-		SimilarityNote note = simRes.iterator().next();
-		FunctionDescription resFunc = note.getFunctionDescription();
-		assertTrue(resFunc.getFunctionName().equals("my_remove_history"));
-		ExecutableRecord resRec = resFunc.getExecutableRecord();
-		assertTrue(resRec.getDate().equals(dt));
-
-		// Restore the original records
-		update = new QueryUpdate();
-		exerec = update.manage.newExecutableRecord(BSimServerTestUtil.LIBREADLINE_MD5,
-			"libreadline.so.7.0", "gcc", "x86:LE:64:default", bash.getDate(),
-			"ghidra://localhost/repo", "/raw", null);
-		catrec = new ArrayList<>();
-		catrec.add(new CategoryRecord("Test Category", "shared"));
-		update.manage.setExeCategories(exerec, catrec);
-		update.manage.newFunctionDescription("remove_history", 0x131c00, exerec);
-		testForError(update.execute(client));
-		if (util.isElasticSearch) {
-			Thread.sleep(2000);		// Give chance for refresh timer to expire
-		}
-	}
-
-	private static void doIngest() throws Exception {
-		if (dumpFile != null && dumpFile.exists()) {
-			return;
-		}
-		runCreateDatabase();
-		if (util.isElasticSearch) {
-			Thread.sleep(2000);		// Give chance for refresh timer to expire
-		}
-		runSetMetaData();
-		runSetExeCategory();
-		runDropIndex();
-		if (util.isElasticSearch) {
-			Thread.sleep(2000);		// Give chance for refresh timer to expire
-		}
-		runCommitSigs();
-		runRebuildIndex();
-		if (util.isElasticSearch) {
-			Thread.sleep(2000);		// Give chance for refresh timer to expire
-		}
-		runDumpFile(BSimServerTestUtil.BASH_MD5);
-		dumpFile =
-			new ResourceFile(new ResourceFile(util.testDir), "sigs_" + BSimServerTestUtil.BASH_MD5);
-		runDelete();
-		if (util.isElasticSearch) {
-			Thread.sleep(2000);		// Give chance for refresh timer to expire
-		}
-	}
-
-	private static void createPropertiesFile() throws IOException {
-		File props = new File(util.xmlDir, PROPERTIES_FILE);
-		FileWriter writer = new FileWriter(props);
-		writer.write("RegressionSignatures: Working directory = " + util.xmlDir + '\n');
-		writer.close();
-	}
-
-	private static void runHeadless() throws IOException {
-		HeadlessAnalyzer analyzer = HeadlessAnalyzer.getInstance();
-		HeadlessOptions options = analyzer.getOptions();
-		List<String> preScripts = new ArrayList<>();
-		List<String> postScripts = new ArrayList<>();
-		List<File> inputFiles = new ArrayList<>();
-
-		options.setPropertiesFileDirectories(util.xmlDir);
-		Path scriptPath = new Path(Path.GHIDRA_HOME + "/Features/BSim/other/testscripts");
-		options.setScriptDirectories(scriptPath.getPath().getAbsolutePath());
-
-		inputFiles.add(new File(util.rawDir));
-		preScripts.add("TailoredAnalysis.java");
-		preScripts.add("InstallMetadataTest.java");
-		postScripts.add("RegressionSignatures.java");
-
-		options.setPreScripts(preScripts);
-		options.setPostScripts(postScripts);
-
-		analyzer.processLocal(util.projectDir, util.repoName, "/", inputFiles);
-	}
-
-	private static void runCreateDatabase() throws Exception {
-		String params[] = new String[3];
-		params[0] = "createdatabase";
-		params[1] = util.bsimURLString;
-		params[2] = "medium_64";
-		bulk.run(params);
-	}
-
-	private static void runSetMetaData() throws Exception {
-		String params[] = new String[5];
-		params[0] = "setmetadata";
-		params[1] = util.bsimURLString;
-		params[2] = "name=TestName";
-		params[3] = "owner=TestOwner";
-		params[4] = "description=TestDescription";
-		bulk.run(params);
-	}
-
-	private static void runSetExeCategory() throws Exception {
-		String params[] = new String[3];
-		params[0] = "addexecategory";
-		params[1] = util.bsimURLString;
-		params[2] = "Test Category";
-		bulk.run(params);
-	}
-
-	private static void runDropIndex() throws Exception {
-		if (util.isH2Database) {
-			return;
-		}
-		String params[] = new String[2];
-		params[0] = "dropindex";
-		params[1] = util.bsimURLString;
-		bulk.run(params);
-	}
-
-	private static void runCommitSigs() throws Exception {
-		String params[] = new String[3];
-		params[0] = "commitsigs";
-		params[1] = util.bsimURLString;
-		params[2] = util.xmlDir;
-		bulk.run(params);
-	}
-
-	private static void runRebuildIndex() throws Exception {
-		if (util.isH2Database) {
-			return;
-		}
-		String params[] = new String[2];
-		params[0] = "rebuildindex";
-		params[1] = util.bsimURLString;
-		bulk.run(params);
-	}
-
-	private static void runDelete() throws Exception {
-		String params[] = new String[3];
-		params[0] = "delete";
-		params[1] = util.bsimURLString;
-		params[2] = "md5=" + BSimServerTestUtil.BASH_MD5;
-		bulk.run(params);
-	}
-
-	private static void runDumpFile(String md5) throws Exception {
-		String params[] = new String[4];
-		params[0] = "dumpsigs";
-		params[1] = util.bsimURLString;
-		params[2] = util.testDir;
-		params[3] = "md5=" + BSimServerTestUtil.BASH_MD5;
-		bulk.run(params);
-	}
-}

Ghidra/Features/BSim/src/test.slow/java/ghidra/features/bsim/query/test/BSimServerTestUtil.java

@@ -1,192 +0,0 @@
-/* ###
- * IP: GHIDRA
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package ghidra.features.bsim.query.test;
-
-import java.io.*;
-
-import ghidra.features.bsim.query.BSimControlLaunchable;
-import ghidra.util.MD5Utilities;
-import utilities.util.FileUtilities;
-
-/**
- * The TEST_DIRECTORY String should be changed to point to a directory that will
- * hold data for the server and for the tests.  To start, this directory should contain
- * a subdirectory "raw", and within this subdirectory should be the following 3 specific binary 
- * executables:
- *    libreadline.so.7.0
- *    libhistory.so.7.0
- *    bash
- *    
- *    all pulled from Ubuntu 18.04.5.
- */
-public class BSimServerTestUtil {
-	private static final String HOST_URL = "postgresql://localhost";
-//	private static final String HOST_URL = "https://localhost:9200";
-//	private static final String HOST_URL = "file:///tmp/bsimtest/db";
-	private static final String TEST_DIRECTORY = "/tmp/bsimtest";
-	public static final String REPO_NAME = "repo";
-	public static final String LIBHISTORY_MD5 = "0a860a716d5bec97c64db652549b72fd";
-	public static final String LIBREADLINE_MD5 = "71b5761b43b840eb88d053790deaf77c";
-	public static final String BASH_MD5 = "557c0271e30cf474e0f46f93721fd1ba";
-	public String repoName;
-	public String bsimURLString = HOST_URL + '/' + REPO_NAME;
-	public String testDir;
-	public String ghidraDir;
-	public String projectDir;
-	public String rawDir;
-	public String xmlDir;
-	public String serverDir;
-	public String serverTouchDir;
-	public boolean isElasticSearch;
-	public boolean isH2Database;
-
-	public BSimServerTestUtil() {
-		testDir = TEST_DIRECTORY;
-		ghidraDir = TEST_DIRECTORY + "/ghidra";
-		repoName = REPO_NAME;
-		projectDir = testDir + "/project";
-		rawDir = testDir + "/raw";
-		xmlDir = testDir + "/xml";
-		serverDir = testDir + "/db";
-		serverTouchDir = testDir + "/servertouch";
-		isElasticSearch = HOST_URL.startsWith("http") || HOST_URL.startsWith("elastic");
-		isH2Database = HOST_URL.startsWith("file");
-	}
-
-	public void verifyDirectories() throws FileNotFoundException {
-		File dir0 = new File(testDir);
-		if (!dir0.exists()) {
-			throw new FileNotFoundException("Could not find test directory");
-		}
-		File dir1 = new File(projectDir);
-		if (!dir1.exists()) {
-			if (!dir1.mkdir()) {
-				throw new FileNotFoundException("Could not create project directory");
-			}
-		}
-		File dir2 = new File(xmlDir);
-		if (!dir2.exists()) {
-			if (!dir2.mkdir()) {
-				throw new FileNotFoundException("Could not create xml directory");
-			}
-		}
-		File dir3 = new File(ghidraDir);
-		if (!dir3.exists()) {
-			if (!dir3.mkdir()) {
-				throw new FileNotFoundException("Could not create ghidra directory");
-			}
-		}
-	}
-
-	public void verifyRaw() throws IOException {
-		File rawDirectory = new File(rawDir);
-		if (!rawDirectory.exists()) {
-			throw new FileNotFoundException(rawDir);
-		}
-		if (!rawDirectory.isDirectory()) {
-			throw new FileNotFoundException("/raw is not a directory");
-		}
-		String[] list = rawDirectory.list();
-		boolean readlinePresent = false;
-		boolean historyPresent = false;
-		boolean bashPresent = false;
-		for (String element : list) {
-			File lib = new File(rawDirectory, element);
-			if (element.equals("libreadline.so.7.0")) {
-				String md5 = MD5Utilities.getMD5Hash(lib);
-				if (md5.equals(LIBREADLINE_MD5)) {
-					readlinePresent = true;
-				}
-				else {
-					throw new FileNotFoundException("libreadline.so.7.0 md5 does not match");
-				}
-			}
-			else if (element.equals("libhistory.so.7.0")) {
-				String md5 = MD5Utilities.getMD5Hash(lib);
-				if (md5.equals(LIBHISTORY_MD5)) {
-					historyPresent = true;
-				}
-				else {
-					throw new FileNotFoundException("libhistory.so.7.0 md5 does not match");
-				}
-			}
-			else if (element.equals("bash")) {
-				String md5 = MD5Utilities.getMD5Hash(lib);
-				if (md5.equals(BASH_MD5)) {
-					bashPresent = true;
-				}
-				else {
-					throw new FileNotFoundException("bash md5 does not match");
-				}
-			}
-		}
-		if (!readlinePresent) {
-			throw new FileNotFoundException("Missing libreadline.so.7.0");
-		}
-		if (!historyPresent) {
-			throw new FileNotFoundException("Missing libhistory.so.7.0");
-		}
-		if (!bashPresent) {
-			throw new FileNotFoundException("Missing bash");
-		}
-	}
-
-	public void startServer() throws Exception {
-		if (isElasticSearch || isH2Database) {
-			return;				// Don't try to start elasticsearch server
-		}
-		File touch = new File(serverTouchDir);
-		if (touch.exists()) {
-			return;
-		}
-		File dir = new File(serverDir);
-		if (dir.isDirectory()) {
-			FileUtilities.deleteDir(dir);
-		}
-		String[] params = new String[2];
-
-		params[0] = "start";
-		params[1] = serverDir;
-		dir.mkdir();			// Create the data directory
-
-		new BSimControlLaunchable().run(params);
-
-		byte[] touchBytes = new byte[2];
-		touchBytes[0] = 'a';
-		touchBytes[1] = 'b';
-		FileUtilities.writeBytes(touch, touchBytes);
-	}
-
-	public void shutdownServer() throws Exception {
-		if (isElasticSearch || isH2Database) {
-			return;
-		}
-		File touch = new File(serverTouchDir);
-		if (!touch.exists()) {
-			return;
-		}
-		String[] params = new String[2];
-
-		params[0] = "stop";
-		params[1] = serverDir;
-		new BSimControlLaunchable().run(params);
-		touch.delete();						// Remove the touch file
-		File dir = new File(serverDir);
-		if (dir.isDirectory()) {
-			FileUtilities.deleteDir(dir);	// Clean up database files
-		}
-	}
-}

GhidraBuild/BuildFiles/Doclets/Module.manifest

@@ -1 +0,0 @@
-MODULE FILE LICENSE: lib/json-simple-1.1.1.jar Apache License 2.0