diff --git a/Ghidra/Features/BSim/Module.manifest b/Ghidra/Features/BSim/Module.manifest index 60e6f9f424..aaf095e736 100755 --- a/Ghidra/Features/BSim/Module.manifest +++ b/Ghidra/Features/BSim/Module.manifest @@ -1,7 +1,6 @@ ##MODULE IP: Oxygen Icons - LGPL 3.0 MODULE FILE LICENSE: postgresql-15.3.tar.gz Postgresql License MODULE FILE LICENSE: lib/postgresql-42.6.2.jar PostgresqlJDBC License -MODULE FILE LICENSE: lib/json-simple-1.1.1.jar Apache License 2.0 MODULE FILE LICENSE: lib/commons-dbcp2-2.9.0.jar Apache License 2.0 MODULE FILE LICENSE: lib/commons-pool2-2.11.1.jar Apache License 2.0 MODULE FILE LICENSE: lib/commons-logging-1.2.jar Apache License 2.0 diff --git a/Ghidra/Features/BSim/build.gradle b/Ghidra/Features/BSim/build.gradle index de0bd157e9..e7550196d5 100755 --- a/Ghidra/Features/BSim/build.gradle +++ b/Ghidra/Features/BSim/build.gradle @@ -33,7 +33,6 @@ dependencies { api project(":CodeCompare") api "org.postgresql:postgresql:42.6.2" - api "com.googlecode.json-simple:json-simple:1.1.1" api "org.apache.commons:commons-dbcp2:2.9.0" api "org.apache.commons:commons-pool2:2.11.1" api "commons-logging:commons-logging:1.2" diff --git a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/filters/ExecutableNameBSimFilterType.java b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/filters/ExecutableNameBSimFilterType.java index e7f0934e00..710e75a253 100644 --- a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/filters/ExecutableNameBSimFilterType.java +++ b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/filters/ExecutableNameBSimFilterType.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,8 +17,6 @@ package ghidra.features.bsim.gui.filters; import java.sql.SQLException; -import org.json.simple.JSONObject; - import ghidra.features.bsim.query.client.IDSQLResolution; import ghidra.features.bsim.query.client.SQLEffects; import ghidra.features.bsim.query.description.ExecutableRecord; @@ -38,7 +36,7 @@ public class ExecutableNameBSimFilterType extends BSimFilterType { @Override public void gatherSQLEffect(SQLEffects effect, FilterAtom atom, IDSQLResolution resolution) - throws SQLException { + throws SQLException { effect.setExeTable(); StringBuilder buf = new StringBuilder(); buf.append("exetable.name_exec = '").append(atom.value).append('\''); @@ -47,10 +45,10 @@ public class ExecutableNameBSimFilterType extends BSimFilterType { @Override public void gatherElasticEffect(ElasticEffects effect, FilterAtom atom, - IDElasticResolution resolution) throws ElasticException { + IDElasticResolution resolution) throws ElasticException { StringBuilder buffer = new StringBuilder(); buffer.append("\"filter\": { \"term\": { \"name_exec\": \""); - buffer.append(JSONObject.escape(atom.value)); + buffer.append(ElasticDatabase.escape(atom.value)); buffer.append("\" } } "); effect.addStandalone(this, buffer.toString()); } diff --git a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/filters/NotExecutableNameBSimFilterType.java b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/filters/NotExecutableNameBSimFilterType.java index ecca1462c0..1d986fcd9f 100644 --- a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/filters/NotExecutableNameBSimFilterType.java +++ b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/filters/NotExecutableNameBSimFilterType.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,8 +17,6 @@ package ghidra.features.bsim.gui.filters; import java.sql.SQLException; -import org.json.simple.JSONObject; - import ghidra.features.bsim.query.client.IDSQLResolution; import ghidra.features.bsim.query.client.SQLEffects; import ghidra.features.bsim.query.description.ExecutableRecord; @@ -37,7 +35,7 @@ public class NotExecutableNameBSimFilterType extends BSimFilterType { @Override public void gatherSQLEffect(SQLEffects effect, FilterAtom atom, IDSQLResolution resolution) - throws SQLException { + throws SQLException { effect.setExeTable(); StringBuilder buf = new StringBuilder(); buf.append("exetable.name_exec != '").append(atom.value).append('\''); @@ -46,10 +44,10 @@ public class NotExecutableNameBSimFilterType extends BSimFilterType { @Override public void gatherElasticEffect(ElasticEffects effect, FilterAtom atom, - IDElasticResolution resolution) throws ElasticException { + IDElasticResolution resolution) throws ElasticException { StringBuilder buffer = new StringBuilder(); buffer.append("\"must_not\": { \"term\": { \"name_exec\": \""); - buffer.append(JSONObject.escape(atom.value)); + buffer.append(ElasticDatabase.escape(atom.value)); buffer.append("\" } } "); effect.addStandalone(this, buffer.toString()); } diff --git a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/filters/PathStartsBSimFilterType.java b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/filters/PathStartsBSimFilterType.java index c047100669..acecb6fa43 100644 --- a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/filters/PathStartsBSimFilterType.java +++ b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/filters/PathStartsBSimFilterType.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -35,7 +35,7 @@ public class PathStartsBSimFilterType extends BSimFilterType { @Override public void gatherSQLEffect(SQLEffects effect, FilterAtom atom, IDSQLResolution resolution) - throws SQLException { + throws SQLException { if (atom.value.length() > 0) { effect.setExeTable(); effect.setPathTable(); @@ -47,11 +47,10 @@ public class PathStartsBSimFilterType extends BSimFilterType { @Override public void gatherElasticEffect(ElasticEffects effect, FilterAtom atom, - IDElasticResolution resolution) throws ElasticException { - effect.addDocValue("String path = doc['path'].value; "); + IDElasticResolution resolution) throws ElasticException { + effect.addDocValue("String path = doc['path'].size() == 0 ? null : doc['path'].value; "); String argName = effect.assignArgument(); - effect.addScriptElement(this, - "(path != null) && path.startsWith(params." + argName + ')'); + effect.addScriptElement(this, "(path != null) && path.startsWith(params." + argName + ')'); effect.addParam(argName, atom.value); } diff --git a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/BSimPostgresDBConnectionManager.java b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/BSimPostgresDBConnectionManager.java index cf912e746a..6728f09201 100644 --- a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/BSimPostgresDBConnectionManager.java +++ b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/BSimPostgresDBConnectionManager.java @@ -284,7 +284,9 @@ public class BSimPostgresDBConnectionManager { String loginError = null; - serverInfo.setUserInfo(bds); + if (bds.getPassword() == null) { + serverInfo.setUserInfo(bds); + } connectionType = serverInfo.hasPassword() ? ConnectionType.SSL_Password_Authentication : ConnectionType.SSL_No_Authentication; diff --git a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/client/AbstractSQLFunctionDatabase.java b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/client/AbstractSQLFunctionDatabase.java index 77a80624ff..daaccebac0 100644 --- a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/client/AbstractSQLFunctionDatabase.java +++ b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/client/AbstractSQLFunctionDatabase.java @@ -186,7 +186,7 @@ public abstract class AbstractSQLFunctionDatabase * @throws SQLException if error occurs obtaining connection */ protected Connection initConnection() throws SQLException { - if (db == null) { + if (db == null || db.isClosed()) { db = ds.getConnection(); } return db; @@ -435,6 +435,12 @@ public abstract class AbstractSQLFunctionDatabase } } + /** + * Drop this database + * @throws SQLException if a database error occured + */ + abstract protected void dropDatabase() throws SQLException; + protected void setConnectionOnTables(Connection db) { weightTable.setConnection(db); @@ -1201,8 +1207,8 @@ public abstract class AbstractSQLFunctionDatabase } else if (msg.contains("authentication failed") || msg.contains("requires a valid client certificate")) { - lasterror = - new BSimError(ErrorCategory.Authentication, "Could not authenticate with database"); + lasterror = new BSimError(ErrorCategory.Authentication, + "Could not authenticate with database"); } else if (msg.contains("does not exist") && !msg.contains(" role ")) { lasterror = new BSimError(ErrorCategory.Nodatabase, cause.getMessage()); @@ -1572,6 +1578,9 @@ public abstract class AbstractSQLFunctionDatabase else if (query instanceof QueryExeCount q) { fdbQueryExeCount(q); } + else if (query instanceof DropDatabase q) { + fdbDatabaseDrop(q); + } else if (query instanceof CreateDatabase q) { fdbDatabaseCreate(q); } @@ -1622,7 +1631,8 @@ public abstract class AbstractSQLFunctionDatabase lasterror = null; try { - if (!(query instanceof CreateDatabase) && !initialize()) { + if (!(query instanceof CreateDatabase) && !(query instanceof DropDatabase) && + !initialize()) { lasterror = new BSimError(ErrorCategory.Nodatabase, "The database does not exist"); return null; } @@ -2087,6 +2097,29 @@ public abstract class AbstractSQLFunctionDatabase } } + private void fdbDatabaseDrop(DropDatabase query) throws LSHException { + ResponseDropDatabase response = query.getResponse(); + if (query.databaseName == null) { + throw new LSHException("Missing databaseName for drop database"); + } + if (!query.databaseName.equals(ds.getServerInfo().getDBName())) { + throw new UnsupportedOperationException("drop database name must match"); + } + response.dropSuccessful = true; // Response parameters assuming success + response.errorMessage = null; + try { + dropDatabase(); + } + catch (SQLException e) { + String msg = e.getMessage(); + if (msg.indexOf("database \"" + query.databaseName + "\" does not exist") > 0) { + return; // missing database + } + response.dropSuccessful = false; + response.errorMessage = e.getMessage(); + } + } + /** * Entry point for the CreateDatabase command * @param query the query to execute diff --git a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/client/PostgresFunctionDatabase.java b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/client/PostgresFunctionDatabase.java index bb3bccc2df..26ca67dd6d 100755 --- a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/client/PostgresFunctionDatabase.java +++ b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/client/PostgresFunctionDatabase.java @@ -31,6 +31,7 @@ import ghidra.features.bsim.query.client.tables.CachedStatement; import ghidra.features.bsim.query.client.tables.SQLStringTable; import ghidra.features.bsim.query.description.*; import ghidra.features.bsim.query.protocol.*; +import ghidra.util.Msg; /** * Defines the BSim {@link FunctionDatabase} backed by a PostgreSQL database. @@ -200,9 +201,6 @@ public final class PostgresFunctionDatabase st.executeUpdate(createdbstring); postgresDs.initializeFrom(defaultDs); } - finally { - defaultDs.dispose(); - } } @Override @@ -244,6 +242,64 @@ public final class PostgresFunctionDatabase } } + @Override + protected void dropDatabase() throws SQLException { + + if (getStatus() == Status.Busy || postgresDs.getActiveConnections() != 0) { + throw new SQLException("database in use"); + } + + BSimServerInfo serverInfo = postgresDs.getServerInfo(); + BSimServerInfo defaultServerInfo = + new BSimServerInfo(DBType.postgres, serverInfo.getUserInfo(), + serverInfo.getServerName(), serverInfo.getPort(), DEFAULT_DATABASE_NAME); + + BSimPostgresDataSource defaultDs = + BSimPostgresDBConnectionManager.getDataSource(defaultServerInfo); + if (getStatus() == Status.Ready) { + defaultDs.initializeFrom(postgresDs); + } + + close(); // close this instance + + try (Connection defaultDb = defaultDs.getConnection(); + Statement defaultSt = defaultDb.createStatement()) { + try (ResultSet rs = defaultSt.executeQuery( + "SELECT 1 FROM pg_database WHERE datname='" + serverInfo.getDBName() + "'")) { + if (!rs.next()) { + return; // database does not exist + } + } + + // Connect to database and examine schema + HashSet tableNames = new HashSet<>(); + postgresDs.initializeFrom(defaultDs); + try (Connection c = initConnection(); Statement st = c.createStatement()) { + try (ResultSet rs = st.executeQuery( + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' ORDER BY table_name")) { + while (rs.next()) { + tableNames.add(rs.getString(1)); + } + } + } + + // Spot check for a few BSim table names that always exist + if (!tableNames.contains("keyvaluetable") || !tableNames.contains("desctable") || + !tableNames.contains("weighttable")) { + throw new SQLException("attempted to drop non-BSim database"); + } + + postgresDs.dispose(); // disconnect before dropping database + + Msg.info(this, "Dropping BSim postgresql database: " + serverInfo); + defaultSt.executeUpdate("DROP DATABASE \"" + serverInfo.getDBName() + '"'); + } + finally { + // ensure + postgresDs.initializeFrom(defaultDs); + } + } + /** * * @throws SQLException if there is a problem creating or executing the query diff --git a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/elastic/ElasticConnection.java b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/elastic/ElasticConnection.java index 473118dc88..9f3496aecc 100755 --- a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/elastic/ElasticConnection.java +++ b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/elastic/ElasticConnection.java @@ -19,9 +19,9 @@ import java.io.*; import java.net.HttpURLConnection; import java.net.URL; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; -import org.json.simple.parser.ParseException; +import com.google.gson.*; + +import ghidra.util.Msg; public class ElasticConnection { public static final String POST = "POST"; @@ -38,24 +38,51 @@ public class ElasticConnection { httpURLbase = url + '/' + repo + '_'; } - public void close() { - // nothing to do - http connections do not persist - } - public boolean lastRequestSuccessful() { return (lastResponseCode >= 200) && (lastResponseCode < 300); } /** - * Assuming the writer has been closed and connection.getResponseCode() is called - * placing the value in lastResponseCode, read the response and parse into a JSONObject - * @return the JSONObject - * @throws IOException for problems with the socket - * @throws ParseException for JSON parse errors + * Get String held by a JsonElement, allowing for a null object. + * @param element is the JsonElement or null + * @return the underlying String or null */ - private JSONObject grabResponse(HttpURLConnection connection) - throws IOException, ParseException { - JSONParser parser = new JSONParser(); + static String convertToString(JsonElement element) { + if (isNull(element)) { + return null; + } + return element.getAsString(); + } + + /** + * Get String held by a JsonElement, allowing for a null object. + * @param element is the JsonElement or null + * @param defaultStr default string to be returned if element or string is null + * @return the underlying String or defaultStr if null + */ + static String convertToString(JsonElement element, String defaultStr) { + String str = convertToString(element); + return str != null ? str : defaultStr; + } + + /** + * Check element for null value + * @param element json element + * @return true if null else false + */ + static boolean isNull(JsonElement element) { + return (element == null || element instanceof JsonNull); + } + + /** + * Assuming the writer has been closed and connection.getResponseCode() is called + * placing the value in lastResponseCode, read the response and parse into a JsonObject + * @return the JsonObject + * @throws IOException for problems with the socket + * @throws JsonParseException for JSON parse errors + */ + private JsonObject grabResponse(HttpURLConnection connection) + throws IOException, JsonParseException { InputStream in; if (lastRequestSuccessful()) { in = connection.getInputStream(); @@ -68,7 +95,7 @@ public class ElasticConnection { throw new IOException(connection.getResponseMessage()); } Reader reader = new InputStreamReader(in); - JSONObject jsonObject = (JSONObject) parser.parse(reader); + JsonObject jsonObject = JsonParser.parseReader(reader).getAsJsonObject(); return jsonObject; } @@ -78,39 +105,86 @@ public class ElasticConnection { * @param resp is the parsed error document * @return the exception String */ - private String parseErrorJSON(JSONObject resp) { + private static String parseErrorJSON(JsonObject resp) { Object errorObj = resp.get("error"); - if (errorObj == null) { - return "Unknown error format"; - } if (errorObj instanceof String) { return (String) errorObj; } - if (!(errorObj instanceof JSONObject)) { + if (!(errorObj instanceof JsonObject err)) { return "Unknown error format"; } - JSONObject jsonObj = (JSONObject) errorObj; - String typeString = (String) jsonObj.get("type"); - String reasonString = (String) jsonObj.get("reason"); - if (typeString == null) { - typeString = "Unknown Error"; - } - if (reasonString == null) { - reasonString = "Unknown reason"; + + String typeString = convertToString(err.get("type"), "Unknown Error"); + if (typeString.endsWith("_exception")) { + // Log elastic exception root cause to assist debug + String errorDetail = parseErrorCause(err); + if (errorDetail.length() != 0) { + Msg.error(ElasticConnection.class, "Elasticsearch exception: " + errorDetail); + } } + + String reasonString = convertToString(err.get("reason"), "Unknown Reason"); return typeString + " : " + reasonString; } + private static StringBuilder conditionalNewLine(StringBuilder buf) { + if (!buf.isEmpty()) { + buf.append("\n"); + } + return buf; + } + + private static String parseErrorCause(JsonObject error) { + + StringBuilder buf = new StringBuilder(); + + JsonElement reason = error.get("reason"); + + String typeString = convertToString(error.get("type")); + if (typeString != null) { + String reasonString = convertToString(reason); // "reason" is string when "type" is present + String errorStr = typeString + " : " + reasonString; + conditionalNewLine(buf).append(errorStr); + } + + JsonElement scriptStack = error.get("script_stack"); + if (scriptStack instanceof JsonArray scriptStackArray) { + scriptStackArray + .forEach(e -> conditionalNewLine(buf).append(" ").append(convertToString(e))); + } + + JsonElement causedBy = error.get("caused_by"); + if (causedBy instanceof JsonObject causedByObject) { + conditionalNewLine(buf).append(" ").append(parseErrorCause(causedByObject)); + } + + JsonElement failedShards = error.get("failed_shards"); + if (failedShards instanceof JsonArray failedShardsArray) { + for (JsonElement failedShardElement : failedShardsArray) { + JsonObject failedShard = (JsonObject) failedShardElement; + String indexStr = convertToString(failedShard.get("index")); + conditionalNewLine(buf).append(" Failed shard index: ").append(indexStr); + conditionalNewLine(buf).append(" ").append(parseErrorCause(failedShard)); + } + } + + if (reason instanceof JsonObject reasonObject) { + conditionalNewLine(buf).append(parseErrorCause(reasonObject)); + } + + return buf.toString(); + } + /** * Send a raw request to the server that is not specific to the repository. * Intended for general configuration or security commands * @param command is the type of command * @param path is the specific URL path receiving the command * @param body is JSON document describing the command - * @return the response as parsed JSONObject + * @return the response as parsed JsonObject * @throws ElasticException for any problems with the connection */ - public JSONObject executeRawStatement(String command, String path, String body) + public JsonObject executeRawStatement(String command, String path, String body) throws ElasticException { HttpURLConnection connection = null; try { @@ -123,7 +197,7 @@ public class ElasticConnection { writer.write(body); } lastResponseCode = connection.getResponseCode(); - JSONObject resp = grabResponse(connection); + JsonObject resp = grabResponse(connection); if (!lastRequestSuccessful()) { throw new ElasticException(parseErrorJSON(resp)); } @@ -132,7 +206,7 @@ public class ElasticConnection { catch (IOException e) { throw new ElasticException("Error sending request: " + e.getMessage()); } - catch (ParseException e) { + catch (JsonParseException e) { throw new ElasticException("Error parsing response: " + e.getMessage()); } finally { @@ -163,7 +237,7 @@ public class ElasticConnection { writer.write(body); } lastResponseCode = connection.getResponseCode(); - JSONObject resp = grabResponse(connection); + JsonObject resp = grabResponse(connection); if (!lastRequestSuccessful()) { throw new ElasticException(parseErrorJSON(resp)); } @@ -171,7 +245,7 @@ public class ElasticConnection { catch (IOException e) { throw new ElasticException("Error sending request: " + e.getMessage()); } - catch (ParseException e) { + catch (JsonParseException e) { throw new ElasticException("Error parsing response: " + e.getMessage()); } finally { @@ -186,10 +260,10 @@ public class ElasticConnection { * @param command is the type of command * @param path is the overarching index/type/ * @param body is JSON document describing the request - * @return the parsed response as a JSONObject + * @return the parsed response as a JsonObject * @throws ElasticException for any problems with the connection */ - public JSONObject executeStatement(String command, String path, String body) + public JsonObject executeStatement(String command, String path, String body) throws ElasticException { HttpURLConnection connection = null; try { @@ -202,7 +276,7 @@ public class ElasticConnection { writer.write(body); } lastResponseCode = connection.getResponseCode(); - JSONObject resp = grabResponse(connection); + JsonObject resp = grabResponse(connection); if (!lastRequestSuccessful()) { throw new ElasticException(parseErrorJSON(resp)); } @@ -211,7 +285,7 @@ public class ElasticConnection { catch (IOException e) { throw new ElasticException("Error sending request: " + e.getMessage()); } - catch (ParseException e) { + catch (JsonParseException e) { throw new ElasticException("Error parsing response: " + e.getMessage()); } finally { @@ -227,10 +301,10 @@ public class ElasticConnection { * @param command is the type of command * @param path is the overarching index/type/ * @param body is JSON document describing the request - * @return the parsed response as a JSONObject + * @return the parsed response as a JsonObject * @throws ElasticException for any problems with the connection */ - public JSONObject executeStatementExpectFailure(String command, String path, String body) + public JsonObject executeStatementExpectFailure(String command, String path, String body) throws ElasticException { HttpURLConnection connection = null; try { @@ -243,13 +317,13 @@ public class ElasticConnection { writer.write(body); } lastResponseCode = connection.getResponseCode(); - JSONObject resp = grabResponse(connection); + JsonObject resp = grabResponse(connection); return resp; } catch (IOException e) { throw new ElasticException("Error sending request: " + e.getMessage()); } - catch (ParseException e) { + catch (JsonParseException e) { throw new ElasticException("Error parsing response: " + e.getMessage()); } finally { @@ -264,10 +338,10 @@ public class ElasticConnection { * and is structured slightly differently from other commands. * @param path is the specific URL path receiving the bulk command * @param body is structured list of JSON commands and source - * @return the response as parsed JSONObject + * @return the response as parsed JsonObject * @throws ElasticException for any problems with the connection */ - public JSONObject executeBulk(String path, String body) throws ElasticException { + public JsonObject executeBulk(String path, String body) throws ElasticException { HttpURLConnection connection = null; try { URL httpURL = new URL(hostURL + path); @@ -279,7 +353,7 @@ public class ElasticConnection { writer.write(body); } lastResponseCode = connection.getResponseCode(); - JSONObject resp = grabResponse(connection); + JsonObject resp = grabResponse(connection); if (!lastRequestSuccessful()) { throw new ElasticException(parseErrorJSON(resp)); } @@ -288,7 +362,7 @@ public class ElasticConnection { catch (IOException e) { throw new ElasticException("Error sending request: " + e.getMessage()); } - catch (ParseException e) { + catch (JsonParseException e) { throw new ElasticException("Error parsing response: " + e.getMessage()); } finally { @@ -298,7 +372,7 @@ public class ElasticConnection { } } - public JSONObject executeURIOnly(String command, String path) throws ElasticException { + public JsonObject executeURIOnly(String command, String path) throws ElasticException { HttpURLConnection connection = null; try { URL httpURL = new URL(httpURLbase + path); @@ -306,7 +380,7 @@ public class ElasticConnection { connection.setRequestMethod(command); connection.setDoOutput(true); lastResponseCode = connection.getResponseCode(); - JSONObject resp = grabResponse(connection); + JsonObject resp = grabResponse(connection); if (!lastRequestSuccessful()) { throw new ElasticException(parseErrorJSON(resp)); } @@ -315,7 +389,7 @@ public class ElasticConnection { catch (IOException e) { throw new ElasticException("Error sending request: " + e.getMessage()); } - catch (ParseException e) { + catch (JsonParseException e) { throw new ElasticException("Error parsing response: " + e.getMessage()); } finally { diff --git a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/elastic/ElasticDatabase.java b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/elastic/ElasticDatabase.java index f5337b22a3..5a2810c2c4 100755 --- a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/elastic/ElasticDatabase.java +++ b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/elastic/ElasticDatabase.java @@ -23,8 +23,8 @@ import java.util.*; import java.util.Map.Entry; import org.apache.commons.lang3.StringUtils; -import org.json.simple.JSONArray; -import org.json.simple.JSONObject; + +import com.google.gson.*; import generic.lsh.vector.*; import ghidra.features.bsim.gui.filters.FunctionTagBSimFilterType; @@ -35,6 +35,7 @@ import ghidra.features.bsim.query.client.tables.ExeTable.ExeTableOrderColumn; import ghidra.features.bsim.query.description.*; import ghidra.features.bsim.query.protocol.*; import ghidra.framework.client.ClientUtil; +import ghidra.util.Msg; import ghidra.util.xml.SpecXmlUtils; /** @@ -70,6 +71,55 @@ public class ElasticDatabase implements FunctionDatabase { private Status status; // status of the connection private boolean initialized; // true if the connection has been successfully initialized + public static String escape(String s) { + final int len = s.length(); + StringBuffer sb = new StringBuffer(); + for (int i = 0; i < len; i++) { + char ch = s.charAt(i); + switch (ch) { + case '"': + sb.append("\\\""); + break; + case '\\': + sb.append("\\\\"); + break; + case '\b': + sb.append("\\b"); + break; + case '\f': + sb.append("\\f"); + break; + case '\n': + sb.append("\\n"); + break; + case '\r': + sb.append("\\r"); + break; + case '\t': + sb.append("\\t"); + break; + case '/': + sb.append("\\/"); + break; + default: + //Reference: http://www.unicode.org/versions/Unicode5.1.0/ + if ((ch >= '\u0000' && ch <= '\u001F') || (ch >= '\u007F' && ch <= '\u009F') || + (ch >= '\u2000' && ch <= '\u20FF')) { + String ss = Integer.toHexString(ch); + sb.append("\\u"); + for (int k = 0; k < 4 - ss.length(); k++) { + sb.append('0'); + } + sb.append(ss.toUpperCase()); + } + else { + sb.append(ch); + } + } + } + return sb.toString(); + } + /** * Append a list of CategoryRecords as (part of) a JSON document to a StringBuilder * Used as part of constructing JSON serialization of ExecutableRecords @@ -90,9 +140,7 @@ public class ElasticDatabase implements FunctionDatabase { buffer.append('\"'); // Append type/value pair as concatendated strings separated by a TAB. // When sorting as strings, this should give the same order as sorted CategoryRecords - buffer.append(catrec.getType()) - .append("\\t") - .append(JSONObject.escape(catrec.getCategory())); + buffer.append(catrec.getType()).append("\\t").append(escape(catrec.getCategory())); buffer.append('\"'); } } @@ -110,9 +158,7 @@ public class ElasticDatabase implements FunctionDatabase { throws ElasticException { StringBuilder builder = new StringBuilder(); builder.append("{ \"md5\": \"").append(exeRecord.getMd5()).append("\", "); - builder.append("\"name_exec\": \"") - .append(JSONObject.escape(exeRecord.getNameExec())) - .append("\", "); + builder.append("\"name_exec\": \"").append(escape(exeRecord.getNameExec())).append("\", "); builder.append("\"architecture\": \"").append(exeRecord.getArchitecture()).append("\", "); builder.append("\"name_compiler\": \"").append(exeRecord.getNameCompiler()).append("\", "); builder.append("\"ingest_date\": ").append(exeRecord.getDate().getTime()).append(", "); @@ -121,16 +167,14 @@ public class ElasticDatabase implements FunctionDatabase { } else { builder.append("\"repository\": \"") - .append(JSONObject.escape(exeRecord.getRepository())) + .append(escape(exeRecord.getRepository())) .append("\", "); } if (exeRecord.getPath() == null) { builder.append("\"path\": null"); } else { - builder.append("\"path\": \"") - .append(JSONObject.escape(exeRecord.getPath())) - .append("\""); + builder.append("\"path\": \"").append(escape(exeRecord.getPath())).append("\""); } List catrecs = exeRecord.getAllCategories(); if (catrecs != null) { @@ -142,15 +186,15 @@ public class ElasticDatabase implements FunctionDatabase { pathbuilder.append("executable/_doc/"); pathbuilder.append(exeId); pathbuilder.append("?op_type=create"); // Do "create" operation, so we fail if document already exists - JSONObject resp = connection.executeStatementExpectFailure(ElasticConnection.PUT, + JsonObject resp = connection.executeStatementExpectFailure(ElasticConnection.PUT, pathbuilder.toString(), builder.toString()); - JSONObject error = (JSONObject) resp.get("error"); + JsonObject error = (JsonObject) resp.get("error"); if (error != null) { - String type = (String) error.get("type"); + String type = error.get("type").getAsString(); if (type.startsWith("version_conflict")) { return false; // Document already inserted } - String reason = (String) error.get("reason"); + String reason = ElasticConnection.convertToString(error.get("reason")); throw new ElasticException(reason); } return true; @@ -222,7 +266,7 @@ public class ElasticDatabase implements FunctionDatabase { builder.append("\", \"routing\": \""); builder.append(exeId).append("\"}}\n"); builder.append("{ \"name_func\": \""); - builder.append(JSONObject.escape(desc.getFunctionName())); + builder.append(escape(desc.getFunctionName())); SignatureRecord sigRec = desc.getSignatureRecord(); long vecid = 0; if (sigRec != null) { @@ -266,21 +310,21 @@ public class ElasticDatabase implements FunctionDatabase { * @return null or the "hit" portion of the response corresponding to the matching document * @throws ElasticException for communication problems with the server */ - private JSONObject queryMd5ExeMatch(String md5) throws ElasticException { + private JsonObject queryMd5ExeMatch(String md5) throws ElasticException { StringBuilder buffer = new StringBuilder(); buffer.append( "{ \"size\": 1, \"query\": { \"bool\": { \"filter\": { \"term\": { \"md5\": \""); buffer.append(md5).append("\" } } } } }"); - JSONObject resp = connection.executeStatement(ElasticConnection.GET, "executable/_search", + JsonObject resp = connection.executeStatement(ElasticConnection.GET, "executable/_search", buffer.toString()); - JSONObject hits = (JSONObject) resp.get("hits"); - JSONObject totalRec = (JSONObject) hits.get("total"); - long total = (Long) totalRec.get("value"); + JsonObject hits = (JsonObject) resp.get("hits"); + JsonObject totalRec = (JsonObject) hits.get("total"); + long total = totalRec.get("value").getAsLong(); if (total == 0) { return null; } - JSONArray hitsArray = (JSONArray) hits.get("hits"); - return (JSONObject) hitsArray.get(0); + JsonArray hitsArray = (JsonArray) hits.get("hits"); + return (JsonObject) hitsArray.get(0); } /** @@ -291,7 +335,7 @@ public class ElasticDatabase implements FunctionDatabase { * @return a list of JSON function documents * @throws ElasticException for communication problems with the server */ - private JSONArray queryFuncNameMatch(String exeId, String functionName, int maxDocuments) + private JsonArray queryFuncNameMatch(String exeId, String functionName, int maxDocuments) throws ElasticException { StringBuilder buffer = new StringBuilder(); buffer.append("{ \"size\": ").append(maxDocuments); @@ -301,21 +345,21 @@ public class ElasticDatabase implements FunctionDatabase { buffer.append(" \"must\": {"); buffer.append(" \"term\": {"); buffer.append(" \"name_func\": \""); - buffer.append(JSONObject.escape(functionName)); + buffer.append(escape(functionName)); buffer.append("\"} },"); buffer.append(" \"filter\": {"); buffer.append(" \"parent_id\": {"); buffer.append(" \"type\": \"function\","); buffer.append(" \"id\": \"").append(exeId); buffer.append("\"} } } } }"); - JSONObject resp = connection.executeStatement(ElasticConnection.GET, "executable/_search", + JsonObject resp = connection.executeStatement(ElasticConnection.GET, "executable/_search", buffer.toString()); - JSONObject baseHits = (JSONObject) resp.get("hits"); - Object hitsArray = baseHits.get("hits"); - if (hitsArray == null) { - return new JSONArray(); + JsonObject baseHits = (JsonObject) resp.get("hits"); + JsonElement hitsArray = baseHits.get("hits"); + if (hitsArray instanceof JsonArray a) { + return a; } - return (JSONArray) hitsArray; + return new JsonArray(); } /** @@ -329,7 +373,7 @@ public class ElasticDatabase implements FunctionDatabase { * @return the JSON function document or null if none match * @throws ElasticException for communication problems with the server */ - private JSONObject queryFuncNameAddress(String exeId, String functionName, long address) + private JsonObject queryFuncNameAddress(String exeId, String functionName, long address) throws ElasticException { StringBuilder buffer = new StringBuilder(); buffer.append("{ \"_source\": { \"excludes\": [ \"childid\" ] }"); @@ -338,7 +382,7 @@ public class ElasticDatabase implements FunctionDatabase { buffer.append(" \"must\": {"); buffer.append(" \"term\": {"); buffer.append(" \"name_func\": \""); - buffer.append(JSONObject.escape(functionName)); + buffer.append(escape(functionName)); buffer.append("\"},"); buffer.append(" \"term\": {"); buffer.append(" \"addr\": ").append(address); @@ -348,16 +392,16 @@ public class ElasticDatabase implements FunctionDatabase { buffer.append(" \"type\": \"function\","); buffer.append(" \"id\": \"").append(exeId); buffer.append("\"} } } } }"); - JSONObject resp = connection.executeStatement(ElasticConnection.GET, "executable/_search", + JsonObject resp = connection.executeStatement(ElasticConnection.GET, "executable/_search", buffer.toString()); - JSONObject baseHits = (JSONObject) resp.get("hits"); - JSONObject totalRec = (JSONObject) baseHits.get("total"); - long total = (Long) totalRec.get("value"); + JsonObject baseHits = (JsonObject) resp.get("hits"); + JsonObject totalRec = (JsonObject) baseHits.get("total"); + long total = totalRec.get("value").getAsLong(); if (total != 1) { return null; } - JSONArray hitsArray = (JSONArray) baseHits.get("hits"); - return (JSONObject) hitsArray.get(0); + JsonArray hitsArray = (JsonArray) baseHits.get("hits"); + return (JsonObject) hitsArray.get(0); } /** @@ -372,7 +416,7 @@ public class ElasticDatabase implements FunctionDatabase { private ExecutableRecord findSingleExecutable(ExeSpecifier specifier, DescriptionManager manager) throws LSHException, ElasticException { if (specifier.exemd5 != null && specifier.exemd5.length() != 0) { - JSONObject row = queryMd5ExeMatch(specifier.exemd5); + JsonObject row = queryMd5ExeMatch(specifier.exemd5); if (row == null) { return null; } @@ -438,7 +482,7 @@ public class ElasticDatabase implements FunctionDatabase { buffer.append("}}, "); if (searchAfter != null) { buffer.append("\"search_after\": [ \""); - buffer.append(JSONObject.escape(searchAfter)); + buffer.append(escape(searchAfter)); buffer.append("\"], "); } if (md5Order) { @@ -447,12 +491,12 @@ public class ElasticDatabase implements FunctionDatabase { else { buffer.append("\"sort\": [ { \"name_exec\": \"asc\" } ] }"); } - JSONObject resp = connection.executeStatement(ElasticConnection.GET, "executable/_search", + JsonObject resp = connection.executeStatement(ElasticConnection.GET, "executable/_search", buffer.toString()); - JSONObject baseHits = (JSONObject) resp.get("hits"); - JSONArray hitsArray = (JSONArray) baseHits.get("hits"); - for (Object element : hitsArray) { - JSONObject exerow = (JSONObject) element; + JsonObject baseHits = (JsonObject) resp.get("hits"); + JsonArray hitsArray = (JsonArray) baseHits.get("hits"); + for (JsonElement element : hitsArray) { + JsonObject exerow = (JsonObject) element; ExecutableRecord exeRecord = makeExecutableRecord(manager, exerow); exeList.add(exeRecord); } @@ -476,9 +520,9 @@ public class ElasticDatabase implements FunctionDatabase { buffer.append(filter); } buffer.append("}}}"); - JSONObject resp = connection.executeStatement(ElasticConnection.GET, "executable/_count", + JsonObject resp = connection.executeStatement(ElasticConnection.GET, "executable/_count", buffer.toString()); - Long res = (Long) resp.get("count"); + Long res = resp.get("count").getAsLong(); return res.intValue(); } @@ -501,7 +545,7 @@ public class ElasticDatabase implements FunctionDatabase { buffer.append(" \"bool\": {"); buffer.append(" \"must\": {"); buffer.append(" \"term\": { \"name_exec\": \"") - .append(JSONObject.escape(exeName)) + .append(escape(exeName)) .append("\" } }"); if (!StringUtils.isEmpty(arch) || !StringUtils.isEmpty(compilerName)) { buffer.append(", \"filter\": {"); @@ -535,16 +579,16 @@ public class ElasticDatabase implements FunctionDatabase { buffer.append("}}}}"); } buffer.append("} } }"); - JSONObject resp = connection.executeStatement(ElasticConnection.GET, "executable/_search", + JsonObject resp = connection.executeStatement(ElasticConnection.GET, "executable/_search", buffer.toString()); - JSONObject baseHits = (JSONObject) resp.get("hits"); - JSONObject totalRec = (JSONObject) baseHits.get("total"); - long total = (Long) totalRec.get("value"); + JsonObject baseHits = (JsonObject) resp.get("hits"); + JsonObject totalRec = (JsonObject) baseHits.get("total"); + long total = totalRec.get("value").getAsLong(); if (total != 1) { return null; // Either no results, or not unique } - JSONArray hitsArray = (JSONArray) baseHits.get("hits"); - JSONObject exerow = (JSONObject) hitsArray.get(0); + JsonArray hitsArray = (JsonArray) baseHits.get("hits"); + JsonObject exerow = (JsonObject) hitsArray.get(0); ExecutableRecord exerec = makeExecutableRecord(manager, exerow); return exerec; } @@ -585,25 +629,25 @@ public class ElasticDatabase implements FunctionDatabase { buffer.append('\"'); } buffer.append(" ] }"); - JSONObject resp = + JsonObject resp = connection.executeStatement(ElasticConnection.GET, "meta/_mget", buffer.toString()); - JSONArray docs = (JSONArray) resp.get("docs"); + JsonArray docs = (JsonArray) resp.get("docs"); for (int i = 0; i < maxDocuments; ++i) { if (!iter2.hasNext()) { break; } vecRes = iter2.next(); - JSONObject oneResp = (JSONObject) docs.get(i); - String matchId = (String) oneResp.get("_id"); + JsonObject oneResp = (JsonObject) docs.get(i); + String matchId = oneResp.get("_id").getAsString(); long matchIdVal = Base64Lite.decodeLongBase64(matchId); - JSONObject source = (JSONObject) oneResp.get("_source"); - if (source == null) { - throw new ElasticException("meta document does not exist for id=" + matchId); - } if (matchIdVal != vecRes.vectorid) { throw new ElasticException("Mismatch in metaid"); } - long count = (Long) source.get("count"); + JsonElement source = oneResp.get("_source"); + if (ElasticConnection.isNull(source)) { + throw new ElasticException("meta document does not exist for id=" + matchId); + } + long count = ((JsonObject) source).get("count").getAsLong(); totalCount += count; vecRes.hitcount = (int) count; } @@ -643,26 +687,27 @@ public class ElasticDatabase implements FunctionDatabase { buffer.append('\"'); } buffer.append(" ] }"); - JSONObject resp = + JsonObject resp = connection.executeStatement(ElasticConnection.GET, "vector/_mget", buffer.toString()); - JSONArray docs = (JSONArray) resp.get("docs"); + JsonArray docs = (JsonArray) resp.get("docs"); char[] vectorDecodeBuffer = Base64VectorFactory.allocateBuffer(); for (int i = 0; i < maxDocuments; ++i) { if (!iter2.hasNext()) { break; } vecRes = iter2.next(); - JSONObject oneResp = (JSONObject) docs.get(i); - String matchId = (String) oneResp.get("_id"); + JsonObject oneResp = (JsonObject) docs.get(i); + String matchId = oneResp.get("_id").getAsString(); long matchIdVal = Base64Lite.decodeLongBase64(matchId); - JSONObject source = (JSONObject) oneResp.get("_source"); - if (source == null) { - throw new ElasticException("vector document does not exist for id=" + matchId); - } if (matchIdVal != vecRes.vectorid) { throw new ElasticException("Mismatch in vectorid"); } - StringReader reader = new StringReader((String) source.get("features")); + JsonElement source = oneResp.get("_source"); + if (ElasticConnection.isNull(source)) { + throw new ElasticException("vector document does not exist for id=" + matchId); + } + StringReader reader = + new StringReader(((JsonObject) source).get("features").getAsString()); try { vecRes.vec = vectorFactory.restoreVectorFromBase64(reader, vectorDecodeBuffer); } @@ -728,7 +773,7 @@ public class ElasticDatabase implements FunctionDatabase { * @return list of matching functions as JSON documents * @throws ElasticException for communication problems with the server */ - private JSONArray queryVectorIdMatch(long vectorId, String filter, int maxDocuments) + private JsonArray queryVectorIdMatch(long vectorId, String filter, int maxDocuments) throws ElasticException { StringBuilder buffer = new StringBuilder(); buffer.append("{ \"size\": ").append(maxDocuments); @@ -743,15 +788,15 @@ public class ElasticDatabase implements FunctionDatabase { buffer.append(filter); } buffer.append("} } }"); - JSONObject resp = connection.executeStatement(ElasticConnection.GET, "executable/_search", + JsonObject resp = connection.executeStatement(ElasticConnection.GET, "executable/_search", buffer.toString()); - JSONObject baseHits = (JSONObject) resp.get("hits"); - JSONObject totalRec = (JSONObject) baseHits.get("total"); - long total = (Long) totalRec.get("value"); + JsonObject baseHits = (JsonObject) resp.get("hits"); + JsonObject totalRec = (JsonObject) baseHits.get("total"); + long total = totalRec.get("value").getAsLong(); if (total == 0) { - return new JSONArray(); + return new JsonArray(); } - JSONArray hitsArray = (JSONArray) baseHits.get("hits"); + JsonArray hitsArray = (JsonArray) baseHits.get("hits"); return hitsArray; } @@ -798,30 +843,30 @@ public class ElasticDatabase implements FunctionDatabase { buffer.append("\", \"simthresh\": ").append(similarityThreshold); buffer.append(", \"sigthresh\": ").append(significanceThreshold); buffer.append(" } } } } ] } } }"); - JSONObject resp = + JsonObject resp = connection.executeStatement(ElasticConnection.GET, "vector/_search", buffer.toString()); - JSONObject baseHits = (JSONObject) resp.get("hits"); + JsonObject baseHits = (JsonObject) resp.get("hits"); if (baseHits == null) { throw new ElasticException("Could not find hits document"); } - JSONObject totalRec = (JSONObject) baseHits.get("total"); - long numHits = (Long) totalRec.get("value"); + JsonObject totalRec = (JsonObject) baseHits.get("total"); + long numHits = totalRec.get("value").getAsLong(); if (numHits == 0) { return 0; } - JSONArray hitsArray = (JSONArray) baseHits.get("hits"); + JsonArray hitsArray = (JsonArray) baseHits.get("hits"); char[] decodeBuffer = Base64VectorFactory.allocateBuffer(); VectorCompare vecCompare = new VectorCompare(); try { int returnedHits = hitsArray.size(); for (int i = 0; i < returnedHits; ++i) { - JSONObject mainHit = (JSONObject) hitsArray.get(i); + JsonObject mainHit = (JsonObject) hitsArray.get(i); VectorResult vecRes = new VectorResult(); - vecRes.vectorid = Base64Lite.decodeLongBase64((String) mainHit.get("_id")); + vecRes.vectorid = Base64Lite.decodeLongBase64(mainHit.get("_id").getAsString()); vecRes.hitcount = -1; // Cannot fill in at this time - vecRes.sim = (Double) mainHit.get("_score"); - JSONObject source = (JSONObject) mainHit.get("_source"); - StringReader reader = new StringReader((String) source.get("features")); + vecRes.sim = mainHit.get("_score").getAsDouble(); + JsonObject source = (JsonObject) mainHit.get("_source"); + StringReader reader = new StringReader(source.get("features").getAsString()); vecRes.vec = vectorFactory.restoreVectorFromBase64(reader, decodeBuffer); vector.compareCounts(vecRes.vec, vecCompare); vecCompare.dotproduct = vecRes.sim * vector.getLength() * vecRes.vec.getLength(); @@ -899,7 +944,7 @@ public class ElasticDatabase implements FunctionDatabase { break; } final SignatureRecord srec = manager.newSignature(dresult.vec, dresult.hitcount); - JSONArray descres; + JsonArray descres; descres = queryVectorIdMatch(dresult.vectorid, filter, query.max - count); if (descres == null) { throw new ElasticException( @@ -1003,10 +1048,10 @@ public class ElasticDatabase implements FunctionDatabase { queryAllFunc(listFunctions, exeRecord, exeId, manager, maxFunctions); } else { - JSONArray hitsarray = queryFuncNameMatch(exeId, functionName, maxFunctions); - JSONObject doc = null; - for (Object element : hitsarray) { - doc = (JSONObject) element; + JsonArray hitsarray = queryFuncNameMatch(exeId, functionName, maxFunctions); + JsonObject doc = null; + for (JsonElement element : hitsarray) { + doc = (JsonObject) element; FunctionDescription funcDesc = convertDescriptionRow(doc, exeRecord, manager, null); listFunctions.add(funcDesc); } @@ -1032,7 +1077,7 @@ public class ElasticDatabase implements FunctionDatabase { throws ElasticException { RowKeyElastic eKey = (RowKeyElastic) exeRecord.getRowId(); String exeId = eKey.generateExeIdString(); - JSONObject doc = queryFuncNameAddress(exeId, functionName, address); + JsonObject doc = queryFuncNameAddress(exeId, functionName, address); if (doc == null) { return null; } @@ -1077,25 +1122,25 @@ public class ElasticDatabase implements FunctionDatabase { break; } } - JSONObject bulkobj = connection.executeBulk(path, buffer.toString()); - JSONArray responses = (JSONArray) bulkobj.get("responses"); + JsonObject bulkobj = connection.executeBulk(path, buffer.toString()); + JsonArray responses = (JsonArray) bulkobj.get("responses"); for (int i = 0; i < count; ++i) { - JSONObject subquery = (JSONObject) responses.get(i); - JSONObject hits = (JSONObject) subquery.get("hits"); - if (hits == null) { + JsonObject subquery = (JsonObject) responses.get(i); + JsonElement hits = subquery.get("hits"); + if (ElasticConnection.isNull(hits)) { throw new ElasticException("Multi-search for exe records failed"); } - JSONObject totalRec = (JSONObject) hits.get("total"); - long total = (Long) totalRec.get("value"); + JsonObject totalRec = (JsonObject) ((JsonObject) hits).get("total"); + long total = totalRec.get("value").getAsLong(); if (total != 1) { throw new ElasticException("Could not recover unique executable via id"); } } for (int i = 0; i < count; ++i) { - JSONObject subquery = (JSONObject) responses.get(i); - JSONObject hits = (JSONObject) subquery.get("hits"); - JSONArray hitsArray = (JSONArray) hits.get("hits"); - hits = (JSONObject) hitsArray.get(0); + JsonObject subquery = (JsonObject) responses.get(i); + JsonObject hits = (JsonObject) subquery.get("hits"); + JsonArray hitsArray = (JsonArray) hits.get("hits"); + hits = (JsonObject) hitsArray.get(0); ExecutableRecord newExe = makeExecutableRecord(manager, hits); RowKey rowKey = iter2.next(); manager.cacheExecutableByRow(newExe, rowKey); @@ -1111,7 +1156,7 @@ public class ElasticDatabase implements FunctionDatabase { * @return the JSON response object * @throws ElasticException for communication problems with the server */ - private JSONObject queryFunctionsOfExeId(String exeId, long maxDocuments, long start) + private JsonObject queryFunctionsOfExeId(String exeId, long maxDocuments, long start) throws ElasticException { StringBuilder buffer = new StringBuilder(); buffer.append("{ \"size\": ").append(maxDocuments); @@ -1146,20 +1191,20 @@ public class ElasticDatabase implements FunctionDatabase { long count = 0; do { int limit = MAX_FUNCTION_WINDOW; - if (maxDocuments != 0 && maxDocuments - start < limit) { - limit = (int) (maxDocuments - start); + if (maxDocuments != 0 && maxDocuments - count < limit) { + limit = (int) (maxDocuments - count); } - JSONObject resp = queryFunctionsOfExeId(exeId, limit, start); - JSONObject hits = (JSONObject) resp.get("hits"); - JSONObject totalRec = (JSONObject) hits.get("total"); - total = (Long) totalRec.get("value"); + JsonObject resp = queryFunctionsOfExeId(exeId, limit, start); + JsonObject hits = (JsonObject) resp.get("hits"); + JsonObject totalRec = (JsonObject) hits.get("total"); + total = totalRec.get("value").getAsLong(); if (maxDocuments != 0 && maxDocuments < total) { total = maxDocuments; } - JSONArray hitsarray = (JSONArray) hits.get("hits"); - JSONObject doc = null; - for (Object element : hitsarray) { - doc = (JSONObject) element; + JsonArray hitsarray = (JsonArray) hits.get("hits"); + JsonObject doc = null; + for (JsonElement element : hitsarray) { + doc = (JsonObject) element; FunctionDescription funcDesc = convertDescriptionRow(doc, exeRecord, manager, null); listFunctions.add(funcDesc); ++count; @@ -1167,8 +1212,8 @@ public class ElasticDatabase implements FunctionDatabase { if (hitsarray.size() == 0) { break; // Shouldn't need this, but just in case } - JSONArray sort = (JSONArray) doc.get("sort"); - start = (Long) sort.get(0); // Sort value for last entry, for passing as search_after parameter + JsonArray sort = (JsonArray) doc.get("sort"); + start = sort.get(0).getAsLong(); // Sort value for last entry, for passing as search_after parameter } while (total > count); return (int) total; @@ -1201,7 +1246,7 @@ public class ElasticDatabase implements FunctionDatabase { if (rec.function_name) { needscomma = true; buffer.append("\"name_func\": \"") - .append(JSONObject.escape(rec.update.getFunctionName())) + .append(escape(rec.update.getFunctionName())) .append('\"'); } if (rec.flags) { @@ -1230,7 +1275,7 @@ public class ElasticDatabase implements FunctionDatabase { if (updateRecord.name_exec) { needscomma = true; buffer.append("\"name_exec\": \"") - .append(JSONObject.escape(updateRecord.update.getNameExec())) + .append(escape(updateRecord.update.getNameExec())) .append('\"'); } if (updateRecord.architecture) { @@ -1313,7 +1358,7 @@ public class ElasticDatabase implements FunctionDatabase { */ private int updateExecutable(DescriptionManager manager, ExecutableRecord exeRecord, List badFunctions) throws ElasticException, LSHException { - JSONObject row = queryMd5ExeMatch(exeRecord.getMd5()); + JsonObject row = queryMd5ExeMatch(exeRecord.getMd5()); if (row == null) { return -1; // Indicate that we couldn't find the executable } @@ -1361,14 +1406,14 @@ public class ElasticDatabase implements FunctionDatabase { * @param source is the exe document * @return the list of CategoryRecords */ - private static List makeCategoryList(JSONObject source) { - JSONArray catArray = (JSONArray) source.get("execategory"); + private static List makeCategoryList(JsonObject source) { + JsonArray catArray = (JsonArray) source.get("execategory"); if (catArray == null || catArray.size() == 0) { return null; } List res = new ArrayList<>(); - for (Object element : catArray) { - String concat = (String) element; + for (JsonElement element : catArray) { + String concat = element.getAsString(); int pos = concat.indexOf('\t'); if (pos > 0) { String type = concat.substring(0, pos); @@ -1386,25 +1431,25 @@ public class ElasticDatabase implements FunctionDatabase { * @param hit is the "hit" document, which should have an "_id" and "_source" property. * @return the new ExecutableRecord parsed from the document */ - private static ExecutableRecord makeExecutableRecordTemp(JSONObject hit) { - RowKeyElastic eKey = RowKeyElastic.parseExeIdString((String) hit.get("_id")); - JSONObject source = (JSONObject) hit.get("_source"); - String md5 = (String) source.get("md5"); - String exename = (String) source.get("name_exec"); - String arch = (String) source.get("architecture"); + private static ExecutableRecord makeExecutableRecordTemp(JsonObject hit) { + RowKeyElastic eKey = RowKeyElastic.parseExeIdString(hit.get("_id").getAsString()); + JsonObject source = (JsonObject) hit.get("_source"); + String md5 = ElasticConnection.convertToString(source.get("md5")); + String exename = ElasticConnection.convertToString(source.get("name_exec")); + String arch = ElasticConnection.convertToString(source.get("architecture")); ExecutableRecord exeres; if (ExecutableRecord.isLibraryHash(md5)) { exeres = new ExecutableRecord(exename, arch, eKey); } else { - String cname = (String) source.get("name_compiler"); - String repo = (String) source.get("repository"); + String cname = ElasticConnection.convertToString(source.get("name_compiler")); + String repo = ElasticConnection.convertToString(source.get("repository")); String path = null; if (repo != null) { - path = (String) source.get("path"); + path = ElasticConnection.convertToString(source.get("path")); } List catrecs = makeCategoryList(source); - long milli = (Long) source.get("ingest_date"); + long milli = source.get("ingest_date").getAsLong(); exeres = new ExecutableRecord(md5, exename, cname, arch, new Date(milli), catrecs, eKey, repo, path); } @@ -1418,27 +1463,27 @@ public class ElasticDatabase implements FunctionDatabase { * @return the new ExecutableRecord parsed from the document * @throws LSHException if the container already contains executable with different metadata */ - private static ExecutableRecord makeExecutableRecord(DescriptionManager manager, JSONObject hit) + private static ExecutableRecord makeExecutableRecord(DescriptionManager manager, JsonObject hit) throws LSHException { - RowKeyElastic eKey = RowKeyElastic.parseExeIdString((String) hit.get("_id")); - JSONObject source = (JSONObject) hit.get("_source"); - String md5 = (String) source.get("md5"); - String exename = (String) source.get("name_exec"); - String arch = (String) source.get("architecture"); + RowKeyElastic eKey = RowKeyElastic.parseExeIdString(hit.get("_id").getAsString()); + JsonObject source = (JsonObject) hit.get("_source"); + String md5 = ElasticConnection.convertToString(source.get("md5")); + String exename = ElasticConnection.convertToString(source.get("name_exec")); + String arch = ElasticConnection.convertToString(source.get("architecture")); ExecutableRecord exerec; if (ExecutableRecord.isLibraryHash(md5)) { exerec = manager.newExecutableLibrary(exename, arch, eKey); } else { - String cname = (String) source.get("name_compiler"); - String repo = (String) source.get("repository"); + String cname = ElasticConnection.convertToString(source.get("name_compiler")); + String repo = ElasticConnection.convertToString(source.get("repository")); String path = null; if (repo != null) { - path = (String) source.get("path"); + path = ElasticConnection.convertToString(source.get("path")); } List catrecs = makeCategoryList(source); - long milli = (Long) source.get("ingest_date"); + long milli = source.get("ingest_date").getAsLong(); Date date = new Date(milli); exerec = manager.newExecutableRecord(md5, exename, cname, arch, date, repo, path, eKey); if (catrecs != null) { @@ -1457,14 +1502,14 @@ public class ElasticDatabase implements FunctionDatabase { * @param sigRecord is the (optional) SignatureRecord to attach to the new function * @return the new FunctionDescription */ - private static FunctionDescription convertDescriptionRow(JSONObject hit, + private static FunctionDescription convertDescriptionRow(JsonObject hit, ExecutableRecord exeRecord, DescriptionManager manager, SignatureRecord sigRecord) { - RowKey rowid = RowKeyElastic.parseFunctionId((String) hit.get("_id")); - JSONObject source = (JSONObject) hit.get("_source"); - String func_name = (String) source.get("name_func"); - long addr = (Long) source.get("addr"); - int flags = ((Long) source.get("flags")).intValue(); - long id_sig = Base64Lite.decodeLongBase64((String) source.get("id_signature")); + RowKey rowid = RowKeyElastic.parseFunctionId(hit.get("_id").getAsString()); + JsonObject source = (JsonObject) hit.get("_source"); + String func_name = source.get("name_func").getAsString(); + long addr = source.get("addr").getAsLong(); + int flags = (int) source.get("flags").getAsLong(); + long id_sig = Base64Lite.decodeLongBase64(source.get("id_signature").getAsString()); FunctionDescription fres = manager.newFunctionDescription(func_name, addr, exeRecord); manager.setFunctionDescriptionId(fres, rowid); manager.setFunctionDescriptionFlags(fres, flags); @@ -1491,7 +1536,7 @@ public class ElasticDatabase implements FunctionDatabase { * @throws ElasticException for communication problems with the server * @throws LSHException for problems adding new records to the container */ - protected void convertDescriptionRows(SimilarityResult similarityResult, JSONArray descRows, + protected void convertDescriptionRows(SimilarityResult similarityResult, JsonArray descRows, VectorResult vectorResult, DescriptionManager manager, SignatureRecord sigRecord) throws ElasticException, LSHException { if (descRows.size() == 0) { @@ -1501,11 +1546,11 @@ public class ElasticDatabase implements FunctionDatabase { Set parents = new TreeSet<>(); RowKeyElastic eKey; String exeid; - for (Object descRow : descRows) { - JSONObject hit = (JSONObject) descRow; - JSONObject source = (JSONObject) hit.get("_source"); - JSONObject joinfield = (JSONObject) source.get("join_field"); - exeid = (String) joinfield.get("parent"); + for (JsonElement descRow : descRows) { + JsonObject hit = (JsonObject) descRow; + JsonObject source = (JsonObject) hit.get("_source"); + JsonObject joinfield = (JsonObject) source.get("join_field"); + exeid = joinfield.get("parent").getAsString(); eKey = RowKeyElastic.parseExeIdString(exeid); parentIds.add(eKey); if (manager.findExecutableByRow(eKey) == null) { @@ -1518,7 +1563,7 @@ public class ElasticDatabase implements FunctionDatabase { queryExecutableRecordById(manager, iter1, iter2, 100); } - JSONObject currow = (JSONObject) descRows.get(0); + JsonObject currow = (JsonObject) descRows.get(0); eKey = parentIds.get(0); ExecutableRecord curexe = manager.findExecutableByRow(eKey); FunctionDescription fres = convertDescriptionRow(currow, curexe, manager, sigRecord); @@ -1526,7 +1571,7 @@ public class ElasticDatabase implements FunctionDatabase { similarityResult.addNote(fres, vectorResult.sim, vectorResult.signif); } for (int i = 1; i < descRows.size(); ++i) { - currow = (JSONObject) descRows.get(i); + currow = (JsonObject) descRows.get(i); eKey = parentIds.get(i); curexe = manager.findExecutableByRow(eKey); fres = convertDescriptionRow(currow, curexe, manager, sigRecord); @@ -1576,7 +1621,7 @@ public class ElasticDatabase implements FunctionDatabase { String exeId = eKey.generateExeIdString(); if (!insertExecutableRecord(exeRecord, exeId)) { // Try to insert the executable - JSONObject exeObj = queryMd5ExeMatch(exeRecord.getMd5()); + JsonObject exeObj = queryMd5ExeMatch(exeRecord.getMd5()); if (exeObj != null) { // Try to retrieve the previous version ExecutableRecord oldrec = makeExecutableRecordTemp(exeObj); int cmp = oldrec.compareMetadata(exeRecord); @@ -1668,11 +1713,11 @@ public class ElasticDatabase implements FunctionDatabase { String body = "{ \"script\": { \"inline\": \"ctx._source.iid += params.bulk_size\", \"params\": { \"bulk_size\": " + Integer.toString(amount) + "}}}"; - JSONObject resp = connection.executeStatement(ElasticConnection.POST, + JsonObject resp = connection.executeStatement(ElasticConnection.POST, "configuration/_update/1?_source=iid&retry_on_conflict=5", body); - JSONObject get = (JSONObject) resp.get("get"); - JSONObject fields = (JSONObject) get.get("_source"); - long res = (Long) fields.get("iid"); + JsonObject get = (JsonObject) resp.get("get"); + JsonObject fields = (JsonObject) get.get("_source"); + long res = fields.get("iid").getAsLong(); return res - amount; } @@ -1716,28 +1761,29 @@ public class ElasticDatabase implements FunctionDatabase { } } while (iter.hasNext()); - JSONObject resp = connection.executeBulk("/_bulk", buffer.toString()); - if ((Boolean) resp.get("errors")) { - JSONArray items = (JSONArray) resp.get("items"); - for (Object item2 : items) { - JSONObject item = (JSONObject) item2; - JSONObject create = (JSONObject) item.get("create"); - JSONObject error = null; + JsonObject resp = connection.executeBulk("/_bulk", buffer.toString()); + if (resp.get("errors").getAsBoolean()) { + JsonArray items = (JsonArray) resp.get("items"); + for (JsonElement elem : items) { + JsonObject item = (JsonObject) elem; + JsonObject create = (JsonObject) item.get("create"); + JsonObject error = null; if (create != null) { - error = (JSONObject) create.get("error"); + error = (JsonObject) create.get("error"); if (error != null) { - String type = (String) error.get("type"); + String type = error.get("type").getAsString(); if (type.startsWith("version_conflict")) { continue; // Normal error, meaning document already exists } } } else { - JSONObject update = (JSONObject) item.get("update"); - error = (JSONObject) update.get("error"); + JsonObject update = (JsonObject) item.get("update"); + error = (JsonObject) update.get("error"); } if (error != null) { - throw new ElasticException((String) error.get("reason")); + throw new ElasticException( + ElasticConnection.convertToString(error.get("reason"))); } } } @@ -1768,20 +1814,20 @@ public class ElasticDatabase implements FunctionDatabase { "{ \"script\": { \"inline\": \"if ((ctx._source.count -= params.count) <=0) { ctx.op = \\\"delete\\\" }\", "); buffer.append("\"params\": { \"count\": ").append(entry.count).append("} } }\n"); } - JSONObject resp = connection.executeBulk("/_bulk", buffer.toString()); - JSONArray items = (JSONArray) resp.get("items"); + JsonObject resp = connection.executeBulk("/_bulk", buffer.toString()); + JsonArray items = (JsonArray) resp.get("items"); for (int i = 0; i < maxVectors; ++i) { if (!iter2.hasNext()) { break; } IdHistogram entry = iter2.next(); - JSONObject item = (JSONObject) items.get(i); - JSONObject update = (JSONObject) item.get("update"); - long id = Base64Lite.decodeLongBase64((String) update.get("_id")); + JsonObject item = (JsonObject) items.get(i); + JsonObject update = (JsonObject) item.get("update"); + long id = Base64Lite.decodeLongBase64(update.get("_id").getAsString()); if (id != entry.id) { throw new ElasticException("Mismatch in decrementVectorCounters"); } - if ("deleted".equals(update.get("result"))) { + if ("deleted".equals(ElasticConnection.convertToString(update.get("result")))) { deleteList.add(entry); // Mark this vector for full deletion } } @@ -1811,8 +1857,8 @@ public class ElasticDatabase implements FunctionDatabase { maxVectors -= 1; } while (iter.hasNext() && maxVectors > 0); - JSONObject resp = connection.executeBulk("/_bulk", buffer.toString()); - if ((Boolean) resp.get("errors")) { + JsonObject resp = connection.executeBulk("/_bulk", buffer.toString()); + if (resp.get("errors").getAsBoolean()) { throw new ElasticException("Error during vector deletion"); } } @@ -1829,9 +1875,9 @@ public class ElasticDatabase implements FunctionDatabase { buffer.append(" \"parent_id\": {"); buffer.append(" \"type\": \"function\","); buffer.append(" \"id\": \"").append(exeId).append("\" } } }"); - JSONObject resp = connection.executeStatement(ElasticConnection.POST, + JsonObject resp = connection.executeStatement(ElasticConnection.POST, "executable/_delete_by_query", buffer.toString()); - long numDocs = (Long) resp.get("deleted"); + long numDocs = resp.get("deleted").getAsLong(); connection.executeURIOnly(ElasticConnection.DELETE, "executable/_doc/" + exeId); return (int) numDocs; } @@ -1886,14 +1932,14 @@ public class ElasticDatabase implements FunctionDatabase { builder.append(refreshRateInSecs).append('s'); } builder.append("\" } }"); - JSONObject resp = connection.executeStatement(ElasticConnection.PUT, index + "/_settings", + JsonObject resp = connection.executeStatement(ElasticConnection.PUT, index + "/_settings", builder.toString()); - Boolean ack = (Boolean) resp.get("acknowledged"); - if (ack == null) { + JsonElement ack = resp.get("acknowledged"); + if (ElasticConnection.isNull(ack)) { throw new ElasticException( "Unknown response trying to adjust number_of_replicas and refresh_interval"); } - if (!ack) { + if (!ack.getAsBoolean()) { throw new ElasticException("Cluster did not accept settings for index: " + index); } } @@ -2048,7 +2094,7 @@ public class ElasticDatabase implements FunctionDatabase { private Map readKeyValues() throws ElasticException, NoDatabaseException { StringBuilder buffer = new StringBuilder(); buffer.append("{ \"size\": 500, \"query\": { \"match_all\": {} } }"); - JSONObject resp; + JsonObject resp; try { resp = connection.executeStatement(ElasticConnection.GET, "configuration/_search", buffer.toString()); @@ -2059,26 +2105,26 @@ public class ElasticDatabase implements FunctionDatabase { } throw ex; } - JSONObject baseHits = (JSONObject) resp.get("hits"); + JsonObject baseHits = (JsonObject) resp.get("hits"); long total = 0; if (baseHits != null) { - JSONObject totalRec = (JSONObject) baseHits.get("total"); - total = (Long) totalRec.get("value"); + JsonObject totalRec = (JsonObject) baseHits.get("total"); + total = totalRec.get("value").getAsLong(); } if (total <= 1) { throw new ElasticException("Unrecoverable error: Could not find configuration"); } HashMap res = new HashMap<>(); - JSONArray hits = (JSONArray) baseHits.get("hits"); - for (Object hit2 : hits) { - JSONObject hit = (JSONObject) hit2; - String key = (String) hit.get("_id"); - JSONObject source = (JSONObject) hit.get("_source"); - Object value = source.get("value"); - if (value == null) { + JsonArray hits = (JsonArray) baseHits.get("hits"); + for (JsonElement hit2 : hits) { + JsonObject hit = (JsonObject) hit2; + String key = hit.get("_id").getAsString(); + JsonObject source = (JsonObject) hit.get("_source"); + JsonElement value = source.get("value"); + if (ElasticConnection.isNull(value)) { continue; // This might be the "sequence" document } - res.put(key, (String) value); + res.put(key, value.getAsString()); } return res; } @@ -2313,7 +2359,7 @@ public class ElasticDatabase implements FunctionDatabase { { StringBuilder buffer = new StringBuilder(); buffer.append("{ \"password\": \""); - buffer.append(JSONObject.escape(String.valueOf(password))); + buffer.append(escape(String.valueOf(password))); buffer.append("\" }"); StringBuilder path = new StringBuilder(); path.append("/_security/user/_password"); // Ignore the username, change for "current" user @@ -2339,7 +2385,7 @@ public class ElasticDatabase implements FunctionDatabase { vectorFactory = new Base64VectorFactory(); config.weightfactory = new WeightFactory(); config.idflookup = new IDFLookup(); - JSONObject res; + JsonObject res; try { res = connection.executeURIOnly(ElasticConnection.GET, "vector/_settings"); } @@ -2349,14 +2395,13 @@ public class ElasticDatabase implements FunctionDatabase { } throw ex; } - JSONObject repo = (JSONObject) res.get(repository + "_vector"); - JSONObject settings = (JSONObject) repo.get("settings"); - JSONObject index = (JSONObject) settings.get("index"); - JSONObject analysis = (JSONObject) index.get("analysis"); - JSONObject tokenizer = (JSONObject) analysis.get("tokenizer"); + JsonObject repo = (JsonObject) res.get(repository + "_vector"); + JsonObject settings = (JsonObject) repo.get("settings"); + JsonObject index = (JsonObject) settings.get("index"); + JsonObject analysis = (JsonObject) index.get("analysis"); + JsonObject tokenizer = (JsonObject) analysis.get("tokenizer"); String tokenizerName = null; - for (Object obj : tokenizer.keySet()) { - String key = (String) obj; + for (String key : tokenizer.keySet()) { if (key.startsWith("lsh_")) { tokenizerName = key; break; @@ -2365,8 +2410,8 @@ public class ElasticDatabase implements FunctionDatabase { if (tokenizerName == null) { throw new ElasticException("Missing tokenizer configuration"); } - JSONObject tokenizerSettings = (JSONObject) tokenizer.get(tokenizerName); - String idfWeights = (String) tokenizerSettings.get(ElasticUtilities.LSH_WEIGHTS); + JsonObject tokenizerSettings = (JsonObject) tokenizer.get(tokenizerName); + String idfWeights = tokenizerSettings.get(ElasticUtilities.LSH_WEIGHTS).getAsString(); String[] split = idfWeights.split(" "); double[] weightArray = new double[split.length]; if (weightArray.length != config.weightfactory.getSize()) { @@ -2377,7 +2422,7 @@ public class ElasticDatabase implements FunctionDatabase { } config.weightfactory.set(weightArray); - String lookup = (String) tokenizerSettings.get(ElasticUtilities.IDF_CONFIG); + String lookup = tokenizerSettings.get(ElasticUtilities.IDF_CONFIG).getAsString(); split = lookup.split(" "); int[] lookupArray = new int[split.length]; for (int i = 0; i < lookupArray.length; ++i) { @@ -2462,10 +2507,7 @@ public class ElasticDatabase implements FunctionDatabase { @Override public void close() { - if (connection != null) { - connection.close(); - connection = null; - } + connection = null; status = Status.Unconnected; initialized = false; info = null; @@ -2478,7 +2520,8 @@ public class ElasticDatabase implements FunctionDatabase { @Override public QueryResponseRecord query(BSimQuery query) { - if ((!isInitialized()) && (!(query instanceof CreateDatabase))) { + if ((!isInitialized()) && !(query instanceof CreateDatabase) && + !(query instanceof DropDatabase)) { lastError = new BSimError(ErrorCategory.Nodatabase, "The database does not exist"); return null; } @@ -2506,6 +2549,9 @@ public class ElasticDatabase implements FunctionDatabase { else if (query instanceof QueryExeCount) { fdbQueryExeCount((QueryExeCount) query); } + else if (query instanceof DropDatabase q) { + fdbDatabaseDrop(q); + } else if (query instanceof CreateDatabase) { fdbDatabaseCreate((CreateDatabase) query); } @@ -2590,6 +2636,36 @@ public class ElasticDatabase implements FunctionDatabase { initialized = true; } + private void dropDatabase() throws ElasticException { + + ElasticConnection c = connection; + if (c != null) { + close(); + } + else { + c = new ElasticConnection(baseURL, repository); + } + + Msg.info(this, "Dropping BSim Elasticsearch database: " + serverInfo); + + dropIndex(c, "configuration"); + dropIndex(c, "executable"); + dropIndex(c, "vector"); + dropIndex(c, "meta"); + } + + private static void dropIndex(ElasticConnection c, String indexName) throws ElasticException { + try { + c.executeURIOnly(ElasticConnection.DELETE, indexName); + } + catch (ElasticException e) { + if (e.getMessage().contains("no such index")) { + return; // ignore + } + throw e; + } + } + /** * Given the name of an executable library, its architecture, and a function name, * return the id of the document describing this specific function. @@ -2603,13 +2679,13 @@ public class ElasticDatabase implements FunctionDatabase { public String recoverExternalFunctionId(String exeName, String funcName, String arch) throws ElasticException { String md5 = ExecutableRecord.calcLibraryMd5Placeholder(exeName, arch); - JSONObject row = queryMd5ExeMatch(md5); + JsonObject row = queryMd5ExeMatch(md5); if (row == null) { throw new ElasticException( "Could not resolve filter specifying executable: " + exeName); } - String exeId = (String) row.get("_id"); - JSONArray descres = queryFuncNameMatch(exeId, funcName, 2); + String exeId = row.get("_id").getAsString(); + JsonArray descres = queryFuncNameMatch(exeId, funcName, 2); if (1 != descres.size()) { throw new ElasticException( "Could not resolve filter specifying function: [" + exeName + "]" + funcName); @@ -2706,7 +2782,7 @@ public class ElasticDatabase implements FunctionDatabase { buffer.append("{ \"wildcard\" : {"); buffer.append(" \"name_exec\" : {"); buffer.append(" \"value\" : \"*"); - buffer.append(JSONObject.escape(filterExeName)); + buffer.append(escape(filterExeName)); buffer.append("*\" }}}"); placedFirst = true; } @@ -3226,7 +3302,7 @@ public class ElasticDatabase implements FunctionDatabase { break; } SignatureRecord srec = manager.newSignature(vecResult.vec, vecResult.hitcount); - JSONArray descres; + JsonArray descres; descres = queryVectorIdMatch(vecResult.vectorid, filter, query.max - count); if (descres == null) { throw new ElasticException( @@ -3258,7 +3334,7 @@ public class ElasticDatabase implements FunctionDatabase { DescriptionManager manager = new DescriptionManager(); ExecutableRecord erec = null; if (spec.exemd5.length() != 0) { - JSONObject row = queryMd5ExeMatch(spec.exemd5); + JsonObject row = queryMd5ExeMatch(spec.exemd5); if (row != null) { erec = makeExecutableRecord(manager, row); } @@ -3317,6 +3393,22 @@ public class ElasticDatabase implements FunctionDatabase { } } + private void fdbDatabaseDrop(DropDatabase query) throws LSHException { + ResponseDropDatabase response = query.getResponse(); + if (!serverInfo.getDBName().equals(query.databaseName)) { + throw new LSHException("Invalid databaseName name"); + } + response.dropSuccessful = true; // Response parameters assuming success + response.errorMessage = null; + try { + dropDatabase(); + } + catch (ElasticException e) { + response.dropSuccessful = false; + response.errorMessage = e.getMessage(); + } + } + /** * Entry point for the Elasticsearch version of PasswordChange command. * @param query is command parameters @@ -3357,19 +3449,19 @@ public class ElasticDatabase implements FunctionDatabase { buffer.append("{ \"query\": { \"ids\": { \"values\": [ \""); buffer.append(rowId); buffer.append("\" ] } } }"); - JSONObject resp = connection.executeStatement(ElasticConnection.GET, "executable/_search", + JsonObject resp = connection.executeStatement(ElasticConnection.GET, "executable/_search", buffer.toString()); - JSONObject hits = (JSONObject) resp.get("hits"); - JSONObject totalRec = (JSONObject) hits.get("total"); - long total = (Long) totalRec.get("value"); + JsonObject hits = (JsonObject) resp.get("hits"); + JsonObject totalRec = (JsonObject) hits.get("total"); + long total = totalRec.get("value").getAsLong(); if (total == 0) { throw new ElasticException("No function documents matching id=" + rowId); } - JSONArray hitsArray = (JSONArray) hits.get("hits"); - JSONObject row = (JSONObject) hitsArray.get(0); - JSONObject source = (JSONObject) row.get("_source"); - JSONObject joinfield = (JSONObject) source.get("join_field"); - String exeId = (String) joinfield.get("parent"); + JsonArray hitsArray = (JsonArray) hits.get("hits"); + JsonObject row = (JsonObject) hitsArray.get(0); + JsonObject source = (JsonObject) row.get("_source"); + JsonObject joinfield = (JsonObject) source.get("join_field"); + String exeId = joinfield.get("parent").getAsString(); RowKeyElastic eKey = RowKeyElastic.parseExeIdString(exeId); ExecutableRecord exeRec = manager.findExecutableByRow(eKey); if (exeRec == null) { @@ -3387,7 +3479,7 @@ public class ElasticDatabase implements FunctionDatabase { * @return the child document ids as an array of JSON strings * @throws ElasticException for communication problems with the server */ - private JSONArray queryCallgraphRows(FunctionDescription funcRecord) throws ElasticException { + private JsonArray queryCallgraphRows(FunctionDescription funcRecord) throws ElasticException { StringBuilder buffer = new StringBuilder(); buffer.append("executable/_doc/"); RowKeyElastic eKey = (RowKeyElastic) funcRecord.getExecutableRecord().getRowId(); @@ -3395,9 +3487,9 @@ public class ElasticDatabase implements FunctionDatabase { buffer.append("?routing="); buffer.append(eKey.generateExeIdString()); buffer.append("&_source_includes=childid"); - JSONObject resp = connection.executeURIOnly(ElasticConnection.GET, buffer.toString()); - JSONObject source = (JSONObject) resp.get("_source"); - JSONArray childid = (JSONArray) source.get("childid"); + JsonObject resp = connection.executeURIOnly(ElasticConnection.GET, buffer.toString()); + JsonObject source = (JsonObject) resp.get("_source"); + JsonArray childid = (JsonArray) source.get("childid"); return childid; } @@ -3417,12 +3509,12 @@ public class ElasticDatabase implements FunctionDatabase { throw new ElasticException( "Elasticsearch database does not have callgraph information enabled"); } - JSONArray callids = queryCallgraphRows(funcRecord); + JsonArray callids = queryCallgraphRows(funcRecord); if (callids == null) { return; // field is not present, meaning children are not present } - for (Object callid : callids) { - String funcId = (String) callid; + for (JsonElement callid : callids) { + String funcId = callid.getAsString(); RowKeyElastic eKey = RowKeyElastic.parseFunctionId(funcId); FunctionDescription fdesc = functionMap.get(eKey); if (fdesc == null) { diff --git a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/file/BSimH2FileDBConnectionManager.java b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/file/BSimH2FileDBConnectionManager.java index 1e3b9ffaca..9ca55562e1 100644 --- a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/file/BSimH2FileDBConnectionManager.java +++ b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/file/BSimH2FileDBConnectionManager.java @@ -172,7 +172,7 @@ public class BSimH2FileDBConnectionManager { dispose(); - if (dbf.isFile()) { + if (!dbf.isFile()) { return true; } @@ -286,6 +286,9 @@ public class BSimH2FileDBConnectionManager { public synchronized Connection getConnection() throws SQLException { if (successfulConnection) { + if (bds.isClosed()) { + bds.restart(); + } return bds.getConnection(); } @@ -317,6 +320,9 @@ public class BSimH2FileDBConnectionManager { * @throws SQLException if connection or authentication error occurs */ private Connection connect() throws SQLException { + if (bds.isClosed()) { + bds.restart(); + } Connection c = bds.getConnection(); successfulConnection = true; return c; diff --git a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/file/H2FileFunctionDatabase.java b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/file/H2FileFunctionDatabase.java index 3343454bfd..e44de67abb 100644 --- a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/file/H2FileFunctionDatabase.java +++ b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/file/H2FileFunctionDatabase.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,13 +22,16 @@ import java.util.*; import generic.concurrent.*; import generic.lsh.vector.LSHVector; import generic.lsh.vector.VectorCompare; -import ghidra.features.bsim.query.BSimServerInfo; -import ghidra.features.bsim.query.LSHException; +import ghidra.features.bsim.query.*; +import ghidra.features.bsim.query.BSimPostgresDBConnectionManager.BSimPostgresDataSource; +import ghidra.features.bsim.query.BSimServerInfo.DBType; +import ghidra.features.bsim.query.FunctionDatabase.Status; import ghidra.features.bsim.query.client.*; import ghidra.features.bsim.query.description.*; import ghidra.features.bsim.query.elastic.Base64VectorFactory; import ghidra.features.bsim.query.file.BSimH2FileDBConnectionManager.BSimH2FileDataSource; import ghidra.features.bsim.query.protocol.*; +import ghidra.util.Msg; import ghidra.util.task.TaskMonitor; public class H2FileFunctionDatabase extends AbstractSQLFunctionDatabase { @@ -120,6 +123,48 @@ public class H2FileFunctionDatabase extends AbstractSQLFunctionDatabase tableNames = new HashSet<>(); + try (Connection c = initConnection(); Statement st = c.createStatement()) { + try (ResultSet rs = st.executeQuery( + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' ORDER BY table_name")) { + while (rs.next()) { + tableNames.add(rs.getString(1)); + } + } + } + + // Spot check for a few BSim table names that always exist + if (!tableNames.contains("keyvaluetable") || !tableNames.contains("desctable") || + !tableNames.contains("weighttable")) { + throw new SQLException("attempted to drop non-BSim database"); + } + + fileDs.dispose(); // disconnect before deleting database + + BSimServerInfo serverInfo = fileDs.getServerInfo(); + if (!fileDs.delete()) { + throw new SQLException("failed to delete H2-file database: " + serverInfo); + } + + Msg.info(this, "Deleted BSim H2-file database: " + serverInfo); + + } + /** * Create vector map which maps vector ID to {@link VectorStoreEntry} * @return vector map @@ -136,7 +181,7 @@ public class H2FileFunctionDatabase extends AbstractSQLFunctionDatabase query, Connection c) - throws SQLException, LSHException, DatabaseNonFatalException { + throws SQLException, LSHException, DatabaseNonFatalException { if (query instanceof PrewarmRequest preWarmRequest) { preWarmRequest.buildResponseTemplate(); @@ -174,8 +219,8 @@ public class H2FileFunctionDatabase extends AbstractSQLFunctionDatabase resultset, LSHVector vec, - double simthresh, double sigthresh, int max) throws SQLException { + protected int queryNearestVector(List resultset, LSHVector vec, double simthresh, + double sigthresh, int max) throws SQLException { VectorCompare comp; List resultsToSort = new ArrayList<>(); for (VectorStoreEntry entry : vectorStore) { @@ -192,7 +237,7 @@ public class H2FileFunctionDatabase extends AbstractSQLFunctionDatabase Double.compare(r2.sim, r1.sim)); int maxResults = Math.min(max, resultsToSort.size()); @@ -220,19 +265,19 @@ public class H2FileFunctionDatabase extends AbstractSQLFunctionDatabase(); ConcurrentQ evalQ = evalBuilder.setThreadPool(threadPool) - .setCollectResults(true) - .setMonitor(TaskMonitor.DUMMY) - .build((fd, m) -> { - List resultset = new ArrayList<>(); - queryNearestVector(resultset, fd.getSignatureRecord().getLSHVector(), + .setCollectResults(true) + .setMonitor(TaskMonitor.DUMMY) + .build((fd, m) -> { + List resultset = new ArrayList<>(); + queryNearestVector(resultset, fd.getSignatureRecord().getLSHVector(), query.thresh, query.signifthresh, vectormax); - if (resultset.isEmpty()) { - return null; - } - SimilarityVectorResult simres = new SimilarityVectorResult(fd); - simres.addNotes(resultset); - return simres; - }); + if (resultset.isEmpty()) { + return null; + } + SimilarityVectorResult simres = new SimilarityVectorResult(fd); + simres.addNotes(resultset); + return simres; + }); evalQ.add(toQuery); try { @@ -257,8 +302,8 @@ public class H2FileFunctionDatabase extends AbstractSQLFunctionDatabase iter) - throws SQLException, LSHException { + DescriptionManager descMgr, Iterator iter) + throws SQLException, LSHException { //TODO: is this what the method should return //TODO: why is iter an argument? Why not just use query.manage.listAllFunctions() diff --git a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/protocol/DropDatabase.java b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/protocol/DropDatabase.java new file mode 100644 index 0000000000..f7effdd6f8 --- /dev/null +++ b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/protocol/DropDatabase.java @@ -0,0 +1,53 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.features.bsim.query.protocol; + +import java.io.IOException; +import java.io.Writer; + +import generic.lsh.vector.LSHVectorFactory; +import ghidra.util.xml.XmlUtilities; +import ghidra.xml.XmlElement; +import ghidra.xml.XmlPullParser; + +public class DropDatabase extends BSimQuery { + public String databaseName; + public ResponseDropDatabase dropResponse; + + public DropDatabase() { + super("dropdatabase"); + } + + @Override + public void buildResponseTemplate() { + if (response == null) + response = dropResponse = new ResponseDropDatabase(); + } + + @Override + public void saveXml(Writer fwrite) throws IOException { + fwrite.append('<').append(XmlUtilities.escapeElementEntities(name)); + fwrite.append(" dbname=\"").append(databaseName).append("\" />\n"); + } + + @Override + public void restoreXml(XmlPullParser parser, LSHVectorFactory vectorFactory) { + XmlElement el = parser.start(name); + databaseName = XmlUtilities.unEscapeElementEntities(el.getAttribute("dbname")); + parser.end(); + } + +} diff --git a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/protocol/ResponseDropDatabase.java b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/protocol/ResponseDropDatabase.java new file mode 100644 index 0000000000..7c33b42f82 --- /dev/null +++ b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/protocol/ResponseDropDatabase.java @@ -0,0 +1,66 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.features.bsim.query.protocol; + +import java.io.IOException; +import java.io.Writer; + +import generic.lsh.vector.LSHVectorFactory; +import ghidra.features.bsim.query.LSHException; +import ghidra.util.xml.SpecXmlUtils; +import ghidra.xml.XmlElement; +import ghidra.xml.XmlPullParser; + +/** + * Response of server indicating whether a password change request ({@link PasswordChange}) succeeded + */ +public class ResponseDropDatabase extends QueryResponseRecord { + + public boolean operationSupported; // true if the back-end supports this operation + public boolean dropSuccessful; // true if drop was successful + public String errorMessage; // Error message if change was not successful + + public ResponseDropDatabase() { + super("responsedropdatabase"); + operationSupported = true; + dropSuccessful = false; + errorMessage = null; + } + + @Override + public void saveXml(Writer fwrite) throws IOException { + fwrite.append('<').append(name); + fwrite.append(" success=\""); + SpecXmlUtils.encodeBoolean(dropSuccessful); + fwrite.append("\">"); + if (errorMessage != null) { + SpecXmlUtils.xmlEscapeWriter(fwrite, errorMessage); + } + fwrite.append("\n"); + } + + @Override + public void restoreXml(XmlPullParser parser, LSHVectorFactory vectorFactory) + throws LSHException { + XmlElement el = parser.start(name); + dropSuccessful = SpecXmlUtils.decodeBoolean(el.getAttribute("success")); + errorMessage = parser.end().getText(); + if (errorMessage != null && errorMessage.length() == 0) { + errorMessage = null; + } + } + +} diff --git a/Ghidra/Features/BSim/src/test.slow/java/ghidra/features/bsim/query/file/BSimH2DatabaseManagerTest.java b/Ghidra/Features/BSim/src/test.slow/java/ghidra/features/bsim/query/file/BSimH2DatabaseManagerTest.java index 32aa009ca5..32b5afd1e9 100644 --- a/Ghidra/Features/BSim/src/test.slow/java/ghidra/features/bsim/query/file/BSimH2DatabaseManagerTest.java +++ b/Ghidra/Features/BSim/src/test.slow/java/ghidra/features/bsim/query/file/BSimH2DatabaseManagerTest.java @@ -23,7 +23,6 @@ import java.util.*; import org.junit.*; import ghidra.features.bsim.query.*; -import ghidra.features.bsim.query.BSimServerInfo.DBType; import ghidra.features.bsim.query.FunctionDatabase.BSimError; import ghidra.features.bsim.query.description.DatabaseInformation; import ghidra.features.bsim.query.file.BSimH2FileDBConnectionManager.BSimH2FileDataSource; diff --git a/Ghidra/Features/BSim/src/test.slow/java/ghidra/features/bsim/query/test/BSimServerTest.java b/Ghidra/Features/BSim/src/test.slow/java/ghidra/features/bsim/query/test/BSimServerTest.java deleted file mode 100755 index 8408f218b5..0000000000 --- a/Ghidra/Features/BSim/src/test.slow/java/ghidra/features/bsim/query/test/BSimServerTest.java +++ /dev/null @@ -1,732 +0,0 @@ -/* ### - * IP: GHIDRA - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package ghidra.features.bsim.query.test; - -import static org.junit.Assert.*; - -import java.io.*; -import java.net.URL; -import java.sql.Date; -import java.time.Instant; -import java.util.*; - -import org.junit.*; -import org.xml.sax.SAXException; - -import generic.jar.ResourceFile; -import generic.lsh.vector.*; -import generic.util.Path; -import ghidra.GhidraTestApplicationLayout; -import ghidra.app.util.headless.HeadlessAnalyzer; -import ghidra.app.util.headless.HeadlessOptions; -import ghidra.features.bsim.gui.filters.ExecutableCategoryBSimFilterType; -import ghidra.features.bsim.gui.filters.HasNamedChildBSimFilterType; -import ghidra.features.bsim.query.*; -import ghidra.features.bsim.query.FunctionDatabase.BSimError; -import ghidra.features.bsim.query.client.tables.ExeTable.ExeTableOrderColumn; -import ghidra.features.bsim.query.description.*; -import ghidra.features.bsim.query.ingest.BSimLaunchable; -import ghidra.features.bsim.query.protocol.*; -import ghidra.framework.*; -import ghidra.framework.client.HeadlessClientAuthenticator; -import ghidra.util.xml.SpecXmlUtils; -import ghidra.xml.NonThreadedXmlPullParserImpl; -import ghidra.xml.XmlPullParser; - -// These tests require specific data files and paths to be set up. See BSimServerTestUtil. -// The "ignore" directive is to prevent these from running as part of the automated nightly tests. -@Ignore -public class BSimServerTest { - private static final String PROPERTIES_FILE = "RegressionSignatures.properties"; - - private static BSimServerTestUtil util; - private static LSHVectorFactory vectorFactory; - private static FunctionDatabase client; - private static BSimLaunchable bulk; - private static ResourceFile dumpFile; - private static DescriptionManager originalBash; - - private static XmlPullParser getParser(ResourceFile file) { - XmlPullParser parser; - try { - InputStream input = file.getInputStream(); - parser = new NonThreadedXmlPullParserImpl(input, "BSim test parser", - SpecXmlUtils.getXmlHandler(), false); - } - catch (SAXException e) { - return null; - } - catch (IOException e) { - return null; - } - return parser; - } - - @BeforeClass - public static void setUp() throws Exception { - util = new BSimServerTestUtil(); - util.verifyDirectories(); - GhidraTestApplicationLayout layout = - new GhidraTestApplicationLayout(new File(util.ghidraDir)); - ApplicationConfiguration config = new HeadlessGhidraApplicationConfiguration(); - Application.initializeApplication(layout, config); - ghidra.framework.protocol.ghidra.Handler.registerHandler(); /// Register ghidra: protocol - ghidra.features.bsim.query.postgresql.Handler.registerHandler(); // Register postgresql: protocol - HeadlessClientAuthenticator.installHeadlessClientAuthenticator(null, null, true); - bulk = new BSimLaunchable(); - - util.verifyRaw(); - File propFile = new File(util.xmlDir, PROPERTIES_FILE); - if (!propFile.isFile()) { - createPropertiesFile(); - runHeadless(); - } - - util.startServer(); - doIngest(); - BSimServerInfo bsimServerInfo; - try { - bsimServerInfo = new BSimServerInfo(new URL(util.bsimURLString)); - } - catch (Exception e) { - throw new AssertionError(e); - } - client = BSimClientFactory.buildClient(bsimServerInfo, false); - if (!client.initialize()) { - throw new IOException("Unable to connect to server"); - } - vectorFactory = client.getLSHVectorFactory(); - - ResourceFile xmlFile = - new ResourceFile(new ResourceFile(util.xmlDir), "sigs_" + BSimServerTestUtil.BASH_MD5); - if (!xmlFile.isFile()) { - throw new IOException("Basic signature generation did not happen"); - } - XmlPullParser parser = getParser(xmlFile); - originalBash = new DescriptionManager(); - originalBash.restoreXml(parser, vectorFactory); - parser.dispose(); - } - - @AfterClass - public static void shutdown() throws Exception { - if (client != null) { - client.close(); - } - util.shutdownServer(); - if (dumpFile != null && dumpFile.exists()) { - dumpFile.delete(); - } - } - - @Test - public void testLibHistoryXml() { - ResourceFile xmlFile = new ResourceFile(new ResourceFile(util.xmlDir), - "sigs_" + BSimServerTestUtil.LIBHISTORY_MD5); - assertTrue(xmlFile.isFile()); - XmlPullParser parser = getParser(xmlFile); - DescriptionManager manager = new DescriptionManager(); - try { - manager.restoreXml(parser, vectorFactory); - assertTrue(manager.getExecutableRecordSet().size() == 2); - ExecutableRecord eRec = manager.findExecutable(BSimServerTestUtil.LIBHISTORY_MD5); - // make sure basic meta-data comes through - assertTrue(eRec.getNameExec().equals("libhistory.so.7.0")); - assertTrue(eRec.getNameCompiler().equals("gcc")); - assertTrue(eRec.getPath().equals("raw")); - assertTrue(eRec.getRepository().equals("ghidra://localhost/repo")); - assertTrue(eRec.hasCategory("Test Category", "shared")); - ExecutableRecord libRec = manager.findExecutable("unknown", "x86:LE:64:default", ""); - assertTrue(libRec.isLibrary()); - FunctionDescription fDesc = manager.findFunctionByName("close", libRec); - assertNotNull(fDesc); - assertEquals(fDesc.getAddress(), -1); - fDesc = manager.findFunctionByName("read_history_range", eRec); - assertNotNull(fDesc); - assertEquals(fDesc.getAddress(), 0x105f60); - FunctionDescription addHistory = null; - FunctionDescription malloc = null; - for (CallgraphEntry entry : fDesc.getCallgraphRecord()) { - String name = entry.getFunctionDescription().getFunctionName(); - if (name.equals("add_history")) { - addHistory = entry.getFunctionDescription(); - } - else if (name.equals("malloc")) { - malloc = entry.getFunctionDescription(); - } - } - assertNotNull(addHistory); - assertNotNull(malloc); - assertEquals(addHistory.getAddress(), 0x102770); - assertEquals(malloc.getAddress(), -1); - assertTrue(addHistory.getExecutableRecord() == eRec); // Should be same object - assertTrue(malloc.getExecutableRecord() == libRec); - } - catch (LSHException e) { - Assert.fail("Failure processing libhistory"); - } - } - - @Test - public void testBashLibReadline() { - try { - ResourceFile xmlFile = new ResourceFile(new ResourceFile(util.xmlDir), - "sigs_" + BSimServerTestUtil.LIBREADLINE_MD5); - XmlPullParser parser = getParser(xmlFile); - DescriptionManager manager = new DescriptionManager(); - manager.restoreXml(parser, vectorFactory); - parser.dispose(); - assertEquals(manager.getExecutableRecordSet().size(), 2); - ExecutableRecord bashRec = originalBash.findExecutable(BSimServerTestUtil.BASH_MD5); - assertTrue(bashRec.hasCategory("Test Category", "static")); - ExecutableRecord readRec = manager.findExecutable(BSimServerTestUtil.LIBREADLINE_MD5); - assertTrue(readRec.hasCategory("Test Category", "shared")); - // Comparing function "history_filename" - FunctionDescription bashFunc = - originalBash.findFunction("FUN_001cc840", 0x1cc840, bashRec); - FunctionDescription readFunc = manager.findFunction("FUN_00134a70", 0x134a70, readRec); - VectorCompare compareData = new VectorCompare(); - double sig = bashFunc.getSignatureRecord() - .getLSHVector() - .compare(readFunc.getSignatureRecord().getLSHVector(), compareData); - assertTrue(sig > 0.99999); - } - catch (LSHException e) { - Assert.fail("Failure processing bash and libreadline"); - } - } - - private static void compareExe(DescriptionManager manager1, DescriptionManager manager2, - String md5) throws Exception { - ExecutableRecord eRec1 = manager1.findExecutable(md5); - ExecutableRecord eRec2 = manager2.findExecutable(md5); - Iterator iter = manager1.listFunctions(eRec1); - while (iter.hasNext()) { - FunctionDescription func1 = iter.next(); - FunctionDescription func2 = - manager2.findFunction(func1.getFunctionName(), func1.getAddress(), eRec2); - assertEquals(func1.getFlags(), func2.getFlags()); - if (func1.getSignatureRecord() == null) { - assertTrue(func2.getSignatureRecord() == null); - } - else { - assertNotNull(func2.getSignatureRecord()); - assertTrue(func1.getSignatureRecord() - .getLSHVector() - .equals(func2.getSignatureRecord().getLSHVector())); - } - if (func1.getCallgraphRecord() == null) { - assertTrue(func2.getCallgraphRecord() == null); - continue; - } - assertNotNull(func2.getCallgraphRecord()); - func1.sortCallgraph(); - func2.sortCallgraph(); - Iterator iter1 = func1.getCallgraphRecord().iterator(); - Iterator iter2 = func2.getCallgraphRecord().iterator(); - while (iter1.hasNext()) { - assertTrue(iter2.hasNext()); - FunctionDescription call1 = iter1.next().getFunctionDescription(); - FunctionDescription call2 = iter2.next().getFunctionDescription(); - assertTrue(call1.equals(call2)); - } - } - } - - @Test - public void testDumpFile() { - try { - assertNotNull(dumpFile); - assertTrue(dumpFile.exists()); - XmlPullParser parser = getParser(dumpFile); - DescriptionManager manager1 = new DescriptionManager(); - manager1.restoreXml(parser, vectorFactory); - parser.dispose(); - compareExe(manager1, originalBash, BSimServerTestUtil.BASH_MD5); - } - catch (Exception e) { - Assert.fail("Failed to perform dumpexexml: " + e.getMessage()); - } - } - - private static void testForError(QueryResponseRecord response) throws LSHException { - if (response == null) { - BSimError lastError = client.getLastError(); - if (lastError == null) { - throw new LSHException("Unknown error"); - } - throw new LSHException(lastError.message); - } - } - - @Test - public void testQueryInfo() throws LSHException { - QueryInfo queryInfo = new QueryInfo(); - ResponseInfo response = queryInfo.execute(client); - testForError(response); - DatabaseInformation info = response.info; - assertTrue(info.databasename.equals("TestName")); - assertTrue(info.owner.equals("TestOwner")); - assertTrue(info.description.equals("TestDescription")); - assertEquals(info.settings, 0x49); - assertTrue(info.trackcallgraph); - assertNotNull(info.execats); - assertEquals(info.execats.size(), 1); - assertTrue(info.execats.get(0).equals("Test Category")); - } - - private void compareExecutableRecords(ExecutableRecord exe1, ExecutableRecord exe2) { - assertEquals(exe1.getMd5(), exe2.getMd5()); - assertEquals(exe1.getNameExec(), exe2.getNameExec()); - assertEquals(exe1.getArchitecture(), exe2.getArchitecture()); - assertEquals(exe1.getNameCompiler(), exe2.getNameCompiler()); - assertEquals(exe1.getPath(), exe2.getPath()); - } - - @Test - public void testQueryExeInfo() throws LSHException { - ExecutableRecord libHistory = new ExecutableRecord(BSimServerTestUtil.LIBHISTORY_MD5, - "libhistory.so.7.0", "gcc", "x86:LE:64:default", null, null, null, "raw"); - ExecutableRecord libReadline = new ExecutableRecord(BSimServerTestUtil.LIBREADLINE_MD5, - "libreadline.so.7.0", "gcc", "x86:LE:64:default", null, null, null, "raw"); - QueryExeInfo queryExeInfo = new QueryExeInfo(); - queryExeInfo.includeFakes = true; - ResponseExe responseExe = queryExeInfo.execute(client); - testForError(responseExe); - assertEquals(responseExe.recordCount, 3); - ExecutableRecord exe1 = responseExe.records.get(0); - compareExecutableRecords(exe1, libHistory); - assertEquals(exe1.getExeCategoryAlphabetic("Test Category"), "shared"); - ExecutableRecord exe2 = responseExe.records.get(1); - compareExecutableRecords(exe2, libReadline); - assertEquals(exe2.getExeCategoryAlphabetic("Test Category"), "shared"); - ExecutableRecord exe3 = responseExe.records.get(2); - assertEquals(exe3.getMd5(), "bbbbbbbbaaaaaaaa4b13cd7905584d9f"); - assertEquals(exe3.getNameExec(), "unknown"); - - queryExeInfo = new QueryExeInfo(); - queryExeInfo.filterMd5 = BSimServerTestUtil.LIBREADLINE_MD5; - responseExe = queryExeInfo.execute(client); - testForError(responseExe); - assertEquals(responseExe.recordCount, 1); - exe1 = responseExe.records.get(0); - compareExecutableRecords(exe1, libReadline); - - queryExeInfo = new QueryExeInfo(); - queryExeInfo.filterMd5 = "0a860"; // Partial md5 - responseExe = queryExeInfo.execute(client); - testForError(responseExe); - assertEquals(responseExe.recordCount, 1); - exe1 = responseExe.getDescriptionManager().findExecutable("libhistory.so.7.0", null, null); - compareExecutableRecords(exe1, libHistory); - - queryExeInfo = new QueryExeInfo(); - queryExeInfo.filterExeName = "lib"; - queryExeInfo.sortColumn = ExeTableOrderColumn.NAME; - responseExe = queryExeInfo.execute(client); - testForError(responseExe); - assertEquals(responseExe.records.size(), 2); - exe1 = responseExe.records.get(0); - exe2 = responseExe.records.get(1); - compareExecutableRecords(exe1, libHistory); - compareExecutableRecords(exe2, libReadline); - - QueryExeCount queryExeCount = new QueryExeCount(); - responseExe = queryExeCount.execute(client); - testForError(responseExe); - assertEquals(responseExe.recordCount, 2); - } - - @Test - public void testQueryName() throws LSHException { - QueryName queryName = new QueryName(); - queryName.spec.exename = "libhistory.so.7.0"; - queryName.funcname = "history_arg_extract"; - ResponseName responseName = queryName.execute(client); - testForError(responseName); - assertTrue(responseName.uniqueexecutable); - ExecutableRecord eRec = - responseName.manage.findExecutable(BSimServerTestUtil.LIBHISTORY_MD5); - assertTrue(eRec.getNameExec().equals("libhistory.so.7.0")); - assertTrue(eRec.getMd5().equals(BSimServerTestUtil.LIBHISTORY_MD5)); - assertTrue(eRec.getNameCompiler().equals("gcc")); - assertTrue(eRec.getPath().equals("raw")); - Iterator iter = responseName.manage.listAllFunctions(); - assertTrue(iter.hasNext()); - FunctionDescription func = iter.next(); - assertFalse(iter.hasNext()); // Should be exactly one function - assertTrue(func.getFunctionName().equals("history_arg_extract")); - assertEquals(func.getAddress(), 0x103d40); - SignatureRecord sigRec = func.getSignatureRecord(); - assertNotNull(sigRec); - assertEquals(sigRec.getCount(), 2); - ExecutableRecord bashRec = originalBash.findExecutable(BSimServerTestUtil.BASH_MD5); - FunctionDescription bashFunc = - originalBash.findFunctionByName("history_arg_extract", bashRec); - assertNotNull(bashFunc); - VectorCompare vectorCompare = new VectorCompare(); - double sim = sigRec.getLSHVector() - .compare(bashFunc.getSignatureRecord().getLSHVector(), vectorCompare); - assertTrue(sim > 0.8); - assertTrue(sim < 0.999); - } - - private static QueryResponseRecord doStagedQuery(BSimQuery query, - StagingManager stagingManager) throws LSHException { - - boolean haveMore = stagingManager.initialize(query); - query.buildResponseTemplate(); - - QueryResponseRecord globalResponse = query.getResponse(); - - while (haveMore) { - // Get the current staged form of the query - BSimQuery stagedQuery = stagingManager.getQuery(); - QueryResponseRecord response = stagedQuery.execute(client); - if (response != null) { - if (globalResponse != response) { - globalResponse.mergeResults(response); // Merge the staged response with the global response - } - - haveMore = stagingManager.nextStage(); - if (haveMore) { - stagedQuery.clearResponse(); // Make space for next stage - } - } - else { - throw new LSHException(client.getLastError().message); - } - } - - return globalResponse; - } - - private static void testSimilarityResult(SimilarityResult simRes) { - assertEquals(simRes.getTotalCount(), 2); - Iterator iter = simRes.iterator(); - SimilarityNote note1 = iter.next(); - SimilarityNote note2 = iter.next(); - FunctionDescription func1 = note1.getFunctionDescription(); - FunctionDescription func2 = note2.getFunctionDescription(); - assertTrue(func1.getExecutableRecord().getNameExec().equals("libhistory.so.7.0")); - assertTrue(func2.getExecutableRecord().getNameExec().equals("libreadline.so.7.0")); - assertNotNull(func1.getSignatureRecord()); - assertNotNull(func2.getSignatureRecord()); - assertNotNull(simRes.getBase().getSignatureRecord()); - LSHVector baseVector = simRes.getBase().getSignatureRecord().getLSHVector(); - VectorCompare vectorCompare = new VectorCompare(); - double sim1 = func1.getSignatureRecord().getLSHVector().compare(baseVector, vectorCompare); - assertEquals(note1.getSimilarity(), sim1, 0.0001); - double sim2 = func2.getSignatureRecord().getLSHVector().compare(baseVector, vectorCompare); - assertEquals(note2.getSimilarity(), sim2, 0.0001); - } - - @Test - public void testQueryNearest() throws LSHException { - QueryNearest queryNearest = new QueryNearest(); - ExecutableRecord bashRec = originalBash.findExecutable(BSimServerTestUtil.BASH_MD5); - FunctionDescription func1 = originalBash.findFunctionByName("_rl_adjust_point", bashRec); - FunctionDescription func2 = originalBash.findFunctionByName("_rl_compare_chars", bashRec); - FunctionDescription func3 = originalBash.findFunctionByName("add_history", bashRec); - FunctionDescription func4 = originalBash.findFunctionByName("get_history_event", bashRec); - queryNearest.manage.transferSettings(originalBash); - queryNearest.manage.transferFunction(func1, true); - queryNearest.manage.transferFunction(func2, true); - queryNearest.manage.transferFunction(func3, true); - queryNearest.manage.transferFunction(func4, true); - StagingManager functionStage = new FunctionStaging(2); - QueryResponseRecord response = doStagedQuery(queryNearest, functionStage); - testForError(response); - ResponseNearest respNearest = (ResponseNearest) response; - respNearest.sort(); - int matchCount = 0; - for (SimilarityResult simRes : respNearest.result) { - if (simRes.getBase().getAddress() == func1.getAddress()) { - assertTrue(func1.equals(simRes.getBase())); - matchCount += 1; - } - else if (simRes.getBase().getAddress() == func2.getAddress()) { - assertTrue(func2.equals(simRes.getBase())); - matchCount += 1; - } - else if (simRes.getBase().getAddress() == func3.getAddress()) { - assertTrue(func3.equals(simRes.getBase())); - matchCount += 1; - } - else if (simRes.getBase().getAddress() == func4.getAddress()) { - assertTrue(func4.equals(simRes.getBase())); - matchCount += 1; - } - testSimilarityResult(simRes); - } - assertEquals(matchCount, 4); // Make sure we hit all functions - } - - @Test - public void testQueryVector() throws LSHException { - QueryNearestVector queryVector = new QueryNearestVector(); - ExecutableRecord bashRec = originalBash.findExecutable(BSimServerTestUtil.BASH_MD5); - FunctionDescription func = originalBash.findFunctionByName("_rl_kill_kbd_macro", bashRec); - queryVector.manage.transferSettings(originalBash); - queryVector.manage.transferFunction(func, true); - ResponseNearestVector respVector = queryVector.execute(client); - testForError(respVector); - respVector.sort(); - assertEquals(respVector.totalvec, 1); - assertEquals(respVector.totalmatch, 2); - assertEquals(respVector.uniquematch, 0); // 1 vector matches 2 functions - Iterator iter = respVector.result.iterator(); - SimilarityVectorResult simVecRes = iter.next(); - assertFalse(iter.hasNext()); - assertTrue(simVecRes.getBase().equals(func)); - assertEquals(simVecRes.getTotalCount(), 2); - Iterator iter2 = simVecRes.iterator(); - VectorResult vec1 = iter2.next(); - VectorResult vec2 = iter2.next(); - assertFalse(iter2.hasNext()); - if (vec1.sim > vec2.sim) { - VectorResult tmp = vec1; - vec1 = vec2; - vec2 = tmp; - } - VectorCompare vectorCompare = new VectorCompare(); - LSHVector baseVector = func.getSignatureRecord().getLSHVector(); - double sim1 = baseVector.compare(vec1.vec, vectorCompare); - assertEquals(sim1, vec1.sim, 0.0001); - assertEquals(vec1.hitcount, 1); - double sim2 = baseVector.compare(vec2.vec, vectorCompare); - assertEquals(sim2, vec2.sim, 0.0001); - assertEquals(vec1.hitcount, 1); - assertTrue(vec2.sim > 0.999); // Second vector should be 1.0 match - } - - @Test - public void testChildFilter() throws LSHException { - QueryNearest query = new QueryNearest(); - query.manage.transferSettings(originalBash); - ExecutableRecord bashRec = originalBash.findExecutable(BSimServerTestUtil.BASH_MD5); - FunctionDescription funcDesc = originalBash.findFunctionByName("_rl_errmsg", bashRec); - query.manage.transferFunction(funcDesc, true); - query.bsimFilter = new BSimFilter(); - query.bsimFilter.addAtom(new HasNamedChildBSimFilterType(), "[unknown]__fprintf_chk"); - ResponseNearest respNearest = query.execute(client); - testForError(respNearest); - assertEquals(respNearest.totalfunc, 1); - assertEquals(respNearest.totalmatch, 1); // Filtered all but one response - assertEquals(respNearest.uniquematch, 1); - ExecutableRecord eRec = respNearest.manage.getExecutableRecordSet().first(); - assertTrue(eRec.getNameExec().equals("libreadline.so.7.0")); - assertTrue(eRec.getMd5().equals(BSimServerTestUtil.LIBREADLINE_MD5)); - assertEquals(respNearest.result.size(), 1); - SimilarityResult simRes = respNearest.result.get(0); - assertEquals(simRes.size(), 1); // Only one function returned - assertEquals(simRes.getTotalCount(), 3); // 3 functions similar to vector - SimilarityNote note = simRes.iterator().next(); - assertTrue(note.getSimilarity() > 0.800); - FunctionDescription resFunc = note.getFunctionDescription(); - assertTrue(resFunc.getFunctionName().equals("FUN_0011ece0")); - assertEquals(resFunc.getAddress(), 0x11ece0); - assertEquals(resFunc.getSignatureRecord().getCount(), 1); // only function with this exact vector - } - - @Test - public void testUpdate() throws Exception { - QueryUpdate update = new QueryUpdate(); - - Date dt = new Date(Instant.parse("2010-12-25T10:15:30.00Z").toEpochMilli()); - ExecutableRecord exerec = update.manage.newExecutableRecord( - BSimServerTestUtil.LIBREADLINE_MD5, "libreadline.so.7.0", "gcc", "x86:LE:64:default", - dt, "ghidra://localhost/repo", "/raw", null); - List catrec = new ArrayList<>(); - catrec.add(new CategoryRecord("Test Category", "SHARED!")); - update.manage.setExeCategories(exerec, catrec); - update.manage.newFunctionDescription("my_remove_history", 0x131c00, exerec); - ResponseUpdate respUpdate = update.execute(client); - testForError(respUpdate); - assertEquals(respUpdate.exeupdate, 1); - assertEquals(respUpdate.funcupdate, 1); - assertTrue(respUpdate.badexe.isEmpty()); - assertTrue(respUpdate.badfunc.isEmpty()); - if (util.isElasticSearch) { - Thread.sleep(2000); // Give chance for refresh timer to expire - } - QueryNearest nearest = new QueryNearest(); - nearest.manage.transferSettings(originalBash); - - ExecutableRecord bash = originalBash.findExecutable(BSimServerTestUtil.BASH_MD5); - FunctionDescription desc = originalBash.findFunctionByName("remove_history", bash); - nearest.manage.transferFunction(desc, true); - nearest.bsimFilter = new BSimFilter(); - nearest.bsimFilter.addAtom(new ExecutableCategoryBSimFilterType("Test Category"), - "SHARED!"); - ResponseNearest respNearest = nearest.execute(client); - testForError(respNearest); - assertEquals(respNearest.totalfunc, 1); - assertEquals(respNearest.totalmatch, 1); - assertEquals(respNearest.uniquematch, 1); - SimilarityResult simRes = respNearest.result.get(0); - assertTrue(simRes.getBase().equals(desc)); // base should match original function - assertEquals(simRes.size(), 1); - assertEquals(simRes.getTotalCount(), 2); // The filtered libhistory version is also counted here - SimilarityNote note = simRes.iterator().next(); - FunctionDescription resFunc = note.getFunctionDescription(); - assertTrue(resFunc.getFunctionName().equals("my_remove_history")); - ExecutableRecord resRec = resFunc.getExecutableRecord(); - assertTrue(resRec.getDate().equals(dt)); - - // Restore the original records - update = new QueryUpdate(); - exerec = update.manage.newExecutableRecord(BSimServerTestUtil.LIBREADLINE_MD5, - "libreadline.so.7.0", "gcc", "x86:LE:64:default", bash.getDate(), - "ghidra://localhost/repo", "/raw", null); - catrec = new ArrayList<>(); - catrec.add(new CategoryRecord("Test Category", "shared")); - update.manage.setExeCategories(exerec, catrec); - update.manage.newFunctionDescription("remove_history", 0x131c00, exerec); - testForError(update.execute(client)); - if (util.isElasticSearch) { - Thread.sleep(2000); // Give chance for refresh timer to expire - } - } - - private static void doIngest() throws Exception { - if (dumpFile != null && dumpFile.exists()) { - return; - } - runCreateDatabase(); - if (util.isElasticSearch) { - Thread.sleep(2000); // Give chance for refresh timer to expire - } - runSetMetaData(); - runSetExeCategory(); - runDropIndex(); - if (util.isElasticSearch) { - Thread.sleep(2000); // Give chance for refresh timer to expire - } - runCommitSigs(); - runRebuildIndex(); - if (util.isElasticSearch) { - Thread.sleep(2000); // Give chance for refresh timer to expire - } - runDumpFile(BSimServerTestUtil.BASH_MD5); - dumpFile = - new ResourceFile(new ResourceFile(util.testDir), "sigs_" + BSimServerTestUtil.BASH_MD5); - runDelete(); - if (util.isElasticSearch) { - Thread.sleep(2000); // Give chance for refresh timer to expire - } - } - - private static void createPropertiesFile() throws IOException { - File props = new File(util.xmlDir, PROPERTIES_FILE); - FileWriter writer = new FileWriter(props); - writer.write("RegressionSignatures: Working directory = " + util.xmlDir + '\n'); - writer.close(); - } - - private static void runHeadless() throws IOException { - HeadlessAnalyzer analyzer = HeadlessAnalyzer.getInstance(); - HeadlessOptions options = analyzer.getOptions(); - List preScripts = new ArrayList<>(); - List postScripts = new ArrayList<>(); - List inputFiles = new ArrayList<>(); - - options.setPropertiesFileDirectories(util.xmlDir); - Path scriptPath = new Path(Path.GHIDRA_HOME + "/Features/BSim/other/testscripts"); - options.setScriptDirectories(scriptPath.getPath().getAbsolutePath()); - - inputFiles.add(new File(util.rawDir)); - preScripts.add("TailoredAnalysis.java"); - preScripts.add("InstallMetadataTest.java"); - postScripts.add("RegressionSignatures.java"); - - options.setPreScripts(preScripts); - options.setPostScripts(postScripts); - - analyzer.processLocal(util.projectDir, util.repoName, "/", inputFiles); - } - - private static void runCreateDatabase() throws Exception { - String params[] = new String[3]; - params[0] = "createdatabase"; - params[1] = util.bsimURLString; - params[2] = "medium_64"; - bulk.run(params); - } - - private static void runSetMetaData() throws Exception { - String params[] = new String[5]; - params[0] = "setmetadata"; - params[1] = util.bsimURLString; - params[2] = "name=TestName"; - params[3] = "owner=TestOwner"; - params[4] = "description=TestDescription"; - bulk.run(params); - } - - private static void runSetExeCategory() throws Exception { - String params[] = new String[3]; - params[0] = "addexecategory"; - params[1] = util.bsimURLString; - params[2] = "Test Category"; - bulk.run(params); - } - - private static void runDropIndex() throws Exception { - if (util.isH2Database) { - return; - } - String params[] = new String[2]; - params[0] = "dropindex"; - params[1] = util.bsimURLString; - bulk.run(params); - } - - private static void runCommitSigs() throws Exception { - String params[] = new String[3]; - params[0] = "commitsigs"; - params[1] = util.bsimURLString; - params[2] = util.xmlDir; - bulk.run(params); - } - - private static void runRebuildIndex() throws Exception { - if (util.isH2Database) { - return; - } - String params[] = new String[2]; - params[0] = "rebuildindex"; - params[1] = util.bsimURLString; - bulk.run(params); - } - - private static void runDelete() throws Exception { - String params[] = new String[3]; - params[0] = "delete"; - params[1] = util.bsimURLString; - params[2] = "md5=" + BSimServerTestUtil.BASH_MD5; - bulk.run(params); - } - - private static void runDumpFile(String md5) throws Exception { - String params[] = new String[4]; - params[0] = "dumpsigs"; - params[1] = util.bsimURLString; - params[2] = util.testDir; - params[3] = "md5=" + BSimServerTestUtil.BASH_MD5; - bulk.run(params); - } -} diff --git a/Ghidra/Features/BSim/src/test.slow/java/ghidra/features/bsim/query/test/BSimServerTestUtil.java b/Ghidra/Features/BSim/src/test.slow/java/ghidra/features/bsim/query/test/BSimServerTestUtil.java deleted file mode 100755 index d1ffa67c49..0000000000 --- a/Ghidra/Features/BSim/src/test.slow/java/ghidra/features/bsim/query/test/BSimServerTestUtil.java +++ /dev/null @@ -1,192 +0,0 @@ -/* ### - * IP: GHIDRA - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package ghidra.features.bsim.query.test; - -import java.io.*; - -import ghidra.features.bsim.query.BSimControlLaunchable; -import ghidra.util.MD5Utilities; -import utilities.util.FileUtilities; - -/** - * The TEST_DIRECTORY String should be changed to point to a directory that will - * hold data for the server and for the tests. To start, this directory should contain - * a subdirectory "raw", and within this subdirectory should be the following 3 specific binary - * executables: - * libreadline.so.7.0 - * libhistory.so.7.0 - * bash - * - * all pulled from Ubuntu 18.04.5. - */ -public class BSimServerTestUtil { - private static final String HOST_URL = "postgresql://localhost"; -// private static final String HOST_URL = "https://localhost:9200"; -// private static final String HOST_URL = "file:///tmp/bsimtest/db"; - private static final String TEST_DIRECTORY = "/tmp/bsimtest"; - public static final String REPO_NAME = "repo"; - public static final String LIBHISTORY_MD5 = "0a860a716d5bec97c64db652549b72fd"; - public static final String LIBREADLINE_MD5 = "71b5761b43b840eb88d053790deaf77c"; - public static final String BASH_MD5 = "557c0271e30cf474e0f46f93721fd1ba"; - public String repoName; - public String bsimURLString = HOST_URL + '/' + REPO_NAME; - public String testDir; - public String ghidraDir; - public String projectDir; - public String rawDir; - public String xmlDir; - public String serverDir; - public String serverTouchDir; - public boolean isElasticSearch; - public boolean isH2Database; - - public BSimServerTestUtil() { - testDir = TEST_DIRECTORY; - ghidraDir = TEST_DIRECTORY + "/ghidra"; - repoName = REPO_NAME; - projectDir = testDir + "/project"; - rawDir = testDir + "/raw"; - xmlDir = testDir + "/xml"; - serverDir = testDir + "/db"; - serverTouchDir = testDir + "/servertouch"; - isElasticSearch = HOST_URL.startsWith("http") || HOST_URL.startsWith("elastic"); - isH2Database = HOST_URL.startsWith("file"); - } - - public void verifyDirectories() throws FileNotFoundException { - File dir0 = new File(testDir); - if (!dir0.exists()) { - throw new FileNotFoundException("Could not find test directory"); - } - File dir1 = new File(projectDir); - if (!dir1.exists()) { - if (!dir1.mkdir()) { - throw new FileNotFoundException("Could not create project directory"); - } - } - File dir2 = new File(xmlDir); - if (!dir2.exists()) { - if (!dir2.mkdir()) { - throw new FileNotFoundException("Could not create xml directory"); - } - } - File dir3 = new File(ghidraDir); - if (!dir3.exists()) { - if (!dir3.mkdir()) { - throw new FileNotFoundException("Could not create ghidra directory"); - } - } - } - - public void verifyRaw() throws IOException { - File rawDirectory = new File(rawDir); - if (!rawDirectory.exists()) { - throw new FileNotFoundException(rawDir); - } - if (!rawDirectory.isDirectory()) { - throw new FileNotFoundException("/raw is not a directory"); - } - String[] list = rawDirectory.list(); - boolean readlinePresent = false; - boolean historyPresent = false; - boolean bashPresent = false; - for (String element : list) { - File lib = new File(rawDirectory, element); - if (element.equals("libreadline.so.7.0")) { - String md5 = MD5Utilities.getMD5Hash(lib); - if (md5.equals(LIBREADLINE_MD5)) { - readlinePresent = true; - } - else { - throw new FileNotFoundException("libreadline.so.7.0 md5 does not match"); - } - } - else if (element.equals("libhistory.so.7.0")) { - String md5 = MD5Utilities.getMD5Hash(lib); - if (md5.equals(LIBHISTORY_MD5)) { - historyPresent = true; - } - else { - throw new FileNotFoundException("libhistory.so.7.0 md5 does not match"); - } - } - else if (element.equals("bash")) { - String md5 = MD5Utilities.getMD5Hash(lib); - if (md5.equals(BASH_MD5)) { - bashPresent = true; - } - else { - throw new FileNotFoundException("bash md5 does not match"); - } - } - } - if (!readlinePresent) { - throw new FileNotFoundException("Missing libreadline.so.7.0"); - } - if (!historyPresent) { - throw new FileNotFoundException("Missing libhistory.so.7.0"); - } - if (!bashPresent) { - throw new FileNotFoundException("Missing bash"); - } - } - - public void startServer() throws Exception { - if (isElasticSearch || isH2Database) { - return; // Don't try to start elasticsearch server - } - File touch = new File(serverTouchDir); - if (touch.exists()) { - return; - } - File dir = new File(serverDir); - if (dir.isDirectory()) { - FileUtilities.deleteDir(dir); - } - String[] params = new String[2]; - - params[0] = "start"; - params[1] = serverDir; - dir.mkdir(); // Create the data directory - - new BSimControlLaunchable().run(params); - - byte[] touchBytes = new byte[2]; - touchBytes[0] = 'a'; - touchBytes[1] = 'b'; - FileUtilities.writeBytes(touch, touchBytes); - } - - public void shutdownServer() throws Exception { - if (isElasticSearch || isH2Database) { - return; - } - File touch = new File(serverTouchDir); - if (!touch.exists()) { - return; - } - String[] params = new String[2]; - - params[0] = "stop"; - params[1] = serverDir; - new BSimControlLaunchable().run(params); - touch.delete(); // Remove the touch file - File dir = new File(serverDir); - if (dir.isDirectory()) { - FileUtilities.deleteDir(dir); // Clean up database files - } - } -} diff --git a/GhidraBuild/BuildFiles/Doclets/Module.manifest b/GhidraBuild/BuildFiles/Doclets/Module.manifest index 4c09989fd8..e69de29bb2 100644 --- a/GhidraBuild/BuildFiles/Doclets/Module.manifest +++ b/GhidraBuild/BuildFiles/Doclets/Module.manifest @@ -1 +0,0 @@ -MODULE FILE LICENSE: lib/json-simple-1.1.1.jar Apache License 2.0