Candidate release of source code.

This commit is contained in:
Dan 2019-03-26 13:45:32 -04:00
parent db81e6b3b0
commit 79d8f164f8
12449 changed files with 2800756 additions and 16 deletions

View file

@ -0,0 +1,69 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic;
import generic.stl.Pair;
/**
* DominantPair is a pair where the key is responsible for equality and
* hashCode (and the value of the pair doesn't matter at all). This is
* useful when you need the pair itself to function as a key in a Map or
* value in a Set.
*
* @param <K>
* @param <V>
*/
public class DominantPair<K, V> extends Pair<K, V> {
public DominantPair(K key, V value) {
super(key, value);
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((first == null) ? 0 : first.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
DominantPair<?, ?> other = (DominantPair<?, ?>) obj;
if (first == null) {
if (other.first != null) {
return false;
}
}
else if (!first.equals(other.first)) {
return false;
}
return true;
}
@Override
public String toString() {
return "(" + first + "," + second + ")";
}
}

View file

@ -0,0 +1,74 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic;
import java.util.Iterator;
import java.util.function.Predicate;
public class FilteredIterator<T> implements Iterator<T>, Iterable<T> {
private Iterator<T> it;
private Predicate<T> filter;
private T nextThing;
/**
* Construct a new FilteredIterator.
* @param it the iterator to filter
* @param filter the filter on T
*/
public FilteredIterator(Iterator<T> it, Predicate<T> filter) {
this.it = it;
this.filter = filter;
}
@Override
public boolean hasNext() {
if (nextThing != null) {
return true;
}
return findNext();
}
@Override
public T next() {
if (hasNext()) {
T t = nextThing;
nextThing = null;
return t;
}
return null;
}
private boolean findNext() {
while (it.hasNext()) {
T t = it.next();
if (filter.test(t)) {
nextThing = t;
return true;
}
}
return false;
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
@Override
public Iterator<T> iterator() {
return this;
}
}

View file

@ -0,0 +1,22 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic;
public class Images {
public static String BOMB = "images/core.png";
public static String BIG_BOMB = "images/core24.png";
}

View file

@ -0,0 +1,53 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.algorithms;
public class CRC64 {
private static final long poly = 0xC96C5795D7870F42L;
private static final long crcTable[] = new long[256];
private long crc = -1;
static {
for (int b = 0; b < crcTable.length; ++b) {
long r = b;
for (int i = 0; i < 8; ++i) {
if ((r & 1) == 1)
r = (r >>> 1) ^ poly;
else
r >>>= 1;
}
crcTable[b] = r;
}
}
public void update(byte[] buf, int off, int len) {
int end = off + len;
while (off < end)
crc = crcTable[(buf[off++] ^ (int) crc) & 0xFF] ^ (crc >>> 8);
}
public long finish() {
long value = ~crc;
crc = -1;
return value;
}
}

View file

@ -0,0 +1,150 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.algorithms;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
import ghidra.util.task.TaskMonitorAdapter;
import java.util.ArrayList;
import java.util.List;
/**
* Abstract class for finding the LCS between two sequences of Matchable
* objects.
*
*
*
* @param <T> the type of the objects being compared.
*/
public abstract class LCS<T> {
private int[][] c;
/**
* Convenient constructor for initializing elements in subclasses
*/
protected LCS() {
}
/**
* @return the length of the X sequence.
*/
protected abstract int lengthOfX();
/**
* @return the length of the Y sequence.
*/
protected abstract int lengthOfY();
/**
* @param index the position of interest in the X sequence.
* @return the value in the X sequence at <code>index</code>.
* Assumes 1-indexing.
*/
protected abstract T valueOfX(int index);
/**
* @param index the position of interest in the Y sequence.
* @return the value in the Y sequence at <code>index</code>.
* Assumes 1-indexing.
*/
protected abstract T valueOfY(int index);
/**
* @param x the X-sequence element of interest
* @param y the Y-sequence element of interest
* @return true if <code>x</code> matches <code>y</code>; false otherwise.
*/
protected abstract boolean matches(T x, T y);
/**
* Compute the LCS
* @param monitor
*/
private void calculateLCS(TaskMonitor monitor) throws CancelledException {
if (c != null) {
return;
}
int[][] tempC = new int[lengthOfX() + 1][];
monitor.setMessage("Calculating LCS...");
monitor.initialize(tempC.length);
for (int i = 0; i < tempC.length; i++) {
// Java int arrays are automatically initialized to 0
tempC[i] = new int[lengthOfY() + 1];
}
for (int i = 1; i < tempC.length; i++) {
monitor.checkCanceled();
for (int j = 1; j < tempC[i].length; j++) {
if (matches(valueOfX(i), valueOfY(j))) {
tempC[i][j] = tempC[i - 1][j - 1] + 1;
}
else {
tempC[i][j] = Math.max(tempC[i][j - 1], tempC[i - 1][j]);
}
}
monitor.incrementProgress(1);
}
c = tempC;
}
/**
* @return a <code>List&ltT&gt</code> of elements in the LCS.
*/
public List<T> getLCS() {
try {
return getLCS(TaskMonitorAdapter.DUMMY_MONITOR);
}
catch (CancelledException e) {
// can't happen with a dummy monitor
}
return null;
}
public List<T> getLCS(TaskMonitor monitor) throws CancelledException {
calculateLCS(monitor);
return getLCSHelperIterative(lengthOfX(), lengthOfY());
}
/**
* Iterative helper function for getLCS().
* @param i the current row index
* @param j the current column index
* @return the LCS after analyzing element c[i, j].
*/
private List<T> getLCSHelperIterative(int i, int j) {
ArrayList<T> result = new ArrayList<T>();
while (i > 0 && j > 0) {
if (c[i][j] == c[i - 1][j - 1] + 1 && matches(valueOfX(i), valueOfY(j))) {
result.add(0, valueOfX(i));
--i;
--j;
}
else if (c[i][j] == c[i - 1][j]) {
--i;
}
else {
--j;
}
}
return result;
}
}

View file

@ -0,0 +1,35 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.cache;
public interface BasicFactory<T> {
/**
* Creates an instance of {@link T}.
*
* @return the new instance of T
* @throws Exception any Exception encountered during creation
*/
public T create() throws Exception;
/**
* Called when clients are finished with the given item and it should be disposed.
*
* @param t the item to dispose.
*/
public void dispose(T t);
}

View file

@ -0,0 +1,114 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.cache;
import ghidra.util.timer.GTimer;
import ghidra.util.timer.GTimerMonitor;
import java.util.ArrayDeque;
import java.util.Deque;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* A thread-safe pool class that knows how to create instances as needed. When clients are done
* with the pooled item they then call {@link #release(Object)}.
*
* @param <T> the type of object to pool
*/
public class CachingPool<T> {
private static final long TIMEOUT = 0;
private AtomicBoolean isDisposed = new AtomicBoolean(false);
private BasicFactory<T> factory;
private Deque<T> cache = new ArrayDeque<T>();
private long disposeTimeout = TIMEOUT;
private GTimerMonitor timerMonitor;
private Runnable cleanupRunnable = new Runnable() {
@Override
public void run() {
synchronized (CachingPool.this) {
for (T t : cache) {
factory.dispose(t);
}
}
}
};
public CachingPool(BasicFactory<T> factory) {
if (factory == null) {
throw new IllegalArgumentException("factory cannot be null");
}
this.factory = factory;
}
/**
* Sets the time to wait for released items to be automatically disposed. The
* default is {@link #TIMEOUT}.
*
* @param timeout the new timeout.
*/
public void setCleanupTimeout(long timeout) {
this.disposeTimeout = timeout;
}
/**
* Returns a cached or new {@link T}
*
* @return a cached or new {@link T}
* @throws Exception if there is a problem instantiating a new instance
*/
public synchronized T get() throws Exception {
cancel();
if (cache.isEmpty()) {
return factory.create();
}
return cache.pop();
}
public synchronized void release(T t) {
restart();
if (isDisposed.get()) {
factory.dispose(t);
return;
}
cache.push(t);
}
public synchronized void dispose() {
cancel();
isDisposed.set(true);
for (T t : cache) {
factory.dispose(t);
}
}
private void cancel() {
if (timerMonitor != null) {
timerMonitor.cancel();
}
}
private void restart() {
if (timerMonitor != null) {
timerMonitor.cancel();
}
timerMonitor = GTimer.scheduleRunnable(disposeTimeout, cleanupRunnable);
}
}

View file

@ -0,0 +1,52 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.cache;
import java.util.concurrent.atomic.AtomicInteger;
public abstract class CountingBasicFactory<T> implements BasicFactory<T> {
/**
* A counter for tracking the number of items that have been created.
*/
protected AtomicInteger counter = new AtomicInteger();
protected AtomicInteger disposedCount = new AtomicInteger();
@Override
public T create() throws Exception {
return doCreate(counter.incrementAndGet());
}
@Override
public void dispose(T t) {
disposedCount.incrementAndGet();
doDispose(t);
}
/**
* The method subclass use to create {@link T}s.
*
* @param itemNumber the number of the item being created--
* <font size="5"><b>one-based</b></font>; the first item
* is item <tt>1</tt>.
* @return a new instance of {@link T}.
* @throws Exception any Exception encountered during creation
*/
public abstract T doCreate(int itemNumber) throws Exception;
public abstract void doDispose(T t);
}

View file

@ -0,0 +1,28 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.cache;
/**
* A simple interface that can build, lookup or otherwise return a value <code>V</code> for a
* key <code>K</code>.
*
* @param <K> the key used to get a value
* @param <V> the value returned for the given key
*/
public interface Factory<K, V> {
public V get(K key);
}

View file

@ -0,0 +1,56 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.cache;
import ghidra.util.datastruct.LRUMap;
/**
* An object that will cache values returned from the given factory. This class lets you combine
* the work of building items as needed with cache maintenance operations, such as get and put
* (and move, in the case of a sized cache).
*
* <P>
* The caching of this class
* is bound by the size parameter of the constructor. Further, the caching strategy is an
* Most Recently Used strategy, meaning that the least accessed cache items will fall off of the
* cache.
*
* @param <K> the key used to get a value
* @param <V> the value returned for the given key
*/
public class FixedSizeMRUCachingFactory<K, V> implements Factory<K, V> {
private LRUMap<K, V> cache;
private Factory<K, V> delegate;
public FixedSizeMRUCachingFactory(Factory<K, V> factory, int size) {
this.delegate = factory;
this.cache = new LRUMap<K, V>(size);
}
@Override
public V get(K key) {
V value = cache.get(key);
if (value != null) {
return value;
}
value = delegate.get(key);
cache.put(key, value);
return value;
}
}

View file

@ -0,0 +1,45 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.complex;
/**
* A complex number a + bi
*
* This doesn't support any actual operations, nor does it implement {@link Comparable}. It's simply
* enough to store and print the number.
*/
public class Complex {
private final double real;
private final double imaginary;
public Complex(double real, double imaginary) {
this.real = real;
this.imaginary = imaginary;
}
public double getReal() {
return real;
}
public double getImaginary() {
return imaginary;
}
@Override
public String toString() {
return String.format("%g + %gi", real, imaginary);
}
}

View file

@ -0,0 +1,62 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.concurrent;
import ghidra.util.graph.DependencyGraph;
import ghidra.util.task.TaskMonitor;
import java.util.Set;
public class ConcurrentGraphQ<I> {
private ConcurrentQ<I, Object> queue;
private DependencyGraph<I> graph;
public ConcurrentGraphQ(QRunnable<I> runnable, DependencyGraph<I> graph, GThreadPool pool,
TaskMonitor monitor) {
this.graph = graph;
// @formatter:off
queue = new ConcurrentQBuilder<I, Object>()
.setCollectResults(false)
.setThreadPool(pool)
.setMonitor(monitor)
.setListener(new MyItemListener())
.build(new QRunnableAdapter<I>(runnable));
// @formatter:on
}
public void execute() throws InterruptedException, Exception {
Set<I> values = graph.getUnvisitedIndependentValues();
queue.add(values);
queue.waitUntilDone();
}
public void dispose() {
queue.dispose();
}
class MyItemListener implements QItemListener<I, Object> {
@Override
public void itemProcessed(QResult<I, Object> result) {
graph.remove(result.getItem());
Set<I> values = graph.getUnvisitedIndependentValues();
queue.add(values);
}
}
}

View file

@ -0,0 +1,47 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.concurrent;
import java.util.Iterator;
import java.util.concurrent.ConcurrentHashMap;
/**
* A listener set that is weakly consistent. This allows for iteration of the set while other
* threads modify the set.
*/
public class ConcurrentListenerSet<T> implements Iterable<T> {
// we use a ConcurrentHashMap because Java has no ConcurrentHashSet
private ConcurrentHashMap<T, T> storage = new ConcurrentHashMap<T, T>();
public void add(T t) {
storage.put(t, t);
}
public void remove(T t) {
storage.remove(t);
}
public void clear() {
storage.clear();
}
@Override
public Iterator<T> iterator() {
return storage.keySet().iterator();
}
}

View file

@ -0,0 +1,808 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.concurrent;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.locks.ReentrantLock;
import ghidra.util.task.CancelledListener;
import ghidra.util.task.TaskMonitor;
/**
* A queue for easily scheduling tasks to be run in parallel (or sequentially)
* via a thread pool. This class provides a clean separation of items that need to
* be processed from the algorithm that does the processing, making it easy to parallelize
* the processing of multiple items. Further, you can control the maximum number of items that
* can be processed concurrently. This is useful to throttle operations that may starve the
* other threads in the system. You may also control how many items get placed into the queue
* at one time, blocking if some threshold is exceeded.
* <p>
* Examples:
* <hr>
* <p>
* <u>Put and Forget:</u>
* <pre>
* QCallback<ITEM, RESULT> callback = new AbstractQCallback<ITEM, RESULT>() {
* public RESULT process(ITEM item, TaskMonitor monitor) {
* // do work here...
* }
* };
*
* ConcurrentQBuilder<ITEM, RESULT> builder = new ConcurrentQBuilder<ITEM, RESULT>();
* builder.setThreadPoolName("Thread Pool Name");
* concurrentQ = builder.getQueue(callback);
* ...
* ...
* concurrentQ.add(item); // where item is one of the instances of ITEM
*
* </pre>
* <hr>
* <p>
* <u>Put Items and Handle Results in Any Order as They Available:</u>
* <pre>
* QCallback<ITEM, RESULT> callback = new AbstractQCallback<ITEM, RESULT>() {
* public RESULT process(ITEM item, TaskMonitor monitor) {
* // do work here...
* }
* };
*
* QItemListener<ITEM, RESULT> itemListener = new QItemListener<ITEM, RESULT>() {
* public void itemProcessed(QResult<ITEM, RESULT> result) {
* RESULT result = result.getResult();
* <font color="blue"><b>// work on my result...</b></font>
* }
* };
*
* ConcurrentQBuilder<ITEM, RESULT> builder = new ConcurrentQBuilder<ITEM, RESULT>();
* builder.setThreadPoolName("Thread Pool Name");
* <font color="blue"><b>builder.setListener(itemListener);</b></font>
* concurrentQ = builder.build(callback);
* ...
* ...
* concurrentQ.add(item); // where item is one of the instances of ITEM
* concurrentQ.add(item);
* concurrentQ.add(item);
*
* </pre>
*
* <hr>
* <p>
* <u>Put Items and Handle Results When All Items Have Been Processed:</u>
* <pre>
* QCallback<ITEM, RESULT> callback = new AbstractQCallback<ITEM, RESULT>() {
* public RESULT process(ITEM item, TaskMonitor monitor) {
* // do work here...
* }
* };
*
* ConcurrentQBuilder<ITEM, RESULT> builder = new ConcurrentQBuilder<ITEM, RESULT>();
* builder.setThreadPoolName("Thread Pool Name");
* <font color="blue"><b>builder.setCollectResults(true);</b></font>
* concurrentQ = builder.getQueue(callback);
* ...
* ...
* concurrentQ.add(item); // where item is one of the instances of ITEM
* concurrentQ.add(item);
* concurrentQ.add(item);
* ...
* <font color="blue"><b>List&lt;QResult&lt;I, R&gt;&gt; results = concurrentQ.waitForResults();</b></font>
* // process the results...
*
* </pre>
* <hr>
* <p>
* <u>Put Items, <b>Blocking While Full</b>, and Handle Results in Any Order as They Available:</u>
* <pre>
* QCallback<ITEM, RESULT> callback = new AbstractQCallback<ITEM, RESULT>() {
* public RESULT process(ITEM item, TaskMonitor monitor) {
* // do work here...
* }
* };
*
* QItemListener<ITEM, RESULT> itemListener = new QItemListener<ITEM, RESULT>() {
* public void itemProcessed(QResult<ITEM, RESULT> result) {
* RESULT result = result.getResult();
* // work on my result...
* }
* };
*
* ConcurrentQBuilder<ITEM, RESULT> builder = new ConcurrentQBuilder<ITEM, RESULT>();
* builder.setThreadPoolName("Thread Pool Name");
* <font color="blue"><b>builder.setQueue(new LinkedBlockingQueue(100));</b></font>
* concurrentQ = builder.getQueue(callback);
* ...
* ...
* Iterator<ITEM> iterator = &lt;get an iterator for 1000s of items somewhere&gt;
* <font color="blue"><b>concurrentQ.offer(iterator); // this call will block when the queue fills up (100 items or more)</b></font>
*
* </pre>
* <hr>
*
* @param <I> The type of the items to be processed.
* @param <R> The type of objects resulting from processing an item; if you don't care about the
* return value, then make this value whatever you want, like <tt>Object</tt> or the
* same value as {@link I} and return null from {@link QCallback#process(Object, TaskMonitor)}.
*/
public class ConcurrentQ<I, R> {
private final Queue<I> queue;
private final GThreadPool threadPool;
private final QCallback<I, R> callback;
private QItemListener<I, R> itemListener;
private QProgressListener<I> progressListener;
private Deque<QResult<I, R>> resultList = new LinkedList<>();
private final Set<FutureTaskMonitor<I, R>> taskSet = new HashSet<>();
private final int maxInProgress;
private final boolean collectResults;
private final boolean jobsReportProgress;
private QMonitorAdapter monitorAdapter;
private Exception unhandledException;
private ProgressTracker tracker;
private ReentrantLock lock = new ReentrantLock(false);
/**
* Creates a ConcurrentQ that will process as many items as the given threadPool can handle
* at one time.
*
* @param name The name of the thread pool that will be created by this constructor.
* @param callback the QWorker object that will be used to process items concurrently.
*/
public ConcurrentQ(String name, QCallback<I, R> callback) {
this(callback, new LinkedList<I>(), GThreadPool.getPrivateThreadPool(name), null, false, 0,
false);
}
/**
* Creates a ConcurrentQ that will process at most maxInProgress items at a time, regardless of
* how many threads are available in the GThreadPool.
*
* @param callback the QWorker object that will be used to process items concurrently.
* @param queue the internal storage queue to use in this concurrent queue.
* @param threadPool the GThreadPool to used for providing the threads for concurrent processing.
* @param listener An optional QItemListener that will be called back with results when the
* item has been processed.
* @param collectResults specifies if this queue should collect the results as items are processed
* so they can be returned in a waitForResults() call.
* @param maxInProgress specifies the maximum number of items that can be process at a time.
* If this is set to 0, then this queue will attempt to execute as many
* items at a time as there are threads in the given threadPool. Setting
* this parameter to 1 will have the effect of guaranteeing that
* all times are processed one at a time in the order they were submitted.
* Any other positive value will run that many items concurrently,
* up to the number of available threads.
* @param jobsReportProgress true signals that jobs wish to report progress via their task
* monitor. The default is false, which triggers this queue to report an
* overall progress for each job that is processed. False is a good default
* for clients that have a finite number of jobs to be done.
*/
public ConcurrentQ(QCallback<I, R> callback, Queue<I> queue, GThreadPool threadPool,
QItemListener<I, R> listener, boolean collectResults, int maxInProgress,
boolean jobsReportProgress) {
this.callback = callback;
this.queue = queue;
this.threadPool = threadPool;
this.itemListener = listener;
this.collectResults = collectResults;
this.jobsReportProgress = jobsReportProgress;
this.maxInProgress = maxInProgress > 0 ? maxInProgress : threadPool.getMaxThreadCount();
this.tracker = new ProgressTracker(lock);
}
/**
* Adds a progress listener for this queue. All the progress and messages reported by a
* QWorker will be routed to these listener.
*
* @param listener the listener for receiving progress and message notifications.
*/
public synchronized void addProgressListener(QProgressListener<I> listener) {
if (progressListener == null) {
progressListener = listener;
}
else {
progressListener = new ChainedProgressListener<>(progressListener, listener);
}
}
/**
* Removes a progress listener from this queue. All the progress and messages reported by a
* QWorker will be routed to this listener.
* @param listener the listener for receiving progress and message notifications.
*/
public synchronized void removeProgressListener(QProgressListener<I> listener) {
if (progressListener == listener) {
progressListener = null;
}
else if (progressListener instanceof ChainedProgressListener) {
progressListener =
((ChainedProgressListener<I>) progressListener).removeListener(listener);
}
}
/**
* Sets the monitor to use with this queue.
*
* @param monitor the monitor to attache to this queue
* @param cancelClearsAllItems if true, cancelling the monitor will cancel all items currently
* being processed by a thread and clear the scheduled
* items that haven't yet run.
* If false, only the items currently being processed will be cancelled.
*/
public void setMonitor(TaskMonitor monitor, boolean cancelClearsAllItems) {
if (monitorAdapter != null) {
monitorAdapter.dispose();
}
if (monitor != null) {
monitorAdapter = new QMonitorAdapter(monitor, cancelClearsAllItems);
}
}
/**
* Adds the list of items to this queue for concurrent processing.
* @param items the items to be scheduled for concurrent processing
*/
public void add(Collection<I> items) {
lock.lock();
try {
queue.addAll(items);
tracker.itemsAdded(items.size());
fillOpenProcessingSlots();
}
finally {
lock.unlock();
}
}
/**
* Adds the items of the given iterator to this queue for concurrent processing.
* @param iterator an iterator from which the items to be scheduled for concurrent processing
* will be taken.
*/
public void add(Iterator<I> iterator) {
lock.lock();
try {
while (iterator.hasNext()) {
queue.add(iterator.next());
tracker.itemsAdded(1);
fillOpenProcessingSlots();
}
}
finally {
lock.unlock();
}
}
/**
* Allows clients to use a bounded queue (such as a {@link LinkedBlockingQueue} to control
* how many items get placed into this queue at one time. Calling the <tt>add</tt> methods
* will place all items into the queue, which for a large number of items, can consume a
* large amount of memory. This method will block once the queue at maximum capacity,
* continuing to add new items as existing items on the queue are processed.
* <p>
* To enable blocking on the queue when it is full, construct this <tt>ConcurrentQ</tt>
* with an instance of {@link BlockingQueue}.
*
* @param iterator An iterator from which items will be taken.
* @throws InterruptedException if this queue is interrupted while waiting to add more items
*/
public void offer(Iterator<I> iterator) throws InterruptedException {
lock.lockInterruptibly();
try {
while (iterator.hasNext()) {
I next = iterator.next();
if (!queue.offer(next)) {
// must be full...wait until space is available
tracker.waitForNext();
queue.offer(next);
}
tracker.itemsAdded(1);
fillOpenProcessingSlots();
}
}
finally {
lock.unlock();
}
}
/**
* Adds the item to this queue for concurrent processing.
* @param item the item to be scheduled for concurrent processing.
*/
public void add(I item) {
lock.lock();
try {
queue.add(item);
tracker.itemsAdded(1);
fillOpenProcessingSlots();
}
finally {
lock.unlock();
}
}
/**
* Returns true if this queue has no items waiting to be processed or currently being processed.
* @return true if this queue has no items waiting to be processed or currently being processed.
*/
public boolean isEmpty() {
return tracker.isDone();
}
/**
* Waits until all scheduled items have been completed or cancelled and returns a list of
* QResults if this queue has been told to collect results.
* <P>
* You can still call this method to wait for items to be processed, even if you did not
* specify to collect results. In that case, the list returned will be empty.
*
* @return the list of QResult objects that have all the results of the completed jobs.
* @throws InterruptedException if this call was interrupted--Note: this interruption only
* happens if the calling thread cannot acquire the lock. If the thread is
* interrupted while waiting for results, then it will try again.
*/
public Collection<QResult<I, R>> waitForResults() throws InterruptedException {
lock.lockInterruptibly();
try {
tracker.waitUntilDone();
Collection<QResult<I, R>> returnValue = resultList;
resultList = new LinkedList<>();
return returnValue;
}
finally {
lock.unlock();
}
}
/**
* Wait until at least one result is available and then return the first result.
*
* @return the first available result
* @throws InterruptedException if interrupted while waiting for a result
* @throws {@link IllegalStateException} if this queue has been set to not collect results
* (see the constructor).
*/
public QResult<I, R> waitForNextResult() throws InterruptedException {
if (!collectResults) {
throw new IllegalStateException(
"Can't wait for next result when not collecting results");
}
lock.lockInterruptibly();
try {
if (resultList.isEmpty()) {
if (isEmpty()) {
return null;
}
tracker.waitForNext();
}
return resultList.pop();
}
finally {
lock.unlock();
}
}
/**
* Waits until all items have been processed <b>OR</b> an Exception happens during the
* processing of <b>ANY item</b>.
* <p>
* <b><u>Note:</u></b>
* If an exception does occur then the remaining items in the
* queue will be cleared and all current items will be cancelled.
* <p>
* If you wish for processing to continue for remaining items when any item encounters an
* exception, then you should instead use {@link #waitForResults()}. That method will return
* all results, both with and without exceptions, which you can then process, including
* checking for exceptions. Note that to use {@link #waitForResults()} to examine exceptions,
* you must have created this queue with <tt>collectResults</tt> as true.
*
* @throws InterruptedException if interrupted while waiting for a result
* @throws Exception any exception encountered while processing an item (this will cancel all
* items in the queue).
*/
public void waitUntilDone() throws InterruptedException, Exception {
lock.lockInterruptibly();
try {
checkException();
while (!isEmpty()) {
tracker.waitForNext();
checkException();
}
}
finally {
lock.unlock();
}
}
private void checkException() throws Exception {
if (unhandledException != null) {
cancelAllTasks(true);
throw unhandledException;
}
}
/**
* Waits up to the specified time for scheduled jobs to complete. The results of all completed
* jobs will be returned if this queue has been told to collect results. At the time that this
* returns, there may still be work to process. The returned list will contain as much work
* as has been processed when the wait has finished. Repeated calls to this method will not
* return results from previous waits.
* <P>
* You can still call this method to wait for items to be processed, even if you did not
* specify to collect results. In that case, the list returned will be empty.
*
* @return the list of QResult objects that have all the results of the completed jobs.
* @throws InterruptedException if this call was interrupted.
*/
public Collection<QResult<I, R>> waitForResults(long timeout, TimeUnit unit)
throws InterruptedException {
lock.lockInterruptibly();
try {
tracker.waitUntilDone(timeout, unit);
Collection<QResult<I, R>> returnValue = resultList;
resultList = new LinkedList<>();
return returnValue;
}
finally {
lock.unlock();
}
}
/**
* Cancels the processing of currently scheduled items in this queue. Any items that haven't
* yet been scheduled on the threadPool are returned immediately from this call. Items that
* are currently being processed will be cancelled and those results will be available on the
* next waitForResults() call and also if there is a QItemListener, it will be called with
* the QResult. There is no guarantee that scheduled tasks will terminate any time soon. If
* they check the isCancelled() state of their QMonitor, it will be true. Setting the
* interruptRunningTasks to true, will result in a thread interrupt to any currently running
* task which might be useful if the task perform waiting operations like I/O.
*
* @param interruptRunningTasks if true, an attempt will be made to interrupt any currently
* processing thread.
* @return a list of all items that have not yet been queued to the threadPool.
*/
public List<I> cancelAllTasks(boolean interruptRunningTasks) {
List<FutureTaskMonitor<I, R>> tasksToBeCancelled = new ArrayList<>();
List<I> nonStartedItems;
lock.lock();
try {
nonStartedItems = removeUnscheduledJobs();
tasksToBeCancelled.addAll(taskSet);
}
finally {
lock.unlock();
}
for (FutureTaskMonitor<I, R> task : tasksToBeCancelled) {
task.cancel(interruptRunningTasks);
}
return nonStartedItems;
}
public List<I> removeUnscheduledJobs() {
List<I> nonStartedItems = new ArrayList<>();
lock.lock();
try {
tracker.neverStartedItemsRemoved(queue.size());
nonStartedItems.addAll(queue);
queue.clear();
}
finally {
lock.unlock();
}
return nonStartedItems;
}
public void cancelScheduledJobs() {
List<FutureTaskMonitor<I, R>> tasksToBeCancelled = new ArrayList<>();
lock.lock();
try {
tasksToBeCancelled.addAll(taskSet);
}
finally {
lock.unlock();
}
for (FutureTaskMonitor<I, R> task : tasksToBeCancelled) {
task.cancel(true);
}
}
/**
* Cancels all running tasks and disposes of the internal thread pool if it is a private
* pool.
*/
public void dispose() {
cancelAllTasks(true);
if (threadPool.isPrivate()) {
threadPool.shutdownNow();
}
}
public boolean waitUntilDone(long timeout, TimeUnit unit) throws InterruptedException {
lock.lockInterruptibly();
try {
tracker.waitUntilDone(timeout, unit);
return tracker.isDone();
}
finally {
lock.unlock();
}
}
// This method adds jobs to the thread pool up to the maximum allowed at a time.
private void fillOpenProcessingSlots() {
while (!queue.isEmpty() && getInProgressCount() < maxInProgress) {
I item = queue.remove();
tracker.itemStarted();
CallbackCallable qCall = new CallbackCallable(item);
FutureTaskMonitor<I, R> task =
new FutureTaskMonitor<>(this, qCall, item, tracker.getNextID());
qCall.setFutureTask(task);
taskSet.add(task);
notifyTaskStarted(task);
threadPool.submit(task);
}
}
private void notifyTaskStarted(FutureTaskMonitor<I, R> task) {
QProgressListener<I> listener = progressListener;
if (listener == null) {
return;
}
lock.unlock();
try {
listener.taskStarted(task.getID(), task.getItem());
}
finally {
lock.lock();
}
}
private long getInProgressCount() {
return tracker.getItemsInProgressCount();
}
void itemProcessed(FutureTaskMonitor<I, R> task, QResult<I, R> result) {
if (itemListener != null) {
itemListener.itemProcessed(result);
}
lock.lock();
try {
taskSet.remove(task);
if (collectResults) {
resultList.add(result);
}
tracker.InProgressitemCompletedOrCancelled();
fillOpenProcessingSlots();
if (result.hasError() && unhandledException == null) {
unhandledException = result.getError();
}
}
finally {
lock.unlock();
}
QProgressListener<I> listener = progressListener;
if (listener != null) {
listener.taskEnded(task.getID(), task.getItem(), tracker.getTotalItemCount(),
tracker.getCompletedItemCount());
}
}
void progressChanged(long id, I item, long currentProgress) {
QProgressListener<I> listener = progressListener;
if (listener != null) {
listener.progressChanged(id, item, currentProgress);
}
}
void maxProgressChanged(long id, I item, long maxProgress) {
QProgressListener<I> listener = progressListener;
if (listener != null) {
listener.maxProgressChanged(id, item, maxProgress);
}
}
void progressModeChanged(long id, I item, boolean indeterminate) {
QProgressListener<I> listener = progressListener;
if (listener != null) {
listener.progressModeChanged(id, item, indeterminate);
}
}
void progressMessageChanged(long id, I item, String message) {
QProgressListener<I> listener = progressListener;
if (listener != null) {
listener.progressMessageChanged(id, item, message);
}
}
private class CallbackCallable implements Callable<R> {
private I item;
private FutureTaskMonitor<I, R> future;
CallbackCallable(I item) {
this.item = item;
}
@Override
public R call() throws Exception {
return callback.process(item, future);
}
void setFutureTask(FutureTaskMonitor<I, R> future) {
this.future = future;
}
}
private static class ChainedProgressListener<I> implements QProgressListener<I> {
private volatile QProgressListener<I> listener1;
private volatile QProgressListener<I> listener2;
ChainedProgressListener(QProgressListener<I> listener1, QProgressListener<I> listener2) {
this.listener1 = listener1;
this.listener2 = listener2;
}
QProgressListener<I> removeListener(QProgressListener<I> listener) {
if (listener1 == listener) {
return listener2;
}
else if (listener2 == listener) {
return listener1;
}
if (listener1 instanceof ChainedProgressListener) {
listener1 = ((ChainedProgressListener<I>) listener1).removeListener(listener);
}
if (listener2 instanceof ChainedProgressListener) {
listener2 = ((ChainedProgressListener<I>) listener2).removeListener(listener);
}
return this;
}
@Override
public void progressChanged(long id, I Item, long currentProgress) {
listener1.progressChanged(id, Item, currentProgress);
listener2.progressChanged(id, Item, currentProgress);
}
@Override
public void taskStarted(long id, I Item) {
listener1.taskStarted(id, Item);
listener2.taskStarted(id, Item);
}
@Override
public void taskEnded(long id, I Item, long totalCount, long completedCount) {
listener1.taskEnded(id, Item, totalCount, completedCount);
listener2.taskEnded(id, Item, totalCount, completedCount);
}
@Override
public void progressModeChanged(long id, I item, boolean indeterminate) {
listener1.progressModeChanged(id, item, indeterminate);
listener2.progressModeChanged(id, item, indeterminate);
}
@Override
public void progressMessageChanged(long id, I item, String message) {
listener1.progressMessageChanged(id, item, message);
listener2.progressMessageChanged(id, item, message);
}
@Override
public void maxProgressChanged(long id, I item, long maxProgress) {
listener1.maxProgressChanged(id, item, maxProgress);
listener2.maxProgressChanged(id, item, maxProgress);
}
}
/**
* Simple connector for traditional TaskMonitor and a task from the ConcurrentQ. This adapter
* adds a cancel listener to the TaskMonitor and when cancelled is called on the monitor,
* it cancels the currently running (scheduled on the thread pool) and leaves the waiting
* tasks alone. It also implements a QProgressListener and adds itselve to the concurrentQ so
* that it gets progress events and messages and sets them on the task monitor.
*
* @param <I>
* @param <R>
*/
private class QMonitorAdapter implements QProgressListener<I>, CancelledListener {
private TaskMonitor monitor;
public final boolean cancelClearsAllJobs;
QMonitorAdapter(TaskMonitor monitor, boolean cancelClearsAll) {
this.monitor = monitor;
cancelClearsAllJobs = cancelClearsAll;
addProgressListener(this);
monitor.addCancelledListener(this);
}
@Override
public void cancelled() {
if (cancelClearsAllJobs) {
cancelAllTasks(true);
}
else {
cancelScheduledJobs();
monitor.clearCanceled();
}
}
@Override
public void progressChanged(long id, I Item, long currentProgress) {
if (jobsReportProgress) {
monitor.setProgress(currentProgress);
}
}
@Override
public void progressModeChanged(long id, I item, boolean indeterminate) {
if (jobsReportProgress) {
monitor.setIndeterminate(indeterminate);
}
}
@Override
public void progressMessageChanged(long id, I item, String message) {
monitor.setMessage(message);
}
@Override
public void maxProgressChanged(long id, I item, long maxProgress) {
if (jobsReportProgress) {
monitor.setMaximum(maxProgress);
}
}
@Override
public void taskStarted(long id, I Item) {
// do nothing
}
@Override
public void taskEnded(long id, I Item, long total, long progress) {
if (!jobsReportProgress) {
if (total != monitor.getMaximum()) {
monitor.setMaximum(total);
}
monitor.setProgress(progress);
}
}
public void dispose() {
removeProgressListener(this);
monitor.removeCancelledListener(this);
monitor = TaskMonitor.DUMMY;
}
}
}

View file

@ -0,0 +1,222 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.concurrent;
import ghidra.util.task.TaskMonitor;
import ghidra.util.task.TaskMonitorAdapter;
import java.util.LinkedList;
import java.util.Queue;
import java.util.concurrent.LinkedBlockingQueue;
/**
* A helper class to build up the potentially complicated {@link ConcurrentQ}.
* <P>
* Note: you must supply either a {@link GThreadPool} instance or a thread pool name. Further,
* if you supply the name of a thread pool, then a private, non-shared pool will be used. If you
* wish to make use of a shared pool, then you need to create that thread pool yourself. See
* {@link GThreadPool#getSharedThreadPool(String)}.
*
* <P>
* Examples:
* <p>
* <pre>
* QCallback<I, R> callback = new AbstractQCallback<I, R>() {
* public R process(I item, TaskMonitor monitor) {
* // do work here...
* }
* };
*
* ConcurrentQBuilder<I, R> builder = new ConcurrentQBuilder<I, R>();
* builder.setThreadPoolName("Thread Pool Name");
* builder.setQueue(new PriorityBlockingQueue());
* concurrentQ = builder.build(callback);
*
* // OR, you can chain the builder calls:
* ConcurrentQBuilder<I, R> builder = new ConcurrentQBuilder<I, R>();
* queue = builder.setThreadPoolName("Thread Pool Name").
* setQueue(new PriorityBlockingQueue()).
* setMaxInProgress(1).
* build(callback);
*
* </pre>
* <p>
*
* Note: if you wish to take advantage of blocking when adding items to the {@link ConcurrentQ},
* see {@link #setQueue(Queue)}.
*
*
*
* @param <I> The type of the items to be processed.
* @param <R> The type of objects resulting from processing an item
*/
public class ConcurrentQBuilder<I, R> {
private Queue<I> queue;
private String threadPoolName;
private GThreadPool threadPool;
private QItemListener<I, R> listener;
private boolean collectResults;
private int maxInProgress;
private boolean jobsReportProgress = false;
private TaskMonitor monitor = TaskMonitorAdapter.DUMMY_MONITOR;
private boolean cancelClearsAllJobs = true;
/**
* Sets the queue to be used by the {@link ConcurrentQ}. If you would like advanced features,
* like a queue that blocks when too many items have been placed in it, then use an
* advanced queue here, such as a {@link LinkedBlockingQueue}.
* <p>
* Note: if you wish to take advantage of blocking when adding items to the {@link ConcurrentQ},
* then be sure to call the appropriate method, such as
* {@link ConcurrentQ#offer(java.util.Iterator)}.
*
* @param queue the queue to be used by the {@link ConcurrentQ}
* @return this builder
*/
public ConcurrentQBuilder<I, R> setQueue(Queue<I> queue) {
this.queue = queue;
return this;
}
/**
* Specifies the maximum number of items that can be process at a time.
* If this is set to 0, then the concurrent queue will attempt to execute as many
* items at a time as there are threads in the given threadPool. Setting
* this parameter to 1 will have the effect of guaranteeing that
* all times are processed one at a time in the order they were submitted.
* Any other positive value will run that many items concurrently,
* up to the number of available threads.
*
* @param max the max number of items to execute at one time; defaults to 0
* @return this builder instance
*/
public ConcurrentQBuilder<I, R> setMaxInProgress(int max) {
this.maxInProgress = max;
return this;
}
/**
* Sets the name to be used when creating a <b>private thread pool</b>. If you wish to use
* a <i>shared thread pool</i>, then you need to create that thread pool youself and call
* {@link #setThreadPool(GThreadPool)}.
*
* @param name the name of the thread pool.
* @return this builder instance
* @see GThreadPool#getSharedThreadPool(String)
*/
public ConcurrentQBuilder<I, R> setThreadPoolName(String name) {
threadPoolName = name;
return this;
}
/**
* Use the given thread pool for processing the work items. If you do not care to configure
* the thread pool used and you do not wish to make use of shared thread pools, then you
* can call {@link #setThreadPoolName(String)} instead of this method.
*
* @param threadPool the thread pool to use
* @return this builder instance
* @see GThreadPool#getSharedThreadPool(String)
*/
public ConcurrentQBuilder<I, R> setThreadPool(GThreadPool threadPool) {
this.threadPool = threadPool;
return this;
}
/**
* Specifies if the concurrent queue should collect the results as items are processed
* so they can be returned in a {@link ConcurrentQ#waitForResults()} call.
* @param collectResults true signals to collect the generated results; defaults to false
* @return this builder instance
*/
public ConcurrentQBuilder<I, R> setCollectResults(boolean collectResults) {
this.collectResults = collectResults;
return this;
}
/**
* True signals that the jobs run by the client wish to report progress. The default value
* is false.
* <p>
* The default of false is good for clients that have a known amount of work to be processed.
* In this case, a total count of work jobs is maintained by the queue. As items are
* completed, the queue will update the monitor provided to it at construction time to reflect
* the number of jobs completed as work is done. On the other hand, some clients have
* known known number of jobs to complete, but simply add work to the queue as it arrives.
* In that case, the client should update its monitor for progress, as the queue cannot
* do so in a meaningful way.
*
* @param reportsProgress true signals that the client will update progress; false signals
* that the queue should do so
* @return this builder instance
*/
public ConcurrentQBuilder<I, R> setJobsReportProgress(boolean reportsProgress) {
this.jobsReportProgress = reportsProgress;
return this;
}
public ConcurrentQBuilder<I, R> setListener(QItemListener<I, R> listener) {
this.listener = listener;
return this;
}
public ConcurrentQBuilder<I, R> setMonitor(TaskMonitor monitor) {
this.monitor = monitor;
return this;
}
/**
* @see {@link ConcurrentQ#setMonitor(TaskMonitor, boolean)}
* <p>
* The default value is <tt>true</tt>.
*/
public ConcurrentQBuilder<I, R> setCancelClearsAllJobs(boolean clearAllJobs) {
this.cancelClearsAllJobs = clearAllJobs;
return this;
}
public ConcurrentQ<I, R> build(QCallback<I, R> callback) {
ConcurrentQ<I, R> concurrentQ =
new ConcurrentQ<I, R>(callback, getQueue(), getThreadPool(), listener, collectResults,
maxInProgress, jobsReportProgress);
if (monitor != null) {
concurrentQ.setMonitor(monitor, cancelClearsAllJobs);
}
return concurrentQ;
}
private GThreadPool getThreadPool() {
if (threadPool != null) {
return threadPool;
}
if (threadPoolName != null) {
return GThreadPool.getPrivateThreadPool(threadPoolName);
}
throw new IllegalStateException("Must either set a GThreadPool or set a thread pool name");
}
private Queue<I> getQueue() {
if (queue != null) {
return queue;
}
return new LinkedList<I>();
}
}

View file

@ -0,0 +1,236 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.concurrent;
import java.util.concurrent.Callable;
import java.util.concurrent.FutureTask;
import ghidra.util.Issue;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.*;
/**
* This is the FutureTask that will be used to call the {@link QCallback} to work on
* an item from a ConcurrentQ. It has been overridden to serve as an individual
* TaskMonitor for the task as well as notifying the ConcurrentQ when a task
* has been completed or cancelled so that additional tasks can be sent to the
* thread pool.
* <P>
* If it was cancelled, then the done() callback will occur in the thread that cancelled this
* task, otherwise it will be called by the thread from the thread pool that
* executed the task. Note that when this task is cancelled, it is up to the
* executing thread to check if it was cancelled and terminate the task execution gracefully.
* Even if the executing task never checks the cancelled and completes the task,
* the return value will be ignored as this task has already been considered done
* and any threads waiting on the return will have already been told it was cancelled.
* <P>
* On ConcurrentQs that only allow one task to run at a time, when a task is cancelled,
* the next task can begin. Most likely, the thread that was running the cancelled
* task won't be free, and a new thread will be used to start running the next task.
*/
class FutureTaskMonitor<I, R> extends FutureTask<R> implements TaskMonitor {
private final ConcurrentQ<I, R> queue;
private final I item;
private final long id;
private volatile long currentProgress;
private volatile long maxProgress;
private volatile CancelledListener cancelledListener;
FutureTaskMonitor(ConcurrentQ<I, R> queue, Callable<R> callable, I item, long id) {
super(callable);
this.queue = queue;
this.id = id;
this.item = item;
}
I getItem() {
return item;
}
long getID() {
return id;
}
@Override
public void run() {
super.run();
QResult<I, R> result = new QResult<>(item, this);
queue.itemProcessed(this, result);
}
@Override
public void setMaximum(long max) {
this.maxProgress = max;
queue.maxProgressChanged(id, item, max);
}
@Override
public void incrementProgress(long incrementAmount) {
currentProgress += incrementAmount;
queue.progressChanged(id, item, currentProgress);
}
@Override
public void setProgress(long value) {
currentProgress = value;
queue.progressChanged(id, item, currentProgress);
}
@Override
public void checkCanceled() throws CancelledException {
if (isCancelled()) {
throw new CancelledException();
}
}
@Override
public void setMessage(String message) {
queue.progressMessageChanged(id, item, message);
}
@Override
public void initialize(long max) {
currentProgress = 0;
maxProgress = max;
queue.maxProgressChanged(id, item, max);
queue.progressChanged(id, item, currentProgress);
}
@Override
public long getMaximum() {
return maxProgress;
}
@Override
public void setShowProgressValue(boolean showProgressValue) {
// nothing to do
}
@Override
public void setIndeterminate(boolean indeterminate) {
queue.progressModeChanged(id, item, indeterminate);
}
@Override
public long getProgress() {
return currentProgress;
}
@Override
public void reportIssue(Issue issue) {
// TODO
}
@Override
public boolean cancel(boolean mayInterruptIfRunning) {
boolean result = super.cancel(mayInterruptIfRunning);
// copy into temp variable so that the null check and the call are atomic.
CancelledListener listener = this.cancelledListener;
if (listener != null) {
listener.cancelled();
}
return result;
}
@Override
public void cancel() {
cancel(true);
}
@Override
public void setCancelEnabled(boolean enable) {
throw new UnsupportedOperationException();
}
@Override
public boolean isCancelEnabled() {
return true;
}
@Override
public void clearCanceled() {
throw new UnsupportedOperationException();
}
@Override
public synchronized void addCancelledListener(CancelledListener listener) {
if (cancelledListener == null) {
cancelledListener = listener;
}
else {
cancelledListener = new ChainedCancelledListener(cancelledListener, listener);
}
}
@Override
public synchronized void removeCancelledListener(CancelledListener listener) {
if (cancelledListener == listener) {
cancelledListener = null;
}
else if (cancelledListener instanceof ChainedCancelledListener) {
cancelledListener =
((ChainedCancelledListener) cancelledListener).removeListener(listener);
}
}
@Override
public void addIssueListener(IssueListener listener) {
throw new UnsupportedOperationException();
}
@Override
public void removeIssueListener(IssueListener listener) {
throw new UnsupportedOperationException();
}
private static class ChainedCancelledListener implements CancelledListener {
private volatile CancelledListener listener1;
private volatile CancelledListener listener2;
public ChainedCancelledListener(CancelledListener listener1, CancelledListener listener2) {
this.listener1 = listener1;
this.listener2 = listener2;
}
public CancelledListener removeListener(CancelledListener listener) {
if (listener1 == listener) {
return listener2;
}
else if (listener2 == listener) {
return listener1;
}
if (listener1 instanceof ChainedCancelledListener) {
listener1 = ((ChainedCancelledListener) listener1).removeListener(listener);
}
if (listener2 instanceof ChainedCancelledListener) {
listener2 = ((ChainedCancelledListener) listener2).removeListener(listener);
}
return this;
}
@Override
public void cancelled() {
listener1.cancelled();
listener2.cancelled();
}
}
}

View file

@ -0,0 +1,254 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.concurrent;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import generic.util.NamedDaemonThreadFactory;
import ghidra.util.SystemUtilities;
/**
* Class for managing and sharing thread pools. The GThreadPool is simplified version of the
* ThreadPoolExecutor, which can be confusing to use with its many configuration parameters.
* The GThreadPool has a simple behavior that is controlled by only two configuration parameters -
* the minimum number of threads and the maximum number of threads.
* <p>
* The simple behavior for when new tasks are submitted:<br>
* 1) If there any idle threads, use that thread.<br>
* 2) If all existing threads are busy and the number of threads is less than max threads, add a
* new thread and use it.<br>
* 3) if all threads are busy and there are max number of threads, queue the item until a thread
* becomes free.<br>
* <p>
* The simple behavior for when tasks are completed by a thread:<br>
* 1) If there are tasks in the queue, start processing a new item in the newly freed thread.<br>
* 2) if there are more threads that min threads, allow this thread to die if no new
* jobs arrive before
* the "KEEP ALIVE" time expires which is currently 15 seconds.<br>
* 3) if there are min threads or less, allow this thread to wait forever for a new job
* to arrive.<br>
*/
public class GThreadPool {
private static final long DEFAULT_KEEP_ALIVE = 15;
private static Map<String, GThreadPool> sharedPoolMap = new HashMap<>();
private final String name;
private final GThreadPoolExecutor executor;
/**
* Creates a new, private thread pool with the given name.
* @param name the name of the thread pool
* @return a private GThreadPool with the given name.
*/
public static GThreadPool getPrivateThreadPool(String name) {
return new GThreadPool(name);
}
/**
* Returns a shared GThreadPool. If a shared GThreadPool already exists with the given name,
* it is returned. Otherwise, a new shared GThreadPool is created and returned.
* @param name the name of the GThreadPool.
* @return a shared GThreadPool with the given name.
*/
public static GThreadPool getSharedThreadPool(String name) {
GThreadPool threadPool = sharedPoolMap.get(name);
if (threadPool == null) {
threadPool = new GThreadPool(name);
}
return threadPool;
}
private GThreadPool(String name) {
this.name = name;
executor = new GThreadPoolExecutor();
sharedPoolMap.put(name, this);
}
/**
* Sets the max number of threads to use in this thread pool. The default is the number
* of processors + 1.
* @param maxThreadCount the maximum number of threads to use in this thread pool.
*/
public void setMaxThreadCount(int maxThreadCount) {
executor.setMaxThreadCount(maxThreadCount);
}
/**
* Returns the minimum number of threads to keep alive in this thread pool.
* @return the minimum number of threads to keep alive in this thread pool.
*/
public int getMinThreadCount() {
return executor.getMinThreadCount();
}
/**
* Sets the minimum number of threads to keep alive in this thread pool.
* @param minThreadCount the minimum number of threads to keep alive in this thread pool.
*/
public void setMinThreadCount(int minThreadCount) {
executor.setMinThreadCount(minThreadCount);
}
/**
* Returns the maximum number of threads to use in this thread pool.
* @return the maximum number of threads to use in this thread pool.
*/
public int getMaxThreadCount() {
return executor.getMaxThreadCount();
}
/**
* Submits a FutreTask to be executed by a thread in this thread pool.
* @param futureTask the future task to be executed.
*/
public void submit(FutureTask<?> futureTask) {
executor.execute(futureTask);
}
/**
* Submits a runnable to be executed by this thread pool.
* @param task the runnable to be executed.
* @return a Future for that runnable.
*/
public Future<?> submit(Runnable task) {
return executor.submit(task);
}
/**
* Submits a runnable to be executed by this thread pool.
* @param task the runnable to be executed.
* @param result the result to be returned after the runnable has executed.
* @return a Future for that runnable.
*/
public <T> Future<T> submit(Runnable task, T result) {
return executor.submit(task, result);
}
/**
* Submits a callable to be executed by this thread pool.
* @param task the callable to be executed.
* @return a Future for that callable.
*/
public <T> Future<T> submit(Callable<T> task) {
return executor.submit(task);
}
public void shutdownNow() {
executor.shutdownNow();
}
/**
* Returns true if this is not a shared thread pool.
*
* @return true if this is not a shared thread pool.
*/
public boolean isPrivate() {
return !sharedPoolMap.containsKey(name);
}
/**
* Returns the {@link Executor} used by this thread pool.
*
* <P>Note: normal usage of this thread pool contraindicates accessing the executor of
* this pool. For managing your own jobs, you should use the method on this class directly.
* The intent of this method is to provide access to the executor so that it may be
* passed to other asynchronous APIs, such as the {@link CompletableFuture}.
*
* @return the executor
*/
public GThreadPoolExecutor getExecutor() {
return executor;
}
//==================================================================================================
// Inner Classes
//==================================================================================================
private class GThreadPoolExecutor extends ThreadPoolExecutor {
private volatile int maxThreadCount = SystemUtilities.getDefaultThreadPoolSize();
private volatile int minThreadCount = 0;
private AtomicInteger taskCount = new AtomicInteger();
public GThreadPoolExecutor() {
super(0, Integer.MAX_VALUE, DEFAULT_KEEP_ALIVE, TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>(), new NamedDaemonThreadFactory(name));
}
public int getMinThreadCount() {
return minThreadCount;
}
public void setMinThreadCount(int minThreadCount) {
this.minThreadCount = Math.max(minThreadCount, 0);
}
public int getMaxThreadCount() {
return maxThreadCount;
}
public void setMaxThreadCount(int maxThreadCount) {
this.maxThreadCount = Math.max(maxThreadCount, 1);
}
@Override
public void execute(Runnable command) {
growPoolIfNeeded();
super.execute(command);
}
private void growPoolIfNeeded() {
int count = taskCount.incrementAndGet();
int corePoolSize = getCorePoolSize();
if (corePoolSize >= maxThreadCount) {
return;
}
if (isCurrentThreadInThisThreadPool()) {
count--;
}
if (count > corePoolSize) {
setCorePoolSize(corePoolSize + 1);
}
}
private boolean isCurrentThreadInThisThreadPool() {
return Thread.currentThread().getName().startsWith(name);
}
private void shrinkPoolIfNotBusy() {
int count = taskCount.decrementAndGet();
int corePoolSize = getCorePoolSize();
if (corePoolSize <= minThreadCount) {
return;
}
if (count < corePoolSize) {
setCorePoolSize(corePoolSize - 1);
}
}
@Override
protected void afterExecute(Runnable r, Throwable t) {
shrinkPoolIfNotBusy();
super.afterExecute(r, t);
}
}
}

View file

@ -0,0 +1,178 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.concurrent;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
/**
* A class to synchronize and track the progress of the items being processed by a concurrentQ. It
* provides various wait methods for when one item is completed or all items are completed.
*/
class ProgressTracker {
private long totalCount;
private long inProgressCount;
private long completedOrCancelledCount;
private long nextID = 0;
private final ReentrantLock lock;
private final Condition done; // notifies when all items have been completed or cancelled
private final Condition itemCompleted; // notifies when an item is processed or cancelled
ProgressTracker(ReentrantLock lock) {
this.lock = lock;
done = lock.newCondition();
itemCompleted = lock.newCondition();
}
void itemsAdded(int n) {
lock.lock();
try {
totalCount += n;
}
finally {
lock.unlock();
}
}
void itemStarted() {
lock.lock();
try {
inProgressCount++;
}
finally {
lock.unlock();
}
}
void InProgressitemCompletedOrCancelled() {
lock.lock();
try {
completedOrCancelledCount++;
inProgressCount--;
if (isDone()) {
done.signalAll();
}
itemCompleted.signalAll();
}
finally {
lock.unlock();
}
}
void neverStartedItemsRemoved(int n) {
lock.lock();
try {
completedOrCancelledCount += n;
if (isDone()) {
done.signalAll();
}
itemCompleted.signalAll();
}
finally {
lock.unlock();
}
}
public long getCompletedItemCount() {
lock.lock();
try {
return completedOrCancelledCount;
}
finally {
lock.unlock();
}
}
public long getTotalItemCount() {
lock.lock();
try {
return totalCount;
}
finally {
lock.unlock();
}
}
public long getItemsInProgressCount() {
lock.lock();
try {
return inProgressCount;
}
finally {
lock.unlock();
}
}
boolean isDone() {
lock.lock();
try {
return completedOrCancelledCount == totalCount;
}
finally {
lock.unlock();
}
}
void waitUntilDone() throws InterruptedException {
lock.lockInterruptibly();
try {
while (!isDone()) {
done.await();
}
}
finally {
lock.unlock();
}
}
void waitForNext() throws InterruptedException {
lock.lockInterruptibly();
try {
if (!isDone()) {
itemCompleted.await();
}
}
finally {
lock.unlock();
}
}
public boolean waitUntilDone(long timeout, TimeUnit unit) throws InterruptedException {
lock.lockInterruptibly();
try {
if (!isDone()) {
done.await(timeout, unit);
}
return isDone();
}
finally {
lock.unlock();
}
}
public long getNextID() {
lock.lock();
try {
return ++nextID;
}
finally {
lock.unlock();
}
}
}

View file

@ -0,0 +1,42 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.concurrent;
import ghidra.util.task.TaskMonitor;
/**
* Interface that defines the callback to work on the items given to the
* {@link ConcurrentQ#add(I)} methods. Each item that is processed will be handed to the
* {@link #process(I, TaskMonitor)} method of the implementing class.
*
* @param <I> The type of the items to be processed.
* @param <R> The type of objects resulting from processing an item; if you don't care about the
* return value, then make this value whatever you want, like <tt>Object</tt> or the
* same value as {@link I} and return null from {@link #process(Object, TaskMonitor)}.
*/
public interface QCallback<I, R> {
/**
* Processes the given item in background thread provided by a GThreadPool.
* @param item the item to process.
* @param monitor a monitor that can be used to check for cancellation and to report progress and
* transient messages.
* @return The return value resulting from processing the item.
*/
public R process(I item, TaskMonitor monitor) throws Exception;
}

View file

@ -0,0 +1,35 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.concurrent;
/**
* Callback for when items have completed processing. It is also called if an item is cancelled
* or had an error condition.
*
* @param <I> The type for the items being processed.
* @param <R> The type for result object returned from the QWorkers process method.
*/
public interface QItemListener<I, R> {
/**
* Callback for when a item has completed processing, regardless of whether or not the item
* process normally, was cancelled, or encountered an error during processing.
* @param result the QResult object.
*/
public void itemProcessed(QResult<I, R> result);
}

View file

@ -0,0 +1,75 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.concurrent;
/**
* Interface for listeners who want progress and transient message information from QWorkers while
* processing items.
*/
public interface QProgressListener<I> {
/**
* Notification that progress has changed during the processing of an item.
* @param id the id of the item being processed. Since multiple items can be processed concurrently,
* the id can be used to "demultiplex" the progress and messages being generated.
* @param currentProgress the current value of the progress for this task.
* @param progressMessage the last message set for this task.s
* @param item the item that was being processed when the worker changed the max progress.
*/
void progressChanged(long id, I Item, long currentProgress);
/**
* Notification that a new task has been generated to process an item.
* @param id the id of the item being processed.
* @param item the item that was being processed when the worker changed the max progress.
*/
void taskStarted(long id, I Item);
/**
* Notification that a new task has completed processing for an item.
* @param id the id of the item that has completed processing.
* @param item the item that was being processed when the worker changed the max progress.
* @param totalCount the total number of items that have been submitted to the ConcurrentQ
* @param completedCount the total number of items that completed processing.
*/
void taskEnded(long id, I Item, long totalCount, long completedCount);
/**
* Notification that the progress mode has changed from/to indeterminate mode
* @param id the id of the item that has completed processing.
* @param item the item that was being processed when the worker changed the max progress.
* @param indeterminate
*/
void progressModeChanged(long id, I item, boolean indeterminate);
/**
*
* @param id the id of the item that has completed processing.
* @param item the item that was being processed when the worker changed the max progress.
* @param message
*/
void progressMessageChanged(long id, I item, String message);
/**
* Notification the the max progress value has changed.
* @param id the id of the item that has completed processing.
* @param item the item that was being processed when the worker changed the max progress.
* @param maxProgress the max value of the progress for this task.
*/
void maxProgressChanged(long id, I item, long maxProgress);
}

View file

@ -0,0 +1,95 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.concurrent;
import java.util.concurrent.CancellationException;
import java.util.concurrent.Future;
/**
* Class for holding the result of processing an Item in a ConcurrentQ.
*
* @param <I> the type of the items in the ConcurrentQ.
* @param <R> the type of objects returned from processing an item in a ConcurrentQ.
*/
public class QResult<I, R> {
private final I item;
private final R result;
private final Exception error;
public QResult(I item, Future<R> future) {
this.item = item;
R tempResult;
Exception tempError;
try {
tempResult = future.get();
tempError = null;
}
catch (Exception e) {
tempError = e;
tempResult = null;
}
result = tempResult;
error = tempError;
}
/**
* Returns the item that was processed.
* @return the item that was processed.
*/
public I getItem() {
return item;
}
/**
* The result from processing the item. Will be null if the item was cancelled or had an error.
*
* @return the result from processing the item or null if it did not complete successfully.
* @throws Exception any exception that was thrown during the processing of the input item
*/
public R getResult() throws Exception {
if (hasError()) {
throw error;
}
return result;
}
/**
* Returns any Exception that was encountered during processing of the item
* @return any Exception that was encountered during processing of the item
*/
public Exception getError() {
return hasError() ? error : null;
}
/**
* Returns true if the item encountered an error while processing the item.
* @return true if the item encountered an error while processing the item.
*/
public boolean hasError() {
return error != null && !(error instanceof CancellationException);
}
/**
* Returns true if the item's processing was cancelled.
* @return true if the item's processing was cancelled.
*/
public boolean isCancelled() {
return error instanceof CancellationException;
}
}

View file

@ -0,0 +1,38 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.concurrent;
import ghidra.util.task.TaskMonitor;
/**
* Interface that defines the Runnable to work on the items given to the
* {@link ConcurrentQ#add(I)} methods. Each item that is processed will be handed to the
* {@link #run(I, TaskMonitor)} method of the implementing class.
*
* @param <I> The type of the items to be processed.
*/
public interface QRunnable<I> {
/**
* Processes the given item in background thread provided by a GThreadPool.
* @param item the item to process.
* @param monitor a monitor that can be used to check for cancellation and to report progress and
* transient messages.
*/
public void run(I item, TaskMonitor monitor) throws Exception;
}

View file

@ -0,0 +1,35 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.concurrent;
import ghidra.util.task.TaskMonitor;
public class QRunnableAdapter<I> implements QCallback<I, Object> {
private QRunnable<I> runnable;
public QRunnableAdapter(QRunnable<I> runnable) {
this.runnable = runnable;
}
@Override
public Object process(I item, TaskMonitor monitor) throws Exception {
runnable.run(item, monitor);
return null;
}
}

View file

@ -0,0 +1,72 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.constraint;
/**
* Constraints are used to make decisions to traverse a decision tree where each node in the
* tree has a constraint that is used to decide if that node is part of the successful decision path.
*
* @param <T> The type of object that decisions will be made.
*/
public abstract class Constraint<T> {
private String name;
/**
* Constructor takes the name of the constraint. This name will be tag used in the XML
* specification file.
* @param name the name of the constraint
*/
public Constraint(String name) {
this.name = name;
}
/**
* Returns the name of the constraint. Note: this name is also the XML tag used in the
* constraints specification files.
* @return the name of the constraint
*/
public final String getName() {
return name;
}
/**
* Returns true if the given object satisfies this constraint.
* @param the object to test this constraint on.
* @return true if the given object satisfies this constraint.
*/
public abstract boolean isSatisfied(T t);
/**
* Initialized this constraint state. Attributes in the xml element with this constaints
* tag name will be extracted into the ConstraintData object for easy retrieval.
* @param data the ConstraintData object used to initialize this constraint.
*/
public abstract void loadConstraintData(ConstraintData data);
@Override
// overridden because it is critical that constraint object override equals.
public abstract boolean equals(Object obj);
/**
* Returns a description of this constraint (with its configuration data) to be used
* to journal the decision path that was taken.
* @return a description of this constraint with its configuration data.
*/
public abstract String getDescription();
}

View file

@ -0,0 +1,111 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.constraint;
import ghidra.util.xml.XmlAttributeException;
import java.util.HashMap;
import java.util.Map;
/**
* Convenience class that converts XML attributes into typed property values.
*/
public class ConstraintData {
private Map<String, String> map = new HashMap<String, String>();
public ConstraintData(Map<String, String> mappings) {
map.putAll(mappings);
}
public String getString(String name) {
return getValue(name, "string");
}
public boolean hasValue(String name) {
return map.containsKey(name);
}
public int getInt(String name) {
String value = getValue(name, "int");
try {
return Integer.parseInt(value);
}
catch (NumberFormatException e) {
throw new XmlAttributeException("Expected int value for attribute \"" + name +
"\", but was \"" + value + "\"");
}
}
public long getLong(String name) {
String value = getValue(name, "long");
try {
return Long.parseLong(value);
}
catch (NumberFormatException e) {
throw new XmlAttributeException("Expected long value for attribute \"" + name +
"\", but was \"" + value + "\"");
}
}
public boolean getBoolean(String name) {
String value = getValue(name, "boolean");
value = value.toLowerCase();
if (value.equals("true")) {
return true;
}
if (value.equals("false")) {
return false;
}
throw new XmlAttributeException("Expected boolean value for attribute \"" + name +
"\", but was \"" + value + "\"");
}
public float getFloat(String name) {
String value = getValue(name, "float");
try {
return Float.parseFloat(value);
}
catch (NumberFormatException e) {
throw new XmlAttributeException("Expected float value for attribute \"" + name +
"\", but was \"" + value + "\"");
}
}
public double getDouble(String name) {
String value = getValue(name, "double");
try {
return Double.parseDouble(value);
}
catch (NumberFormatException e) {
throw new XmlAttributeException("Expected double value for attribute \"" + name +
"\", but was \"" + value + "\"");
}
}
private String getValue(String name, String type) {
String value = map.get(name);
if (value == null) {
throw new XmlAttributeException("Missing " + type + " value for attribute \"" + name +
"\"");
}
return value;
}
}

View file

@ -0,0 +1,78 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.constraint;
import java.util.List;
/**
* Result object from getting values that match the constraints for given test object.
*/
public class Decision {
private String value;
private List<String> decisionPath;
private String source;
public Decision(String value, List<String> decisionPath, String source) {
this.value = value;
this.decisionPath = decisionPath;
this.source = source;
}
/**
* Returns the value of the property for which this decision matched the constraints
* @return the value of the property for which this decision matched the constraints
*/
public String getValue() {
return value;
}
/**
* Returns the constraint source file that added the value for this decision.
* @return the constraint source file that added the value for this decision.
*/
public String getSource() {
return source;
}
/**
* Returns a list of strings where each string is a description of the constraint that passed
* to reach this decision.
* @return a list of strings where each string is a description of the constraint that passed
* to reach this decision.
*/
public List<String> getDecisionPath() {
return decisionPath;
}
/**
* Returns a string that is a description of the constraints that passed
* to reach this decision.
* @return a string that is a description of the constraints that passed
* to reach this decision.
*/
public String getDescisionPathString() {
StringBuilder builder = new StringBuilder();
for (String string : decisionPath) {
builder.append(string);
builder.append("\n");
}
return builder.toString();
}
}

View file

@ -0,0 +1,109 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.constraint;
import java.util.*;
import org.apache.commons.collections4.map.HashedMap;
import ghidra.xml.XmlParseException;
/**
* A node in a decision tree. Each node contains exactly one constraint and a map of property
* values.
*
* @param <T> the type of objects that the constraint operates on.
*/
public class DecisionNode<T> {
private Map<String, PropertyValue> propertyMap = new HashedMap<String, PropertyValue>();
private Constraint<T> constraint;
private List<DecisionNode<T>> children = new ArrayList<DecisionNode<T>>();
private DecisionNode<T> parent;
public DecisionNode(Constraint<T> constraint, DecisionNode<T> parent) {
this.constraint = constraint;
this.parent = parent;
}
public DecisionNode<T> getOrCreateNodeForContraint(Constraint<T> newConstraint) {
for (DecisionNode<T> child : children) {
if (newConstraint.equals(child.constraint)) {
return child;
}
}
DecisionNode<T> newChild = new DecisionNode<T>(newConstraint, this);
children.add(newChild);
return newChild;
}
public void setProperty(String propertyName, String value, String source)
throws XmlParseException {
if (propertyMap.containsKey(propertyName)) {
throw new XmlParseException("Attempted to overwrite property value for " +
propertyName + " in contraint node: " + this);
}
propertyMap.put(propertyName, new PropertyValue(value, source));
}
public boolean populateDecisions(T t, DecisionSet decisionSet, String propertyName) {
if (!constraint.isSatisfied(t)) {
return false;
}
boolean decisionFound = false;
for (DecisionNode<T> child : children) {
decisionFound |= child.populateDecisions(t, decisionSet, propertyName);
}
// if no child found a more specific decision, see if we have a value for the property
if (!decisionFound && propertyMap.containsKey(propertyName)) {
PropertyValue value = propertyMap.get(propertyName);
List<String> decisionPath = getDecisionPath();
decisionSet.addDecision(new Decision(value.value, decisionPath, value.source));
decisionFound = true;
}
return decisionFound;
}
protected List<String> getDecisionPath() {
List<String> decisionPath = parent.getDecisionPath();
decisionPath.add(constraint.getDescription());
return decisionPath;
}
@Override
public String toString() {
StringBuffer buf = new StringBuffer();
List<String> decisionPath = getDecisionPath();
for (String string : decisionPath) {
buf.append("/");
buf.append(string);
}
return buf.toString();
}
static class PropertyValue {
String value;
String source;
PropertyValue(String value, String source) {
this.value = value;
this.source = source;
}
}
}

View file

@ -0,0 +1,76 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.constraint;
import java.util.ArrayList;
import java.util.List;
/**
* The result object returned from a scan of a decision tree looking for property values that
* match the constrains for some test object.
*/
public class DecisionSet {
private List<Decision> decisionList = new ArrayList<Decision>();
private String propertyName;
public DecisionSet(String propertyName) {
this.propertyName = propertyName;
}
/**
* Returns a list of all the decisions whose descision path constraints matched the given
* test object.
* @return a list of all the decisions whose descision path constraints matched the given
* test object.
*/
public List<Decision> getDecisions() {
return decisionList;
}
/**
* Returns a list of property values from decision paths that matched the constraints.
* @return a list of property values from decision paths that matched the constraints.
*/
public List<String> getValues() {
List<String> values = new ArrayList<String>(decisionList.size());
for (Decision decision : decisionList) {
values.add(decision.getValue());
}
return values;
}
/**
* Returns the name of the property that was scanned for in the decision tree.
* @return the name of the property that was scanned for in the decision tree.
*/
public String getDecisionPropertyName() {
return propertyName;
}
void addDecision(Decision decision) {
decisionList.add(decision);
}
/**
* Returns true if this decisionSet has no results.
* @return true if this decisionSet has no results.
*/
public boolean isEmpty() {
return decisionList.isEmpty();
}
}

View file

@ -0,0 +1,204 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.constraint;
import java.io.*;
import java.util.*;
import org.xml.sax.*;
import generic.jar.ResourceFile;
import ghidra.xml.*;
/**
* A decisionTree is used to find property values that are determined by traversing a tree
* of constraints. Each node in the tree has an associated constraint. If the constraint is
* satisfied for a given test object, then its child nodes are tested to find more and more
* specific results. When either there are no children in a node or none of the children's
* constraints are satisfied or by traversing those that are satisfied did not result in find
* a property match, the current node is check to see if it has a value for the property being
* search. If so, that result is added as a Decision.
*
* <P> There can be multiple paths where all constraints a matched resulting in multiple possible
* decisions.</P>
* <P> A non-leaf node can have properties as well, that serve as a default if it's constraint
* is satisfied, but not of its children is satisfied or resulted in a decision.</P>
*
* @param <T> the type of object that the constraints are checked against.
*/
public class DecisionTree<T> {
private DecisionNode<T> root;
private Map<String, Class<? extends Constraint<T>>> constraintClassMap;
private Set<String> propertyNameSet;
public DecisionTree() {
root = new RootDecisionNode<T>();
constraintClassMap = new HashMap<String, Class<? extends Constraint<T>>>();
propertyNameSet = new HashSet<String>();
}
/**
* Searches the decision tree for values of given property name that match the constraints
* within this tree.
* @param testObject the object that the constraints are test against.
* @param propertyName the name of the property whose values are being collected.
* @return a DecisionSet containing all the values of the given property whose path in the
* tree matched all the constraints for the given test object.
*/
public DecisionSet getDecisionsSet(T testObject, String propertyName) {
DecisionSet decisionSet = new DecisionSet(propertyName);
root.populateDecisions(testObject, decisionSet, propertyName);
return decisionSet;
}
/**
* Registers a constraint class to be recognized from an xml constraint specification file.
* @param name the name of the constraint which is also the xml tag value.
* @param constraintClass the constraint type which will be initialized from the xml constraint
* specification file.
*/
public void registerConstraintType(String name, Class<? extends Constraint<T>> constraintClass) {
constraintClassMap.put(name, constraintClass);
}
/**
* Registers a property name. Every tag in an xml constraint file (except the root tag which
* is unused) must be either a constraint name or a property name.
* @param propertyName the name of a valid property to be expected in an xml constraints file.
*/
public void registerPropertyName(String propertyName) {
propertyNameSet.add(propertyName);
}
/**
* Loads the tree from an xml data contained within an input stream. Note: this method can be
* called multiple times, with each call appending to the existing tree.
* @param name the name of the input source so that decisions can be traced back to
* the appropriate xml constraints source.
* @param stream the InputStream from which to read an xml constraints specification.
* @throws IOException if an I/O problem occurs reading from the stream.
* @throws XmlParseException if the XML is not property formatted or a tag that is not
* a constraint name or property name is encountered.
*/
public void loadConstraints(String name, InputStream stream) throws IOException,
XmlParseException {
XmlPullParser parser;
try {
parser = new NonThreadedXmlPullParserImpl(stream, name, new XMLErrorHandler(), false);
}
catch (SAXException e) {
throw new XmlParseException("Sax Exception", e);
}
parser.next(); // skip root element start
processSubContraintsAndProperties(root, parser);
}
/**
* Loads the tree from an xml constraint file. Note: this method can be called multiple times,
* with each call appending to the existing tree.
* @param file the file that contains the xml for the constraint.
* @throws IOException if an I/O problem occurs reading from the stream.
* @throws XmlParseException if the XML is not property formatted or a tag that is not
* a constraint name or property name is encountered.
*/
public void loadConstraints(ResourceFile file) throws FileNotFoundException, IOException,
XmlParseException {
InputStream inputStream = file.getInputStream();
String name = file.getName();
loadConstraints(name, inputStream);
inputStream.close();
}
private void processSubContraintsAndProperties(DecisionNode<T> parent, XmlPullParser parser)
throws XmlParseException {
XmlElement element = parser.next();
while (!element.isEnd()) {
Constraint<T> constraint = readConstraint(element);
if (constraint != null) {
DecisionNode<T> node = parent.getOrCreateNodeForContraint(constraint);
processSubContraintsAndProperties(node, parser);
}
else if (propertyNameSet.contains(element.getName())) {
processPropertyElement(parent, element, parser);
}
else {
throw new XmlParseException("Unknown element tag: " + element.getName());
}
element = parser.next();
}
}
private Constraint<T> getConstraint(String name) throws XmlParseException {
Class<? extends Constraint<T>> constraintClass = constraintClassMap.get(name);
if (constraintClass == null) {
return null;
}
try {
return constraintClass.newInstance();
}
catch (Exception e) {
throw new XmlParseException(
"Can't create constraint instance for class " + constraintClass.getName(), e);
}
}
private void processPropertyElement(DecisionNode<T> node, XmlElement element,
XmlPullParser parser) throws XmlParseException {
String propertyName = element.getName();
XmlElement nextElement = parser.next();
if (!nextElement.isEnd()) {
throw new XmlParseException("Expected end tag for property " + propertyName);
}
node.setProperty(propertyName, nextElement.getText(), parser.getName());
}
private Constraint<T> readConstraint(XmlElement element) throws XmlParseException {
String name = element.getName();
Constraint<T> constraint = getConstraint(name);
if (constraint == null) {
return null;
}
constraint.loadConstraintData(new ConstraintData(element.getAttributes()));
return constraint;
}
private static class XMLErrorHandler implements ErrorHandler {
@Override
public void error(SAXParseException exception) throws SAXException {
throw new SAXException("Error: " + exception);
}
@Override
public void fatalError(SAXParseException exception) throws SAXException {
throw new SAXException("Fatal error: " + exception);
}
@Override
public void warning(SAXParseException exception) throws SAXException {
throw new SAXException("Warning: " + exception);
}
}
}

View file

@ -0,0 +1,66 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.constraint;
import java.util.ArrayList;
import java.util.List;
/**
* Special root node for a decision tree. Root nodes don't have a real constraint, so
* a dummy constraint that is always satisfied is used.
*
* @param <T> the type of object used to test constraints.s
*/
public class RootDecisionNode<T> extends DecisionNode<T> {
public RootDecisionNode() {
super(new DummyConstraint<T>(), null);
}
@Override
protected List<String> getDecisionPath() {
return new ArrayList<String>();
}
private static class DummyConstraint<T> extends Constraint<T> {
public DummyConstraint() {
super("");
}
@Override
public boolean isSatisfied(T t) {
return true;
}
@Override
public void loadConstraintData(ConstraintData data) {
// nothing to load
}
@Override
public boolean equals(Object obj) {
return this == obj;
}
@Override
public String getDescription() {
return null;
}
}
}

View file

@ -0,0 +1,69 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.continues;
import java.lang.reflect.Constructor;
import net.sf.cglib.proxy.Enhancer;
public class ContinuesFactory implements GenericFactory {
private static final String CONTINUES_INTERCEPTOR_DISABLED_PROPERTY =
"ContinuesInterceptor.disabled";
private static final boolean disabled =
System.getProperty(CONTINUES_INTERCEPTOR_DISABLED_PROPERTY) != null;
private ExceptionHandler exceptionHandler;
public ContinuesFactory(ExceptionHandler exceptionHandler) {
if (exceptionHandler == null) {
throw new IllegalArgumentException("exceptionHandler == null not allowed");
}
this.exceptionHandler = exceptionHandler;
}
@Override
public Object create(Class<?> type, Object... args) {
try {
Object thing;
if (disabled) {
Constructor<?> c = type.getConstructor(new Class<?>[0]);
thing = c.newInstance(args);
}
else {
ContinuesInterceptor interceptor = new ContinuesInterceptor(exceptionHandler);
Enhancer e = new Enhancer();
e.setSuperclass(type);
e.setCallback(interceptor);
thing = e.create();
}
return thing;
}
catch (Throwable e) {
try {
exceptionHandler.handle(e);
}
catch (Throwable t) {
// let the handler supplant the original exception if need be
e = t;
}
// wrap so clients don't need try/catch everywhere
throw new RuntimeException(e);
}
}
}

View file

@ -0,0 +1,47 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.continues;
import java.lang.reflect.Method;
import net.sf.cglib.proxy.MethodInterceptor;
import net.sf.cglib.proxy.MethodProxy;
class ContinuesInterceptor implements MethodInterceptor {
private final ExceptionHandler handler;
ContinuesInterceptor(ExceptionHandler handler) {
this.handler = handler;
}
@Override
public Object intercept(Object obj, Method method, Object[] args,
MethodProxy proxy) throws Throwable {
Object retValFromSuper = null;
if (method.getAnnotation(DoNotContinue.class) != null) {
retValFromSuper = proxy.invokeSuper(obj, args);
} else {
try {
retValFromSuper = proxy.invokeSuper(obj, args);
}
catch (Exception e) {
handler.handle(e);
}
}
return retValFromSuper;
}
}

View file

@ -0,0 +1,28 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.continues;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.METHOD)
public @interface DoNotContinue {
// marker interface
}

View file

@ -0,0 +1,21 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.continues;
public interface ExceptionHandler {
public void handle(Throwable e) throws Throwable;
}

View file

@ -0,0 +1,21 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.continues;
public interface GenericFactory {
public Object create(Class<?> type, Object... args);
}

View file

@ -0,0 +1,25 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.continues;
public class RethrowContinuesFactory extends ContinuesFactory {
public static final RethrowContinuesFactory INSTANCE = new RethrowContinuesFactory();
private RethrowContinuesFactory() {
super(RethrowExceptionHandler.INSTANCE);
}
}

View file

@ -0,0 +1,28 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.continues;
public class RethrowExceptionHandler implements ExceptionHandler {
public static final RethrowExceptionHandler INSTANCE = new RethrowExceptionHandler();
private RethrowExceptionHandler() {}
@Override
public void handle(Throwable e) throws Throwable {
throw e;
}
}

View file

@ -0,0 +1,105 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.hash;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
public abstract class AbstractMessageDigest implements MessageDigest {
public final String algorithm;
public final int digestLength;
protected AbstractMessageDigest(String algorithm, int digestLength) {
this.algorithm = algorithm;
this.digestLength = digestLength;
}
@Override
public final String getAlgorithm() {
return algorithm;
}
@Override
public final int getDigestLength() {
return digestLength;
}
@Override
public final byte[] digest() {
byte[] results = new byte[digestLength];
digest(results, 0, digestLength);
return results;
}
@Override
public final void update(byte[] input, TaskMonitor monitor) throws CancelledException {
update(input, 0, input.length, monitor);
}
@Override
public final void update(byte[] input) {
update(input, 0, input.length);
}
@Override
public void update(short input) {
update((byte) ((input >> 8) & 0xff));
update((byte) (input & 0xff));
}
@Override
public void update(int input) {
update((byte) ((input >> 24) & 0xff));
update((byte) ((input >> 16) & 0xff));
update((byte) ((input >> 8) & 0xff));
update((byte) (input & 0xff));
}
@Override
public void update(long input) {
update((byte) ((input >> 56) & 0xff));
update((byte) ((input >> 48) & 0xff));
update((byte) ((input >> 40) & 0xff));
update((byte) ((input >> 32) & 0xff));
update((byte) ((input >> 24) & 0xff));
update((byte) ((input >> 16) & 0xff));
update((byte) ((input >> 8) & 0xff));
update((byte) (input & 0xff));
}
/**
* You REALLY want to override this method.
*/
@Override
public void update(byte[] input, int offset, int len) {
for (int ii = 0; ii < len; ++ii) {
update(input[offset++]);
}
}
/**
* You REALLY want to override this method too.
* @throws CancelledException
*/
@Override
public void update(byte[] input, int offset, int len, TaskMonitor monitor)
throws CancelledException {
for (int ii = 0; ii < len; ++ii) {
monitor.checkCanceled();
update(input[offset++]);
}
}
}

View file

@ -0,0 +1,102 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.hash;
import ghidra.util.task.TaskMonitor;
public class FNV1a32MessageDigest extends AbstractMessageDigest {
public static final int FNV_32_OFFSET_BASIS = 0x811c9dc5;
public static final int FNV_32_PRIME = 16777619;
private int hashvalue;
public FNV1a32MessageDigest(int initialVector) {
super("FNV-1a", 4);
hashvalue = initialVector;
}
public FNV1a32MessageDigest() {
super("FNV-1a", 4);
init();
}
private void init() {
hashvalue = FNV_32_OFFSET_BASIS;
}
@Override
public void update(byte[] input, int offset, int len) {
for (int ii = 0; ii < len; ++ii) {
hashvalue ^= (input[offset++] & 0xff);
hashvalue *= FNV_32_PRIME;
}
}
@Override
public void update(byte[] input, int offset, int len, TaskMonitor monitor) {
for (int ii = 0; ii < len; ++ii) {
if (ii % 1000000 == 0 && monitor.isCancelled()) {
break;
}
hashvalue ^= (input[offset++] & 0xff);
hashvalue *= FNV_32_PRIME;
}
}
@Override
public void update(byte input) {
hashvalue ^= input & 0xff;
hashvalue *= FNV_32_PRIME;
}
@Override
public int digest(byte[] buf, int offset, int len) {
if (buf.length < 4 || len < 4) {
offset += len - 1;
hashvalue >>= 8 * (4 - len);
for (int ii = 0; ii < len; ++ii) {
buf[offset--] = (byte) (hashvalue & 0xff);
hashvalue >>= 8;
}
init();
return len;
}
// unwind the loop
offset += 3;
buf[offset--] = (byte) (hashvalue & 0xff);
hashvalue >>= 8;
buf[offset--] = (byte) (hashvalue & 0xff);
hashvalue >>= 8;
buf[offset--] = (byte) (hashvalue & 0xff);
hashvalue >>= 8;
buf[offset--] = (byte) (hashvalue & 0xff);
init();
return 4;
}
@Override
public long digestLong() {
long result = hashvalue & 0x00000000ffffffffL;
init();
return result;
}
@Override
public void reset() {
init();
}
}

View file

@ -0,0 +1,24 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.hash;
public class FNV1a32MessageDigestFactory implements MessageDigestFactory {
@Override
public MessageDigest createDigest() {
return new FNV1a32MessageDigest();
}
}

View file

@ -0,0 +1,110 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.hash;
import ghidra.util.task.TaskMonitor;
public class FNV1a64MessageDigest extends AbstractMessageDigest {
public static final long FNV_64_OFFSET_BASIS = 0xcbf29ce484222325L;
public static final long FNV_64_PRIME = 1099511628211L;
private long hashvalue;
public FNV1a64MessageDigest() {
super("FNV-1a", 8);
init();
}
public FNV1a64MessageDigest(long initialVector) {
super("FNV-1a", 8);
hashvalue = initialVector;
}
private void init() {
hashvalue = FNV_64_OFFSET_BASIS;
}
@Override
public void update(byte[] input, int offset, int len) {
for (int ii = 0; ii < len; ++ii) {
hashvalue ^= (input[offset++] & 0xff);
hashvalue *= FNV_64_PRIME;
}
}
@Override
public void update(byte[] input, int offset, int len, TaskMonitor monitor) {
for (int ii = 0; ii < len; ++ii) {
if (ii % 1000000 == 0 && monitor.isCancelled()) {
break;
}
hashvalue ^= (input[offset++] & 0xff);
hashvalue *= FNV_64_PRIME;
}
}
@Override
public void update(byte input) {
hashvalue ^= input & 0xff;
hashvalue *= FNV_64_PRIME;
}
@Override
public int digest(byte[] buf, int offset, int len) {
if (buf.length < 8 || len < 8) {
offset += len - 1;
hashvalue >>= 8 * (8 - len);
for (int ii = 0; ii < len; ++ii) {
buf[offset--] = (byte) (hashvalue & 0xff);
hashvalue >>= 8;
}
init();
return len;
}
// unwind the loop
offset += 7;
buf[offset--] = (byte) (hashvalue & 0xff);
hashvalue >>= 8;
buf[offset--] = (byte) (hashvalue & 0xff);
hashvalue >>= 8;
buf[offset--] = (byte) (hashvalue & 0xff);
hashvalue >>= 8;
buf[offset--] = (byte) (hashvalue & 0xff);
hashvalue >>= 8;
buf[offset--] = (byte) (hashvalue & 0xff);
hashvalue >>= 8;
buf[offset--] = (byte) (hashvalue & 0xff);
hashvalue >>= 8;
buf[offset--] = (byte) (hashvalue & 0xff);
hashvalue >>= 8;
buf[offset--] = (byte) (hashvalue & 0xff);
init();
return 8;
}
@Override
public long digestLong() {
long result = hashvalue;
init();
return result;
}
@Override
public void reset() {
init();
}
}

View file

@ -0,0 +1,24 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.hash;
public class FNV1a64MessageDigestFactory implements MessageDigestFactory {
@Override
public MessageDigest createDigest() {
return new FNV1a64MessageDigest();
}
}

View file

@ -0,0 +1,123 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.hash;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
public interface MessageDigest {
/**
* Returns a string that identifies the algorithm, independent of
* implementation details.
* @return the name of the algorithm
*/
public String getAlgorithm();
/**
* Returns the length of the digest in bytes.
* @return the digest length in bytes
*/
public int getDigestLength();
/**
* Updates the digest using the specified byte.
* @param input the byte with which to update the digest
*/
public void update(byte input);
/**
* Updates the digest using the specified short.
* @param input the short with which to update the digest (big endian)
*/
public void update(short input);
/**
* Updates the digest using the specified int.
* @param input the int with which to update the digest (big endian)
*/
public void update(int input);
/**
* Updates the digest using the specified long.
* @param input the long with which to update the digest (big endian)
*/
public void update(long input);
/**
* Updates the digest using the specified array of bytes. Do not use a monitor
* @param input the array of bytes
*/
public void update(byte[] input);
/**
* Updates the digest using the specified array of bytes, starting at the
* specified offset (and for the specified length). Do not use a monitor.
* @param input the array of bytes
* @param offset the offset to start from in the array of bytes
* @param len the number of bytes to use, starting at offset
*/
public void update(byte[] input, int offset, int len);
/**
* Updates the digest using the specified array of bytes.
* @param input the array of bytes
* @param monitor the monitor to check during loops
* @throws CancelledException
*/
public void update(byte[] input, TaskMonitor monitor) throws CancelledException;
/**
* Updates the digest using the specified array of bytes, starting at the
* specified offset (and for the specified length).
* @param input the array of bytes
* @param offset the offset to start from in the array of bytes
* @param len the number of bytes to use, starting at offset
* @param monitor the monitor to check during loops
* @throws CancelledException
*/
public void update(byte[] input, int offset, int len, TaskMonitor monitor)
throws CancelledException;
/**
* Completes the hash computation by performing final operations such as
* padding. The digest is reset after this call is made.
* @return the array of bytes for the resulting hash value
*/
public byte[] digest();
/**
* Completes the hash computation by performing final operations such as
* padding, and returns (up to) the first 8 bytes as a big-endian long
* value. The digest is reset after this call is made.
* @return the digest value as a long value
*/
public long digestLong();
/**
* Completes the hash computation by performing final operations such as
* padding. The digest is reset after this call is made.
* @param buf output buffer for the computed digest
* @param offset offset into the output buffer to begin storing the digest
* @param len number of bytes within buf allocated for the digest
* @return the number of bytes placed into buf
*/
public int digest(byte[] buf, int offset, int len);
/**
* Resets the digest for further use.
*/
public void reset();
}

View file

@ -0,0 +1,20 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.hash;
public interface MessageDigestFactory {
public MessageDigest createDigest();
}

View file

@ -0,0 +1,78 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.hash;
public final class SimpleCRC32 {
public static final int crc32tab[] = {
0,1996959894,-301047508,-1727442502,124634137,
1886057615,-379345611,-1637575261,249268274,2044508324,
-522852066,-1747789432,162941995,2125561021,-407360249,
-1866523247,498536548,1789927666,-205950648,-2067906082,
450548861,1843258603,-187386543,-2083289657,325883990,
1684777152,-43845254,-1973040660,335633487,1661365465,
-99664541,-1928851979,997073096,1281953886,-715111964,
-1570279054,1006888145,1258607687,-770865667,-1526024853,
901097722,1119000684,-608450090,-1396901568,853044451,
1172266101,-589951537,-1412350631,651767980,1373503546,
-925412992,-1076862698,565507253,1454621731,-809855591,
-1195530993,671266974,1594198024,-972236366,-1324619484,
795835527,1483230225,-1050600021,-1234817731,1994146192,
31158534,-1731059524,-271249366,1907459465,112637215,
-1614814043,-390540237,2013776290,251722036,-1777751922,
-519137256,2137656763,141376813,-1855689577,-429695999,
1802195444,476864866,-2056965928,-228458418,1812370925,
453092731,-2113342271,-183516073,1706088902,314042704,
-1950435094,-54949764,1658658271,366619977,-1932296973,
-69972891,1303535960,984961486,-1547960204,-725929758,
1256170817,1037604311,-1529756563,-740887301,1131014506,
879679996,-1385723834,-631195440,1141124467,855842277,
-1442165665,-586318647,1342533948,654459306,-1106571248,
-921952122,1466479909,544179635,-1184443383,-832445281,
1591671054,702138776,-1328506846,-942167884,1504918807,
783551873,-1212326853,-1061524307,-306674912,-1698712650,
62317068,1957810842,-355121351,-1647151185,81470997,
1943803523,-480048366,-1805370492,225274430,2053790376,
-468791541,-1828061283,167816743,2097651377,-267414716,
-2029476910,503444072,1762050814,-144550051,-2140837941,
426522225,1852507879,-19653770,-1982649376,282753626,
1742555852,-105259153,-1900089351,397917763,1622183637,
-690576408,-1580100738,953729732,1340076626,-776247311,
-1497606297,1068828381,1219638859,-670225446,-1358292148,
906185462,1090812512,-547295293,-1469587627,829329135,
1181335161,-882789492,-1134132454,628085408,1382605366,
-871598187,-1156888829,570562233,1426400815,-977650754,
-1296233688,733239954,1555261956,-1026031705,-1244606671,
752459403,1541320221,-1687895376,-328994266,1969922972,
40735498,-1677130071,-351390145,1913087877,83908371,
-1782625662,-491226604,2075208622,213261112,-1831694693,
-438977011,2094854071,198958881,-2032938284,-237706686,
1759359992,534414190,-2118248755,-155638181,1873836001,
414664567,-2012718362,-15766928,1711684554,285281116,
-1889165569,-127750551,1634467795,376229701,-1609899400,
-686959890,1308918612,956543938,-1486412191,-799009033,
1231636301,1047427035,-1362007478,-640263460,1088359270,
936918000,-1447252397,-558129467,1202900863,817233897,
-1111625188,-893730166,1404277552,615818150,-1160759803,
-841546093,1423857449,601450431,-1285129682,-1000256840,
1567103746,711928724,-1274298825,-1022587231,1510334235,
755167117
};
public static final int hashOneByte(int hashcode,int val) {
return crc32tab[(hashcode ^ val)&0xff] ^ (hashcode>>>8);
}
}

View file

@ -0,0 +1,28 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.init;
import ghidra.framework.Application;
import utility.application.ApplicationSettings;
import java.io.File;
public class GenericApplicationSettings extends ApplicationSettings {
@Override
protected File doGetUserApplicationSettingsDirectory() {
return Application.getUserSettingsDirectory();
}
}

View file

@ -0,0 +1,35 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.init;
import ghidra.framework.ModuleInitializer;
import ghidra.framework.PluggableServiceRegistry;
import utility.application.ApplicationSettings;
public class GenericInitializer implements ModuleInitializer {
@Override
public void run() {
PluggableServiceRegistry.registerPluggableService(ApplicationSettings.class,
new GenericApplicationSettings());
}
@Override
public String getName() {
return "Generic Module";
}
}

View file

@ -0,0 +1,149 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.io;
import ghidra.util.Msg;
import ghidra.util.task.TaskMonitor;
import java.io.*;
import java.util.jar.JarEntry;
import java.util.jar.JarInputStream;
import utilities.util.FileUtilities;
/**
* JarReader is a class for reading from a jar input stream.
*/
public class JarReader {
protected JarInputStream jarIn;
/** Creates a JarReader
* @param jarIn the the jar file input stream the zip entries are
* read from.
*/
public JarReader(JarInputStream jarIn) {
this.jarIn = jarIn;
}
/**
* Recursively reads the files from the jar input stream and creates the
* respetive directories and files in the file system.
* <P>It effectively unzips the Jar file.
* Warning: This will overwrite any files that already exist on the file
* system as it outputs the jar contents.
*
* @param basePath the base path for where to output the JarInputStream
* file contents to.
*
* @throws FileNotFoundException if the file exists but is a directory
* rather than a regular file, does not exist but cannot
* be created, or cannot be opened for any other reason
* @throws IOException if it can't read the jar file or output one
* or more of its files.
*/
public void createRecursively(String basePath, TaskMonitor monitor)
throws FileNotFoundException, IOException {
boolean done = false;
while (!done && !monitor.isCancelled()) {
//Get the zip entry.
JarEntry entry = jarIn.getNextJarEntry();
if (entry == null) {
done = true;
break;
}
String name = entry.getName();
long modTime = entry.getTime();
// long size = entry.getSize();
// String comment = entry.getComment();
// Create the output file.
String filePath = basePath+name;
// replace any embedded separator characters
// with the separator char on this platform
filePath = filePath.replace('/', File.separatorChar);
filePath = filePath.replace('\\', File.separatorChar);
long lastIndex = filePath.lastIndexOf(File.separatorChar);
String dirPath = filePath.substring(0, (int)lastIndex);
// String fileName = filePath.substring((int)(lastIndex+1));
File dir = new File(dirPath);
FileUtilities.mkdirs(dir);
File file = new File(filePath);
if (!file.createNewFile() && !file.exists()) {
throw new IOException("Couldn't create file "+file.getAbsolutePath());
}
// Write it out to the file along with its data.
FileOutputStream out = null;
out = new FileOutputStream(file);
byte[] bytes = new byte[4096];
int numRead = 0;
try {
while ((numRead = jarIn.read(bytes)) != -1 &&
!monitor.isCancelled()) {
out.write(bytes, 0 , numRead);
}
} finally {
try {
out.close();
} catch (IOException ioe) {
Msg.error(this, "Unexpected Exception: " + ioe.getMessage(), ioe);
}
}
if (modTime > 0 && file.isFile()) {
file.setLastModified(modTime);
}
}
// Fix directory times
fixDirModifiedTimes(new File(basePath));
}
private void fixDirModifiedTimes(File dir) {
long modTime = 0;
File[] files = dir.listFiles();
for (File f : files) {
if (f.isDirectory()) {
fixDirModifiedTimes(f);
}
long t = f.lastModified();
if (t > modTime) {
modTime = t;
}
}
if (modTime > 0) {
dir.setLastModified(modTime);
}
}
/**
* Return the jar input stream being used by this JarReader.
*/
public JarInputStream getJarInputStream() {
return jarIn;
}
}

View file

@ -0,0 +1,193 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.io;
import ghidra.util.Msg;
import ghidra.util.task.TaskMonitor;
import ghidra.util.task.TaskMonitorAdapter;
import java.io.*;
import java.util.jar.JarOutputStream;
import java.util.zip.ZipEntry;
/**
* JarWriter is a class for writing to a jar output stream.
*/
public class JarWriter {
protected JarOutputStream jarOut;
private String[] excludedExtensions;
/**
* @param jarOut the the jar file output stream the zip entries are
* to be written to.
*/
public JarWriter(JarOutputStream jarOut) {
this(jarOut, new String[0]);
}
public JarWriter(JarOutputStream jarOut, String[] excludedExtensions) {
this.jarOut = jarOut;
this.excludedExtensions = excludedExtensions;
}
/**
* Outputs an individual file to the jar.
*
* @param baseFile the file to be output
* @param jarPath the base path to prepend to the file as it is written
* to the jar output stream.
* @param monitor cancellable task monitor
* @return true if file is output to the jar file successfully.
*/
public boolean outputFile(File baseFile, String jarPath, TaskMonitor monitor) {
if (baseFile.isDirectory()) {
return false;
}
FileInputStream in = null;
try {
in = new FileInputStream(baseFile);
}
catch (FileNotFoundException fnfe) {
Msg.error(this, "Unexpected Exception: " + fnfe.getMessage(), fnfe);
return false;
}
try {
return outputEntry(jarPath + baseFile.getName(), baseFile.lastModified(), in, monitor);
}
finally {
try {
in.close();
}
catch (IOException ioe) {
Msg.error(this, "Unexpected Exception: " + ioe.getMessage(), ioe);
}
}
}
/**
* Outputs an individual entry to the jar. The data input stream will be read until and EOF is read.
* @param path entry path within the jar file
* @param time entry time
* @param in data input stream
* @param monitor cancellable task monitor
* @return true if entry is output to the jar file successfully.
*/
public boolean outputEntry(String path, long time, InputStream in, TaskMonitor monitor) {
byte[] bytes = new byte[4096];
int numRead = 0;
//Create a zip entry and write it out along with its data.
ZipEntry entry = new ZipEntry(path);
entry.setTime(time);
try {
monitor.setMessage("Writing " + path);
jarOut.putNextEntry(entry);
try {
while ((numRead = in.read(bytes)) != -1) {
if (monitor.isCancelled()) {
return false;
}
jarOut.write(bytes, 0, numRead);
}
return true;
}
catch (IOException ioe) {
Msg.error(this, "Unexpected Exception: " + ioe.getMessage(), ioe);
}
finally {
jarOut.closeEntry();
}
}
catch (IOException ioe) {
Msg.error(this, "Unexpected Exception: " + ioe.getMessage(), ioe);
}
return false;
}
/**
* Recursively outputs a directory to the jar output stream
* If baseFile is a file then it is simply output to the jar.
*
* @param baseFile the file or directory to be output
* @param jarPath the base path to prepend to the files as they are written
* to the jar output stream.
*
* @return true if all files are recursively output to the jar file.
*/
public boolean outputRecursively(File baseFile, String jarPath, TaskMonitor monitor) {
boolean succeeded = true;
File[] subFiles = new File[0];
if (baseFile.isDirectory()) {
subFiles = baseFile.listFiles();
for (int i = 0; i < subFiles.length; i++) {
if (monitor.isCancelled()) {
break;
}
String newPath = jarPath + baseFile.getName() + File.separator;
succeeded = outputRecursively(subFiles[i], newPath, monitor) && succeeded;
}
}
else {
String name = baseFile.getName();
for (int i = 0; i < excludedExtensions.length; i++) {
if (name.endsWith(excludedExtensions[i])) {
return true;
}
}
succeeded = outputFile(baseFile, jarPath, monitor);
}
return succeeded;
}
/**
* Return the jar output stream being used by this JarWriter.
*/
public JarOutputStream getJarOutputStream() {
return jarOut;
}
/**
* Simple test for the JarWriter
* @param args args[0] is the source directory, args[1] is the output filename
*/
public static void main(String[] args) {
if (args.length != 2) {
System.out.println("Usage: java JarWriter [sourceDir] [outputFilename]");
System.exit(0);
}
try {
JarOutputStream jarOut = new JarOutputStream(new FileOutputStream(args[1]));
JarWriter writer = new JarWriter(jarOut);
writer.outputRecursively(new File(args[0]), "", TaskMonitorAdapter.DUMMY_MONITOR);
jarOut.close();
}
catch (FileNotFoundException e) {
e.printStackTrace();
}
catch (IOException e) {
e.printStackTrace();
}
}
}

View file

@ -0,0 +1,26 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.io;
import java.io.PrintWriter;
public class NullPrintWriter extends PrintWriter {
public NullPrintWriter() {
super(new NullWriter());
}
}

View file

@ -0,0 +1,78 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.io;
import java.io.IOException;
import java.io.Writer;
/**
* An implementation of {@link Writer} to use when you wish to not use any writing, but to also
* avoid null checks.
*/
public class NullWriter extends Writer {
@Override
public void write(char[] cbuf, int off, int len) throws IOException {
// stub
}
@Override
public void write(int c) throws IOException {
// stub
}
@Override
public void write(char[] cbuf) throws IOException {
// stub
}
@Override
public void write(String str) throws IOException {
// stub
}
@Override
public void write(String str, int off, int len) throws IOException {
// stub
}
@Override
public Writer append(CharSequence csq) throws IOException {
return this;
}
@Override
public Writer append(CharSequence csq, int start, int end) throws IOException {
return this;
}
@Override
public Writer append(char c) throws IOException {
return this;
}
@Override
public void flush() throws IOException {
// stub
}
@Override
public void close() throws IOException {
// stub
}
}

View file

@ -0,0 +1,134 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.jar;
import java.io.File;
import java.io.IOException;
import java.util.*;
import ghidra.framework.Application;
import ghidra.util.exception.AssertException;
import utility.module.ModuleManifestFile;
public class ApplicationModule implements Comparable<ApplicationModule> {
private File moduleDir;
private String relativePath;
private File applicationRoot;
public ApplicationModule(File applicationRoot, File moduleDir) {
this.applicationRoot = applicationRoot;
this.moduleDir = moduleDir;
String filePath = moduleDir.getAbsolutePath();
String rootPath = applicationRoot.getAbsolutePath();
if (!filePath.startsWith(rootPath)) {
throw new AssertException("ApplicationRoot is not in the parent path of moduleDir!");
}
relativePath = filePath.substring(rootPath.length() + 1);
}
public String getName() {
return moduleDir.getName();
}
public File getModuleDir() {
return moduleDir;
}
public File getApplicationRoot() {
return applicationRoot;
}
public String getRelativePath() {
return relativePath;
}
public boolean isExtension() {
return moduleDir.getParentFile().getName().equalsIgnoreCase("Extensions");
}
public boolean isFramework() {
return moduleDir.getParentFile().getName().equalsIgnoreCase("Framework");
}
public boolean isProcessor() {
return moduleDir.getParentFile().getName().equalsIgnoreCase("Processors");
}
public boolean isFeature() {
return moduleDir.getParentFile().getName().equalsIgnoreCase("Features");
}
public boolean isConfiguration() {
return moduleDir.getParentFile().getName().equalsIgnoreCase("Configurations");
}
public boolean isGPL() {
return moduleDir.getParentFile().getName().equalsIgnoreCase("GPL");
}
@Override
public int compareTo(ApplicationModule o) {
int myRank = getRank();
int otherRank = o.getRank();
int result = myRank - otherRank;
if (result == 0) {
result = getName().compareTo(o.getName());
}
return result;
}
@Override
public String toString() {
return getName();
}
private int getRank() {
if (getName().equals("RenoirGraph")) {
return 10; // renoir is always last
}
if (isFramework()) {
return 1;
}
if (isFeature()) {
return 2;
}
if (isProcessor()) {
return 3;
}
return 4;
}
public boolean excludeFromGhidraJar() {
try {
Collection<ResourceFile> applicationRoots = Application.getApplicationRootDirectories();
// multiple dirs during development (repo dirs); single dir in installation (install dir)
Set<File> rootDirParents = new HashSet<File>();
for (ResourceFile root : applicationRoots) {
rootDirParents.add(root.getParentFile().getFile(true));
}
ModuleManifestFile moduleManifestFile = new ModuleManifestFile(moduleDir);
return moduleManifestFile.excludeFromGhidraJar();
}
catch (IOException e) {
return false;
}
}
}

View file

@ -0,0 +1,17 @@
/* ###
* IP: MIT
*/
package generic.json;
public enum JSONError {
/* Everything was fine */
JSMN_SUCCESS,
/* Not enough tokens were provided */
JSMN_ERROR_NOMEM,
/* Invalid character inside JSON string */
JSMN_ERROR_INVAL,
/* The string is not a full JSON packet, more bytes expected */
JSMN_ERROR_PART
}

View file

@ -0,0 +1,511 @@
/* ###
* IP: MIT
*/
package generic.json;
import java.util.*;
import ghidra.util.Msg;
import ghidra.util.NumericUtilities;
/*
this file is a modification of jsmn.
its copyright and MIT license follow.
Copyright (c) 2010 Serge A. Zaitsev
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
public class JSONParser {
/**
* JSON parser. Contains an array of token blocks available. Also stores
* the string being parsed now and current position in that string
*/
static boolean JSMN_STRICT = false;
int pos; /* offset in the JSON string */
int toknext; /* next token to allocate */
int toksuper; /* suporior token node, e.g parent object or array */
int ndx = 0;
/**
* Create JSON parser over an array of tokens
*/
//static void jsmn_init(jsmn_parser *parser);
/**
* Run JSON parser. It parses a JSON data string into and array of tokens, each describing
* a single JSON object.
*/
//static jsmnerr_t jsmn_parse(jsmn_parser *parser, const char *js,
// jsmntok_t *tokens, unsigned int num_tokens);
/**
* Creates a new parser based over a given buffer with an array of tokens
* available.
*/
public JSONParser() {
pos = 0;
toknext = 0;
toksuper = -1;
}
/**
* Allocates a fresh unused token from the token pull.
*/
JSONToken allocateToken(List<JSONToken> tokens, JSONType type, int start, int end) {
JSONToken token = new JSONToken(type, start, end);
tokens.add(token);
toknext = tokens.size();
return token;
}
/**
* Fills next available token with JSON primitive.
*/
JSONError parsePrimitive(char [] js, List<JSONToken> tokens) {
int start;
start = pos;
boolean found = false;
if (JSMN_STRICT) {
for (; js[pos] != '\0'; pos++) {
switch (js[pos]) {
case ',' :
case ']' :
case '}' :
found = true;
break;
}
if (found) break;
if (js[pos] < 32) {
pos = start;
return JSONError.JSMN_ERROR_INVAL;
}
}
if (!found) {
/* In strict mode primitive must be followed by a comma/object/array */
pos = start;
return JSONError.JSMN_ERROR_PART;
}
} else {
for (; js[pos] != '\0'; pos++) {
switch (js[pos]) {
/* In strict mode primitive must be followed by "," or "}" or "]" */
case '\t' :
case '\r' :
case '\n' :
case ' ' :
case ':':
case ',' :
case ']' :
case '}' :
found = true;
break;
}
if (found) break;
if (js[pos] < 32) {
pos = start;
return JSONError.JSMN_ERROR_INVAL;
}
}
}
allocateToken(tokens, JSONType.JSMN_PRIMITIVE, start, pos);
pos--;
return JSONError.JSMN_SUCCESS;
}
/**
* Filsl next token with JSON string.
*/
JSONError parseString(char [] js, List<JSONToken> tokens) {
int i;
int start = pos;
pos++;
/* Skip starting quote */
for (; pos < js.length; pos++) {
char c = js[pos];
/* Quote: end of string */
if (c == '\"') {
allocateToken(tokens, JSONType.JSMN_STRING, start+1, pos);
return JSONError.JSMN_SUCCESS;
}
/* Backslash: Quoted symbol expected */
if (c == '\\') {
pos++;
switch (js[pos]) {
/* Allowed escaped symbols */
case '\"': case '/' : case '\\' : case 'b' :
case 'f' : case 'r' : case 'n' : case 't' :
break;
/* Allows escaped symbol XXXX */
case 'u':
for(i = 0; i < 4; i++){
pos++;
if(!isxdigit(js[pos])){
pos = start;
return JSONError.JSMN_ERROR_INVAL;
}
}
break;
/* Unexpected symbol */
default:
pos = start;
return JSONError.JSMN_ERROR_INVAL;
}
}
}
pos = start;
return JSONError.JSMN_ERROR_PART;
}
/**
* Parse JSON string and fill tokens.
*/
public JSONError parse(char [] js, List<JSONToken> tokens) {
JSONError r;
int i;
JSONToken token;
for (; pos < js.length; pos++) {
char c ;
JSONType type;
c = js[pos];
switch (c) {
case '{': case '[':
token = allocateToken(tokens,
c == '{' ? JSONType.JSMN_OBJECT : JSONType.JSMN_ARRAY,
pos, -1);
if (toksuper != -1) {
tokens.get(toksuper).incSize();
}
toksuper = toknext - 1;
break;
case '}': case ']':
type = (c == '}' ? JSONType.JSMN_OBJECT : JSONType.JSMN_ARRAY);
for (i = toknext - 1; i >= 0; i--) {
token = tokens.get(i);
if (token.start != -1 && token.end == -1) {
if (token.type != type) {
return JSONError.JSMN_ERROR_INVAL;
}
toksuper = -1;
token.end = pos + 1;
break;
}
}
/* Error if unmatched closing bracket */
if (i == -1) {
return JSONError.JSMN_ERROR_INVAL;
}
for (; i >= 0; i--) {
token = tokens.get(i);
if (token.start != -1 && token.end == -1) {
toksuper = i;
break;
}
}
break;
case '\"':
r = parseString(js, tokens);
if (r != JSONError.JSMN_SUCCESS) return r;
if (toksuper != -1) {
tokens.get(toksuper).incSize();
}
break;
case '\t' : case '\r' : case '\n' : case ':' : case ',': case ' ':
break;
/* In strict mode primitives are: numbers and booleans */
case '-': case '0': case '1' : case '2': case '3' : case '4':
case '5': case '6': case '7' : case '8': case '9':
case 't': case 'f': case 'n' :
r = parsePrimitive(js, tokens);
if (r != JSONError.JSMN_SUCCESS) return r;
if (toksuper != -1)
tokens.get(toksuper).incSize();
break;
/* In non-strict mode every unquoted value is a primitive */
default:
if (JSMN_STRICT) {
r = parsePrimitive(js, tokens);
if (r != JSONError.JSMN_SUCCESS) return r;
if (toksuper != -1) {
tokens.get(toksuper).incSize();
}
} else {
return JSONError.JSMN_ERROR_INVAL;
}
break;
}
}
for (i = toknext - 1; i >= 0; i--) {
/* Unmatched opened object or array */
JSONToken test = tokens.get(i);
if (test.start != -1 && test.end == -1) {
return JSONError.JSMN_ERROR_PART;
}
}
return JSONError.JSMN_SUCCESS;
}
String expands(String s) {
// if (s.contains("\\")) {
// Msg.error(this, "hmmm");
// }
return s;
}
// int c;
// char *r, *w, *z;
// unsigned long nlen;
// unsigned short u;
// unsigned i;
// Str *rv;
//
// rv = mkstr(s, len);
// r = s;
// w = strdata(rv);
// z = s+len;
// while(r < z){
// if(r[0] != '\\'){
// *w++ = *r++;
// continue;
// }
//
// /* escape sequence */
// r++;
// switch(*r){
// case '"':
// c = '\"';
// r++;
// break;
// case '\\':
// c = '\\';
// r++;
// break;
// case '/':
// c = '/';
// r++;
// break;
// case 'b':
// c = '\b';
// r++;
// break;
// case 'f':
// c = '\f';
// r++;
// break;
// case 'n':
// c = '\n';
// r++;
// break;
// case 'r':
// c = '\r';
// r++;
// break;
// case 't':
// c = '\t';
// r++;
// break;
// case 'u':
// /* assume jsmn_parse verified we have 4 hex digits */
// r++;
// u = 0;
// for(i = 0; i < 4; i++){
// u <<= 4;
// if(*r >= 'A' && *r <= 'F')
// u += *r-'A'+10;
// else if(*r >= 'a' && *r <= 'f')
// u += *r-'a'+10;
// else
// u += *r-'0';
// r++;
// }
// if(u > 255)
// return 0;
// c = (char)u;
// break;
// default:
// return 0;
// }
// *w++ = c;
// }
// nlen = w-strdata(rv);
// rv = mkstr(strdata(rv), nlen);
// return mkvalstr(rv);
public Object convert(char [] s, List<JSONToken> t)
{
Object rv = null, k, v;
int i;
JSONToken tp;
if (ndx == t.size()) {
System.out.println("array overflow in JSON parser");
}
tp = t.get(ndx++);
String tstr = new String(s, tp.start, tp.end-tp.start);
switch(tp.type){
case JSMN_OBJECT:
HashMap<Object, Object> tab = new HashMap<Object, Object>();
if(tp.size%2 != 0) {
Msg.error(this, "invalid json object");
return null;
}
for(i = 0; i < tp.size/2; i++){
k = convert(s, t);
v = convert(s, t);
tab.put(k, v);
}
rv = tab;
break;
case JSMN_ARRAY:
List<Object> l = new ArrayList<Object>();
for(i = 0; i < tp.size; i++)
l.add(convert(s, t));
rv = l;
break;
case JSMN_PRIMITIVE:
i = tp.start;
switch(s[tp.start]){
case 't':
Msg.error(this, "what is this? "+tstr);
//rv = mkvalcval2(cval1);
break;
case 'f':
Msg.error(this, "what is this? "+tstr);
//rv = mkvalcval2(cval0);
break;
case 'n':
//rv = null;
break;
case '-':
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
rv = NumericUtilities.parseLong(tstr);
break;
default:
Msg.error(this, "invalid json primitive: "+tstr);
return null;
}
break;
case JSMN_STRING:
rv = expands(tstr);
if (rv == null){
Msg.error(this, "invalid json string: "+tstr);
}
break;
default:
throw new RuntimeException("invalid json type: "+tp.type);
}
return rv;
}
private static boolean isxdigit(char b) {
switch (b) {
case '0': case '1' : case '2': case '3' : case '4':
case '5': case '6': case '7' : case '8': case '9':
case 'A': case 'B': case 'C' : case 'D': case 'E' : case 'F':
case 'a': case 'b': case 'c' : case 'd': case 'e' : case 'f':
return true;
default:
return false;
}
}
// public static void main(String[] args) throws IOException {
//
// GhidraApplication.initialize( new HeadlessGhidraApplicationConfiguration() );
//
// JSONParser parser = new JSONParser();
// BufferedReader in = openReader();
//
// int n = 4096;
// char [] cbuf = new char[n];
// int read = in.read(cbuf);
// if (read < 2) {
// Msg.error(null, "No input found");
// return;
// }
//
// List<Object> objs = new ArrayList<Object>();
// List<JSONToken> tokens = new ArrayList<JSONToken>();
//
// JSONError r = parser.parse(cbuf, tokens);
//
// switch(r){
// case JSMN_SUCCESS:
// break;
// case JSMN_ERROR_NOMEM:
// Msg.error(null, "out of memory");
// return;
// case JSMN_ERROR_INVAL:
// Msg.error(null, "invalid json input");
// return;
// case JSMN_ERROR_PART:
// Msg.error(null, "incomplete json input");
// return;
// default:
// Msg.error(null, "json parser returned undefined status");
// return;
// }
// if(tokens.get(0).start == -1){
// Msg.error(null, "invalid json input");
// return;
// }
// ndx = 0;
// while (ndx <tokens.size()) {
// Object obj = parser.convert(cbuf, tokens);
// objs.add(obj);
// }
// System.out.println("end of parsing");
// return;
// }
//
// public static BufferedReader openReader() {
// return new BufferedReader(new InputStreamReader(System.in));
// }
}

View file

@ -0,0 +1,54 @@
/* ###
* IP: MIT
*/
package generic.json;
public class JSONToken {
/**
* JSON token description.
* @param type type (object, array, string etc.)
* @param start start position in JSON data string
* @param end end position in JSON data string
*/
public JSONType type;
public int start;
public int end;
public int size;
public JSONToken(JSONType type, int start, int end) {
setType(type);
setStart(start);
setEnd(end);
setSize(0);
}
public void setType(JSONType type) {
this.type = type;
}
public JSONType getType() {
return type;
}
public void setStart(int start) {
this.start = start;
}
public int getStart() {
return start;
}
public void setEnd(int end) {
this.end = end;
}
public int getEnd() {
return end;
}
public void setSize(int size) {
this.size = size;
}
public int getSize() {
return size;
}
public void incSize() {
size++;
}
}

View file

@ -0,0 +1,21 @@
/* ###
* IP: MIT
*/
package generic.json;
public enum JSONType {
/**
* JSON type identifier. Basic types are:
* o Object
* o Array
* o String
* o Other primitive: number, boolean (true/false) or null
*/
JSMN_PRIMITIVE,
JSMN_OBJECT,
JSMN_ARRAY,
JSMN_STRING
}

View file

@ -0,0 +1,104 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
* NOTE: Locality Sensitive Hashing
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.lsh;
import static java.lang.Math.PI;
import static java.lang.Math.acos;
import static java.lang.Math.pow;
import java.io.PrintStream;
import java.math.BigDecimal;
import java.math.BigInteger;
/**
* Translated from the C++ version.
*
*/
public class KandL {
// Input desired tau threshold for cosine similarity,
// P1 probability lower bound for meeting this threshold,
// Spits out sample hash size, k and number of tables L giving
// table size and expected query time for various data sizes (n).
static double probOfHashMatch(double tau) {
// Given a lower bound on cosine similarity, -tau-, calculate the probability of a random
// projection, splitting two vectors whose similarity lies within this bound
//
// Pr[ h(v) = h(w) ] = 1 - \theta / pi, where \theta is the angle between v and w
//
double thetabound = acos(tau);
// taubound <= tau => theta <= thetabound
// => probofmatch = 1 -theta/pi >= 1 -thetabound/pi
double probbound = 1.0 - thetabound / PI;
return probbound;
}
public static int memoryModelToL(LSHMemoryModel model) {
return kToL(model.getK(), model.getTauBound(), model.getProbabilityThreshold());
}
public static int kToL(int k, double taubound, double probthresh) {
double P1 = probOfHashMatch(taubound);
// Given hash size and probability of match for a single match, calculate number of tables
// to achieve desired probability threshold
double prob_k_matches = pow(P1, k);
double prob_nomatch = 1.0 - prob_k_matches;
int L = 1;
double prob_nomatch_n = prob_nomatch;
while (1.0 - prob_nomatch_n < probthresh) {
L += 1;
prob_nomatch_n *= prob_nomatch; // Probability of no match after L tables
}
return L;
}
static double binHits(int k, int L, BigInteger n) {
// Expected number of vectors in (one of the) same bins as query (random) vector
BigInteger numbins = new BigInteger(new byte[] { 1 });
numbins = numbins.shiftLeft(k);
double hitsperbin = new BigDecimal(n).divide(new BigDecimal(numbins)).doubleValue();// Expected number of elements per bin
double numcompare = hitsperbin * L;
return numcompare;
}
static void print_result(PrintStream out, int k, int L, BigInteger n, double qt) {
out.println(String.format("k=%d L=%d n=%s bin hits=%f k*L=%d", k, L, n.toString(), qt, k *
L));
}
static void process_n(PrintStream out, BigInteger n, double taubound, double probthresh) {
for (int k = 10; k <= 30; ++k) {
int L = kToL(k, taubound, probthresh);
double qt = binHits(k, L, n);
print_result(out, k, L, n, qt);
}
}
public static void main(String[] args) {
try {
BigInteger n = new BigInteger(args[0]);
double taubound = Double.parseDouble(args[1]);
double probthresh = Double.parseDouble(args[2]);
process_n(System.out, n, taubound, probthresh);
}
catch (Exception e) {
System.err.println("caught " + e.getClass().getName() + ": " + e.getLocalizedMessage());
System.err.println("USAGE: KandL n taulowerbound probthresh");
}
}
}

View file

@ -0,0 +1,57 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.lsh;
public enum LSHMemoryModel {
SMALL("Small (slower)", 10, 0.97, 0.75),
MEDIUM("Medium", 13, 0.97, 0.75),
LARGE("Large (faster)", 16, 0.97, 0.75);
private String label;
//k = #of hyperplanes comprising the each binning.
private int k;
private double probabilityThreshold;
private double tauBound;
private LSHMemoryModel(String label, int k, double probabilityThreshold, double tauBound) {
this.label = label;
this.k = k;
this.probabilityThreshold = probabilityThreshold;
this.tauBound = tauBound;
}
public String getLabel() {
return label;
}
public int getK() {
return k;
}
public double getProbabilityThreshold() {
return probabilityThreshold;
}
public double getTauBound() {
return tauBound;
}
@Override
public String toString() {
return label;
}
}

View file

@ -0,0 +1,87 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.lsh;
import generic.lsh.vector.HashEntry;
public class Partition {
private Partition() {
// non-instantiable class
}
private static final int FNV_32_BIT_OFFSET_BASIS = 0x811C9DC5;
private static final int FNV_32_BIT_PRIME = 0x1000193;
private static boolean partition(final int identity, final int value) {
int hash = FNV_32_BIT_OFFSET_BASIS;
int blender = value;
hash ^= blender & 0xff;
hash *= FNV_32_BIT_PRIME;
blender >>>= 8;
hash ^= blender & 0xff;
hash *= FNV_32_BIT_PRIME;
blender >>>= 8;
hash ^= blender & 0xff;
hash *= FNV_32_BIT_PRIME;
blender >>>= 8;
hash ^= blender & 0xff;
hash *= FNV_32_BIT_PRIME;
blender = identity;
hash ^= blender & 0xff;
hash *= FNV_32_BIT_PRIME;
blender >>>= 8;
hash ^= blender & 0xff;
hash *= FNV_32_BIT_PRIME;
blender >>>= 8;
hash ^= blender & 0xff;
hash *= FNV_32_BIT_PRIME;
blender >>>= 8;
hash ^= blender & 0xff;
hash *= FNV_32_BIT_PRIME;
final int bitCount = Integer.bitCount(hash);
return bitCount % 2 == 0;
}
private static int partition(final int identity, final HashEntry[] values) {
float total = 0;
for (int i=0;i<values.length;++i) {
HashEntry entry = values[i];
if (partition(identity, entry.getHash()))
total += entry.getCoeff();
else
total -= entry.getCoeff();
}
return total < 0 ? 0 : 1;
}
public static int hash(final int[] partitionIdentities, final HashEntry[] values) {
int result = 0;
int bit = 1;
for (int identity : partitionIdentities) {
if (partition(identity, values) == 1) {
result |= bit;
}
bit <<= 1;
}
return result;
}
}

View file

@ -0,0 +1,269 @@
/* ###
* IP: GHIDRA
* NOTE: Locality Sensitive Hashing
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.lsh.vector;
import java.io.IOException;
import java.io.Writer;
import ghidra.util.xml.SpecXmlUtils;
import ghidra.xml.XmlElement;
import ghidra.xml.XmlPullParser;
public class HashEntry {
private int hash;// A specific hash
private short tf;// Associated hash(term) frequency (minus one) within the vector
private short idf;// Inverse Document Frequency (a lookup index for "information" in this hash)
private double coeff;// The actual weight of this hash as a coefficient
public HashEntry() {// For use with restoreXml
}
/**
* Create a hash entry with an explicit weight
* @param h is the 32-bit hash
* @param tcnt is the (optional) term-frequency count (set to 1 if not using)
* @param weight is the weight associated with the hash
*/
public HashEntry(int h, int tcnt, double weight) {
hash = h;
tf = (short) ((tcnt > 63) ? 63 : tcnt - 1);
idf = 1;
coeff = weight;
}
/**
* Create a hash entry with a weight calculated from its term frequency and idf frequency
* @param h is the 32-bit hash
* @param tcnt is the term frequency count
* @param dcnt is the (normalized) idf frequency (should be generated by an IDFLookup)
* @param w is the factory used to generate the final weight
*/
public HashEntry(int h, int tcnt, int dcnt, WeightFactory w) {
hash = h;
tf = (short) ((tcnt > 63) ? 63 : tcnt - 1);
idf = (short) ((dcnt > 511) ? 511 : dcnt);
coeff = w.getCoeff(idf, tf);
}
/**
* Eclipse-generated hash function.
*
* @return
*/
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + hash;
result = prime * result + tf;
return result;
}
/**
* Eclipse-generated equals function.
*
* @param obj
* @return
*/
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (!(obj instanceof HashEntry)) {
return false;
}
HashEntry other = (HashEntry) obj;
if (hash != other.hash) {
return false;
}
if (tf != other.tf) {
return false;
}
return true;
}
public int getHash() {
return hash;
}
public short getTF() {
return (short) (tf + 1);
}
public short getIDF() {
return idf;
}
public double getCoeff() {
return coeff;
}
public void saveXml(Writer fwrite) throws IOException {
StringBuilder buf = new StringBuilder();
buf.append(" <hash");
if (tf != 0) {
SpecXmlUtils.encodeSignedIntegerAttribute(buf, "tf", tf + 1);
}
// if (idf != 0) {
// SpecXmlUtils.encodeSignedIntegerAttribute(buf, "idf", idf);
// }
buf.append('>');
buf.append(SpecXmlUtils.encodeUnsignedInteger((hash) & 0xffffffffL));
buf.append("</hash>\n");
fwrite.append(buf.toString());
}
public void saveSQL(StringBuilder buf) {
buf.append(Integer.toHexString(tf + 1));
buf.append(':');
buf.append(Integer.toHexString(hash));
}
public void restoreXml(XmlPullParser parser, WeightFactory w) {
tf = 0;
idf = 0;
XmlElement el = parser.start("hash");
String str = el.getAttribute("tf");
if (str != null) {
tf = (short) SpecXmlUtils.decodeInt(str);
tf -= 1;
}
str = el.getAttribute("idf");
if (str != null) {
idf = (short) SpecXmlUtils.decodeInt(str);
}
hash = SpecXmlUtils.decodeInt(parser.end().getText());
coeff = w.getCoeff(idf, tf);
}
/**
* Restore entry but recalculate the idf
* @param parser // xml state
* @param w // weight factory to calculate coefficient with
* @param lookup // lookup object to recalculate idf
*/
public void restoreXml(XmlPullParser parser, WeightFactory w, IDFLookup lookup) {
tf = 0;
XmlElement el = parser.start("hash");
String str = el.getAttribute("tf");
if (str != null) {
tf = (short) SpecXmlUtils.decodeInt(str);
tf -= 1;
}
hash = SpecXmlUtils.decodeInt(parser.end().getText());
idf = (short) lookup.getCount(hash);
coeff = w.getCoeff(idf, tf);
}
private int parseHash(String sql, int start) throws IOException {
hash = 0;
for (;;) {
if (start >= sql.length()) {
throw new IOException("Parsing hashentry with no terminator");
}
int tok = sql.charAt(start);
if (tok < '0') {
return start;
}
if (tok <= '9') {
hash <<= 4;
hash += (tok - '0');
}
else if (tok < 'A') {
return start;
}
else if (tok <= 'F') {
hash <<= 4;
hash += ((tok - 'A') + 10);
}
else if (tok < 'a') {
return start;
}
else if (tok <= 'f') {
hash <<= 4;
hash += ((tok - 'a') + 10);
}
else {
return start;
}
start += 1;
}
}
public int restoreSQL(String sql, int start, WeightFactory w, IDFLookup lookup)
throws IOException {
hash = 0;
start = parseHash(sql, start);
if ((hash == 0) || (sql.charAt(start) != ':')) {
throw new IOException("Error parsing HashEntry");
}
tf = (short) (hash - 1);
start = parseHash(sql, start + 1);
idf = (short) lookup.getCount(hash);
coeff = w.getCoeff(idf, tf);
return start;
}
public boolean restoreBase64(char[] buffer,int offset,int[] decoder,WeightFactory w,IDFLookup lookup) {
tf = (short)decoder[buffer[offset]]; // Value between 0 and 63
if (tf < 0) {
return false; // Check for bad character
}
int val = decoder[buffer[offset+1]];
val <<= 6;
val |= decoder[buffer[offset+2]];
val <<= 6;
val |= decoder[buffer[offset+3]];
val <<= 6;
val |= decoder[buffer[offset+4]];
val <<= 6;
val |= decoder[buffer[offset+5]];
if (val < 0) {
return false; // Only 30-bits read so far, should be positive
}
int rem1 = decoder[buffer[offset+6]];
val <<= 2;
hash = val | (rem1 & 3); // Final 2 bits of 32-bit hash
if (rem1 > 3) {
return false; // Remaining 4-bits should be zero
}
idf = (short) lookup.getCount(hash);
coeff = w.getCoeff(idf, tf);
return true;
}
public void saveBase64(char[] buffer,int offset,char[] encoder) {
buffer[offset] = encoder[tf];
int val = hash;
buffer[offset+6] = encoder[ val & 3 ]; // Final 2 bits
val >>>= 2;
buffer[offset+5] = encoder[ val & 0x3f ];
val >>>= 6;
buffer[offset+4] = encoder[ val & 0x3f ];
val >>>= 6;
buffer[offset+3] = encoder[ val & 0x3f ];
val >>>= 6;
buffer[offset+2] = encoder[ val & 0x3f ];
val >>>= 6;
buffer[offset+1] = encoder[ val & 0x3f ];
}
}

View file

@ -0,0 +1,170 @@
/* ###
* IP: GHIDRA
* NOTE: Locality Sensitive Hashing
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.lsh.vector;
import ghidra.util.xml.SpecXmlUtils;
import ghidra.xml.XmlElement;
import ghidra.xml.XmlPullParser;
import java.io.IOException;
import java.io.Writer;
public class IDFLookup {
public class IDFEntry {
public int hash;
public int count;
}
private int size; // Number of entries in table
private int mask;
private IDFEntry[] hashtable;
private void initializeTable() {
mask = 1;
while(mask < size) // Find first power of two greater than or equal to size
mask <<= 1;
mask <<= 1;
hashtable = new IDFEntry[ mask ];
for(int i=0;i<mask;++i) {
hashtable[i] = new IDFEntry();
hashtable[i].count = 0xffffffff; // Mark all the slots as empty
}
mask -= 1;
}
public IDFLookup() {
hashtable = null;
mask = 0;
}
public boolean empty() {
return (hashtable==null);
}
public int getCount(int hash) {
if (mask==0) return 0;
int val = hash & mask;
IDFEntry entry = hashtable[val];
while(entry.count != 0xffffffff) {
if (entry.hash == hash)
return entry.count;
val = (val+1)&mask;
entry = hashtable[val];
}
return 0;
}
public int getCapacity() {
return mask;
}
public int getRawHash(int pos) {
return hashtable[pos].hash;
}
public int getRawCount(int pos) {
return hashtable[pos].count;
}
private void insertHash(int hash,int count) {
IDFEntry entry;
int val = hash & mask;
for(;;) {
entry = hashtable[val];
if (entry.count == 0xffffffff) // An empty slot
break;
val = (val+1)&mask;
}
entry.hash = hash;
entry.count = count;
}
public void saveXml(Writer fwrite) throws IOException {
if (empty()) {
fwrite.append("<idflookup/>\n");
return;
}
StringBuilder buf = new StringBuilder();
buf.append("<idflookup");
SpecXmlUtils.encodeSignedIntegerAttribute(buf, "size", size);
buf.append(">\n");
int sz = mask + 1;
for(int i=0;i<sz;++i) {
IDFEntry entry = hashtable[i];
if (entry.count == 0xffffffff) continue;
buf.append("<hash");
SpecXmlUtils.encodeSignedIntegerAttribute(buf, "count", entry.count);
buf.append('>');
buf.append(SpecXmlUtils.encodeUnsignedInteger(entry.hash));
buf.append("</hash>\n");
}
buf.append("</idflookup>\n");
fwrite.append(buf.toString());
}
public void restoreXml(XmlPullParser parser) {
XmlElement el = parser.start("idflookup");
if (!el.hasAttribute("size"))
return; // Empty table
size = SpecXmlUtils.decodeInt(el.getAttribute("size"));
initializeTable();
while(parser.peek().isStart()) {
XmlElement subel = parser.start("hash");
int count = SpecXmlUtils.decodeInt(subel.getAttribute("count"));
int hash = SpecXmlUtils.decodeInt(parser.end().getText());
insertHash(hash,count);
}
parser.end(el);
}
/**
* Collapse IDFLookup into an int array, suitable for storage
* @return int[]
*/
public int[] toArray() {
int count = 0;
for(int i=0;i<hashtable.length;++i)
if (hashtable[i].count != 0xffffffff) // If not empty
count += 1; // count it
int[] res = new int[count * 2];
int pos = 0;
for(int i=0;i<hashtable.length;++i) {
if (hashtable[i].count == 0xffffffff) continue;
res[pos] = hashtable[i].hash;
pos += 1;
res[pos] = hashtable[i].count;
pos += 1;
}
return res;
}
/**
* Set from an array of hash/count pairs. Every even index is a hash, every odd index is a count
* @param hashCountPair is the pair array
*/
public void set(int[] hashCountPair) {
size = hashCountPair.length/2;
initializeTable();
for(int i=0;i<hashCountPair.length;i+=2)
insertHash(hashCountPair[i],hashCountPair[i+1]);
}
}

View file

@ -0,0 +1,568 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.lsh.vector;
import java.io.IOException;
import java.io.Reader;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Arrays;
import generic.hash.SimpleCRC32;
import ghidra.xml.XmlPullParser;
public class LSHCosineVector implements LSHVector {
private static final HashEntry[] EMPTY = new HashEntry[0];
private HashEntry[] hash = EMPTY;// Sorted list of hash values and their counts
private double length;// Length of vector
private int hashcount;// Total number of hashes (counting multiplicity)
public LSHCosineVector() {// For use as a template
length = 0.0;
hashcount = 0;
}
/**
* Install a set of features as an int[]. Each integer is a hash. The integers MUST already be sorted.
* The same integer can occur more than once in the array (term frequency (TF) > 1).
* Weights are determined by TF and Inverse Document Frequency (IDF) of individual features
* @param feature is the sorted array of integer hashes
* @param wfactory is the container of weighting information
* @param idflookup is the container of IDF information
*/
public LSHCosineVector(int[] feature,WeightFactory wfactory,IDFLookup idflookup) {
installFeatures(feature,wfactory,idflookup);
calcLength();
}
/**
* Uses the existing {@link #calcUniqueHash()} method to determine hash value.
*
* @return
*/
@Override
public int hashCode() {
return (int) calcUniqueHash();
}
/**
* Eclipse-generated equals method. Only the hash attribute is necessary.
*
* @param obj
* @return
*/
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (!(obj instanceof LSHCosineVector)) {
return false;
}
LSHCosineVector other = (LSHCosineVector) obj;
if (!Arrays.equals(hash, other.hash)) {
return false;
}
return true;
}
/**
* Install hashes and weights directly. Length is automatically calculated.
* The entries must already be sorted on the hash
* @param entries
*/
public void setHashEntries(HashEntry[] entries) {
hash = entries;
calcLength();
}
private void calcLength() {
length = 0.0;
hashcount = 0;
for (int i = 0; i < hash.length; ++i) {
if (hash[i] == null) {
continue;
}
double coeff = hash[i].getCoeff();
length += coeff * coeff;
hashcount += hash[i].getTF();
}
length = Math.sqrt(length);
}
/**
* Assuming -feature- is sorted and -hash- is empty, count the features and populate -hash- and -tf-
* For every unique feature, look up its idf via -idflookup-
* @param feature is the list of sorted hash features
* @param wfactory is the WeightFactory used to decide feature weights
* @param idflookup is the IDFLookup used to decide relative frequency of individual features
*/
private void installFeatures(int[] feature,WeightFactory wfactory,IDFLookup idflookup) {
if (feature.length == 0) {
return;// No features
}
int lasthash = feature[0];
int idf;
int count = 1;
int sz = 1;
for (int i = 0; i < feature.length; ++i) {
if (feature[i] != lasthash) {
lasthash = feature[i];
sz += 1;
}
}
hash = new HashEntry[sz];
lasthash = feature[0];
sz = 0;
if (!idflookup.empty()) {
idf = idflookup.getCount(lasthash);
for (int i = 1; i < feature.length; ++i) {
int featurei = feature[i];
if (featurei != lasthash) {
hash[sz] = new HashEntry(lasthash, count, idf, wfactory);
sz += 1;
lasthash = featurei;
count = 1;
idf = idflookup.getCount(lasthash);
}
else {
count += 1;
}
}
}
else {
idf = 0;
for (int i = 1; i < feature.length; ++i) {
int featurei = feature[i];
if (featurei != lasthash) {
hash[sz] = new HashEntry(lasthash, count, idf, wfactory);
sz += 1;
lasthash = featurei;
count = 1;
}
else {
count += 1;
}
}
}
hash[sz] = new HashEntry(lasthash, count, idf, wfactory);
}
@Override
public int numEntries() {
return hash.length;
}
@Override
public HashEntry getEntry(int i) {
return hash[i];
}
@Override
public HashEntry[] getEntries() {
return hash;
}
@Override
public double compare(LSHVector op2, VectorCompare data) {
int iter, enditer, iter2, enditer2;
LSHCosineVector op = (LSHCosineVector) op2;
iter = 0;
enditer = hash.length;
iter2 = 0;
enditer2 = op.hash.length;
double res = 0.0;
int intersectcount = 0;
int hash1, hash2;
if ((iter != enditer) && (iter2 != enditer2)) {
hash1 = hash[iter].getHash();
hash2 = op.hash[iter2].getHash();
for (;;) {
if (hash1 == hash2) {
int t1 = hash[iter].getTF();
int t2 = op.hash[iter2].getTF();
if (t1 < t2) {
double w1 = hash[iter].getCoeff();
res += w1 * w1;
intersectcount += t1;
}
else {
double w2 = op.hash[iter2].getCoeff();
res += w2 * w2;
intersectcount += t2;
}
++iter;
++iter2;
if (iter == enditer) {
break;
}
if (iter2 == enditer2) {
break;
}
hash1 = hash[iter].getHash();
hash2 = op.hash[iter2].getHash();
}
else if (hash1 + 0x80000000 < hash2 + 0x80000000) {// This needs to be an UNSIGNED comparison of hash1 and hash2
++iter;
if (iter == enditer) {
break;
}
hash1 = hash[iter].getHash();
}
else {// hash1 > hash2
++iter2;
if (iter2 == enditer2) {
break;
}
hash2 = op.hash[iter2].getHash();
}
}
data.dotproduct = res;
res /= (length * op.length);
}
else {
data.dotproduct = res;
}
data.intersectcount = intersectcount;
data.acount = hashcount;
data.bcount = op.hashcount;
return res;
}
@Override
public void compareCounts(LSHVector op2, VectorCompare data) {
int iter, enditer, iter2, enditer2;
LSHCosineVector op = (LSHCosineVector) op2;
iter = 0;
enditer = hash.length;
iter2 = 0;
enditer2 = op.hash.length;
int intersectcount = 0;
int hash1, hash2;
if ((iter != enditer) && (iter2 != enditer2)) {
hash1 = hash[iter].getHash();
hash2 = op.hash[iter2].getHash();
for (;;) {
if (hash1 == hash2) {
int t1 = hash[iter].getTF();
int t2 = op.hash[iter2].getTF();
intersectcount += (t1 < t2) ? t1 : t2;
++iter;
++iter2;
if (iter == enditer) {
break;
}
if (iter2 == enditer2) {
break;
}
hash1 = hash[iter].getHash();
hash2 = op.hash[iter2].getHash();
}
else if (hash1 + 0x80000000 < hash2 + 0x80000000) {// This needs to be an UNSIGNED comparison of hash1 and hash2
++iter;
if (iter == enditer) {
break;
}
hash1 = hash[iter].getHash();
}
else {// hash1 > hash2
++iter2;
if (iter2 == enditer2) {
break;
}
hash2 = op.hash[iter2].getHash();
}
}
}
data.intersectcount = intersectcount;
data.acount = hashcount;
data.bcount = op.hashcount;
}
private void writeOnlyList(ArrayList<HashEntry> only, StringBuilder buf) {
for (int i = 0; i < only.size(); ++i) {
HashEntry entry = only.get(i);
buf.append(Integer.toHexString(entry.getHash()));
buf.append(' ').append(entry.getTF());
buf.append(' ').append(entry.getCoeff());
buf.append('\n');
}
}
private void writeBothList(ArrayList<HashEntry> both, StringBuilder buf) {
for (int i = 0; i < both.size(); i += 2) {
HashEntry entry1 = both.get(i);
HashEntry entry2 = both.get(i + 1);
buf.append(Integer.toHexString(entry1.getHash()));
buf.append(" (").append(entry1.getTF()).append(',').append(entry2.getTF()).append(
") (");
buf.append(entry1.getCoeff()).append(',').append(entry2.getCoeff()).append(")\n");
}
}
@Override
public double compareDetail(LSHVector op2, StringBuilder buf) {
int iter, enditer, iter2, enditer2;
LSHCosineVector op = (LSHCosineVector) op2;
ArrayList<HashEntry> a_only = new ArrayList<HashEntry>();
ArrayList<HashEntry> b_only = new ArrayList<HashEntry>();
ArrayList<HashEntry> ab_both = new ArrayList<HashEntry>();
buf.append("lena=").append(getLength()).append('\n');
buf.append("lenb=").append(op2.getLength()).append('\n');
iter = 0;
enditer = hash.length;
iter2 = 0;
enditer2 = op.hash.length;
double res = 0.0;
int intersectcount = 0;
int hash1, hash2;
if ((iter != enditer) && (iter2 != enditer2)) {
hash1 = hash[iter].getHash();
hash2 = op.hash[iter2].getHash();
for (;;) {
if (hash1 == hash2) {
ab_both.add(hash[iter]);
ab_both.add(op.hash[iter2]);
int t1 = hash[iter].getTF();
int t2 = op.hash[iter2].getTF();
if (t1 < t2) {
double w1 = hash[iter].getCoeff();
res += w1 * w1;
intersectcount += t1;
}
else {
double w2 = op.hash[iter2].getCoeff();
res += w2 * w2;
intersectcount += t2;
}
++iter;
++iter2;
if (iter == enditer) {
break;
}
if (iter2 == enditer2) {
break;
}
hash1 = hash[iter].getHash();
hash2 = op.hash[iter2].getHash();
}
else if (hash1 + 0x80000000 < hash2 + 0x80000000) {// This needs to be an UNSIGNED comparison of hash1 and hash2
a_only.add(hash[iter]);
++iter;
if (iter == enditer) {
break;
}
hash1 = hash[iter].getHash();
}
else {// hash1 > hash2
b_only.add(op.hash[iter2]);
++iter2;
if (iter2 == enditer2) {
break;
}
hash2 = op.hash[iter2].getHash();
}
}
buf.append("dotproduct=").append(res).append('\n');
buf.append("intersect=").append(intersectcount).append('\n');
res /= (length * op.length);
}
while (iter != enditer) {
a_only.add(hash[iter]);
++iter;
}
while (iter2 != enditer2) {
b_only.add(op.hash[iter2]);
++iter2;
}
writeOnlyList(a_only, buf);
buf.append('\n');
writeBothList(ab_both, buf);
buf.append('\n');
writeOnlyList(b_only, buf);
return res;
}
@Override
public double getLength() {
return length;
}
@Override
public void restoreXml(XmlPullParser parser,WeightFactory wfactory,IDFLookup idflookup) {
parser.start("lshcosine");
ArrayList<HashEntry> hashlist = new ArrayList<HashEntry>();
if (idflookup.empty()) {
while (parser.peek().isStart()) {
HashEntry entry = new HashEntry();
hashlist.add(entry);
entry.restoreXml(parser, wfactory);
}
}
else {
while (parser.peek().isStart()) {
HashEntry entry = new HashEntry();
hashlist.add(entry);
entry.restoreXml(parser, wfactory, idflookup);
}
}
parser.end();
hash = new HashEntry[hashlist.size()];
hashlist.toArray(hash);
calcLength();// The length is not stored as part of XML
}
@Override
public void restoreSQL(String sql,WeightFactory wfactory,IDFLookup idflookup) throws IOException {
ArrayList<HashEntry> hashlist = new ArrayList<HashEntry>();
char tok;
int start;
if (sql.length() < 2) {
throw new IOException("Empty lshvector SQL");
}
if (sql.charAt(0) != '(') {
throw new IOException("Missing '(' while parsing lshvector SQL");
}
start = 1;
tok = sql.charAt(1);
if (tok != ')') {
do {
HashEntry entry = new HashEntry();
hashlist.add(entry);
start = entry.restoreSQL(sql, start, wfactory, idflookup);
tok = sql.charAt(start);
start += 1;
}
while (tok == ',');
}
if (tok != ')') {
throw new IOException("Missing ')' while parsing lshvector SQL");
}
hash = new HashEntry[hashlist.size()];
hashlist.toArray(hash);
calcLength();
}
@Override
public void restoreBase64(Reader input,char[] buffer,WeightFactory wfactory,IDFLookup idflookup,int[] decode) throws IOException {
ArrayList<HashEntry> hashlist = new ArrayList<HashEntry>();
int returned;
do {
returned = input.read(buffer,0,112);
for(int i=0;i<returned;i+=7) {
HashEntry entry = new HashEntry();
if (!entry.restoreBase64(buffer, i, decode, wfactory, idflookup))
throw new IOException("Bad base64 encoding of LSHCosine vector");
hashlist.add(entry);
}
} while( returned == 112);
hash = new HashEntry[hashlist.size()];
hashlist.toArray(hash);
calcLength();
}
@Override
public void saveXml(Writer fwrite) throws IOException {
StringBuilder buf = new StringBuilder();
buf.append("<lshcosine>\n");
fwrite.append(buf.toString());
// The length is not stored as part of XML
for (int i = 0; i < hash.length; ++i) {
hash[i].saveXml(fwrite);
}
fwrite.append("</lshcosine>\n");
}
@Override
public String saveSQL() {
StringBuilder buf = new StringBuilder();
buf.append('(');
if (hash.length == 0) {
buf.append(')');
return buf.toString();
}
hash[0].saveSQL(buf);
for (int i = 1; i < hash.length; ++i) {
buf.append(',');
hash[i].saveSQL(buf);
}
buf.append(')');
return buf.toString();
}
@Override
public void saveBase64(StringBuilder buffer, char[] encoder) {
if (hash.length == 0)
return;
char[] charBuf = new char[70];
int i = 0;
int charpos = 0;
for (;;) {
hash[i].saveBase64(charBuf, charpos, encoder);
i += 1;
charpos += 7;
if (i >= hash.length)
break;
if (charpos == 70) {
buffer.append(charBuf);
charpos = 0;
}
}
if (charpos != 0)
buffer.append(charBuf, 0, charpos);
}
/* (non-Javadoc)
* @see ghidra.query.vector.LSHVector#calcUniqueHash()
*/
@Override
public long calcUniqueHash() {
int reg1 = 0x12CF93AB;
int reg2 = 0xEE39B2D6;
for (int i = 0; i < hash.length; ++i) {
HashEntry entry = hash[i];
int curtf = entry.getTF();
int curhash = entry.getHash();
int oldreg1 = reg1;
reg1 = SimpleCRC32.hashOneByte(reg1, curtf);
reg1 = SimpleCRC32.hashOneByte(reg1, curhash);
reg1 = SimpleCRC32.hashOneByte(reg1, (reg2 >>> 24));
reg2 = SimpleCRC32.hashOneByte(reg2, (oldreg1 >>> 24));
reg2 = SimpleCRC32.hashOneByte(reg2, (curhash >>> 8));
reg2 = SimpleCRC32.hashOneByte(reg2, (curhash >>> 16));
reg2 = SimpleCRC32.hashOneByte(reg2, (curhash >>> 24));
}
long res = reg1;
long res2 = reg2;
res2 <<= 32;// Make sure we don't sign extend, casting from int to long
res2 >>>= 32;
res <<= 32;
res |= res2;
return res;
}
}

View file

@ -0,0 +1,124 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.lsh.vector;
import java.util.TreeSet;
/**
* A cosine vector where we can accumulate the (feature,weight) pairs over time
* using the addHash method. Once either the getLength or compare methods is
* called the vector becomes "finalized" and acts as an ordinary LSHCosineVector
*/
public class LSHCosineVectorAccum extends LSHCosineVector {
public static class Entry implements Comparable<Entry> {
public final int hash;
public final double weight;
public Entry(int hash, double weight) {
this.hash = hash;
this.weight = weight;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + hash;
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
Entry other = (Entry) obj;
if (hash != other.hash)
return false;
return true;
}
/**
* Comparison must be UNSIGNED!!
*/
@Override
public int compareTo(Entry o) {
if (hash < 0) {
if (o.hash >= 0)
return 1;
}
else if (o.hash < 0)
return -1;
return (hash - o.hash);
}
@Override
public String toString() {
return hash + "(" + weight + ")";
}
}
private TreeSet<Entry> treehash;
private boolean finalized = false;
public LSHCosineVectorAccum() {
super();
treehash = new TreeSet<Entry>();
}
public void addHash(int h, double w) {
if (finalized) {
throw new RuntimeException("already finalized");
}
treehash.add(new Entry(h, w));
}
public void doFinalize() {
if (finalized)
return;
HashEntry[] entries = new HashEntry[treehash.size()];
int count = 0;
for (Entry entry : treehash) {
HashEntry h = new HashEntry(entry.hash, 1, entry.weight);
entries[count] = h;
count += 1;
}
setHashEntries(entries);
treehash = null; // Allow the accumulator to be reclaimed
finalized = true;
}
@Override
public double getLength() {
doFinalize();
return super.getLength();
}
@Override
public double compare(LSHVector op2, VectorCompare data) {
doFinalize();
((LSHCosineVectorAccum) op2).doFinalize(); // Trigger finalization
return super.compare(op2, data);
}
@Override
public int numEntries() {
return treehash.size();
}
}

View file

@ -0,0 +1,54 @@
/* ###
* IP: GHIDRA
* NOTE: Locality Sensitive Hashing
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.lsh.vector;
import ghidra.xml.XmlPullParser;
import java.io.IOException;
import java.io.Reader;
import java.io.Writer;
public interface LSHVector {
public int numEntries();
public HashEntry getEntry(int i);
public HashEntry[] getEntries();
public double getLength();
public double compare(LSHVector op2,VectorCompare data);
public void compareCounts(LSHVector op2, VectorCompare data);
public double compareDetail(LSHVector op2, StringBuilder buf);
public void saveXml(Writer fwrite) throws IOException;
public String saveSQL();
public void saveBase64(StringBuilder buffer,char[] encoder);
public void restoreXml(XmlPullParser parser,WeightFactory weightFactory,IDFLookup idfLookup);
public void restoreSQL(String sql,WeightFactory weightFactory,IDFLookup idfLookup) throws IOException;
public void restoreBase64(Reader input,char[] buffer,WeightFactory wfactory,IDFLookup idflookup,int[] decode) throws IOException;
public long calcUniqueHash();
}

View file

@ -0,0 +1,163 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.lsh.vector;
import java.io.IOException;
import org.xml.sax.SAXException;
import ghidra.xml.XmlElement;
import ghidra.xml.XmlPullParser;
public abstract class LSHVectorFactory {
protected WeightFactory weightFactory = null; // Container for vector weighting information and score normalization
protected IDFLookup idfLookup = null; // Container for Inverse Document Frequency (IDF) information
protected int settings; // Settings used to generate weights and lookup hashes
/**
* Generate vector with all coefficients zero.
* @return the zero vector
*/
public abstract LSHVector buildZeroVector();
/**
* Generate an LSHVector from a feature set, individual features are integer hashes.
* The integers MUST already be sorted.
* The same integer can occur more than once in the array (term frequency (TF) > 1).
* The factory decides internally how to create weights based on term frequency and any
* knowledge of Inverse Document Frequency (IDF)
* @param feature is the sorted array of integer features
* @return the newly minted LSHVector
*/
public abstract LSHVector buildVector(int[] feature);
/**
* Generate an LSHVector based on XML tag seen by pull parser.
* Factory generates weights based on term frequency info in the XML tag and its internal IDF knowledge
* @param parser is the XML parser
* @return the newly minted LSHVector
*/
public abstract LSHVector restoreVectorFromXml(XmlPullParser parser);
/**
* Generate an LSHVector based on string returned from SQL query
* Factory generates weights based on term frequency info in the string and its internal IDF knowledge
* @param sql is the column data string returned by an SQL query
* @return the newly minted LSHVector
* @throws IOException
*/
public abstract LSHVector restoreVectorFromSql(String sql) throws IOException;
/**
* Load the factory with weights and the feature map
* @param wFactory is the weight table of IDF and TF weights
* @param iLookup is the map from features int the weight table
* @param settings is an integer id for this particular weighting scheme
*/
public void set(WeightFactory wFactory, IDFLookup iLookup, int settings) {
weightFactory = wFactory;
idfLookup = iLookup;
this.settings = settings;
}
/**
* @return true if this factory has weights and lookup loaded
*/
public boolean isLoaded() {
return ((idfLookup != null) && (!idfLookup.empty()));
}
/**
* @return the weighttable's significance scale for this factory
*/
public double getSignificanceScale() {
return weightFactory.getScale();
}
/**
* @return the weighttable's significance addend for this factory
*/
public double getSignificanceAddend() {
return weightFactory.getAddend();
}
/**
* @return settings ID used to generate factory's current weights
*/
public int getSettings() {
return settings;
}
/**
* Calculate a vector's significance as compared to itself, normalized for this factory's specific weight settings
* @param vector is the LSHVector
* @return the vector's significance score
*/
public double getSelfSignificance(LSHVector vector) {
return vector.getLength() * vector.getLength() + weightFactory.getAddend();
}
/**
* Given comparison data generated by the LSHVector.compare() method,
* calculate the significance of any similarity between the two vectors,
* normalized for this factory's specific weight settings
* @param data is the comparison object produced when comparing two LSHVectors
* @return the significance score
*/
public double calculateSignificance(VectorCompare data) {
data.fillOut();
return data.dotproduct -
data.numflip *
(weightFactory.getFlipNorm0() + weightFactory.getFlipNorm1() / data.max) -
data.diff * (weightFactory.getDiffNorm0() + weightFactory.getDiffNorm1() / data.max) +
weightFactory.getAddend();
}
/**
* Read both the weights and the lookup hashes from an XML stream
* @param parser is the XML parser
* @throws SAXException
*/
public void readWeights(XmlPullParser parser) throws SAXException {
weightFactory = new WeightFactory(); // Allocate new weight factory we will read into
idfLookup = new IDFLookup(); // Allocate new IDF object we will read into
boolean foundweights = false;
boolean foundlookup = false;
XmlElement el = parser.start();
settings = Integer.decode(el.getAttribute("settings"));
el = parser.peek(); // The <weightfactory> and <idflookup> tags must be at the second level of the xml
while(el.isStart()) {
if (el.getName().equals("weightfactory")) {
weightFactory.restoreXml(parser);
foundweights = true;
}
else if (el.getName().equals("idflookup")) {
idfLookup.restoreXml(parser);
foundlookup = true;
}
else {
parser.discardSubTree();
}
el = parser.peek();
}
if (!foundweights) {
throw new SAXException("Could not find <weightfactory> tag in configuration");
}
if (!foundlookup) {
throw new SAXException("Could not find <idflookup> tag in configuration");
}
}
}

View file

@ -0,0 +1,67 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.lsh.vector;
/**
* Class for containing intermediate results of doing the LSHVector compare operation
*
*/
public class VectorCompare {
public double dotproduct; // Result of the dot product
public int acount; // total number of hashes in first vector
public int bcount; // total number of hashes in second vector
public int intersectcount; // total number of hashes in common
public int min; // Minimum vector count
public int max; // Maximum vector count
public int numflip; // Number of hashes flipped
public int diff; // Difference in number of hashes
/**
* Assume the dotproduct, acount, bcount, and intersectcount are filled in
* Calculate the remaining values: min, max, numflip, and diff
* Assume small vector is produced by flipping and removing hashes from big vector
* Calculate the number of flipped hashes (numflip) from a VectorCompare result
* Calculate the difference in the number of hashes (diff) from a VectorCompare result
*/
public void fillOut() {
if (acount < bcount) {
min = acount; // Smallest vector is a
max = bcount;
}
else {
min = bcount; // Smallest vector is b
max = acount;
}
diff = max - min; // Subtract to get a positive difference
numflip = min - intersectcount; // Number of hashes in smallest vector not in intersection
}
@Override
public String toString() {
StringBuffer buffer = new StringBuffer();
buffer.append("\nVectorCompare: ");
buffer.append("\n Result of the dot product = " + dotproduct);
buffer.append("\n # of hashes in first vector = " + acount);
buffer.append("\n # of hashes in second vector = " + bcount);
buffer.append("\n # of hashes in common = " + intersectcount);
buffer.append("\n Minimum vector count = " + min);
buffer.append("\n Maximum vector count = " + max);
buffer.append("\n Number of hashes flipped = " + numflip);
buffer.append("\n Difference in # of hashes = " + diff);
return buffer.toString();
}
}

View file

@ -0,0 +1,267 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.lsh.vector;
import java.io.IOException;
import java.io.Writer;
import ghidra.xml.XmlElement;
import ghidra.xml.XmlPullParser;
public class WeightFactory {
private double idfweight[] = new double[512]; // Weights associated with (normalized) idf counts
private double tfweight[] = new double[64]; // Weights associated with tf (term frequency) counts
private double weightnorm; // Scale to which idf weights are normalized = -log2( probability of 1000th most common hash)
private double probflip0; // Hash flipping probability in causal model, param0
private double probflip1; // Hash flipping probability in causal model, param1
private double probdiff0; // Hash addition/removal probability, param0
private double probdiff1; // Hash addition/removal probability, param1
private double scale; // Final scaling to all weights
private double addend; // Final correction to score
private double probflip0_norm;
private double probflip1_norm;
private double probdiff0_norm;
private double probdiff1_norm;
private void updateNorms() {
probflip0_norm = probflip0 * scale;
probflip1_norm = probflip1 * scale;
probdiff0_norm = probdiff0 * scale;
probdiff1_norm = probdiff1 * scale;
}
/**
* @return number of weights in the IDF portion of the table
*/
public final int getIDFSize() {
return idfweight.length;
}
/**
* @return number of weights in the TF portion of the table
*/
public final int getTFSize() {
return tfweight.length;
}
/**
* @return number of floating-point entries needed to serialize the factory
*/
public final int getSize() {
return idfweight.length + tfweight.length + 7;
}
/**
* @param val
* @return the IDF weight at the given position
*/
public final double getIDFWeight(short val) {
return idfweight[val];
}
/**
* @param val is the term count (-1)
* @return the TF weight for the given count
*/
public final double getTFWeight(short val) {
return tfweight[val];
}
/**
* Given an IDF position and a TF count, build the feature coefficient
* @param i is the IDF position
* @param t is the TF count
* @return the feature coefficient
*/
public final double getCoeff(short i, short t) {
return idfweight[i] * tfweight[t];
}
/**
* @return the weight normalization factor
*/
public final double getWeightNorm() {
return weightnorm;
}
/**
* @return the first feature flip penalty parameter
*/
public final double getFlipNorm0() {
return probflip0_norm;
}
/**
* @return the first feature drop penalty parameter
*/
public final double getDiffNorm0() {
return probdiff0_norm;
}
/**
* @return the second feature flip penalty parameter
*/
public final double getFlipNorm1() {
return probflip1_norm;
}
/**
* @return the second feature drop penalty parameter
*/
public final double getDiffNorm1() {
return probdiff1_norm;
}
/**
* @return the final score scaling factor
*/
public final double getScale() {
return scale;
}
/**
* @return the final score addend
*/
public final double getAddend() {
return addend;
}
public void setLogarithmicTFWeights() {
double log2 = Math.log(2.0);
for (int i = 0; i < tfweight.length; ++i) {
tfweight[i] = Math.sqrt(1.0 + Math.log( i + 1) / log2);
}
}
/**
* Serialize this object as XML to a Writer
* @param fwrite is the Writer
* @throws IOException
*/
public void saveXml(Writer fwrite) throws IOException {
fwrite.append("<weightfactory scale=\"");
fwrite.append(Double.toString(scale));
fwrite.append("\" addend=\"");
fwrite.append(Double.toString(addend));
fwrite.append("\">\n");
double scale_sqrt = Math.sqrt(scale);
for (double element : idfweight) {
fwrite.append(" <idf>");
fwrite.append(Double.toString(element / scale_sqrt));
fwrite.append("</idf>\n");
}
for (double element : tfweight) {
fwrite.append(" <tf>");
fwrite.append(Double.toString(element));
fwrite.append("</tf>\n");
}
fwrite.append(" <weightnorm>").append(Double.toString(weightnorm * scale)).append("</weightnorm>\n");
fwrite.append(" <probflip0>").append(Double.toString(probflip0)).append("</probflip0>\n");
fwrite.append(" <probflip1>").append(Double.toString(probflip1)).append("</probflip1>\n");
fwrite.append(" <probdiff0>").append(Double.toString(probdiff0)).append("</probdiff0>\n");
fwrite.append(" <probdiff1>").append(Double.toString(probdiff1)).append("</probdiff1>\n");
fwrite.append("<weightfactory>\n");
}
/**
* Condense weight table down to array of doubles
* @return array of doubles
*/
public double[] toArray() {
int numrows = getSize();
double[] res = new double[ numrows ];
double scaleSqrt = Math.sqrt(scale);
for (int i = 0; i < idfweight.length; ++i) {
res[i] = idfweight[i] / scaleSqrt;
}
for (int i = 0; i < tfweight.length; ++i) {
res[i + idfweight.length] = tfweight[i];
}
res[numrows - 7] = weightnorm * scale;
res[numrows - 6] = probflip0;
res[numrows - 5] = probflip1;
res[numrows - 4] = probdiff0;
res[numrows - 3] = probdiff1;
res[numrows - 2] = scale;
res[numrows - 1] = addend;
return res;
}
/**
* Initialize the WeightTable from an array of doubles
* @param weightArray
*/
public void set(double[] weightArray) {
int numrows = weightArray.length;
if (numrows != getSize()) {
throw new NumberFormatException("Not enough values in double array");
}
scale = weightArray[numrows - 2];
addend = weightArray[numrows - 1];
weightnorm = weightArray[numrows - 7] / scale;
probflip0 = weightArray[numrows - 6];
probflip1 = weightArray[numrows - 5];
probdiff0 = weightArray[numrows - 4];
probdiff1 = weightArray[numrows - 3];
double sqrtScale = Math.sqrt(scale);
for (int i = 0; i < idfweight.length; ++i) {
idfweight[i] = weightArray[i] * sqrtScale;
}
for (int i = 0; i < tfweight.length; ++i) {
tfweight[i] = weightArray[i + idfweight.length];
}
updateNorms();
}
/**
* Build (deserialize) this object from an XML stream
* @param parser is the XML parser
*/
public void restoreXml(XmlPullParser parser) {
XmlElement el = parser.start("weightfactory");
scale = Double.parseDouble(el.getAttribute("scale"));
addend = Double.parseDouble(el.getAttribute("addend"));
double scale_sqrt = Math.sqrt(scale);
for(int i=0;i<idfweight.length;++i) {
parser.start("idf");
double val = Double.parseDouble(parser.end().getText());
idfweight[i] = val * scale_sqrt;
}
for(int i=0;i<tfweight.length;++i) {
parser.start("tf");
double val = Double.parseDouble(parser.end().getText());
tfweight[i] = val;
}
parser.start("weightnorm");
weightnorm = Double.parseDouble(parser.end().getText());
weightnorm /= scale;
parser.start("probflip0");
probflip0 = Double.parseDouble(parser.end().getText());
parser.start("probflip1");
probflip1 = Double.parseDouble(parser.end().getText());
parser.start("probdiff0");
probdiff0 = Double.parseDouble(parser.end().getText());
parser.start("probdiff1");
probdiff1 = Double.parseDouble(parser.end().getText());
parser.end(el);
updateNorms();
}
}

View file

@ -0,0 +1,47 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.lsh.vector;
import java.io.IOException;
import ghidra.xml.XmlPullParser;
public class WeightedLSHCosineVectorFactory extends LSHVectorFactory {
@Override
public LSHVector buildZeroVector() {
return new LSHCosineVector();
}
@Override
public LSHVector buildVector(int[] feature) {
return new LSHCosineVector(feature,weightFactory,idfLookup);
}
@Override
public LSHVector restoreVectorFromXml(XmlPullParser parser) {
LSHCosineVector vector = new LSHCosineVector();
vector.restoreXml(parser, weightFactory, idfLookup);
return vector;
}
@Override
public LSHVector restoreVectorFromSql(String sql) throws IOException {
LSHCosineVector vector = new LSHCosineVector();
vector.restoreSQL(sql, weightFactory, idfLookup);
return vector;
}
}

View file

@ -0,0 +1,43 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.platform;
import java.lang.reflect.InvocationHandler;
import org.apache.commons.lang3.reflect.MethodUtils;
/**
* A general interface for handle Mac Application callbacks. Some possible callbacks are:
* <ul>
* <li>quit</li>
* <li>about</li>
* <li>preferences</li>
* <li>file handling</li>
* </ul>
*
* see com.apple.eawt.Application
*/
abstract class AbstractMacHandler implements InvocationHandler {
protected Object getApplication() throws Exception {
Class<?> clazz = Class.forName("com.apple.eawt.Application");
Object application = MethodUtils.invokeExactStaticMethod(clazz, "getApplication");
return application;
}
}

View file

@ -0,0 +1,77 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.platform;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import org.apache.commons.lang3.reflect.MethodUtils;
import ghidra.framework.OperatingSystem;
import ghidra.framework.Platform;
import ghidra.util.Msg;
/**
* A base implementation for creating an 'About' menu action callback. This is executed when
* the user presses the Dock's 'Ghidra->About' menu action.
* <p>
* Simply constructing this class will register it.
* <p>
* See
* com.apple.eawt.Application.setAboutHandler(AboutHandler)
* com.apple.eawt.AboutHandler.handleAbout(AboutEvent)
*/
public abstract class MacAboutHandler extends AbstractMacHandler {
public MacAboutHandler() {
addAboutApplicationListener();
}
public abstract void about();
private void addAboutApplicationListener() {
if (Platform.CURRENT_PLATFORM.getOperatingSystem() != OperatingSystem.MAC_OS_X) {
return;
}
try {
Object application = getApplication();
setAboutHandler(application);
}
catch (Exception e) {
Msg.error(this, "Unable to install Mac quit handler", e);
}
}
private void setAboutHandler(Object application) throws Exception {
Class<?> aboutHandlerClass = Class.forName("com.apple.eawt.AboutHandler");
Object aboutHandler = Proxy.newProxyInstance(getClass().getClassLoader(),
new Class[] { aboutHandlerClass }, this);
MethodUtils.invokeMethod(application, "setAboutHandler", aboutHandler);
}
@Override
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
// Args: AboutEvent
about(); // call our about() callback, ignoring the Application API
// the handleAbout() is void--return null
return null;
}
}

View file

@ -0,0 +1,83 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.platform;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import org.apache.commons.lang3.reflect.MethodUtils;
import ghidra.framework.OperatingSystem;
import ghidra.framework.Platform;
import ghidra.util.Msg;
/**
* A base implementation for creating an 'Quit' menu action callback. This is executed when
* the user presses the Dock's 'Ghidra->Quit' menu action.
* <p>
* Simply constructing this class will register it.
* <p>
* See
* com.apple.eawt.Application.setQuitHandler(QuitHandler)
* com.apple.eawt.AboutHandler.handleQuitRequestWith(QuitEvent, QuitResponse)
*/
public abstract class MacQuitHandler extends AbstractMacHandler {
public MacQuitHandler() {
addQuitApplicationListener(this);
}
public abstract void quit();
private void addQuitApplicationListener(MacQuitHandler macQuitHandler) {
if (Platform.CURRENT_PLATFORM.getOperatingSystem() != OperatingSystem.MAC_OS_X) {
return;
}
try {
Object application = getApplication();
setQuitHandler(application);
}
catch (Exception e) {
Msg.error(this, "Unable to install Mac quit handler", e);
}
}
private void setQuitHandler(Object application) throws Exception {
Class<?> quitHandlerClass = Class.forName("com.apple.eawt.QuitHandler");
Object quitHandler = Proxy.newProxyInstance(getClass().getClassLoader(),
new Class[] { quitHandlerClass }, this);
MethodUtils.invokeMethod(application, "setQuitHandler", quitHandler);
}
@Override
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
// Args: QuitEvent event, QuitResponse response
// Call QuitResponse.cancelQuit(), as we will allow our tool to quit the application
// instead of the OS.
Object response = args[1];
MethodUtils.invokeExactMethod(response, "cancelQuit");
quit();
// the handleQuitRequestWith() is void--return null
return null;
}
}

View file

@ -0,0 +1,66 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.random;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import ghidra.util.Msg;
/**
* <code>SecureRandomFactory</code> provides a static singleton instance of SecureRandom
*/
public class SecureRandomFactory {
private static SecureRandom random;
private static final String[] preferredAlgorithm =
new String[] { "NativePRNGNonBlocking", "SHA1PRNG" };
private static synchronized void initialize() {
if (random != null) {
return;
}
Msg.info(SecureRandomFactory.class, "Initializing Random Number Generator...");
for (String algorithm : preferredAlgorithm) {
try {
random = SecureRandom.getInstance(algorithm);
Msg.info(SecureRandomFactory.class,
"Random Number Generator initialization complete: " + algorithm);
return;
}
catch (NoSuchAlgorithmException e) {
// ignore
}
}
random = new SecureRandom();
Msg.info(SecureRandomFactory.class,
"Random Number Generator initialization complete: default");
}
public static SecureRandom getSecureRandom() {
initialize();
return random;
}
/**
* Construction not allowed
*/
private SecureRandomFactory() {
// not used
}
}

View file

@ -0,0 +1,49 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
public class Algorithms {
public static <T> IteratorSTL<T> lower_bound(IteratorSTL<T> start, IteratorSTL<T> end, T key) {
if (!(key instanceof Comparable)) {
throw new IllegalArgumentException("Element must implement Comparable");
}
Comparable<T> comparableKey = (Comparable<T>)key;
IteratorSTL<T> cur = start.copy();
for(;!cur.equals(end);cur.increment()) {
int result = comparableKey.compareTo( cur.get() );
if (result <= 0) {
return cur;
}
}
return end;
}
public static <T> IteratorSTL<T> upper_bound(IteratorSTL<T> start, IteratorSTL<T> end, T key) {
if (!(key instanceof Comparable)) {
throw new IllegalArgumentException("Element must implement Comparable");
}
Comparable<T> comparableKey = (Comparable<T>)key;
IteratorSTL<T> cur = start.copy();
for(;!cur.equals(end);cur.increment()) {
int result = comparableKey.compareTo( cur.get() );
if (result < 0) {
return cur;
}
}
return end;
}
}

View file

@ -0,0 +1,24 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
public class ComparableMapSTL<K extends Comparable<K>, V> extends MapSTL<K, V> {
public ComparableMapSTL() {
super(new SelfComparator<K>());
}
}

View file

@ -0,0 +1,24 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
public class ComparableMultiMapSTL<K extends Comparable<K>, V> extends MultiMapSTL<K, V> {
public ComparableMultiMapSTL() {
super(new SelfComparator<K>());
}
}

View file

@ -0,0 +1,24 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
public class ComparableMultiSetSTL<T extends Comparable<T>> extends MultiSetSTL<T> {
public ComparableMultiSetSTL() {
super(new SelfComparator<T>());
}
}

View file

@ -0,0 +1,24 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
public class ComparableSetSTL<T extends Comparable<T>> extends SetSTL<T> {
public ComparableSetSTL() {
super(new SelfComparator<T>());
}
}

View file

@ -0,0 +1,81 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
public class EmptyIteratorSTL<T> implements IteratorSTL<T> {
public IteratorSTL<T> copy() {
return this;
}
public void delete() {
throw new UnsupportedOperationException();
}
public void delete( int count ) {
throw new UnsupportedOperationException();
}
public T get() {
return null;
}
public IteratorSTL<T> increment() {
throw new IndexOutOfBoundsException();
}
public IteratorSTL<T> decrement() {
throw new IndexOutOfBoundsException();
}
public void insert( T value ) {
throw new UnsupportedOperationException();
}
public boolean isBegin() {
return true;
}
public boolean isEnd() {
return true;
}
public void set( T value ) {
throw new UnsupportedOperationException();
}
public IteratorSTL<T> decrement( int n ) {
throw new UnsupportedOperationException();
}
public IteratorSTL<T> increment( int n ) {
throw new UnsupportedOperationException();
}
public boolean isRBegin() {
return true;
}
public boolean isREnd() {
return true;
}
public void assign( IteratorSTL<T> otherIterator ) {
throw new UnsupportedOperationException();
}
}

View file

@ -0,0 +1,101 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
public interface IteratorSTL<T> {
/**
* Returns the current value of the iterator.
* @return the current value of the iterator.
* @throws IndexOutOfBoundsException if the iterator is positioned before the first value or
* after the last value.
*/
T get();
/**
* Sets the current value of the iterator to the given value.
* @param value the value to set at the iterator position
* @throws IndexOutOfBoundsException if the iterator is positioned befor the first value or
* after the last value.
*/
void set(T value);
/**
* Advances the iterator to the next position.
* @return a reference to the iterator itself
* @throws IndexOutOfBoundsException if the the iterator is already past the last element.
*/
IteratorSTL<T> increment();
/**
* Advances the iterator n positions.
* @return a reference to the iterator itself
* @throws IndexOutOfBoundsException if the n value pushes past the end of the collection.
*/
IteratorSTL<T> increment(int n);
/**
* Devance the iterator to the previous position. This method is only supported in
* bidirectional iterators.
* @return a reference to the iterator itself
*/
IteratorSTL<T> decrement();
/**
* Devances the iterator n positions.
* @return a reference to the iterator itself
* @throws IndexOutOfBoundsException if the n value pushes past the beginning of the collection
*/
IteratorSTL<T> decrement(int n);
/**
* Returns true if the iterator is positioned on the first first element of the collection. If the
* collection is empty, this will always return false.
* @return true if the iterator is positioned on the first element of the collection.
*/
boolean isBegin();
/**
* Returns true if the iterator is positioned past the last element of the collection. If the
* collection is empty, this will always return true.
* @return true if the iterator is positioned past the last element of the collection.
*/
boolean isEnd();
/**
* Inserts the given value at the current position (the current value will be pushed to the next value).
* The iterator will be positioned on the new value.
* @param value the value to insert into the collection.
* @throws IndexOutOfBoundsException if the iterator is positioned before the first item.
*/
void insert(T value);
/**
* Creates a copy of this iterator.
* @return a copy of this iterator.
*/
IteratorSTL<T> copy();
/**
* 'Assigns' this iterator to be equivalent to the given iterator. This is equivalent to
* C++'s '=' overloading mechanism
* @param otherIterator The iterator to copy
*/
void assign( IteratorSTL<T> otherIterator );
}

View file

@ -0,0 +1,117 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
public class ListIterator<T> implements IteratorSTL<T> {
ListSTL<T> list;
ListNodeSTL<T> root;
protected ListNodeSTL<T> node;
public StackTraceElement[] stackUse;
ListIterator(ListSTL<T> list, ListNodeSTL<T> root, ListNodeSTL<T> node) {
this.list = list;
this.root = root;
this.node = node;
}
public void assign( IteratorSTL<T> otherIterator ) {
ListIterator<T> other = (ListIterator<T>)otherIterator;
this.list = other.list;
this.root = other.root;
this.node = other.node;
}
public IteratorSTL<T> copy() {
return new ListIterator<T>( list, root, node );
}
public boolean isBegin() {
return node == root.next;
}
public boolean isEnd() {
return node == root;
}
public IteratorSTL<T> decrement() {
if (node.prev == root) {
throw new IndexOutOfBoundsException();
}
node = node.prev;
return this;
}
public T get() {
if (node == root) {
throw new IndexOutOfBoundsException();
}
return node.value;
}
public IteratorSTL<T> increment() {
node = node.next;
return this;
}
public IteratorSTL<T> increment(int count) {
for(int i=0;i<count;i++) {
increment();
}
return this;
}
public IteratorSTL<T> decrement( int n ) {
throw new UnsupportedOperationException();
}
public void insert(T value) {
ListNodeSTL<T> newNode = new ListNodeSTL<T>(node.prev, node, value);
node.prev.next = newNode;
node.prev = newNode;
node = newNode;
list.adjustSize(1);
}
public void set(T value) {
if (root == node) {
throw new IndexOutOfBoundsException();
}
node.value = value;
}
protected ListNodeSTL<T> getNode(){
return node;
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (obj == this) {
return true;
}
if (!(obj instanceof ListIterator)) {
return false;
}
ListIterator<?> other = (ListIterator)obj;
return list == other.list && node == other.node;
}
@Override
public int hashCode() {
return list.hashCode();
}
}

View file

@ -0,0 +1,34 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
public class ListNodeSTL<T> {
ListNodeSTL<T> next;
ListNodeSTL<T> prev;
public T value;
public StackTraceElement[] stackUse;
public ListNodeSTL(ListNodeSTL<T> prev, ListNodeSTL<T> next, T value) {
this.prev = prev;
this.next = next;
this.value = value;
}
public ListNodeSTL() {
next = this;
prev = this;
value = null;
}
}

View file

@ -0,0 +1,354 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
import java.util.Comparator;
public class ListSTL<T> {
public static final String EOL = System.getProperty( "line.separator" );
private ListNodeSTL<T> root = new ListNodeSTL<> ();
private int size;
@Override
public String toString() {
StringBuffer buffy = new StringBuffer( "ListSTL[size=" + size + "]\n" );
int showSize = Math.min( 20, size() );
ListNodeSTL<T> current = root.next;
for ( int i = 0; i < showSize; i++ ) {
buffy.append( "\t[" + i + "]=" + current.value ).append( EOL );
current = current.next;
}
return buffy.toString();
}
// for debug
public void printDebug() {
StringBuffer buffy = new StringBuffer();
IteratorSTL<T> begin = begin();
while ( !begin.isEnd() ) {
T t = begin.get();
begin.increment();
String value = t == null ? "null" : t.toString();
buffy.append( value ).append( EOL );
}
System.err.println( buffy.toString() );
}
public IteratorSTL<T> begin() {
return new ListIterator<>(this, root, root.next);
}
public IteratorSTL<T> end() {
return new ListIterator<>(this, root, root);
}
public IteratorSTL<T> rBegin() {
return new ReverseListIterator<>(this, root, root.prev);
}
public IteratorSTL<T> rEnd() {
return new ReverseListIterator<>(this, root, root);
}
public int size() {
return size;
}
public void clear() {
size = 0;
root = new ListNodeSTL<>();
}
public boolean isEmpty() {
return size == 0;
}
public T front() {
if (isEmpty()) {
throw new IndexOutOfBoundsException();
}
return root.next.value;
}
public T back() {
if (isEmpty()) {
throw new IndexOutOfBoundsException();
}
return root.prev.value;
}
public void push_back(T value) {
ListNodeSTL<T> newNode = new ListNodeSTL<>(root.prev, root, value);
root.prev.next = newNode;
root.prev = newNode;
size++;
}
public void push_front(T value) {
ListNodeSTL<T> newNode = new ListNodeSTL<>(root, root.next, value);
root.next.prev = newNode;
root.next = newNode;
size++;
}
public IteratorSTL<T> insert(IteratorSTL<T> position, T value) {
ListNodeSTL<T> newNode = new ListNodeSTL<>(root.prev, root, value);
ListIterator<T> listIterator = (ListIterator<T>)position;
newNode.next = listIterator.getNode();
newNode.prev = listIterator.getNode().prev;
newNode.prev.next = newNode;
newNode.next.prev = newNode;
size++;
// if (size != computeSize()) {
// throw new RuntimeException("Bad list state");
// }
return new ListIterator<>(this, root, newNode);
}
// private int computeSize() {
// int computedSize = 0;
// ListNodeSTL<T> node = root.next;
// while(node != root) {
// node = node.next;
// computedSize++;
// }
// return computedSize;
// }
public void erase(IteratorSTL<T> position) {
ListIterator<T> insertPos = (ListIterator<T>)position;
if (this != insertPos.list) {
throw new RuntimeException("Attempting to erase using an iterator from a different list");
}
ListNodeSTL<T> node = insertPos.getNode();
node.prev.next = node.next;
node.next.prev = node.prev;
size--;
// if (size != computeSize()) {
// throw new RuntimeException("Bad list state");
// }
}
public T pop_front() {
if (isEmpty()) {
throw new IndexOutOfBoundsException();
}
ListNodeSTL<T> node = root.next;
node.next.prev = root;
root.next = node.next;
node.next = null;
node.prev = null;
size--;
return node.value;
}
public T pop_back() {
if (isEmpty()) {
throw new IndexOutOfBoundsException();
}
ListNodeSTL<T> node = root.prev;
node.prev.next = root;
root.prev = node.prev;
node.next = null;
node.prev = null;
size--;
return node.value;
}
void adjustSize(int count) {
size += count;
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (obj == this) {
return true;
}
if (!(obj instanceof ListSTL)) {
return false;
}
ListSTL<?> other = (ListSTL<?>)obj;
if (size != other.size) {
return false;
}
IteratorSTL<?> thisIt = begin();
IteratorSTL<?> otherIt = other.begin();
while(!thisIt.isEnd()) {
Object thisValue = thisIt.get();
Object otherValue = otherIt.get();
if (!thisValue.equals(otherValue)) {
return false;
}
thisIt.increment();
otherIt.increment();
}
return true;
}
public void sort(Comparator<T> comparator) {
ListNodeSTL<T> TERMINAL = new ListNodeSTL<>();
if (size <= 1) {
return;
}
root.prev.next = TERMINAL;
root.next = mergeSort(root.next, comparator, TERMINAL);
ListNodeSTL<T> node = root.next;
ListNodeSTL<T> prevNode = root;
while(node != TERMINAL) {
node.prev = prevNode;
prevNode = node;
node = node.next;
}
prevNode.next = root;
root.prev = prevNode;
}
private static <T> ListNodeSTL<T> mergeSort(ListNodeSTL<T> c, Comparator<T> comparator, ListNodeSTL<T> TERMINAL) {
ListNodeSTL<T> a;
ListNodeSTL<T> b;
if (c.next != TERMINAL) {
a = c;
b = c.next.next.next;
while(b != TERMINAL) {
c = c.next;
b = b.next.next;
}
b = c.next;
c.next = TERMINAL;
a = mergeSort(a, comparator, TERMINAL);
b = mergeSort(b, comparator, TERMINAL);
return merge(a, b, comparator, TERMINAL);
}
return c;
}
/**
* moves a single element, decreasing the length of list by 1 and increasing this list by 1.
* @param position the position into this list where the element is to be inserted
* @param list the list from which the element is removed.
* @param listPosition the postion of the element to be removed.
*/
public void splice( IteratorSTL<T> position, ListSTL<T> list, IteratorSTL<T> listPosition ) {
ListIterator<T> toPosition = (ListIterator<T>)position;
ListIterator<T> fromPosition = (ListIterator<T>)listPosition;
ListNodeSTL<T> node = fromPosition.getNode();
ListNodeSTL<T> insertAtNode = toPosition.getNode();
node.prev.next = node.next;
node.next.prev = node.prev;
list.size--;
node.next = insertAtNode;
node.prev = insertAtNode.prev;
node.prev.next = node;
node.next.prev = node;
size++;
}
private static <T> ListNodeSTL<T> merge(ListNodeSTL<T> a, ListNodeSTL<T> b, Comparator<T> comparator, ListNodeSTL<T> TERMINAL) {
ListNodeSTL<T> head = new ListNodeSTL<>();
ListNodeSTL<T> c = head;
do {
if (b == TERMINAL || ((a != TERMINAL) && comparator.compare(a.value, b.value)<= 0)) {
c.next = a;
c = a;
a = a.next;
}
else {
c.next = b;
c = b;
b = b.next;
}
}
while(c != TERMINAL);
return head.next;
}
// public static void main(String[] args) {
// ListSTL<Integer> list = new ListSTL<Integer>();
// list.push_back(5);
// list.push_back(10);
// list.push_back(3);
// list.push_back(7);
// list.push_back(6);
//
//System.err.println(" ONE");
// BidirectionalIteratorSTL<Integer> it = list.begin();
// while(!it.isEnd()) {
// System.err.println("value = "+it.getAndIncrement());
// }
//System.err.println(" TWO");
// list.sort();
// it = list.begin();
// while(!it.isEnd()) {
// System.err.println("value = "+it.getAndIncrement());
// }
//
//System.err.println(" THREE");
// IteratorSTL<Integer> it2 = list.rBegin();
// while(!it2.isEnd()) {
// System.err.println("value = "+it2.getAndIncrement());
// }
//
// ListSTL<Integer> list2 = new ListSTL<Integer>();
// list2.push_back(1000);
// list2.push_back(1001);
// list2.push_back(1002);
// list2.push_back(1003);
// list2.push_back(1004);
//
// BidirectionalIteratorSTL<Integer> it3 = list2.begin();
// it3.increment();
// it3.increment();
//
// list.splice( list.end(), list2, it3 );
//
// System.err.println("old list");
// IteratorSTL<Integer> it4 = list.begin();
// while(!it4.isEnd()) {
// System.err.println("value = "+it4.getAndIncrement());
// }
// System.err.println("new list");
// it4 = list2.begin();
// while(!it4.isEnd()) {
// System.err.println("value = "+it4.getAndIncrement());
// }
//
// it3 = list2.begin();
// it3.increment();
// it3.increment();
// System.err.println("it3 = "+it3.get());
//
// list2.push_front(1002);
// BidirectionalIteratorSTL<Integer> it5 = list2.begin();
//
// System.err.println("pre-splice list");
// it4 = list2.begin();
// while(!it4.isEnd()) {
// System.err.println("value = "+it4.getAndIncrement());
// }
//
// list2.splice( it3, list2, it5 );
//
// System.err.println("repaired list");
// it4 = list2.begin();
// while(!it4.isEnd()) {
// System.err.println("value = "+it4.getAndIncrement());
// }
//
// }
}

View file

@ -0,0 +1,125 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
public class MapIteratorSTL<K, V> implements IteratorSTL<Pair<K, V>> {
protected RedBlackTree<K, V> tree;
protected RedBlackNode<K, V> node;
protected boolean erased;
MapIteratorSTL(RedBlackTree<K,V> tree, RedBlackNode<K,V> node, boolean erased) {
this.tree = tree;
this.node = node;
this.erased = erased;
}
MapIteratorSTL(RedBlackTree<K,V> tree, RedBlackNode<K,V> node) {
this.tree = tree;
this.node = node;
}
public void assign( IteratorSTL<Pair<K,V>> otherIterator ) {
MapIteratorSTL<K,V> other = (MapIteratorSTL<K, V>)otherIterator;
this.tree = other.tree;
this.node = other.node;
this.erased = other.erased;
}
public IteratorSTL<Pair<K,V>> decrement() {
if (node == null && tree.isEmpty()) {
throw new IndexOutOfBoundsException();
}
if (node == null) {
node = tree.getLast();
}
else {
node = node.getPredecessor();
}
erased = false;
return this;
}
public Pair<K, V> get() {
if (erased) {
throw new IndexOutOfBoundsException("element erased");
}
if (node == null) {
throw new IndexOutOfBoundsException();
}
return new Pair<K,V>(node.getKey(), node.getValue());
}
public IteratorSTL<Pair<K,V>> increment() {
if (!erased && node == null) {
throw new IndexOutOfBoundsException();
}
if (!erased) { // erased nodes already point to the successor
node = node.getSuccessor();
}
erased = false;
return this;
}
public void insert(Pair<K, V> value) {
throw new UnsupportedOperationException();
}
public boolean isBegin() {
if (erased) {
throw new RuntimeException("Iterater in invalid state");
}
return node == tree.getFirst();
}
public boolean isEnd() {
if (erased) {
throw new RuntimeException("Iterater in invalid state");
}
return node == null;
}
public void set(Pair<K, V> value) {
throw new UnsupportedOperationException();
}
public IteratorSTL<Pair<K, V>> copy() {
return new MapIteratorSTL<K, V>(tree, node);
}
public IteratorSTL<Pair<K, V>> decrement( int n ) {
throw new UnsupportedOperationException();
}
public IteratorSTL<Pair<K, V>> increment( int n ) {
throw new UnsupportedOperationException();
}
@SuppressWarnings("unchecked")
@Override
public boolean equals( Object obj ) {
if (obj == this) {
return true;
}
if (obj == null) {
return false;
}
if (obj.getClass() != getClass() ) {
return false;
}
MapIteratorSTL<K, V> other = (MapIteratorSTL<K, V>) obj;
return tree == other.tree && node == other.node && erased == other.erased;
}
}

View file

@ -0,0 +1,150 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
import java.util.Comparator;
public class MapSTL<K,V> {
public static final String EOL = System.getProperty( "line.separator" );
RedBlackTree<K,V> rbTree;
public MapSTL(Comparator<K> comparator) {
rbTree = new RedBlackTree<>(comparator, false);
}
@Override
public String toString() {
StringBuffer buffy = new StringBuffer("{");
IteratorSTL<Pair<K, V>> begin = begin();
while ( !begin.isEnd() ) {
Pair<K, V> pair = begin.get();
begin.increment();
buffy.append( pair.toString() ).append( ", " ).append( EOL );
}
buffy.append("}");
return buffy.toString();
}
public void put( K key, V value ) {
rbTree.put( key, value );
}
public boolean add(K key, V value) {
if (rbTree.containsKey( key )) {
return false;
}
rbTree.put(key, value);
return true;
}
public boolean contains(K key) {
return rbTree.containsKey(key);
}
public V remove(K key) {
return rbTree.remove(key);
}
public IteratorSTL<Pair<K,V>> begin() {
return new MapIteratorSTL<>(rbTree, rbTree.getFirst());
}
public IteratorSTL<Pair<K,V>> end() {
return new MapIteratorSTL<>(rbTree, null);
}
public IteratorSTL<Pair<K,V>> rBegin() {
return new ReverseMapIteratorSTL<>(rbTree, rbTree.getLast());
}
public IteratorSTL<Pair<K,V>> rEnd() {
return new ReverseMapIteratorSTL<>(rbTree, null);
}
public V erase(K key) {
return remove(key);
}
public boolean empty() {
return rbTree.isEmpty();
}
public IteratorSTL<Pair<K,V>> lower_bound( K key ) {
RedBlackNode<K, V> node = rbTree.lowerBound( key );
return new MapIteratorSTL<>(rbTree, node);
}
public IteratorSTL<Pair<K,V>> upper_bound( K key ) {
RedBlackNode<K, V> node = rbTree.upperBound( key );
MapIteratorSTL<K, V> it = new MapIteratorSTL<>(rbTree, node);
return it;
}
public boolean isEmpty() {
return rbTree.isEmpty();
}
public void clear() {
rbTree.removeAll();
}
public void erase( IteratorSTL<Pair<K,V>> iter ) {
MapIteratorSTL<K,V> it = ((MapIteratorSTL<K,V>)iter);
RedBlackNode<K, V> node = it.node;
if (node == null) {
throw new IndexOutOfBoundsException();
}
it.node = node.getSuccessor();
it.erased = true;
rbTree.deleteEntry(node);
}
public void erase( IteratorSTL<Pair<K,V>> start, IteratorSTL<Pair<K,V>> end ) {
while ( !start.equals( end ) ) {
erase(start);
start.increment();
}
}
public V get( K key ) {
RedBlackNode<K, V> node = rbTree.findFirstNode( key );
if ( node == null ) {
return null;
}
return node.value;
}
public IteratorSTL<Pair<K,V>> find(K key) {
if (rbTree.containsKey(key)) {
return lower_bound(key);
}
return end();
}
public int size() {
return rbTree.size();
}
public void insert(IteratorSTL<Pair<K,V>> start, IteratorSTL<Pair<K,V>> end ) {
while(!start.equals(end)) {
add(start.get().first, start.get().second);
start.increment();
}
}
}

View file

@ -0,0 +1,102 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
import java.util.Comparator;
public class MultiMapSTL<K,V> {
RedBlackTree<K,V> rbTree;
public MultiMapSTL(Comparator<K> comparator) {
rbTree = new RedBlackTree<>(comparator, true);
}
public void add(K key, V value) {
rbTree.put(key, value);
}
public boolean contains(K key) {
return rbTree.containsKey(key);
}
public V remove(K key) {
return rbTree.remove(key);
}
public void erase( IteratorSTL<Pair<K,V>> iter ) {
MapIteratorSTL<K,V> it = ((MapIteratorSTL<K,V>)iter);
RedBlackNode<K, V> node = it.node;
if (node == null) {
throw new IndexOutOfBoundsException();
}
it.node = node.getSuccessor();
it.erased = true;
rbTree.deleteEntry(node);
}
public IteratorSTL<Pair<K,V>> begin() {
return new MapIteratorSTL<>(rbTree, rbTree.getFirst());
}
public IteratorSTL<Pair<K,V>> end() {
return new MapIteratorSTL<>(rbTree, null);
}
public IteratorSTL<Pair<K,V>> rBegin() {
return new ReverseMapIteratorSTL<>(rbTree, rbTree.getLast());
}
public IteratorSTL<Pair<K,V>> rEnd() {
return new ReverseMapIteratorSTL<>(rbTree, null);
}
public IteratorSTL<Pair<K,V>> lower_bound( K key ) {
RedBlackNode<K, V> node = rbTree.lowerBound( key );
return new MapIteratorSTL<>(rbTree, node);
}
public IteratorSTL<Pair<K,V>> upper_bound( K key ) {
RedBlackNode<K, V> node = rbTree.upperBound( key );
MapIteratorSTL<K, V> it = new MapIteratorSTL<>(rbTree, node);
return it;
}
public static void main(String[] args) {
MultiMapSTL<Integer, String> set = new ComparableMultiMapSTL<>();
set.add(7, "dog");
set.add(3, "blue");
set.add(9, "elf");
set.add(20,"gate");
set.add(15, "fog");
set.add(1, "apple");
set.add(20, "hog");
set.add(20,"indian");
set.add(4, "cat");
set.add(50, "jump");
// IteratorSTL<Pair<Integer, String>> it = set.begin();
// while(!it.isEnd()) {
// System.out.println("value = "+it.getAndIncrement());
// }
//System.out.println(" ----");
// it = set.rBegin();
// while(!it.isEnd()) {
// System.out.println("value = "+it.getAndIncrement());
// }
//
}
}

View file

@ -0,0 +1,115 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
import java.util.Comparator;
public class MultiSetSTL<K> {
RedBlackTree<K,K> rbTree;
public MultiSetSTL(Comparator<K> comparator) {
rbTree = new RedBlackTree<>(comparator, true);
}
public void insert(K key) {
rbTree.put(key, key);
}
public boolean contains(K key) {
return rbTree.containsKey(key);
}
public boolean remove(K key) {
return rbTree.remove(key) != null;
}
public IteratorSTL<K> begin() {
return new SetIterator<>(rbTree, rbTree.getFirst());
}
public IteratorSTL<K> end() {
return new SetIterator<>(rbTree, null);
}
public IteratorSTL<K> rBegin() {
return new ReverseSetIterator<>(rbTree, rbTree.getLast());
}
public IteratorSTL<K> rEnd() {
return new ReverseSetIterator<>(rbTree, null);
}
public IteratorSTL<K> lower_bound( K key ) {
RedBlackNode<K,K> node = rbTree.lowerBound( key );
return new SetIterator<>(rbTree, node);
}
public IteratorSTL<K> upper_bound( K key ) {
RedBlackNode<K,K> node = rbTree.upperBound( key );
SetIterator<K> it = new SetIterator<>(rbTree, node);
return it;
}
public static void main(String[] args) {
MultiSetSTL<Integer> set = new ComparableMultiSetSTL<Integer>();
set.insert(7);
set.insert(3);
set.insert(9);
set.insert(20);
set.insert(15);
set.insert(1);
set.insert(20);
set.insert(20);
set.insert(4);
set.insert(50);
// IteratorSTL<Integer> it = set.begin();
// while(!it.isEnd()) {
// System.out.println("value = "+it.getAndIncrement());
// }
//System.out.println(" ----");
// it = set.rBegin();
// while(!it.isEnd()) {
// System.out.println("value = "+it.getAndIncrement());
// }
//
// it = set.lower_bound( 20 );
// while(!it.isEnd()) {
// System.out.println("value = "+it.getAndIncrement());
// }
}
//TODO can we make this faster using the hint?
public IteratorSTL<K> insert( IteratorSTL<K> low, K key ) {
Pair<RedBlackNode<K, K>, Boolean> pair = rbTree.put(key, key);
return new SetIterator<>(rbTree, pair.first);
}
public void erase( IteratorSTL<K> position ) {
SetIterator<K> setIterator = (SetIterator<K>) position;
RedBlackNode<K, K> node = setIterator.node;
if (node == null) {
throw new IndexOutOfBoundsException();
}
setIterator.node = node.getSuccessor();
setIterator.erased = true;
rbTree.deleteEntry(node);
}
}

View file

@ -0,0 +1,68 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
import ghidra.util.SystemUtilities;
public class Pair<T1, T2> {
public final T1 first;
public final T2 second;
public static <T1, T2> Pair<T1, T2> emptyPair() {
return new Pair<T1, T2>(null, null);
}
public Pair(T1 key, T2 value) {
this.first = key;
this.second = value;
}
@Override
public String toString() {
return "<" + first + "," + second + ">";
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((first == null) ? 0 : first.hashCode());
result = prime * result + ((second == null) ? 0 : second.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
@SuppressWarnings("unchecked")
Pair<T1, T2> other = (Pair<T1, T2>) obj;
if (!SystemUtilities.isEqual(first, other.first)) {
return false;
}
return SystemUtilities.isEqual(second, other.second);
}
}

View file

@ -0,0 +1,31 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
public class Quad<T1, T2, T3, T4> {
public T1 first;
public T2 second;
public T3 third;
public T4 fourth;
public Quad(T1 first, T2 second, T3 third, T4 fourth) {
this.first = first;
this.second = second;
this.third = third;
this.fourth = fourth;
}
}

View file

@ -0,0 +1,102 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
// RedBlack Tree node
public class RedBlackNode<K, V> {
enum NodeColor{
RED, BLACK
}
K key;
V value;
NodeColor color;
RedBlackNode<K,V> parent;
RedBlackNode<K,V> left;
RedBlackNode<K,V> right;
RedBlackNode(K key, V value, RedBlackNode<K,V> parent) {
this.key = key;
this.value = value;
this.parent = parent;
this.color = NodeColor.BLACK;
}
@Override
public String toString() {
return "" + value;
}
public V getValue() {
return value;
}
public void setValue(V value) {
this.value = value;
}
public K getKey() {
return key;
}
public RedBlackNode<K, V> getSuccessor() {
if (right != null) {
RedBlackNode<K,V> node = right;
while(node.left != null) {
node = node.left;
}
return node;
}
RedBlackNode<K, V> node = this;
while(node.parent != null) {
if (node.isLeftChild()) {
return node.parent;
}
node = node.parent;
}
return null;
}
public RedBlackNode<K, V> getPredecessor() {
if (left != null) {
RedBlackNode<K,V> node = left;
while(node.right != null) {
node = node.right;
}
return node;
}
RedBlackNode<K, V> node = this;
while(node.parent != null) {
if (!node.isLeftChild()) {
return node.parent;
}
node = node.parent;
}
return null;
}
boolean isLeftChild() {
return parent.left == this;
}
boolean isRightChild() {
return parent.right == this;
}
}

View file

@ -0,0 +1,645 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
import static generic.stl.RedBlackNode.NodeColor.BLACK;
import static generic.stl.RedBlackNode.NodeColor.RED;
import java.util.Comparator;
/**
* A RedBlack Tree implementation with K type keys and place to store V type values.
*/
public class RedBlackTree<K,V> {
public static final String EOL = System.getProperty( "line.separator" );
private RedBlackNode<K,V> root;
private int size;
private final Comparator<K> comparator;
private final boolean allowDuplicateKeys;
/**
* Creates a new RedBlackKeySet that can store keys between 0 and n.
* @param n the maximum key for this set.
*/
public RedBlackTree(Comparator<K> comparator, boolean allowDuplicateKeys) {
this.comparator = comparator;
this.allowDuplicateKeys = allowDuplicateKeys;
}
public RedBlackTree(RedBlackTree<K, V> tree) {
this.comparator = tree.comparator;
this.allowDuplicateKeys = tree.allowDuplicateKeys;
RedBlackNode<K, V> node = tree.getFirst();
while(node != null) {
put(node.key, node.value);
node = node.getSuccessor();
}
}
@Override
public String toString() {
StringBuffer buffy = new StringBuffer( "RedBlackTree[size=" + size + "]\n" );
int showSize = Math.min( 20, size() );
RedBlackNode<K, V> current = getFirst();
for ( int i = 0; i < showSize; i++ ) {
buffy.append( "\t[" ).append( i ).append( "]=(" ).append( current.key ).append(
"," ).append( current.value ).append( ")" ).append( EOL );
current = current.getSuccessor();
}
return buffy.toString();
}
/**
* Returns the number keys in this set.
*/
public int size() {
return size;
}
/**
* Returns true if the key is in the set.
* @param key the key whose presence is to be tested.
*/
public boolean containsKey(K key) {
RedBlackNode<K,V> node = root;
while(node != null) {
int comp = comparator.compare(key, node.key);
if (comp == 0 ){
return true;
}
if (comp < 0) {
node = node.left;
}
else {
node = node.right;
}
}
return false;
}
/**
* Returns the first entry in this set.
*/
public RedBlackNode<K,V> getFirst() {
if (root == null) {
return null;
}
RedBlackNode<K,V> node = root;
while(node.left != null) {
node = node.left;
}
return node;
}
/**
* Returns the last entry in this set.
*/
public RedBlackNode<K,V> getLast() {
if (root == null) {
return null;
}
RedBlackNode<K,V> node = root;
while(node.right != null) {
node = node.right;
}
return node;
}
/**
* Finds the node with the lowest key that is >= to the given key. Returns null if all nodes
* in the tree have keys less than the given key.
* @param key the key to search for.
* @return the node with the lowest key that is >= to the given key or null if no such key exists.
*/
public RedBlackNode<K,V> lowerBound(K key) {
RedBlackNode<K,V> bestNode = null;
RedBlackNode<K,V> node = root;
while (node != null) {
int result = comparator.compare(key, node.key);
if (result <= 0) {
bestNode = node;
node = node.left;
}
else {
node = node.right;
}
}
return bestNode;
}
/**
* Finds the node with the lowest key that is > the given key. Returns null if all nodes
* in the tree have keys less than or equal to the given key.
* @param key the key to search for.
* @return the node with the lowest key that is > to the given key or null if no such key exists.
*/
public RedBlackNode<K,V> upperBound(K key){
RedBlackNode<K,V> bestNode = null;
RedBlackNode<K,V> node = root;
while (node != null) {
int result = comparator.compare(key, node.key);
if (result < 0) {
bestNode = node;
node = node.left;
}
else {
node = node.right;
}
}
return bestNode;
}
/**
* Adds the given key,value to the map. If the map does not allow duplicate keys and a key
* already exists, the old value will be replaced by the new value and the old value will be
* returned.
* @param key the key to add to the set.
* @return the old value associated with the key, or null if the key was not previously in the map.
*/
public Pair<RedBlackNode<K, V>, Boolean> put(K key, V value) {
if (root == null) {
size++;
root = new RedBlackNode<K,V>(key,value,null);
return new Pair<RedBlackNode<K, V>, Boolean>( root, Boolean.TRUE );
}
RedBlackNode<K,V> node = root;
while (true) {
int comp = comparator.compare(key, node.key);
if (comp == 0 && !allowDuplicateKeys) {
node.value = value;
return new Pair<RedBlackNode<K, V>, Boolean>( node, Boolean.FALSE );
}
else if (comp < 0) {
if (node.left != null) {
node = node.left;
}
else {
size++;
RedBlackNode<K, V> newNode = new RedBlackNode<K,V>(key, value, node);
node.left = newNode;
fixAfterInsertion(newNode);
return new Pair<RedBlackNode<K, V>, Boolean>( newNode, Boolean.TRUE );
}
}
else {
if (node.right != null) {
node = node.right;
}
else {
size++;
RedBlackNode<K, V> newNode = new RedBlackNode<K,V>(key, value, node);
node.right = newNode;
fixAfterInsertion(newNode);
return new Pair<RedBlackNode<K, V>, Boolean>( newNode, Boolean.TRUE );
}
}
}
}
public RedBlackNode<K,V> findFirstNode(K key) {
RedBlackNode<K,V> node = root;
RedBlackNode<K,V> bestNode = null;
while(node != null) {
int comp = comparator.compare(key, node.key);
if (comp == 0) {
bestNode = node;
}
if (comp <= 0) {
node = node.left;
}
else {
node = node.right;
}
}
return bestNode;
}
public RedBlackNode<K,V> findLastNode(K key) {
RedBlackNode<K,V> node = root;
RedBlackNode<K,V> bestNode = null;
while(node != null) {
int comp = comparator.compare(key, node.key);
if (comp == 0) {
bestNode = node;
}
if (comp < 0) {
node = node.left;
}
else {
node = node.right;
}
}
return bestNode;
}
/**
* Removes the given key (first if duplicates are allowed) from the set.
* @param key the key to remove from the set.
* @return the value associated with the key removed or null if the key not found.
*/
public V remove(K key) {
RedBlackNode<K,V> node = findFirstNode(key);
if (node == null) {
return null;
}
V value = node.value;
deleteEntry(node);
return value;
}
/**
* Removes all entrys from the set.
*/
public void removeAll() {
size = 0;
root = null;
}
/**
* Test if the set is empty.
*@return true if the set is empty.
*/
public boolean isEmpty() {
return size == 0;
}
/**
* Balancing operations.
*
* Implementations of rebalancings during insertion and deletion are
* slightly different than the CLR version. Rather than using dummy
* nilnodes, we use a set of accessors that deal properly with null. They
* are used to avoid messiness surrounding nullness checks in the main
* algorithms.
*/
/**
* Returns the color of the given node.
*/
private static <K,V> RedBlackNode.NodeColor colorOf(RedBlackNode<K,V> p) {
return (p == null ? BLACK : p.color);
}
/**
* Returns the parent of the given node.
*/
private static <K,V> RedBlackNode<K,V> parentOf(RedBlackNode<K,V> p) {
return (p == null ? null: p.parent);
}
/**
* Sets the color of the given node to the given color.
*/
private static <K,V> void setColor(RedBlackNode<K,V> p, RedBlackNode.NodeColor c) {
if (p != null) p.color = c;
}
/**
* Returns the left child of the given node.
*/
private static <K,V> RedBlackNode<K,V> leftOf(RedBlackNode<K,V> p) {
return (p == null)? null: p.left;
}
/**
* Returns the right child of the given node.
*/
private static <K,V> RedBlackNode<K,V> rightOf(RedBlackNode<K,V> p) {
return (p == null)? null: p.right;
}
/** From CLR **/
private void rotateLeft(RedBlackNode<K,V> p) {
RedBlackNode<K,V> r = p.right;
p.right = r.left;
if (r.left != null) {
r.left.parent = p;
}
r.parent = p.parent;
if (p.parent == null) {
root = r;
}
else if (p.parent.left == p) {
p.parent.left = r;
}
else {
p.parent.right = r;
}
r.left = p;
p.parent = r;
}
/** From CLR **/
private void rotateRight(RedBlackNode<K,V> p) {
RedBlackNode<K,V> l = p.left;
p.left = l.right;
if (l.right != null) {
l.right.parent = p;
}
l.parent = p.parent;
if (p.parent == null) {
root = l;
}
else if (p.parent.right == p) {
p.parent.right = l;
}
else {
p.parent.left = l;
}
l.right = p;
p.parent = l;
}
/** From CLR **/
private void fixAfterInsertion(RedBlackNode<K,V> x) {
x.color = RED;
while (x != null && x != root && x.parent.color == RED) {
if (parentOf(x) == leftOf(parentOf(parentOf(x)))) {
RedBlackNode<K,V> y = rightOf(parentOf(parentOf(x)));
if (colorOf(y) == RED) {
setColor(parentOf(x), BLACK);
setColor(y, BLACK);
setColor(parentOf(parentOf(x)), RED);
x = parentOf(parentOf(x));
}
else {
if (x == rightOf(parentOf(x))) {
x = parentOf(x);
rotateLeft(x);
}
setColor(parentOf(x), BLACK);
setColor(parentOf(parentOf(x)), RED);
if (parentOf(parentOf(x)) != null) {
rotateRight(parentOf(parentOf(x)));
}
}
}
else {
RedBlackNode<K,V> y = leftOf(parentOf(parentOf(x)));
if (colorOf(y) == RED) {
setColor(parentOf(x), BLACK);
setColor(y, BLACK);
setColor(parentOf(parentOf(x)), RED);
x = parentOf(parentOf(x));
}
else {
if (x == leftOf(parentOf(x))) {
x = parentOf(x);
rotateRight(x);
}
setColor(parentOf(x), BLACK);
setColor(parentOf(parentOf(x)), RED);
if (parentOf(parentOf(x)) != null) {
rotateLeft(parentOf(parentOf(x)));
}
}
}
}
root.color = BLACK;
}
/**
* Delete node p, and then rebalance the tree.
*/
public void deleteEntry(RedBlackNode<K,V> p) {
size--;
// If strictly internal, first swap position with successor.
if (p.left != null && p.right != null) {
RedBlackNode<K,V> node = p.getSuccessor();
swapPosition(node, p);
}
// Start fixup at replacement node, if it exists.
RedBlackNode<K,V> replacement = (p.left != null ? p.left : p.right);
if (replacement != null) {
// Link replacement to parent
replacement.parent = p.parent;
if (p.parent == null) {
root = replacement;
}
else if (p.isLeftChild()) {
p.parent.left = replacement;
}
else {
p.parent.right = replacement;
}
// Null out links so they are OK to use by fixAfterDeletion.
p.left = p.right = p.parent = null;
// Fix replacement
if (p.color == BLACK) {
fixAfterDeletion(replacement);
}
}
else if (p.parent == null) { // return if we are the only node.
root = null;
}
else { // No children. Use self as phantom replacement and unlink.
if (p.color == BLACK) {
fixAfterDeletion(p);
}
if (p.parent != null) {
if (p.isLeftChild()) {
p.parent.left = null;
}
else if (p == p.parent.right) {
p.parent.right = null;
}
p.parent = null;
}
}
}
/** From CLR **/
private void fixAfterDeletion(RedBlackNode<K,V> x) {
while (x != root && colorOf(x) == BLACK) {
if (x == leftOf(parentOf(x))) {
RedBlackNode<K,V> sib = rightOf(parentOf(x));
if (colorOf(sib) == RED) {
setColor(sib, BLACK);
setColor(parentOf(x), RED);
rotateLeft(parentOf(x));
sib = rightOf(parentOf(x));
}
if (colorOf(leftOf(sib)) == BLACK &&
colorOf(rightOf(sib)) == BLACK) {
setColor(sib, RED);
x = parentOf(x);
}
else {
if (colorOf(rightOf(sib)) == BLACK) {
setColor(leftOf(sib), BLACK);
setColor(sib, RED);
rotateRight(sib);
sib = rightOf(parentOf(x));
}
setColor(sib, colorOf(parentOf(x)));
setColor(parentOf(x), BLACK);
setColor(rightOf(sib), BLACK);
rotateLeft(parentOf(x));
x = root;
}
}
else { // symmetric
RedBlackNode<K,V> sib = leftOf(parentOf(x));
if (colorOf(sib) == RED) {
setColor(sib, BLACK);
setColor(parentOf(x), RED);
rotateRight(parentOf(x));
sib = leftOf(parentOf(x));
}
if (colorOf(rightOf(sib)) == BLACK &&
colorOf(leftOf(sib)) == BLACK) {
setColor(sib, RED);
x = parentOf(x);
}
else {
if (colorOf(leftOf(sib)) == BLACK) {
setColor(rightOf(sib), BLACK);
setColor(sib, RED);
rotateLeft(sib);
sib = leftOf(parentOf(x));
}
setColor(sib, colorOf(parentOf(x)));
setColor(parentOf(x), BLACK);
setColor(leftOf(sib), BLACK);
rotateRight(parentOf(x));
x = root;
}
}
}
setColor(x, BLACK);
}
/**
* Swap the linkages of two nodes in a tree.
*/
private void swapPosition(RedBlackNode<K,V> x, RedBlackNode<K,V> y) {
// Save initial values.
RedBlackNode<K,V> px = x.parent, lx = x.left, rx = x.right;
RedBlackNode<K,V> py = y.parent, ly = y.left, ry = y.right;
boolean xWasLeftChild = px != null && x == px.left;
boolean yWasLeftChild = py != null && y == py.left;
// Swap, handling special cases of one being the other's parent.
if (x == py) { // x was y's parent
x.parent = y;
if (yWasLeftChild) {
y.left = x;
y.right = rx;
}
else {
y.right = x;
y.left = lx;
}
}
else {
x.parent = py;
if (py != null) {
if (yWasLeftChild) {
py.left = x;
}
else {
py.right = x;
}
}
y.left = lx;
y.right = rx;
}
if (y == px) { // y was x's parent
y.parent = x;
if (xWasLeftChild) {
x.left = y;
x.right = ry;
}
else {
x.right = y;
x.left = ly;
}
}
else {
y.parent = px;
if (px != null) {
if (xWasLeftChild) {
px.left = y;
}
else {
px.right = y;
}
}
x.left = ly;
x.right = ry;
}
// Fix children's parent pointers
if (x.left != null) {
x.left.parent = x;
}
if (x.right != null) {
x.right.parent = x;
}
if (y.left != null) {
y.left.parent = y;
}
if (y.right != null) {
y.right.parent = y;
}
// Swap colors
RedBlackNode.NodeColor c = x.color;
x.color = y.color;
y.color = c;
// Check if root changed
if (root == x) {
root = y;
}
else if (root == y) {
root = x;
}
}
}

View file

@ -0,0 +1,78 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
class ReverseListIterator<T> extends ListIterator<T> {
ReverseListIterator(ListSTL<T> list, ListNodeSTL<T> root, ListNodeSTL<T> node) {
super(list, root, node);
}
@Override
public IteratorSTL<T> copy() {
return new ReverseListIterator<T>( list, root, node );
}
@Override
public boolean isBegin() {
return node == root.prev;
}
@Override
public IteratorSTL<T> decrement() {
if (node.prev == root) {
throw new IndexOutOfBoundsException();
}
node = node.next;
return this;
}
@Override
public IteratorSTL<T> increment() {
node = node.prev;
return this;
}
@Override
public void insert(T value) {
ListNodeSTL<T> newNode = new ListNodeSTL<T>(node, node.next, value);
node.next.prev = newNode;
node.next = newNode;
node = newNode;
list.adjustSize(1);
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (obj == this) {
return true;
}
if (obj.getClass() != this.getClass()) {
return false;
}
ReverseListIterator<?> other = (ReverseListIterator)obj;
return list == other.list && node == other.node;
}
@Override
public int hashCode() {
return list.hashCode();
}
}

View file

@ -0,0 +1,90 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
public class ReverseMapIteratorSTL<K, V> extends MapIteratorSTL<K, V> {
ReverseMapIteratorSTL(RedBlackTree<K,V> tree, RedBlackNode<K,V> node) {
super( tree, node);
}
@Override
public IteratorSTL<Pair<K,V>> increment() {
if (node == null) {
throw new IndexOutOfBoundsException();
}
node = node.getPredecessor();
return this;
}
@Override
public IteratorSTL<Pair<K,V>> decrement() {
if (node == null && tree.isEmpty()) {
throw new IndexOutOfBoundsException();
}
if (node == null) {
node = tree.getFirst();
}
else {
node = node.getSuccessor();
}
return this;
}
public void delete() {
if (node == null) {
throw new IndexOutOfBoundsException();
}
RedBlackNode<K, V> nextNode = node.getPredecessor();
tree.deleteEntry(node);
node = nextNode;
}
public void delete(int count) {
throw new UnsupportedOperationException();
}
@Override
public void insert(Pair<K, V> value) {
throw new UnsupportedOperationException();
}
@Override
public boolean isBegin() {
return node == tree.getLast();
}
@Override
public IteratorSTL<Pair<K, V>> copy() {
return new ReverseMapIteratorSTL<K, V>(tree, node);
}
@SuppressWarnings("unchecked")
@Override
public boolean equals( Object obj ) {
if (obj == this) {
return true;
}
if (obj == null) {
return false;
}
if (obj.getClass() != getClass() ) {
return false;
}
ReverseMapIteratorSTL<K, V> other = (ReverseMapIteratorSTL<K, V>) obj;
return tree == other.tree && node == other.node;
}
}

View file

@ -0,0 +1,86 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
public class ReverseSetIterator<T> extends SetIterator<T> {
ReverseSetIterator(RedBlackTree<T,T> tree, RedBlackNode<T,T> node) {
super(tree, node);
}
@Override
public IteratorSTL<T> copy() {
return new ReverseSetIterator<T>( tree, node);
}
@Override
public IteratorSTL<T> increment() {
if (node == null) {
throw new IndexOutOfBoundsException();
}
node = node.getPredecessor();
return this;
}
@Override
public IteratorSTL<T> decrement() {
if (node == null && tree.isEmpty()) {
throw new IndexOutOfBoundsException();
}
if (node == null) {
node = tree.getFirst();
}
else {
node = node.getSuccessor();
}
return this;
}
public void delete() {
if (node == null) {
throw new IndexOutOfBoundsException();
}
RedBlackNode<T, T> nextNode = node.getPredecessor();
tree.deleteEntry(node);
node = nextNode;
}
@Override
public boolean isBegin() {
return node == tree.getLast();
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (obj == this) {
return true;
}
if (obj.getClass() != this.getClass()) {
return false;
}
ReverseSetIterator<?> other = (ReverseSetIterator)obj;
return tree == other.tree && node == other.node;
}
@Override
public int hashCode() {
return tree.hashCode();
}
}

View file

@ -0,0 +1,111 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
import java.util.*;
class ReverseVectorIterator<T> extends VectorIterator<T> {
public ReverseVectorIterator(ArrayList<T> data, int index) {
super(data, index);
}
@Override
public boolean isBegin() {
return index == (data.size()-1);
}
@Override
public boolean isEnd() {
return index < 0;
}
@Override
public T get(int offset) {
return data.get(index-offset);
}
@Override
public IteratorSTL<T> decrement() {
if (index == data.size()-1) {
throw new IndexOutOfBoundsException();
}
index++;
return this;
}
public void delete(int count) {
if (index < count-1) {
throw new IndexOutOfBoundsException();
}
data.subList(index-count+1, count).clear();
}
@Override
public IteratorSTL<T> increment() {
if (index < 0) {
throw new IndexOutOfBoundsException();
}
index--;
return this;
}
@Override
public IteratorSTL<T> increment(int count) {
if (index-count < -1) {
throw new IndexOutOfBoundsException();
}
index -= count;
return this;
}
@Override
public IteratorSTL<T> decrement(int count) {
if (index+count >= data.size()) {
throw new IndexOutOfBoundsException();
}
index += count;
return this;
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (obj == this) {
return true;
}
if (obj.getClass() != this.getClass()) {
return false;
}
ReverseVectorIterator<?> other = (ReverseVectorIterator)obj;
return data == other.data && index == other.index;
}
@Override
public int hashCode() {
return data.hashCode();
}
@Override
public IteratorSTL<T> copy() {
return new ReverseVectorIterator<T>(data, index);
}
@Override
public int getIndex() {
return index;
}
}

View file

@ -0,0 +1,27 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
import java.util.Comparator;
public class SelfComparator<T extends Comparable<T>> implements Comparator<T> {
@Override
public int compare(T o1, T o2) {
return o1.compareTo(o2);
}
}

View file

@ -0,0 +1,128 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
public class SetIterator<T> implements IteratorSTL<T> {
protected RedBlackTree<T, T> tree;
protected RedBlackNode<T, T> node;
public boolean erased = false;
SetIterator(RedBlackTree<T,T> tree, RedBlackNode<T,T> node) {
this.tree = tree;
this.node = node;
}
SetIterator(RedBlackTree<T,T> tree, RedBlackNode<T,T> node, boolean erased) {
this.tree = tree;
this.node = node;
this.erased = erased;
}
public void assign( IteratorSTL<T> otherIterator ) {
SetIterator<T> other = (SetIterator<T>) otherIterator;
this.tree = other.tree;
this.node = other.node;
this.erased = other.erased;
}
public IteratorSTL<T> copy() {
return new SetIterator<T>( tree, node, erased);
}
public IteratorSTL<T> decrement() {
if (node == null && tree.isEmpty()) {
throw new IndexOutOfBoundsException();
}
if (node == null) {
node = tree.getLast();
}
else {
node = node.getPredecessor();
}
erased = false;
return this;
}
public T get() {
if (erased) {
throw new IndexOutOfBoundsException("element erased");
}
if (node == null) {
throw new IndexOutOfBoundsException();
}
return node.getKey();
}
public IteratorSTL<T> increment() {
if (!erased && node == null) {
throw new IndexOutOfBoundsException();
}
if (!erased) { // erased nodes already point to the successor
node = node.getSuccessor();
}
erased = false;
return this;
}
public void insert(T value) {
throw new UnsupportedOperationException();
}
public boolean isBegin() {
if (erased) {
throw new RuntimeException("Iterater in invalid state");
}
return node == tree.getFirst();
}
public boolean isEnd() {
if (erased) {
throw new RuntimeException("Iterater in invalid state");
}
return node == null;
}
public void set(T value) {
throw new UnsupportedOperationException();
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (obj == this) {
return true;
}
if (obj.getClass() != this.getClass()) {
return false;
}
SetIterator<?> other = (SetIterator)obj;
return tree == other.tree && node == other.node && erased == other.erased;
}
@Override
public int hashCode() {
return tree.hashCode();
}
public IteratorSTL<T> decrement( int n ) {
throw new UnsupportedOperationException();
}
public IteratorSTL<T> increment( int n ) {
throw new UnsupportedOperationException();
}
}

View file

@ -0,0 +1,118 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
import java.util.Comparator;
public class SetSTL<K> {
RedBlackTree<K,K> rbTree;
public SetSTL(Comparator<K> comparator) {
rbTree = new RedBlackTree<>(comparator, false);
}
public SetSTL(SetSTL<K> set) {
rbTree = new RedBlackTree<>(set.rbTree);
}
public Pair<IteratorSTL<K>, Boolean> insert(K key) {
Pair<RedBlackNode<K, K>, Boolean> result = rbTree.put(key, key);
return new Pair<>( new SetIterator<>( rbTree, result.first ),
result.second );
}
public boolean contains(K key) {
return rbTree.containsKey(key);
}
public boolean remove(K key) {
return rbTree.remove(key) != null;
}
public IteratorSTL<K> find( K key ) {
RedBlackNode<K, K> node = rbTree.findFirstNode( key );
if ( node == null ) {
return end();
}
return new SetIterator<>( rbTree, node );
}
public void erase( IteratorSTL<K> iterator ) {
SetIterator<K> it = (SetIterator<K>)iterator;
RedBlackNode<K, K> node = it.node;
if (node == null) {
throw new IndexOutOfBoundsException();
}
it.node = node.getSuccessor();
it.erased = true;
rbTree.deleteEntry(node);
}
public IteratorSTL<K> begin() {
return new SetIterator<>(rbTree, rbTree.getFirst());
}
public IteratorSTL<K> end() {
return new SetIterator<>(rbTree, null);
}
public IteratorSTL<K> rBegin() {
return new ReverseSetIterator<>(rbTree, rbTree.getLast());
}
public IteratorSTL<K> rEnd() {
return new ReverseSetIterator<>(rbTree, null);
}
public IteratorSTL<K> lower_bound( K key ) {
RedBlackNode<K,K> node = rbTree.lowerBound( key );
return new SetIterator<>(rbTree, node);
}
public IteratorSTL<K> upper_bound( K key ) {
RedBlackNode<K,K> node = rbTree.upperBound( key );
SetIterator<K> it = new SetIterator<>(rbTree, node);
return it;
}
public void erase( K key ) {
rbTree.remove( key );
}
public void clear() {
rbTree.removeAll();
}
public boolean isEmpty() {
return rbTree.isEmpty();
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("{ ");
IteratorSTL<K> ii = begin();
while (!ii.isEnd()) {
K thing = ii.get();
sb.append(thing.toString());
sb.append(" ");
ii.increment();
}
sb.append("}");
return sb.toString();
}
}

View file

@ -0,0 +1,72 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
/**
* This wrapper class is used to detect cases where code is
* modifying iterators that shouldn't change.
*/
public class UnmodifiableListIteratorSTL<T> extends ListIterator<T> {
public UnmodifiableListIteratorSTL( ListIterator<T> iterator ) {
super( iterator.list, iterator.root, iterator.node );
}
@Override
public void assign( IteratorSTL<T> otherIterator ) {
throw new UnsupportedOperationException( "Cannot modify this iterator!" );
}
@Override
public IteratorSTL<T> decrement() {
throw new UnsupportedOperationException( "Cannot modify this iterator!" );
}
@Override
public IteratorSTL<T> decrement( int n ) {
throw new UnsupportedOperationException( "Cannot modify this iterator!" );
}
public void delete() {
throw new UnsupportedOperationException( "Cannot modify this iterator!" );
}
public void delete( int count ) {
throw new UnsupportedOperationException( "Cannot modify this iterator!" );
}
@Override
public IteratorSTL<T> increment() {
throw new UnsupportedOperationException( "Cannot modify this iterator!" );
}
@Override
public IteratorSTL<T> increment( int n ) {
throw new UnsupportedOperationException( "Cannot modify this iterator!" );
}
@Override
public void insert( T value ) {
throw new UnsupportedOperationException( "Cannot modify this iterator!" );
}
@Override
public void set( Object value ) {
throw new UnsupportedOperationException( "Cannot modify this iterator!" );
}
}

View file

@ -0,0 +1,123 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic.stl;
import java.util.ArrayList;
public class VectorIterator<T> implements IteratorSTL<T> {
protected int index;
protected ArrayList<T> data;
public VectorIterator(ArrayList<T> data, int index) {
this.data = data;
this.index = index;
}
@Override
public String toString() {
T value = index >= data.size() ? null : data.get( index );
return "VectorIterator: [index=" + index + " - " + value + "]";
}
public void assign( IteratorSTL<T> otherIterator ) {
VectorIterator<T> other = (VectorIterator<T>) otherIterator;
this.index = other.index;
this.data = other.data;
}
public boolean isBegin() {
return index == 0;
}
public boolean isEnd() {
return index >= data.size();
}
public T get() {
return data.get(index);
}
public void set(T value) {
data.set(index, value);
}
public T get(int offset) {
return data.get(index+offset);
}
public IteratorSTL<T> decrement() {
if (index == 0) {
throw new IndexOutOfBoundsException();
}
index--;
return this;
}
public IteratorSTL<T> increment() {
if (index >= data.size()) {
throw new IndexOutOfBoundsException();
}
index++;
return this;
}
public IteratorSTL<T> increment(int count) {
if (index+count > data.size()) {
throw new IndexOutOfBoundsException();
}
index += count;
return this;
}
public IteratorSTL<T> decrement(int count) {
if (index-count < 0) {
throw new IndexOutOfBoundsException();
}
index -= count;
return this;
}
public void insert(T value) {
data.add(index, value);
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (obj == this) {
return true;
}
if (obj.getClass() != this.getClass()) {
return false;
}
VectorIterator<?> other = (VectorIterator)obj;
return data == other.data && index == other.index;
}
@Override
public int hashCode() {
return data.hashCode();
}
public IteratorSTL<T> copy() {
return new VectorIterator<T>(data, index);
}
public int getIndex() {
return index;
}
}

Some files were not shown because too many files have changed in this diff Show more