yetangitu/ghidra
1
0
Fork 0
mirror of https://github.com/NationalSecurityAgency/ghidra.git synced 2025-10-05 10:49:34 +02:00
Code Issues Releases Wiki Activity

GP-0 fix string representation rendering changes in tests

Browse source
This commit is contained in:
dev747368 2023-12-27 21:52:27 +00:00
parent a806848b31
commit eb01064e3e
6 changed files with 130 additions and 84 deletions
Download patch file Download diff file Expand all files Collapse all files

4
Ghidra/Features/Base/src/test.slow/java/ghidra/app/plugin/core/data/DataAction4Test.java
View file

@ -816,7 +816,7 @@ public class DataAction4Test extends AbstractDataActionTest {
d = getContextData(); d = getContextData();
assertEquals("ChooseFontW\0\u0015\0ReplaceTextW\0\0\u0004", d.getValue()); assertEquals("ChooseFontW\0\u0015\0ReplaceTextW\0\0\u0004", d.getValue());
assertEquals("\"ChooseFontW\\0\",15h,\"\\0ReplaceTextW\\0\\0\",04h", assertEquals("\"ChooseFontW\\0\",15h,00h,\"ReplaceTextW\\0\\0\",04h",
d.getDefaultValueRepresentation()); d.getDefaultValueRepresentation());
} }
@ -888,7 +888,7 @@ public class DataAction4Test extends AbstractDataActionTest {
checkOnDefined(null, UnicodeDataType.class); checkOnDefined(null, UnicodeDataType.class);
d = getContextData(); d = getContextData();
assertEquals("01h,00h,\"\\0Sample\"", d.getDefaultValueRepresentation()); assertEquals("01h,00h,00h,00h,\"Sample\"", d.getDefaultValueRepresentation());
assertEquals("\1\0Sample", d.getValue()); assertEquals("\1\0Sample", d.getValue());
} }

2
Ghidra/Features/Base/src/test.slow/java/ghidra/app/plugin/core/datapreview/DataTypePreviewPluginTest.java
View file

@ -118,7 +118,7 @@ public class DataTypePreviewPluginTest extends AbstractGhidraHeadedIntegrationTe
assertEquals("6.119088925166103E-308", model.getValueAt(9, DTPPTableModel.PREVIEW_COL)); assertEquals("6.119088925166103E-308", model.getValueAt(9, DTPPTableModel.PREVIEW_COL));
assertEquals("2600h", model.getValueAt(10, DTPPTableModel.PREVIEW_COL)); assertEquals("2600h", model.getValueAt(10, DTPPTableModel.PREVIEW_COL));
assertEquals("7000h", model.getValueAt(11, DTPPTableModel.PREVIEW_COL)); assertEquals("7000h", model.getValueAt(11, DTPPTableModel.PREVIEW_COL));
assertEquals("\"\\0\\0\\0\\0\\0\\0\\0\",0Eh,\"\\0f\"", assertEquals("00h,00h,00h,00h,00h,00h,00h,0Eh,00h,\"f\"",
model.getValueAt(12, DTPPTableModel.PREVIEW_COL)); model.getValueAt(12, DTPPTableModel.PREVIEW_COL));
assertEquals(14, model.getRowCount()); assertEquals(14, model.getRowCount());

40
Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/data/StringDataInstance.java
View file

@ -579,14 +579,17 @@ public class StringDataInstance {
if (stringBytes == null) { if (stringBytes == null) {
return UNKNOWN_DOT_DOT_DOT; return UNKNOWN_DOT_DOT_DOT;
} }
ByteBuffer bb = ByteBuffer.wrap(stringBytes); ByteBuffer bb = ByteBuffer.wrap(stringBytes);
String adjustedCharsetName = getAdjustedCharsetInfo(bb); // force BE or LE variants of UTF charsets
// force BE or LE variants of UTF charsets, consume any BOM
String adjustedCharsetName = getAdjustedCharsetInfo(bb);
if (!Charset.isSupported(adjustedCharsetName)) { if (!Charset.isSupported(adjustedCharsetName)) {
return UNKNOWN_DOT_DOT_DOT; return UNKNOWN_DOT_DOT_DOT;
} }
Charset cs = Charset.forName(adjustedCharsetName); Charset cs = Charset.forName(adjustedCharsetName);
return new String(stringBytes, cs); return new String(stringBytes, bb.position(), bb.remaining(), cs);
} }
private byte[] getStringBytes() { private byte[] getStringBytes() {
@ -780,14 +783,17 @@ public class StringDataInstance {
bb.get(0, bytes); bb.get(0, bytes);
int be_val = (int) BigEndianDataConverter.INSTANCE.getValue(bytes, charSize); int be_val = (int) BigEndianDataConverter.INSTANCE.getValue(bytes, charSize);
switch (be_val) { Endian result = switch (be_val) {
case StringUtilities.UNICODE_BE_BYTE_ORDER_MARK: case StringUtilities.UNICODE_BE_BYTE_ORDER_MARK -> Endian.BIG;
return Endian.BIG; case StringUtilities.UNICODE_LE16_BYTE_ORDER_MARK,
case StringUtilities.UNICODE_LE16_BYTE_ORDER_MARK: StringUtilities.UNICODE_LE32_BYTE_ORDER_MARK -> Endian.LITTLE;
case StringUtilities.UNICODE_LE32_BYTE_ORDER_MARK: default -> null;
return Endian.LITTLE; };
if (result != null) {
// consume the BOM bytes
bb.position(charSize);
} }
return null; return result;
} }
/** /**
@ -842,17 +848,17 @@ public class StringDataInstance {
} }
ByteBuffer bb = ByteBuffer.wrap(stringBytes); ByteBuffer bb = ByteBuffer.wrap(stringBytes);
String adjustedCharsetName = getAdjustedCharsetInfo(bb); // force BE or LE variants of UTF charsets
StringRenderBuilder renderer = // force BE or LE variants of UTF charsets, consume any BOM
new StringRenderBuilder(adjustedCharsetName.startsWith("UTF"), charSize, quoteChar); String adjustedCharsetName = getAdjustedCharsetInfo(bb);
if (!Charset.isSupported(adjustedCharsetName)) { if (!Charset.isSupported(adjustedCharsetName)) {
return UNKNOWN_DOT_DOT_DOT; return UNKNOWN_DOT_DOT_DOT;
} }
Charset cs = Charset.forName(adjustedCharsetName); Charset cs = Charset.forName(adjustedCharsetName);
renderer.decodeBytesUsingCharset(bb, cs, renderSetting,
stringLayout.shouldTrimTrailingNulls()); StringRenderBuilder renderer = new StringRenderBuilder(cs, charSize, quoteChar);
renderer.decodeBytesUsingCharset(bb, renderSetting, stringLayout.shouldTrimTrailingNulls());
String result = renderer.build(); String result = renderer.build();
return result; return result;
@ -917,12 +923,8 @@ public class StringDataInstance {
return UNKNOWN_DOT_DOT_DOT; return UNKNOWN_DOT_DOT_DOT;
} }
// if the charset's charsize is bigger than the number of bytes we have,
// discard the charset and fall back to US-ASCII
String newCSName = (length < charSize) ? DEFAULT_CHARSET_NAME : charsetName;
StringDataInstance charseqSDI = StringDataInstance charseqSDI =
new StringDataInstance(this, StringLayoutEnum.CHAR_SEQ, buf, length, newCSName); new StringDataInstance(this, StringLayoutEnum.CHAR_SEQ, buf, length, charsetName);
char quoteChar = length == charSize char quoteChar = length == charSize
? StringRenderBuilder.SINGLE_QUOTE ? StringRenderBuilder.SINGLE_QUOTE

67
Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/data/StringRenderBuilder.java
View file

@ -37,19 +37,21 @@ public class StringRenderBuilder {
private static final int MAX_ASCII = 0x80; private static final int MAX_ASCII = 0x80;
private final StringBuilder sb = new StringBuilder(); private final StringBuilder sb = new StringBuilder();
private final Charset cs;
private final int charSize; private final int charSize;
private final boolean utfCharset; private final boolean utfCharset;
private final char quoteChar; private final char quoteChar;
private boolean byteMode = true; private boolean byteMode = true;
public StringRenderBuilder(boolean utfCharset, int charSize) { public StringRenderBuilder(Charset cs, int charSize) {
this(utfCharset, charSize, DOUBLE_QUOTE); this(cs, charSize, DOUBLE_QUOTE);
} }
public StringRenderBuilder(boolean utfCharset, int charSize, char quoteChar) { public StringRenderBuilder(Charset cs, int charSize, char quoteChar) {
this.cs = cs;
this.charSize = charSize; this.charSize = charSize;
this.utfCharset = utfCharset;
this.quoteChar = quoteChar; this.quoteChar = quoteChar;
this.utfCharset = cs.name().startsWith("UTF");
} }
/** /**
@ -87,12 +89,16 @@ public class StringRenderBuilder {
* @param trimTrailingNulls boolean flag, if true trailing null bytes will not be included * @param trimTrailingNulls boolean flag, if true trailing null bytes will not be included
* in the rendered output * in the rendered output
*/ */
public void decodeBytesUsingCharset(ByteBuffer bb, Charset cs, RENDER_ENUM renderSetting, public void decodeBytesUsingCharset(ByteBuffer bb, RENDER_ENUM renderSetting,
boolean trimTrailingNulls) { boolean trimTrailingNulls) {
if (!bb.hasRemaining()) {
// early exit avoids problems trying to flush un-initialized codec later
return;
}
CharsetDecoder codec = cs.newDecoder() CharsetDecoder codec = cs.newDecoder()
.onMalformedInput(CodingErrorAction.REPORT) .onMalformedInput(CodingErrorAction.REPORT)
.onUnmappableCharacter(CodingErrorAction.REPORT); .onUnmappableCharacter(CodingErrorAction.REPORT);
CharBuffer cb = CharBuffer.allocate(Math.min(10, bb.remaining())); CharBuffer cb = CharBuffer.allocate(Math.max(10, bb.remaining() + (bb.remaining() / 2)));
while (bb.hasRemaining()) { while (bb.hasRemaining()) {
CoderResult cr = codec.decode(bb, cb, true); CoderResult cr = codec.decode(bb, cb, true);
if (!bb.hasRemaining() && trimTrailingNulls) { if (!bb.hasRemaining() && trimTrailingNulls) {
@ -143,6 +149,14 @@ public class StringRenderBuilder {
} }
} }
private void addByteSeq(int codePoint) {
// NOTE: this is a bit of a hack. We assume we can run the codepoint back thru the
// charset and get the original bytes
CharBuffer cb = CharBuffer.wrap(new String(new int[] { codePoint }, 0, 1));
ByteBuffer bb = cs.encode(cb);
addByteSeq(bb, bb.limit());
}
private void trimTrailingNulls(CharBuffer cb) { private void trimTrailingNulls(CharBuffer cb) {
while (cb.position() > 0 && cb.get(cb.position() - 1) == 0) { while (cb.position() > 0 && cb.get(cb.position() - 1) == 0) {
cb.position(cb.position() - 1); cb.position(cb.position() - 1);
@ -159,27 +173,40 @@ public class StringRenderBuilder {
for (int i = 0, strLength = stringValue.length(); i < strLength;) { for (int i = 0, strLength = stringValue.length(); i < strLength;) {
int codePoint = Character.codePointAt(stringValue, i); int codePoint = Character.codePointAt(stringValue, i);
if (StringUtilities.isDisplayable(codePoint)) { if (StringUtilities.isControlCharacterOrBackslash(codePoint)) {
addCodePointChar(codePoint);
}
else if (codePoint == 0) {
// TODO: there is an opportunity to make this smarter by not switching from
// byte mode to string mode to add nulls.
addString("\\0");
}
else if (StringUtilities.isControlCharacterOrBackslash(codePoint)) {
addString(StringUtilities.convertCodePointToEscapeSequence(codePoint)); addString(StringUtilities.convertCodePointToEscapeSequence(codePoint));
} }
else if (Character.isISOControl(codePoint) || !Character.isDefined(codePoint) || else if (codePoint == 0) {
codePoint == StringUtilities.UNICODE_BE_BYTE_ORDER_MARK) { if (byteMode) {
addEscapedCodePoint(codePoint); addByteSeq(0);
}
else {
addString("\\0");
}
} }
else if (renderSetting == RENDER_ENUM.ALL) { else if (Character.isISOControl(codePoint) || !Character.isDefined(codePoint)) {
addByteSeq(codePoint);
}
else if (StringUtilities.isDisplayable(codePoint)) {
addCodePointChar(codePoint); addCodePointChar(codePoint);
} }
else { else if (codePoint == StringUtilities.UNICODE_BE_BYTE_ORDER_MARK) {
addEscapedCodePoint(codePoint); addEscapedCodePoint(codePoint);
} }
else {
switch (renderSetting) {
default:
case ALL:
addCodePointChar(codePoint);
break;
case BYTE_SEQ:
addByteSeq(codePoint);
break;
case ESC_SEQ:
addEscapedCodePoint(codePoint);
break;
}
}
i += Character.charCount(codePoint); i += Character.charCount(codePoint);
} }

6
Ghidra/Framework/SoftwareModeling/src/test/java/ghidra/program/model/data/StringDataTypeTest.java
View file

@ -457,8 +457,7 @@ public class StringDataTypeTest extends AbstractGTest {
newset().set(StandardCharsets.US_ASCII), buf.getLength()); newset().set(StandardCharsets.US_ASCII), buf.getLength());
//@formatter:off //@formatter:off
String expected = String expected =
"\"\\0\"," + "00h,01h,02h,03h,04h,05h,06h,"+
"01h,02h,03h,04h,05h,06h,"+
"\"\\a\\b\\t\\n\\v\\f\\r\",0Eh,0Fh,10h,11h,12h,13h,14h,15h,16h,17h,18h,19h,1Ah,1Bh,1Ch,1Dh,1Eh,1Fh,\""+ "\"\\a\\b\\t\\n\\v\\f\\r\",0Eh,0Fh,10h,11h,12h,13h,14h,15h,16h,17h,18h,19h,1Ah,1Bh,1Ch,1Dh,1Eh,1Fh,\""+
" !\\\"#$%&'()*+,-./0123456789:;<=>?@"+ " !\\\"#$%&'()*+,-./0123456789:;<=>?@"+
"ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`"+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`"+
@ -484,8 +483,7 @@ public class StringDataTypeTest extends AbstractGTest {
allAsciiChars[i] = (byte) i; allAsciiChars[i] = (byte) i;
} }
String repr = "" + String repr = "" +
"\"\\0\"," + "00h,01h,02h,03h,04h,05h,06h," +
"01h,02h,03h,04h,05h,06h," +
"\"\\a\\b\\t\\n\\v\\f\\r\",0Eh,0Fh,10h,11h,12h,13h,14h,15h,16h,17h,18h,19h,1Ah,1Bh,1Ch,1Dh,1Eh,1Fh,\"" + "\"\\a\\b\\t\\n\\v\\f\\r\",0Eh,0Fh,10h,11h,12h,13h,14h,15h,16h,17h,18h,19h,1Ah,1Bh,1Ch,1Dh,1Eh,1Fh,\"" +
" !\\\"#$%&'()*+,-./0123456789:;<=>?@" + " !\\\"#$%&'()*+,-./0123456789:;<=>?@" +
"ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`" +

95
Ghidra/Framework/SoftwareModeling/src/test/java/ghidra/program/model/data/StringRenderBuilderTest.java
View file

@ -18,6 +18,7 @@ package ghidra.program.model.data;
import static org.junit.Assert.*; import static org.junit.Assert.*;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import org.junit.Test; import org.junit.Test;
@ -26,6 +27,10 @@ import generic.test.AbstractGTest;
import ghidra.program.model.data.RenderUnicodeSettingsDefinition.RENDER_ENUM; import ghidra.program.model.data.RenderUnicodeSettingsDefinition.RENDER_ENUM;
public class StringRenderBuilderTest extends AbstractGTest { public class StringRenderBuilderTest extends AbstractGTest {
private static final Charset US_ASCII = StandardCharsets.US_ASCII;
private static final Charset UTF8 = StandardCharsets.UTF_8;
private static final Charset UTF16 = StandardCharsets.UTF_16;
private static final Charset UTF32 = Charset.forName("UTF-32");
private ByteBuffer bb(int... values) { private ByteBuffer bb(int... values) {
return ByteBuffer.wrap(bytes(values)); return ByteBuffer.wrap(bytes(values));
@ -34,7 +39,7 @@ public class StringRenderBuilderTest extends AbstractGTest {
@Test @Test
public void testEmptyString() { public void testEmptyString() {
StringRenderBuilder srb = StringRenderBuilder srb =
new StringRenderBuilder(false, 1, StringRenderBuilder.DOUBLE_QUOTE); new StringRenderBuilder(US_ASCII, 1, StringRenderBuilder.DOUBLE_QUOTE);
String emptyString = srb.build(); String emptyString = srb.build();
assertEquals("\"\"", emptyString); assertEquals("\"\"", emptyString);
} }
@ -42,7 +47,7 @@ public class StringRenderBuilderTest extends AbstractGTest {
@Test @Test
public void testEmptyWChar2String() { public void testEmptyWChar2String() {
StringRenderBuilder srb = StringRenderBuilder srb =
new StringRenderBuilder(true, 2, StringRenderBuilder.DOUBLE_QUOTE); new StringRenderBuilder(UTF16, 2, StringRenderBuilder.DOUBLE_QUOTE);
String emptyString = srb.build(); String emptyString = srb.build();
assertEquals("u\"\"", emptyString); assertEquals("u\"\"", emptyString);
} }
@ -50,7 +55,7 @@ public class StringRenderBuilderTest extends AbstractGTest {
@Test @Test
public void testEmptyWChar4String() { public void testEmptyWChar4String() {
StringRenderBuilder srb = StringRenderBuilder srb =
new StringRenderBuilder(true, 4, StringRenderBuilder.DOUBLE_QUOTE); new StringRenderBuilder(UTF32, 4, StringRenderBuilder.DOUBLE_QUOTE);
String emptyString = srb.build(); String emptyString = srb.build();
assertEquals("U\"\"", emptyString); assertEquals("U\"\"", emptyString);
} }
@ -59,8 +64,8 @@ public class StringRenderBuilderTest extends AbstractGTest {
public void testEmptyStringWithNulls() { public void testEmptyStringWithNulls() {
ByteBuffer bb = bb(0, 0, 0); ByteBuffer bb = bb(0, 0, 0);
StringRenderBuilder srb = StringRenderBuilder srb =
new StringRenderBuilder(false, 1, StringRenderBuilder.DOUBLE_QUOTE); new StringRenderBuilder(US_ASCII, 1, StringRenderBuilder.DOUBLE_QUOTE);
srb.decodeBytesUsingCharset(bb, StandardCharsets.US_ASCII, RENDER_ENUM.ALL, true); srb.decodeBytesUsingCharset(bb, RENDER_ENUM.ALL, true);
String emptyString = srb.build(); String emptyString = srb.build();
assertEquals("\"\"", emptyString); assertEquals("\"\"", emptyString);
} }
@ -69,18 +74,18 @@ public class StringRenderBuilderTest extends AbstractGTest {
public void testEmptyStringWithNullsNoTrim() { public void testEmptyStringWithNullsNoTrim() {
ByteBuffer bb = bb(0, 0, 0); ByteBuffer bb = bb(0, 0, 0);
StringRenderBuilder srb = StringRenderBuilder srb =
new StringRenderBuilder(false, 1, StringRenderBuilder.DOUBLE_QUOTE); new StringRenderBuilder(US_ASCII, 1, StringRenderBuilder.DOUBLE_QUOTE);
srb.decodeBytesUsingCharset(bb, StandardCharsets.US_ASCII, RENDER_ENUM.ALL, false); srb.decodeBytesUsingCharset(bb, RENDER_ENUM.ALL, false);
String s = srb.build(); String s = srb.build();
assertEquals("\"\\0\\0\\0\"", s); assertEquals("00h,00h,00h", s);
} }
@Test @Test
public void testInteriorNulls() { public void testInteriorNulls() {
ByteBuffer bb = bb('t', 'e', 0, 's', 't', 0); ByteBuffer bb = bb('t', 'e', 0, 's', 't', 0);
StringRenderBuilder srb = StringRenderBuilder srb =
new StringRenderBuilder(false, 1, StringRenderBuilder.DOUBLE_QUOTE); new StringRenderBuilder(US_ASCII, 1, StringRenderBuilder.DOUBLE_QUOTE);
srb.decodeBytesUsingCharset(bb, StandardCharsets.US_ASCII, RENDER_ENUM.ALL, true); srb.decodeBytesUsingCharset(bb, RENDER_ENUM.ALL, true);
String s = srb.build(); String s = srb.build();
assertEquals("\"te\\0st\"", s); assertEquals("\"te\\0st\"", s);
} }
@ -89,8 +94,8 @@ public class StringRenderBuilderTest extends AbstractGTest {
public void testSimpleString() { public void testSimpleString() {
ByteBuffer bb = bb('t', 'e', 's', 't'); ByteBuffer bb = bb('t', 'e', 's', 't');
StringRenderBuilder srb = StringRenderBuilder srb =
new StringRenderBuilder(false, 1, StringRenderBuilder.DOUBLE_QUOTE); new StringRenderBuilder(US_ASCII, 1, StringRenderBuilder.DOUBLE_QUOTE);
srb.decodeBytesUsingCharset(bb, StandardCharsets.US_ASCII, RENDER_ENUM.ALL, true); srb.decodeBytesUsingCharset(bb, RENDER_ENUM.ALL, true);
String s = srb.build(); String s = srb.build();
assertEquals("\"test\"", s); assertEquals("\"test\"", s);
} }
@ -99,8 +104,8 @@ public class StringRenderBuilderTest extends AbstractGTest {
public void testStandardEscapedChars() { public void testStandardEscapedChars() {
ByteBuffer bb = bb('t', 'e', 's', 't', '\n', '\t', '\r'); ByteBuffer bb = bb('t', 'e', 's', 't', '\n', '\t', '\r');
StringRenderBuilder srb = StringRenderBuilder srb =
new StringRenderBuilder(false, 1, StringRenderBuilder.DOUBLE_QUOTE); new StringRenderBuilder(US_ASCII, 1, StringRenderBuilder.DOUBLE_QUOTE);
srb.decodeBytesUsingCharset(bb, StandardCharsets.US_ASCII, RENDER_ENUM.ALL, true); srb.decodeBytesUsingCharset(bb, RENDER_ENUM.ALL, true);
String s = srb.build(); String s = srb.build();
assertEquals("\"test\\n\\t\\r\"", s); assertEquals("\"test\\n\\t\\r\"", s);
} }
@ -109,8 +114,8 @@ public class StringRenderBuilderTest extends AbstractGTest {
public void testQuotedQuotesChars() { public void testQuotedQuotesChars() {
ByteBuffer bb = bb('t', 'e', 's', 't', '"', '1', '2', '3'); ByteBuffer bb = bb('t', 'e', 's', 't', '"', '1', '2', '3');
StringRenderBuilder srb = StringRenderBuilder srb =
new StringRenderBuilder(false, 1, StringRenderBuilder.DOUBLE_QUOTE); new StringRenderBuilder(US_ASCII, 1, StringRenderBuilder.DOUBLE_QUOTE);
srb.decodeBytesUsingCharset(bb, StandardCharsets.US_ASCII, RENDER_ENUM.ALL, true); srb.decodeBytesUsingCharset(bb, RENDER_ENUM.ALL, true);
String s = srb.build(); String s = srb.build();
assertEquals("\"test\\\"123\"", s); assertEquals("\"test\\\"123\"", s);
} }
@ -119,8 +124,8 @@ public class StringRenderBuilderTest extends AbstractGTest {
public void testSingleQuoteChars() { public void testSingleQuoteChars() {
ByteBuffer bb = bb('t', 'e', 's', 't', '\'', '1', '2', '3'); ByteBuffer bb = bb('t', 'e', 's', 't', '\'', '1', '2', '3');
StringRenderBuilder srb = StringRenderBuilder srb =
new StringRenderBuilder(false, 1, StringRenderBuilder.DOUBLE_QUOTE); new StringRenderBuilder(US_ASCII, 1, StringRenderBuilder.DOUBLE_QUOTE);
srb.decodeBytesUsingCharset(bb, StandardCharsets.US_ASCII, RENDER_ENUM.ALL, true); srb.decodeBytesUsingCharset(bb, RENDER_ENUM.ALL, true);
String s = srb.build(); String s = srb.build();
assertEquals("\"test'123\"", s); assertEquals("\"test'123\"", s);
} }
@ -129,8 +134,8 @@ public class StringRenderBuilderTest extends AbstractGTest {
public void testSimpleStringWithTrailingNulls() { public void testSimpleStringWithTrailingNulls() {
ByteBuffer bb = bb('t', 'e', 's', 't', 0, 0, 0); ByteBuffer bb = bb('t', 'e', 's', 't', 0, 0, 0);
StringRenderBuilder srb = StringRenderBuilder srb =
new StringRenderBuilder(false, 1, StringRenderBuilder.DOUBLE_QUOTE); new StringRenderBuilder(US_ASCII, 1, StringRenderBuilder.DOUBLE_QUOTE);
srb.decodeBytesUsingCharset(bb, StandardCharsets.US_ASCII, RENDER_ENUM.ALL, true); srb.decodeBytesUsingCharset(bb, RENDER_ENUM.ALL, true);
String s = srb.build(); String s = srb.build();
assertEquals("\"test\"", s); assertEquals("\"test\"", s);
} }
@ -139,8 +144,8 @@ public class StringRenderBuilderTest extends AbstractGTest {
public void testSimpleStringWithTrailingNullsNoTrim() { public void testSimpleStringWithTrailingNullsNoTrim() {
ByteBuffer bb = bb('t', 'e', 's', 't', 0, 0, 0); ByteBuffer bb = bb('t', 'e', 's', 't', 0, 0, 0);
StringRenderBuilder srb = StringRenderBuilder srb =
new StringRenderBuilder(false, 1, StringRenderBuilder.DOUBLE_QUOTE); new StringRenderBuilder(US_ASCII, 1, StringRenderBuilder.DOUBLE_QUOTE);
srb.decodeBytesUsingCharset(bb, StandardCharsets.US_ASCII, RENDER_ENUM.ALL, false); srb.decodeBytesUsingCharset(bb, RENDER_ENUM.ALL, false);
String s = srb.build(); String s = srb.build();
assertEquals("\"test\\0\\0\\0\"", s); assertEquals("\"test\\0\\0\\0\"", s);
} }
@ -149,8 +154,8 @@ public class StringRenderBuilderTest extends AbstractGTest {
public void testUtf8String() { public void testUtf8String() {
ByteBuffer bb = bb(0xE1, 0x84, 0xA2); // should decode to \u1122 ByteBuffer bb = bb(0xE1, 0x84, 0xA2); // should decode to \u1122
StringRenderBuilder srb = StringRenderBuilder srb =
new StringRenderBuilder(true, 1, StringRenderBuilder.DOUBLE_QUOTE); new StringRenderBuilder(UTF8, 1, StringRenderBuilder.DOUBLE_QUOTE);
srb.decodeBytesUsingCharset(bb, StandardCharsets.UTF_8, RENDER_ENUM.ALL, true); srb.decodeBytesUsingCharset(bb, RENDER_ENUM.ALL, true);
String s = srb.build(); String s = srb.build();
assertEquals("u8\"\u1122\"", s); assertEquals("u8\"\u1122\"", s);
} }
@ -159,8 +164,8 @@ public class StringRenderBuilderTest extends AbstractGTest {
public void testUtf8NoRenderNonLatinString() { public void testUtf8NoRenderNonLatinString() {
ByteBuffer bb = bb(0xE1, 0x84, 0xA2); // should decode to \u1122 ByteBuffer bb = bb(0xE1, 0x84, 0xA2); // should decode to \u1122
StringRenderBuilder srb = StringRenderBuilder srb =
new StringRenderBuilder(true, 1, StringRenderBuilder.DOUBLE_QUOTE); new StringRenderBuilder(UTF8, 1, StringRenderBuilder.DOUBLE_QUOTE);
srb.decodeBytesUsingCharset(bb, StandardCharsets.UTF_8, RENDER_ENUM.ESC_SEQ, true); srb.decodeBytesUsingCharset(bb, RENDER_ENUM.ESC_SEQ, true);
String s = srb.build(); String s = srb.build();
assertEquals("u8\"\\u1122\"", s); // <- result is \ u 1122 assertEquals("u8\"\\u1122\"", s); // <- result is \ u 1122
} }
@ -169,8 +174,8 @@ public class StringRenderBuilderTest extends AbstractGTest {
public void testBadBytes_USASCII() { public void testBadBytes_USASCII() {
ByteBuffer bb = bb('t', 'e', 's', 't', 0x80); ByteBuffer bb = bb('t', 'e', 's', 't', 0x80);
StringRenderBuilder srb = StringRenderBuilder srb =
new StringRenderBuilder(false, 1, StringRenderBuilder.DOUBLE_QUOTE); new StringRenderBuilder(US_ASCII, 1, StringRenderBuilder.DOUBLE_QUOTE);
srb.decodeBytesUsingCharset(bb, StandardCharsets.US_ASCII, RENDER_ENUM.ALL, true); srb.decodeBytesUsingCharset(bb, RENDER_ENUM.ALL, true);
String s = srb.build(); String s = srb.build();
assertEquals("\"test\",80h", s); assertEquals("\"test\",80h", s);
} }
@ -180,8 +185,8 @@ public class StringRenderBuilderTest extends AbstractGTest {
// bad bytes in interior of string, switching modes // bad bytes in interior of string, switching modes
ByteBuffer bb = bb('t', 'e', 0x80, 's', 't'); ByteBuffer bb = bb('t', 'e', 0x80, 's', 't');
StringRenderBuilder srb = StringRenderBuilder srb =
new StringRenderBuilder(false, 1, StringRenderBuilder.DOUBLE_QUOTE); new StringRenderBuilder(US_ASCII, 1, StringRenderBuilder.DOUBLE_QUOTE);
srb.decodeBytesUsingCharset(bb, StandardCharsets.US_ASCII, RENDER_ENUM.ALL, true); srb.decodeBytesUsingCharset(bb, RENDER_ENUM.ALL, true);
String s = srb.build(); String s = srb.build();
assertEquals("\"te\",80h,\"st\"", s); assertEquals("\"te\",80h,\"st\"", s);
} }
@ -191,8 +196,8 @@ public class StringRenderBuilderTest extends AbstractGTest {
// bad bytes at beginning of string // bad bytes at beginning of string
ByteBuffer bb = bb(0x80, 't', 'e', 's', 't'); ByteBuffer bb = bb(0x80, 't', 'e', 's', 't');
StringRenderBuilder srb = StringRenderBuilder srb =
new StringRenderBuilder(false, 1, StringRenderBuilder.DOUBLE_QUOTE); new StringRenderBuilder(US_ASCII, 1, StringRenderBuilder.DOUBLE_QUOTE);
srb.decodeBytesUsingCharset(bb, StandardCharsets.US_ASCII, RENDER_ENUM.ALL, true); srb.decodeBytesUsingCharset(bb, RENDER_ENUM.ALL, true);
String s = srb.build(); String s = srb.build();
assertEquals("80h,\"test\"", s); assertEquals("80h,\"test\"", s);
} }
@ -201,8 +206,8 @@ public class StringRenderBuilderTest extends AbstractGTest {
public void testTruncatedUtf8() { public void testTruncatedUtf8() {
ByteBuffer bb = bb('t', 'e', 's', 't', 0xE1, 0x84); ByteBuffer bb = bb('t', 'e', 's', 't', 0xE1, 0x84);
StringRenderBuilder srb = StringRenderBuilder srb =
new StringRenderBuilder(true, 1, StringRenderBuilder.DOUBLE_QUOTE); new StringRenderBuilder(UTF8, 1, StringRenderBuilder.DOUBLE_QUOTE);
srb.decodeBytesUsingCharset(bb, StandardCharsets.UTF_8, RENDER_ENUM.ALL, true); srb.decodeBytesUsingCharset(bb, RENDER_ENUM.ALL, true);
String s = srb.build(); String s = srb.build();
assertEquals("u8\"test\",E1h,84h", s); assertEquals("u8\"test\",E1h,84h", s);
} }
@ -211,8 +216,8 @@ public class StringRenderBuilderTest extends AbstractGTest {
public void testUtf16() { public void testUtf16() {
ByteBuffer bb = bb('t', 0, 'e', 0, 's', 0, 't', 0); ByteBuffer bb = bb('t', 0, 'e', 0, 's', 0, 't', 0);
StringRenderBuilder srb = StringRenderBuilder srb =
new StringRenderBuilder(true, 2, StringRenderBuilder.DOUBLE_QUOTE); new StringRenderBuilder(StandardCharsets.UTF_16LE, 2, StringRenderBuilder.DOUBLE_QUOTE);
srb.decodeBytesUsingCharset(bb, StandardCharsets.UTF_16LE, RENDER_ENUM.ALL, true); srb.decodeBytesUsingCharset(bb, RENDER_ENUM.ALL, true);
String s = srb.build(); String s = srb.build();
assertEquals("u\"test\"", s); assertEquals("u\"test\"", s);
} }
@ -221,9 +226,23 @@ public class StringRenderBuilderTest extends AbstractGTest {
public void testUtf16BOM_LE() { public void testUtf16BOM_LE() {
ByteBuffer bb = bb(0xff, 0xfe, 't', 0, 'e', 0, 's', 0, 't', 0); ByteBuffer bb = bb(0xff, 0xfe, 't', 0, 'e', 0, 's', 0, 't', 0);
StringRenderBuilder srb = StringRenderBuilder srb =
new StringRenderBuilder(true, 2, StringRenderBuilder.DOUBLE_QUOTE); new StringRenderBuilder(StandardCharsets.UTF_16LE, 2, StringRenderBuilder.DOUBLE_QUOTE);
srb.decodeBytesUsingCharset(bb, StandardCharsets.UTF_16LE, RENDER_ENUM.ALL, true); srb.decodeBytesUsingCharset(bb, RENDER_ENUM.ALL, true);
String s = srb.build(); String s = srb.build();
assertEquals("u\"\\uFEFFtest\"", s); assertEquals("u\"\\uFEFFtest\"", s);
} }
@Test
public void testUtf32BOM_LE() {
// This test demonstrates the inconsistency of decoding a BOM in UTF-16 vs UTF-32
// UTF-16 charset impls will preserve the BOM in the result, whereas the UTF-32 does not.
// This probably isn't a big deal as BOMs aren't used frequently (?)
ByteBuffer bb =
bb(0xff, 0xfe, 0, 0, 't', 0, 0, 0, 'e', 0, 0, 0, 's', 0, 0, 0, 't', 0, 0, 0);
StringRenderBuilder srb = new StringRenderBuilder(Charset.forName("UTF-32LE"), 4,
StringRenderBuilder.DOUBLE_QUOTE);
srb.decodeBytesUsingCharset(bb, RENDER_ENUM.ALL, true);
String s = srb.build();
assertEquals("U\"test\"", s);
}
} }