mirror of
https://git.huckle.dev/Huckles-Minecraft-Archive/jpexs-decompiler.git
synced 2026-09-26 13:41:10 +00:00
Character sets are limited to GB2312, Shift_JIS, Unicode, and all singlebyte.
Fixed font code points - proper characters display, adding, etc.
This commit is contained in:
Jindra Petřík
@@ -16,10 +16,21 @@
|
||||
*/
|
||||
package com.jpexs.helpers.utf8;
|
||||
|
||||
import com.jpexs.helpers.Helper;
|
||||
import com.jpexs.helpers.utf8.charset.Gb2312;
|
||||
import com.jpexs.helpers.utf8.charset.ShiftJis;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.URLDecoder;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.CharBuffer;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
/**
|
||||
*
|
||||
@@ -28,8 +39,44 @@ import java.nio.charset.Charset;
|
||||
public class Utf8Helper {
|
||||
|
||||
public static String charsetName = "UTF-8";
|
||||
|
||||
|
||||
public static Charset charset = Charset.forName("UTF-8");
|
||||
|
||||
private static List<String> allowedVariableLengthCharsets = Arrays.asList(
|
||||
"GB2312", "Shift_JIS", "UTF-8", "UTF-16", "UTF16-BE", "UTF-16-LE", "UTF-32", "UTF-32LE", "UTF-32BE");
|
||||
|
||||
/**
|
||||
* Allowed charsets. They are limited to single byte charsets + allowedVariableLengthCharsets
|
||||
*/
|
||||
public static List<String> allowedCharsets = new ArrayList<>();
|
||||
|
||||
static {
|
||||
Map<String, Charset> charsets = Charset.availableCharsets();
|
||||
for (String s : charsets.keySet()) {
|
||||
Charset charset = charsets.get(s);
|
||||
int maxLen = 0;
|
||||
int minLen = Integer.MAX_VALUE;
|
||||
try {
|
||||
for (int i = 0; i < 65536; i++) {
|
||||
|
||||
ByteBuffer buf = charset.encode("" + (char) i);
|
||||
int len = buf.remaining();
|
||||
if (len > maxLen) {
|
||||
maxLen = len;
|
||||
}
|
||||
if (len < minLen) {
|
||||
minLen = len;
|
||||
}
|
||||
}
|
||||
if ((minLen == maxLen && minLen == 1) || allowedVariableLengthCharsets.contains(s)) {
|
||||
allowedCharsets.add(s);
|
||||
}
|
||||
} catch (UnsupportedOperationException ex) {
|
||||
//System.out.println(s + " ... ERROR");
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public static String urlDecode(String s) {
|
||||
try {
|
||||
@@ -55,4 +102,73 @@ public class Utf8Helper {
|
||||
// todo: make it faster without actually writing it to an array
|
||||
return string.getBytes(charset).length;
|
||||
}
|
||||
|
||||
public static char codePointToChar(int codePoint, String charsetName) {
|
||||
int newCodePoint;
|
||||
switch (charsetName) {
|
||||
case "GB2312":
|
||||
newCodePoint = new Gb2312().toUnicode(codePoint);
|
||||
break;
|
||||
case "Shift_JIS":
|
||||
newCodePoint = new ShiftJis().toUnicode(codePoint);
|
||||
break;
|
||||
case "UTF-8":
|
||||
case "UTF-16":
|
||||
case "UTF-16BE":
|
||||
case "UTF-16LE":
|
||||
case "UTF-32":
|
||||
case "UTF-32BE":
|
||||
case "UTF-32LE":
|
||||
newCodePoint = codePoint;
|
||||
break;
|
||||
default: {
|
||||
//Assuming single byte - ANSI
|
||||
newCodePoint = -1;
|
||||
try {
|
||||
newCodePoint = new String(new byte[]{(byte) codePoint}, charsetName).codePointAt(0);
|
||||
} catch (UnsupportedEncodingException ex) {
|
||||
Logger.getLogger(Utf8Helper.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (newCodePoint >= 0) {
|
||||
return (char) newCodePoint;
|
||||
}
|
||||
return '?';
|
||||
}
|
||||
|
||||
public static int charToCodePoint(char character, String charsetName) {
|
||||
int unicodeCodePoint = (int) character;
|
||||
int codePoint;
|
||||
switch (charsetName) {
|
||||
case "GB2312":
|
||||
codePoint = new Gb2312().fromUnicode(unicodeCodePoint);
|
||||
break;
|
||||
case "Shift_JIS":
|
||||
codePoint = new ShiftJis().fromUnicode(unicodeCodePoint);
|
||||
break;
|
||||
case "UTF-8":
|
||||
case "UTF-16":
|
||||
case "UTF-16BE":
|
||||
case "UTF-16LE":
|
||||
case "UTF-32":
|
||||
case "UTF-32BE":
|
||||
case "UTF-32LE":
|
||||
codePoint = unicodeCodePoint;
|
||||
break;
|
||||
default: {
|
||||
codePoint = -1;
|
||||
try {
|
||||
//assuming single byte ANSI
|
||||
codePoint = ("" + character).getBytes(charsetName)[0] & 0xff;
|
||||
} catch (UnsupportedEncodingException ex) {
|
||||
Logger.getLogger(Utf8Helper.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return codePoint;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
package com.jpexs.helpers.utf8.charset;
|
||||
|
||||
import com.jpexs.decompiler.flash.action.parser.ActionParseException;
|
||||
import com.jpexs.decompiler.flash.action.parser.script.ActionScriptLexer;
|
||||
import com.jpexs.decompiler.flash.action.parser.script.ParsedSymbol;
|
||||
import com.jpexs.decompiler.flash.action.parser.script.SymbolType;
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author JPEXS
|
||||
*/
|
||||
public abstract class AbstractCharsetConverter {
|
||||
|
||||
protected static void readMap(Map<Integer, Integer> data, ActionScriptLexer lexer) throws IOException, ActionParseException {
|
||||
ParsedSymbol s;
|
||||
lexer.lex(); //identifier;
|
||||
lexer.lex(); //=
|
||||
lexer.lex(); // {
|
||||
int pos1 = 0;
|
||||
do {
|
||||
s = lexer.lex(); //{
|
||||
if (s.type == SymbolType.CURLY_CLOSE) {
|
||||
break;
|
||||
}
|
||||
s = lexer.lex();
|
||||
int key = (int) (long) (Long) s.value;
|
||||
lexer.lex(); //,
|
||||
s = lexer.lex();
|
||||
int value = (int) (long) (Long) s.value;
|
||||
data.put(key, value);
|
||||
s = lexer.lex(); //}
|
||||
s = lexer.lex();
|
||||
pos1++;
|
||||
} while ((s.type == SymbolType.COMMA));
|
||||
lexer.lex(); //;
|
||||
}
|
||||
|
||||
protected static void readOneDimensionalInt(int data[], ActionScriptLexer lexer) throws IOException, ActionParseException {
|
||||
ParsedSymbol s;
|
||||
lexer.lex(); //identifier
|
||||
lexer.lex(); //=
|
||||
lexer.lex(); // {
|
||||
int pos = 0;
|
||||
do {
|
||||
s = lexer.lex();
|
||||
if (s.type == SymbolType.CURLY_CLOSE) {
|
||||
break;
|
||||
}
|
||||
boolean negative = false;
|
||||
if (s.type == SymbolType.MINUS) {
|
||||
negative = true;
|
||||
s = lexer.lex();
|
||||
}
|
||||
data[pos] = (int) (long) (Long) s.value;
|
||||
if (negative) {
|
||||
data[pos] = -data[pos];
|
||||
}
|
||||
s = lexer.lex();
|
||||
pos++;
|
||||
} while (s.type == SymbolType.COMMA);
|
||||
lexer.lex(); //;
|
||||
}
|
||||
|
||||
protected static void readTwoDimensionalInt(int data[][], ActionScriptLexer lexer) throws IOException, ActionParseException {
|
||||
ParsedSymbol s;
|
||||
lexer.lex(); //identifier;
|
||||
lexer.lex(); //=
|
||||
lexer.lex(); // {
|
||||
int pos1 = 0;
|
||||
do {
|
||||
s = lexer.lex(); //{
|
||||
int pos2 = 0;
|
||||
do {
|
||||
s = lexer.lex();
|
||||
if (s.type == SymbolType.CURLY_CLOSE) {
|
||||
break;
|
||||
}
|
||||
boolean negative = false;
|
||||
if (s.type == SymbolType.MINUS) {
|
||||
negative = true;
|
||||
s = lexer.lex();
|
||||
}
|
||||
data[pos1][pos2] = (int) (long) (Long) s.value;
|
||||
if (negative) {
|
||||
data[pos1][pos2] = -data[pos1][pos2];
|
||||
}
|
||||
s = lexer.lex();
|
||||
pos2++;
|
||||
} while (s.type == SymbolType.COMMA);
|
||||
s = lexer.lex();
|
||||
pos1++;
|
||||
} while ((s.type == SymbolType.COMMA));
|
||||
//lexer.lex(); // }
|
||||
lexer.lex(); //;
|
||||
}
|
||||
|
||||
public abstract int toUnicode(int codePoint);
|
||||
|
||||
public abstract int fromUnicode(int codePoint);
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package com.jpexs.helpers.utf8.charset;
|
||||
|
||||
import com.jpexs.decompiler.flash.action.parser.ActionParseException;
|
||||
import com.jpexs.decompiler.flash.action.parser.script.ActionScriptLexer;
|
||||
import com.jpexs.decompiler.flash.action.parser.script.ParsedSymbol;
|
||||
import static com.jpexs.helpers.utf8.charset.AbstractCharsetConverter.readOneDimensionalInt;
|
||||
import static com.jpexs.helpers.utf8.charset.AbstractCharsetConverter.readTwoDimensionalInt;
|
||||
import com.jpexs.helpers.utf8.Utf8Helper;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author JPEXS
|
||||
*/
|
||||
public class Gb18030 extends AbstractCharsetConverter {
|
||||
|
||||
private static int[][] gb18030_index_to_cp_len2_record = new int[126][191];
|
||||
|
||||
private static final int GB18030_NULL = 0;
|
||||
|
||||
private static int[] gb18030_len4_record_shift = new int[]{0, -1546, -2806, -4066, -5326, -6586,
|
||||
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, -41987, -43247, -44507, -45767, -47027, -48287, -49547, -50807,
|
||||
-52067, -53327, 11, -59963, -61223, -62483, 12, 13};
|
||||
|
||||
private static int[][] gb18030_index_to_cp_len4_record = new int[14][1260];
|
||||
|
||||
private static int[] gb18030_cp_to_index_record = new int[61339];
|
||||
|
||||
static {
|
||||
//Since data is too long to save it directly into Java source, load it from bin
|
||||
InputStream is = Gb18030.class.getResourceAsStream("/com/jpexs/helpers/utf8/charset/Gb18030data.bin");
|
||||
if (is == null) {
|
||||
System.exit(0);
|
||||
}
|
||||
ActionScriptLexer lexer = new ActionScriptLexer(new InputStreamReader(is, Utf8Helper.charset));
|
||||
try {
|
||||
ParsedSymbol s;
|
||||
readTwoDimensionalInt(gb18030_index_to_cp_len2_record, lexer);
|
||||
readTwoDimensionalInt(gb18030_index_to_cp_len4_record, lexer);
|
||||
readOneDimensionalInt(gb18030_cp_to_index_record, lexer);
|
||||
} catch (IOException | ActionParseException ex) {
|
||||
Logger.getLogger(Gb2312.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
|
||||
public static int gb18030_index_to_cp_len2(int byte1, int byte2) {
|
||||
if (0x81 <= byte1 && byte1 <= 0xfe && 0x40 <= byte2 && byte2 <= 0xfe) {
|
||||
return gb18030_index_to_cp_len2_record[byte1 - 0x81][byte2 - 0x40];
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
public static int gb18030_index_to_cp_len4(int byte1, int byte2, int byte3, int byte4) {
|
||||
int pos_1, pos_2;
|
||||
byte1 -= 0x81;
|
||||
byte2 -= 0x30;
|
||||
byte3 -= 0x81;
|
||||
byte4 -= 0x30;
|
||||
pos_1 = byte1 * 10 + byte2;
|
||||
pos_2 = byte3 * 10 + byte4;
|
||||
if (pos_1 <= 31 && pos_2 <= 1259) {
|
||||
if (gb18030_len4_record_shift[pos_1] < 0) {
|
||||
return pos_2 - gb18030_len4_record_shift[pos_1];
|
||||
} else {
|
||||
return gb18030_index_to_cp_len4_record[gb18030_len4_record_shift[pos_1]][pos_2];
|
||||
}
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param codepoint
|
||||
* @return
|
||||
*/
|
||||
@Override
|
||||
public int toUnicode(int codepoint) {
|
||||
int result = 0;
|
||||
if (0 <= codepoint && codepoint <= 55295) {
|
||||
result = (int) gb18030_cp_to_index_record[codepoint];
|
||||
} else if (59493 <= codepoint && codepoint <= 65535) {
|
||||
result = (int) gb18030_cp_to_index_record[codepoint - 4197];
|
||||
}
|
||||
return result == 0 ? GB18030_NULL : result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int fromUnicode(int codePoint) {
|
||||
throw new UnsupportedOperationException("Not supported yet.");
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,88 @@
|
||||
package com.jpexs.helpers.utf8.charset;
|
||||
|
||||
import com.jpexs.decompiler.flash.action.parser.ActionParseException;
|
||||
import com.jpexs.decompiler.flash.action.parser.script.ActionScriptLexer;
|
||||
import com.jpexs.decompiler.flash.action.parser.script.ParsedSymbol;
|
||||
import com.jpexs.decompiler.flash.action.parser.script.SymbolType;
|
||||
import com.jpexs.helpers.utf8.Utf8Helper;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
/**
|
||||
*
|
||||
* GB2312 to unicode and back conversion.
|
||||
* Based on https://github.com/MoarVM/MoarVM/blob/master/src/strings/gb2312_codeindex.h
|
||||
*/
|
||||
public class Gb2312 extends AbstractCharsetConverter {
|
||||
static final int GB2312_NULL = -1;
|
||||
|
||||
/* Conversion tables are generated according to mapping from
|
||||
* unicode.org-mappings/EASTASIA/GB/GB2312.TXT
|
||||
* at https://haible.de/bruno/charsets/conversion-tables/GB2312.html
|
||||
* The following tables use EUC form for GB2312 characters.
|
||||
|
||||
* Unicode indexes 1106 - 8212, 9795 - 12287, 12842 - 19967,
|
||||
* and 40865 - 65280 don't correspond to gb2312 codepoint.
|
||||
* To reduce code length and save memory, these intervals are omitted
|
||||
* in the conversion table and indexes are shifted in the function. */
|
||||
|
||||
private static int[][] gb2312_index_to_cp_record = new int[87][94];
|
||||
|
||||
private static int[] gb2312_cp_to_index_record = new int [24380];
|
||||
|
||||
static {
|
||||
//Since data is too long to save it directly into Java source, load it from bin
|
||||
InputStream is = Gb2312.class.getResourceAsStream("/com/jpexs/helpers/utf8/charset/Gb2312data.bin");
|
||||
if (is == null) {
|
||||
System.exit(0);
|
||||
}
|
||||
ActionScriptLexer lexer = new ActionScriptLexer(new InputStreamReader(is, Utf8Helper.charset));
|
||||
try {
|
||||
ParsedSymbol s;
|
||||
readTwoDimensionalInt(gb2312_index_to_cp_record, lexer);
|
||||
readOneDimensionalInt(gb2312_cp_to_index_record, lexer);
|
||||
} catch (IOException | ActionParseException ex) {
|
||||
Logger.getLogger(Gb2312.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int toUnicode(int codePoint) {
|
||||
if (codePoint < 128) {
|
||||
return codePoint;
|
||||
}
|
||||
|
||||
int zone = codePoint / 256 - 161;
|
||||
int point = codePoint % 256 - 161;
|
||||
if (0 <= zone && zone < 87 && 0 <= point && point < 94) {
|
||||
return gb2312_index_to_cp_record[zone][point];
|
||||
} else {
|
||||
return GB2312_NULL;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int fromUnicode(int codePoint) {
|
||||
|
||||
int result = 0;
|
||||
if (0 <= codePoint && codePoint <= 1105) {
|
||||
result = gb2312_cp_to_index_record[codePoint];
|
||||
}
|
||||
else if (8213 <= codePoint && codePoint <= 9794) {
|
||||
result = gb2312_cp_to_index_record[codePoint - 7107];
|
||||
}
|
||||
else if (12288 <= codePoint && codePoint <= 12841) {
|
||||
result = gb2312_cp_to_index_record[codePoint - 9600];
|
||||
}
|
||||
else if (19968 <= codePoint && codePoint <= 40864) {
|
||||
result = gb2312_cp_to_index_record[codePoint - 16726];
|
||||
}
|
||||
else if (65281 <= codePoint && codePoint <= 65510) {
|
||||
result = gb2312_cp_to_index_record[codePoint - 41142];
|
||||
}
|
||||
return result == 0 ? GB2312_NULL : result;
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,92 @@
|
||||
package com.jpexs.helpers.utf8.charset;
|
||||
|
||||
import com.jpexs.decompiler.flash.action.parser.ActionParseException;
|
||||
import com.jpexs.decompiler.flash.action.parser.script.ActionScriptLexer;
|
||||
import com.jpexs.decompiler.flash.action.parser.script.ParsedSymbol;
|
||||
import com.jpexs.decompiler.flash.action.parser.script.SymbolType;
|
||||
import com.jpexs.helpers.utf8.Utf8Helper;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
/**
|
||||
* ShiftJis to unicode and back conversion.
|
||||
* Based on https://github.com/MoarVM/MoarVM/blob/master/src/strings/shiftjis_codeindex.h
|
||||
*/
|
||||
public class ShiftJis extends AbstractCharsetConverter {
|
||||
|
||||
public static final int[][] shiftjis_offset_values = {
|
||||
{107, 11},
|
||||
{126, 8},
|
||||
{141, 11},
|
||||
{167, 7},
|
||||
{182, 4},
|
||||
{187, 15},
|
||||
{212, 7},
|
||||
{245, 6},
|
||||
{277, 4},
|
||||
{364, 11},
|
||||
{461, 8},
|
||||
{493, 8},
|
||||
{525, 38},
|
||||
{596, 15},
|
||||
{644, 13},
|
||||
{689, 438},
|
||||
{1157, 1},
|
||||
{1181, 8},
|
||||
{1219, 190},
|
||||
{4374, 43},
|
||||
{7807, 2908}
|
||||
};
|
||||
|
||||
public static final int SHIFTJIS_OFFSET_VALUES_ELEMS = 21;
|
||||
public static final int SHIFTJIS_INDEX_TO_CP_CODEPOINTS_ELEMS = 7350;
|
||||
public static final int SHIFTJIS_MAX_INDEX = 11103;
|
||||
|
||||
private static final int SHIFTJIS_NULL = -1;
|
||||
|
||||
private static int[] shiftjis_index_to_cp_codepoints = new int[7350];
|
||||
private static Map<Integer, Integer> shiftjis_cp_to_index = new HashMap<>();
|
||||
|
||||
static {
|
||||
//Since data is too long to save it directly into Java source, load it from bin
|
||||
|
||||
InputStream is = Gb2312.class.getResourceAsStream("/com/jpexs/helpers/utf8/charset/ShiftJisdata.bin");
|
||||
if (is == null) {
|
||||
System.exit(0);
|
||||
}
|
||||
ActionScriptLexer lexer = new ActionScriptLexer(new InputStreamReader(is, Utf8Helper.charset));
|
||||
try {
|
||||
ParsedSymbol s;
|
||||
readOneDimensionalInt(shiftjis_index_to_cp_codepoints, lexer);
|
||||
readMap(shiftjis_cp_to_index, lexer);
|
||||
|
||||
} catch (IOException | ActionParseException ex) {
|
||||
Logger.getLogger(ShiftJis.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int toUnicode(int codePoint) {
|
||||
if (codePoint < 128) {
|
||||
return codePoint;
|
||||
}
|
||||
|
||||
if (shiftjis_cp_to_index.containsKey(codePoint)) {
|
||||
return shiftjis_cp_to_index.get(codePoint);
|
||||
}
|
||||
return SHIFTJIS_NULL;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int fromUnicode(int codePoint) {
|
||||
if (codePoint < shiftjis_index_to_cp_codepoints.length) {
|
||||
return shiftjis_index_to_cp_codepoints[codePoint];
|
||||
}
|
||||
return SHIFTJIS_NULL;
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user