fix: a bunch of iggy string encoding UB and use simdutf

.gitignore

@@ -13,6 +13,7 @@
 !/subprojects/
 /subprojects/*
 !/subprojects/*.wrap
+!/subprojects/packagefiles
 /scripts/__pycache__/
 !/.clang-format
 !/.git-blame-ignore-revs

Minecraft.Assets/Common/Media/en-EN.lang

@@ -478,7 +478,7 @@ If "Host Privileges" is enabled the host player can modify some privileges for t
     <value>HUD</value>
   </data>
       <data name="IDS_HOW_TO_PLAY_MENU_INVENTORY">
-    <value>Inventory</value>
+    <value>hi</value>
   </data>
       <data name="IDS_HOW_TO_PLAY_MENU_CHESTS">
     <value>Chests</value>
@@ -2596,7 +2596,7 @@ Can also be used for low-level lighting.</value>
     <value>Jump/Fly Up</value>
   </data>
   <data name="IDS_CONTROLS_INVENTORY">
-    <value>Inventory</value>
+    <value>hi</value>
   </data>
   <data name="IDS_CONTROLS_HELDITEM">
     <value>Cycle Held Item</value>
@@ -4416,7 +4416,7 @@ Would you like to unlock the full game?</value>
   </data>
 
     <data name="IDS_INVENTORY">
-    <value>Inventory</value>
+    <value>hi</value>
   </data>
     <data name="IDS_INGREDIENTS">
     <value>Ingredients</value>

Minecraft.Client/Platform/Common/UI/UIComponent_Tooltips.cpp

@@ -307,7 +307,7 @@ void UIComponent_Tooltips::_SetTooltip(unsigned int iToolTipId, UIString label,
     value[0].type = IGGY_DATATYPE_number;
     value[0].number = iToolTipId;
 
-    const std::u16string convLabel = convWstringToU16string(label.getString());
+    const std::u16string convLabel = wstring_to_u16string(label.getString());
 
     value[1].type = IGGY_DATATYPE_string_UTF16;
     IggyStringUTF16 stringVal;

Minecraft.Client/Platform/Common/UI/UIControl_Base.cpp

@@ -29,7 +29,7 @@ void UIControl_Base::tick() {
         // app.DebugPrintf("Calling SetLabel - '%ls'\n", m_label.c_str());
         m_bLabelChanged = false;
 
-        const std::u16string convLabel = convWstringToU16string(m_label.getString());
+        const std::u16string convLabel = wstring_to_u16string(m_label.getString());
 
         IggyDataValue result;
         IggyDataValue value[1];
@@ -57,7 +57,7 @@ void UIControl_Base::setLabel(UIString label, bool instant, bool force) {
     if (m_bLabelChanged && instant) {
         m_bLabelChanged = false;
 
-        const std::u16string convLabel = convWstringToU16string(m_label.getString());
+        const std::u16string convLabel = wstring_to_u16string(m_label.getString());
 
         IggyDataValue result;
         IggyDataValue value[1];
@@ -81,8 +81,7 @@ const wchar_t* UIControl_Base::getLabel() {
                                getIggyValuePath(), m_funcGetLabel, 0, NULL);
 
     if (result.type == IGGY_DATATYPE_string_UTF16) {
-        m_label = std::wstring((wchar_t*)result.string16.string,
+        m_label = u16string_to_wstring(result.string16.string);
-                               result.string16.length);
     }
 
     return m_label.c_str();
@@ -98,7 +97,7 @@ void UIControl_Base::setAllPossibleLabels(int labelCount,
     conv.reserve(labelCount);
 
     for (int i = 0; i < labelCount; ++i) {
-        conv.push_back(convWstringToU16string(labels[i]));
+        conv.push_back(wstring_to_u16string(labels[i]));
         stringVal[i].string = conv[i].c_str();
         stringVal[i].length = (S32)conv[i].length();
         value[i].type = IGGY_DATATYPE_string_UTF16;

Minecraft.Client/Platform/Common/UI/UIControl_BitmapIcon.cpp

@@ -17,7 +17,7 @@ void UIControl_BitmapIcon::setTextureName(const std::wstring& iconName) {
     IggyDataValue result;
     IggyDataValue value[1];
 
-    const std::u16string convName = convWstringToU16string(iconName);
+    const std::u16string convName = wstring_to_u16string(iconName);
 
     IggyStringUTF16 stringVal;
     stringVal.string = convName.c_str();

Minecraft.Client/Platform/Common/UI/UIControl_Button.cpp

@@ -19,7 +19,7 @@ void UIControl_Button::init(UIString label, int id) {
     m_label = label;
     m_id = id;
 
-    const std::u16string convLabel = convWstringToU16string(label.getString());
+    const std::u16string convLabel = wstring_to_u16string(label.getString());
 
     IggyDataValue result;
     IggyDataValue value[2];

Minecraft.Client/Platform/Common/UI/UIControl_ButtonList.cpp

@@ -95,7 +95,7 @@ void UIControl_ButtonList::addItem(const std::wstring& label, int data) {
     IggyDataValue result;
     IggyDataValue value[2];
 
-    const std::u16string convLabel = convWstringToU16string(label);
+    const std::u16string convLabel = wstring_to_u16string(label);
 
     IggyStringUTF16 stringVal;
     stringVal.string = convLabel.c_str();
@@ -150,7 +150,7 @@ void UIControl_ButtonList::setButtonLabel(int iButtonId,
     value[0].type = IGGY_DATATYPE_number;
     value[0].number = iButtonId;
 
-    const std::u16string convLabel = convWstringToU16string(label);
+    const std::u16string convLabel = wstring_to_u16string(label);
 
     IggyStringUTF16 stringVal;
     stringVal.string = convLabel.c_str();

Minecraft.Client/Platform/Common/UI/UIControl_CheckBox.cpp

@@ -24,7 +24,7 @@ void UIControl_CheckBox::init(UIString label, int id, bool checked) {
     m_id = id;
     m_bChecked = checked;
 
-    const std::u16string convLabel = convWstringToU16string(label.getString());
+    const std::u16string convLabel = wstring_to_u16string(label.getString());
 
     IggyDataValue result;
     IggyDataValue value[3];

Minecraft.Client/Platform/Common/UI/UIControl_DLCList.cpp

@@ -42,7 +42,7 @@ void UIControl_DLCList::addItem(const std::wstring& label, bool showTick,
     IggyDataValue result;
     IggyDataValue value[3];
 
-    const std::u16string convLabel = convWstringToU16string(label);
+    const std::u16string convLabel = wstring_to_u16string(label);
 
     IggyStringUTF16 stringVal;
     stringVal.string = convLabel.c_str();

Minecraft.Client/Platform/Common/UI/UIControl_DynamicLabel.cpp

@@ -20,7 +20,7 @@ bool UIControl_DynamicLabel::setupControl(UIScene* scene, IggyValuePath* parent,
 
 void UIControl_DynamicLabel::addText(const std::wstring& text,
                                      bool bLastEntry) {
-    const std::u16string convText = convWstringToU16string(text);
+    const std::u16string convText = wstring_to_u16string(text);
 
     IggyDataValue result;
     IggyDataValue value[2];

Minecraft.Client/Platform/Common/UI/UIControl_Label.cpp

@@ -18,13 +18,16 @@ bool UIControl_Label::setupControl(UIScene* scene, IggyValuePath* parent,
 void UIControl_Label::init(UIString label) {
     m_label = label;
 
+    const std::u16string convLabel =
+        wstring_to_u16string(label.getString());
+
     IggyDataValue result;
     IggyDataValue value[1];
     value[0].type = IGGY_DATATYPE_string_UTF16;
     IggyStringUTF16 stringVal;
 
-    stringVal.string = (IggyUTF16*)label.c_str();
+    stringVal.string = convLabel.c_str();
-    stringVal.length = label.length();
+    stringVal.length = convLabel.length();
     value[0].string16 = stringVal;
     IggyResult out =
         IggyPlayerCallMethodRS(m_parentScene->getMovie(), &result,

Minecraft.Client/Platform/Common/UI/UIControl_LeaderboardList.cpp

@@ -53,7 +53,7 @@ void UIControl_LeaderboardList::setupTitles(const std::wstring& rank,
     IggyDataValue result;
     IggyDataValue value[2];
 
-    const std::u16string convRank = convWstringToU16string(rank);
+    const std::u16string convRank = wstring_to_u16string(rank);
 
     IggyStringUTF16 stringVal0;
     stringVal0.string = convRank.c_str();
@@ -61,7 +61,7 @@ void UIControl_LeaderboardList::setupTitles(const std::wstring& rank,
     value[0].type = IGGY_DATATYPE_string_UTF16;
     value[0].string16 = stringVal0;
 
-    const std::u16string convGamertag = convWstringToU16string(gamertag);
+    const std::u16string convGamertag = wstring_to_u16string(gamertag);
 
     IggyStringUTF16 stringVal1;
     stringVal1.string = convGamertag.c_str();
@@ -137,7 +137,7 @@ void UIControl_LeaderboardList::addDataSet(
     value[2].type = IGGY_DATATYPE_number;
     value[2].number = iRank;
 
-    const std::u16string convGamertag = convWstringToU16string(gamertag);
+    const std::u16string convGamertag = wstring_to_u16string(gamertag);
 
     IggyStringUTF16 stringVal0;
     stringVal0.string = convGamertag.c_str();
@@ -148,7 +148,7 @@ void UIControl_LeaderboardList::addDataSet(
     value[4].type = IGGY_DATATYPE_boolean;
     value[4].boolval = bDisplayMessage;
 
-    const std::u16string convCol0 = convWstringToU16string(col0);
+    const std::u16string convCol0 = wstring_to_u16string(col0);
 
     IggyStringUTF16 stringVal1;
     stringVal1.string = convCol0.c_str();
@@ -159,7 +159,7 @@ void UIControl_LeaderboardList::addDataSet(
     if (col1.empty()) {
         value[6].type = IGGY_DATATYPE_null;
     } else {
-        const std::u16string convCol1 = convWstringToU16string(col1);
+        const std::u16string convCol1 = wstring_to_u16string(col1);
 
         IggyStringUTF16 stringVal2;
         stringVal2.string = convCol1.c_str();
@@ -171,7 +171,7 @@ void UIControl_LeaderboardList::addDataSet(
     if (col2.empty()) {
         value[7].type = IGGY_DATATYPE_null;
     } else {
-        const std::u16string convCol2 = convWstringToU16string(col2);
+        const std::u16string convCol2 = wstring_to_u16string(col2);
 
         IggyStringUTF16 stringVal3;
         stringVal3.string = convCol2.c_str();
@@ -183,7 +183,7 @@ void UIControl_LeaderboardList::addDataSet(
     if (col3.empty()) {
         value[8].type = IGGY_DATATYPE_null;
     } else {
-        const std::u16string convCol3 = convWstringToU16string(col3);
+        const std::u16string convCol3 = wstring_to_u16string(col3);
 
         IggyStringUTF16 stringVal4;
         stringVal4.string = convCol3.c_str();
@@ -195,7 +195,7 @@ void UIControl_LeaderboardList::addDataSet(
     if (col4.empty()) {
         value[9].type = IGGY_DATATYPE_null;
     } else {
-        const std::u16string convCol4 = convWstringToU16string(col4);
+        const std::u16string convCol4 = wstring_to_u16string(col4);
 
         IggyStringUTF16 stringVal5;
         stringVal5.string = convCol4.c_str();
@@ -207,7 +207,7 @@ void UIControl_LeaderboardList::addDataSet(
     if (col5.empty()) {
         value[10].type = IGGY_DATATYPE_null;
     } else {
-        const std::u16string convCol5 = convWstringToU16string(col5);
+        const std::u16string convCol5 = wstring_to_u16string(col5);
 
         IggyStringUTF16 stringVal6;
         stringVal6.string = convCol5.c_str();
@@ -219,7 +219,7 @@ void UIControl_LeaderboardList::addDataSet(
     if (col6.empty()) {
         value[11].type = IGGY_DATATYPE_null;
     } else {
-        const std::u16string convCol6 = convWstringToU16string(col6);
+        const std::u16string convCol6 = wstring_to_u16string(col6);
 
         IggyStringUTF16 stringVal7;
         stringVal7.string = convCol6.c_str();

Minecraft.Client/Platform/Common/UI/UIControl_PlayerList.cpp

@@ -20,7 +20,7 @@ void UIControl_PlayerList::addItem(const std::wstring& label, int iPlayerIcon,
     IggyDataValue result;
     IggyDataValue value[4];
 
-    const std::u16string convLabel = convWstringToU16string(label);
+    const std::u16string convLabel = wstring_to_u16string(label);
 
     IggyStringUTF16 stringVal;
     stringVal.string = convLabel.c_str();

Minecraft.Client/Platform/Common/UI/UIControl_Progress.cpp

@@ -30,7 +30,7 @@ void UIControl_Progress::init(UIString label, int id, int min, int max,
     m_max = max;
     m_current = current;
 
-    const std::u16string convLabel = convWstringToU16string(label.getString());
+    const std::u16string convLabel = wstring_to_u16string(label.getString());
 
     IggyDataValue result;
     IggyDataValue value[1];

Minecraft.Client/Platform/Common/UI/UIControl_SaveList.cpp

@@ -56,7 +56,7 @@ void UIControl_SaveList::addItem(const std::string& label,
     value[1].type = IGGY_DATATYPE_number;
     value[1].number = m_itemCount;
 
-    const std::u16string convName = convWstringToU16string(iconName);
+    const std::u16string convName = wstring_to_u16string(iconName);
 
     IggyStringUTF16 stringVal2;
     stringVal2.string = convName.c_str();
@@ -73,7 +73,7 @@ void UIControl_SaveList::addItem(const std::wstring& label,
     IggyDataValue result;
     IggyDataValue value[3];
 
-    const std::u16string convLabel = convWstringToU16string(label);
+    const std::u16string convLabel = wstring_to_u16string(label);
 
     IggyStringUTF16 stringVal;
     stringVal.string = convLabel.c_str();
@@ -84,7 +84,7 @@ void UIControl_SaveList::addItem(const std::wstring& label,
     value[1].type = IGGY_DATATYPE_number;
     value[1].number = m_itemCount;
 
-    const std::u16string convName = convWstringToU16string(iconName);
+    const std::u16string convName = wstring_to_u16string(iconName);
 
     IggyStringUTF16 stringVal2;
     stringVal2.string = convName.c_str();
@@ -103,7 +103,7 @@ void UIControl_SaveList::setTextureName(int iId, const std::wstring& iconName) {
     value[0].type = IGGY_DATATYPE_number;
     value[0].number = iId;
 
-    const std::u16string convName = convWstringToU16string(iconName);
+    const std::u16string convName = wstring_to_u16string(iconName);
 
     IggyStringUTF16 stringVal;
     stringVal.string = convName.c_str();

Minecraft.Client/Platform/Common/UI/UIControl_Slider.cpp

@@ -29,7 +29,7 @@ void UIControl_Slider::init(UIString label, int id, int min, int max,
     m_max = max;
     m_current = current;
 
-    const std::u16string convLabel = convWstringToU16string(label.getString());
+    const std::u16string convLabel = wstring_to_u16string(label.getString());
 
     IggyDataValue result;
     IggyDataValue value[5];

Minecraft.Client/Platform/Common/UI/UIControl_SpaceIndicatorBar.cpp

@@ -31,13 +31,15 @@ void UIControl_SpaceIndicatorBar::init(UIString label, int id, int64_t min,
     m_min = min;
     m_max = max;
 
+    const std::u16string convLabel = wstring_to_u16string(label.getString());
+    
     IggyDataValue result;
     IggyDataValue value[1];
     value[0].type = IGGY_DATATYPE_string_UTF16;
     IggyStringUTF16 stringVal;
 
-    stringVal.string = (IggyUTF16*)label.c_str();
+    stringVal.string = convLabel.c_str();
-    stringVal.length = label.length();
+    stringVal.length = convLabel.length();
     value[0].string16 = stringVal;
 
     IggyResult out =

Minecraft.Client/Platform/Common/UI/UIControl_TextInput.cpp

@@ -22,7 +22,7 @@ void UIControl_TextInput::init(UIString label, int id) {
     m_label = label;
     m_id = id;
 
-    const std::u16string convLabel = convWstringToU16string(label.getString());
+    const std::u16string convLabel = wstring_to_u16string(label.getString());
 
     IggyDataValue result;
     IggyDataValue value[2];

Minecraft.Client/Platform/Common/UI/UIControl_TexturePackList.cpp

@@ -26,7 +26,7 @@ void UIControl_TexturePackList::init(const std::wstring& label, int id) {
     m_label = label;
     m_id = id;
 
-    const std::u16string convLabel = convWstringToU16string(label);
+    const std::u16string convLabel = wstring_to_u16string(label);
 
     IggyDataValue result;
     IggyDataValue value[2];
@@ -58,7 +58,7 @@ void UIControl_TexturePackList::init(const std::wstring& label, int id) {
 
 void UIControl_TexturePackList::addPack(int id,
                                         const std::wstring& textureName) {
-    const std::u16string convName = convWstringToU16string(textureName);
+    const std::u16string convName = wstring_to_u16string(textureName);
 
     IggyDataValue result;
     IggyDataValue value[2];

Minecraft.Client/Platform/Common/UI/UIController.cpp

@@ -613,7 +613,7 @@ IggyLibrary UIController::loadSkin(const std::wstring& skinPath,
     // normal skin requires some elements from the platform skin
     if (!skinPath.empty() && app.hasArchiveFile(skinPath)) {
         byteArray baFile = app.getArchiveFile(skinPath);
-        const std::u16string convSkinName = convWstringToU16string(skinName);
+        const std::u16string convSkinName = wstring_to_u16string(skinName);
 
         lib = IggyLibraryCreateFromMemoryUTF16(
             convSkinName.data(), (void*)baFile.data, baFile.length, NULL);

Minecraft.Client/Platform/Common/UI/UIScene.cpp

@@ -472,7 +472,7 @@ IggyName UIScene::registerFastName(const std::wstring& name) {
     if (it != m_fastNames.end()) {
         var = it->second;
     } else {
-        const std::u16string convName = convWstringToU16string(name);
+        const std::u16string convName = wstring_to_u16string(name);
         var = IggyPlayerCreateFastName(getMovie(), (IggyUTF16*)convName.c_str(),
                                        -1);
 

Minecraft.Client/Platform/Common/UI/UIScene_AbstractContainerMenu.cpp

@@ -230,7 +230,7 @@ void UIScene_AbstractContainerMenu::customDraw(
         pMinecraft->localgameModes[m_iPad] == NULL)
         return;
 
-        int slotId = parseSlotId(region->name);
+    int slotId = parseSlotId(region->name);
     std::shared_ptr<ItemInstance> item = nullptr;
     if (std::char_traits<char16_t>::compare(region->name, u"pointerIcon", 11) ==
         0) {

Minecraft.Client/Platform/Common/UI/UIScene_AnvilMenu.cpp

@@ -348,7 +348,7 @@ void UIScene_AnvilMenu::setCostLabel(const std::wstring& label,
     IggyDataValue result;
     IggyDataValue value[2];
 
-    const std::u16string convLabel = convWstringToU16string(label);
+    const std::u16string convLabel = wstring_to_u16string(label);
 
     IggyStringUTF16 stringVal;
     stringVal.string = convLabel.c_str();

Minecraft.Client/Platform/Common/UI/UIScene_Credits.cpp

@@ -836,7 +836,7 @@ void UIScene_Credits::setNextLabel(const std::wstring& label,
     IggyDataValue result;
     IggyDataValue value[3];
 
-    const std::u16string convLabel = convWstringToU16string(label);
+    const std::u16string convLabel = wstring_to_u16string(label);
 
     IggyStringUTF16 stringVal;
     stringVal.string = convLabel.c_str();

Minecraft.Client/Platform/Common/UI/UIScene_EndPoem.cpp

@@ -122,7 +122,7 @@ void UIScene_EndPoem::tick() {
         IggyDataValue result;
         IggyDataValue value[3];
 
-        const std::u16string convLabel = convWstringToU16string(label);
+        const std::u16string convLabel = wstring_to_u16string(label);
 
         IggyStringUTF16 stringVal;
         stringVal.string = convLabel.c_str();

Minecraft.Client/Platform/Common/UI/UIScene_HUD.cpp

@@ -529,7 +529,7 @@ void UIScene_HUD::SetDragonHealth(float health) {
 void UIScene_HUD::SetDragonLabel(const std::wstring& label) {
     IggyDataValue result;
     IggyDataValue value[1];
-    const std::u16string convLabel = convWstringToU16string(label);
+    const std::u16string convLabel = wstring_to_u16string(label);
     IggyStringUTF16 stringVal;
     stringVal.string = convLabel.c_str();
     stringVal.length = convLabel.length();
@@ -565,7 +565,7 @@ void UIScene_HUD::SetSelectedLabel(const std::wstring& label) {
 
     IggyDataValue result;
     IggyDataValue value[1];
-    const std::u16string convLabel = convWstringToU16string(label);
+    const std::u16string convLabel = wstring_to_u16string(label);
     IggyStringUTF16 stringVal;
     stringVal.string = convLabel.c_str();
     stringVal.length = convLabel.length();
@@ -805,7 +805,7 @@ void UIScene_HUD::SetDisplayName(const std::wstring& displayName) {
         IggyDataValue result;
         IggyDataValue value[1];
         IggyStringUTF16 stringVal;
-        const std::u16string convName = convWstringToU16string(displayName);
+        const std::u16string convName = wstring_to_u16string(displayName);
         stringVal.string = convName.c_str();
         stringVal.length = convName.length();
         value[0].type = IGGY_DATATYPE_string_UTF16;

Minecraft.Client/Platform/Common/UI/UIScene_HorseInventoryMenu.cpp

@@ -285,7 +285,7 @@ void UIScene_HorseInventoryMenu::customDraw(
         pMinecraft->localgameModes[m_iPad] == NULL)
         return;
 
-    if (wcscmp((wchar_t*)region->name, L"horse") == 0) {
+    if (std::char_traits<char16_t>::compare(region->name, u"horse", 5) == 0) {
         // Setup GDraw, normal game render states and matrices
         CustomDrawData* customDrawRegion = ui.setupCustomDraw(this, region);
         delete customDrawRegion;

Minecraft.Client/Platform/Common/UI/UIScene_HowToPlay.cpp

@@ -342,8 +342,10 @@ void UIScene_HowToPlay::StartPage(EHowToPlayPage ePage) {
     value[0].number = gs_pageToFlashMapping[(int)ePage];
 
     for (unsigned int i = 0; i < paragraphs.size(); ++i) {
-        stringVal[i].string = (IggyUTF16*)paragraphs[i].c_str();
+        const std::u16string convParagraph = wstring_to_u16string(paragraphs[i]);
-        stringVal[i].length = paragraphs[i].length();
+
+        stringVal[i].string = convParagraph.c_str();
+        stringVal[i].length = convParagraph.length();
         value[i + 1].type = IGGY_DATATYPE_string_UTF16;
         value[i + 1].string16 = stringVal[i];
     }

Minecraft.Client/Platform/Common/UI/UIScene_InventoryMenu.cpp

@@ -300,7 +300,7 @@ void UIScene_InventoryMenu::updateEffectsDisplay() {
             value[0].number = icon;
 
             const std::u16string convString =
-                convWstringToU16string(effectString);
+                wstring_to_u16string(effectString);
             IggyStringUTF16 stringVal;
             stringVal.string = convString.c_str();
             stringVal.length = convString.length();

Minecraft.Client/Platform/Common/UI/UIScene_Keyboard.cpp

@@ -29,7 +29,7 @@ UIScene_Keyboard::UIScene_Keyboard(int iPad, void* initData,
     // string
     std::wstring label = L"Abc";
     IggyStringUTF16 stringVal;
-    const std::u16string convLabel = convWstringToU16string(label);
+    const std::u16string convLabel = wstring_to_u16string(label);
     stringVal.string = convLabel.c_str();
     stringVal.length = convLabel.length();
 

Minecraft.Client/Platform/Common/UI/UIScene_SkinSelectMenu.cpp

@@ -1202,7 +1202,7 @@ void UIScene_SkinSelectMenu::setLeftLabel(const std::wstring& label) {
     if (label.compare(m_leftLabel) != 0) {
         m_leftLabel = label;
 
-        const std::u16string convLabel = convWstringToU16string(label);
+        const std::u16string convLabel = wstring_to_u16string(label);
 
         IggyDataValue result;
         IggyDataValue value[1];
@@ -1223,7 +1223,7 @@ void UIScene_SkinSelectMenu::setCentreLabel(const std::wstring& label) {
     if (label.compare(m_centreLabel) != 0) {
         m_centreLabel = label;
 
-        const std::u16string convLabel = convWstringToU16string(label);
+        const std::u16string convLabel = wstring_to_u16string(label);
 
         IggyDataValue result;
         IggyDataValue value[1];
@@ -1244,7 +1244,7 @@ void UIScene_SkinSelectMenu::setRightLabel(const std::wstring& label) {
     if (label.compare(m_rightLabel) != 0) {
         m_rightLabel = label;
 
-        const std::u16string convLabel = convWstringToU16string(label);
+        const std::u16string convLabel = wstring_to_u16string(label);
 
         IggyDataValue result;
         IggyDataValue value[1];

Minecraft.Client/Platform/Common/UI/UIScene_TradingMenu.cpp

@@ -258,7 +258,7 @@ void UIScene_TradingMenu::setTradeRedBox(int index, bool show) {
 
 void UIScene_TradingMenu::setOfferDescription(std::vector<HtmlString>* description) {
 	std::wstring descriptionStr = HtmlString::Compose(description);
-    const std::u16string conv = convWstringToU16string(descriptionStr);
+    const std::u16string conv = wstring_to_u16string(descriptionStr);
 
     IggyDataValue result;
     IggyDataValue value[1];

Minecraft.Client/Rendering/GameRenderer.cpp

@@ -804,7 +804,7 @@ void GameRenderer::turnOnLightLayer(double alpha) {  // 4J - TODO
 		glActiveTexture(GL_TEXTURE0);
 	}
 #endif
-    // update light texture
+    // 4jcraft: update light texture
     // todo: check implementation of getLightTexture.
     RenderManager.TextureBindVertex(
         getLightTexture(mc->player->GetXboxPad(), mc->level));

Minecraft.World/Util/StringHelpers.cpp

@@ -1,4 +1,5 @@
 #include "../Platform/stdafx.h"
+#include "simdutf.h"
 
 std::wstring toLower(const std::wstring& a) {
     std::wstring out = std::wstring(a);
@@ -41,36 +42,45 @@ std::wstring convStringToWstring(const std::string& converting) {
     return converted;
 }
 
-std::u16string convWstringToU16string(const std::wstring& converting) {
+std::wstring u16string_to_wstring(const std::u16string& converting) {
-    std::u16string out;
+#if WCHAR_MAX == 0xFFFF
-    out.reserve(converting.size());
+    // on windows, wchar_t is UTF-16 so we can get away with just a type
+    // transmutation
+    return std::wstring(reinterpret_cast<const wchar_t*>(converting.data()),
+                        converting.size());
+#else
+    // POSIX has wchar_t as UTF-32 instead so simdutf time :>>>
+    if (converting.empty()) return {};
 
-    if constexpr (sizeof(wchar_t) == sizeof(char16_t)) {
+    std::wstring result(
-        // wchar_t is UTF-16: direct copy
+        simdutf::utf32_length_from_utf16(converting.data(), converting.size()),
-        out.assign(converting.begin(), converting.end());
+        L'\0');
-    } else {
+    simdutf::convert_utf16_to_utf32(converting.data(), converting.size(),
-        // wchar_t is UTF-32: encode to UTF-16
+                                    reinterpret_cast<char32_t*>(result.data()));
-        for (wchar_t wc : converting) {
-            uint32_t cp = static_cast<uint32_t>(wc);
 
-            if (cp <= 0xFFFF) {
+    return result;
-                // Avoid producing UTF-16 surrogate code points directly
+#endif
-                if (cp >= 0xD800 && cp <= 0xDFFF) {
+}
-                    out.push_back(u'\uFFFD');  // replacement char
-                } else {
-                    out.push_back(static_cast<char16_t>(cp));
-                }
-            } else if (cp <= 0x10FFFF) {
-                cp -= 0x10000;
-                out.push_back(static_cast<char16_t>(0xD800 + (cp >> 10)));
-                out.push_back(static_cast<char16_t>(0xDC00 + (cp & 0x3FF)));
-            } else {
-                out.push_back(u'\uFFFD');  // invalid code point
-            }
-        }
-    }
 
-    return out;
+std::u16string wstring_to_u16string(const std::wstring& converting) {
+    #if WCHAR_MAX == 0xFFFF
+    // windows, UTF-16
+    return std::u16string(reinterpret_cast<const char16_t*>(converting.data()),
+                          converting.size());
+#else
+    // POSIX, UTF-32
+    if (converting.empty()) return {};
+
+    const char32_t* data32 =
+        reinterpret_cast<const char32_t*>(converting.data());
+    const std::size_t len32 = converting.size();
+
+    std::u16string result(simdutf::utf16_length_from_utf32(data32, len32),
+                          u'\0');
+    simdutf::convert_utf32_to_utf16(data32, len32, result.data());
+
+    return result;
+#endif
 }
 
 // Convert for filename std::wstrings to a straight character pointer for Xbox

Minecraft.World/Util/StringHelpers.h

@@ -30,7 +30,8 @@ T _fromHEXString(const std::wstring& s) {
 }
 
 std::wstring convStringToWstring(const std::string& converting);
-std::u16string convWstringToU16string(const std::wstring& converting);
+std::wstring wstring_to_u16string(const std::u16string& converting);
+std::u16string wstring_to_u16string(const std::wstring& converting);
 const char* wstringtofilename(const std::wstring& name);
 std::wstring filenametowstring(const char* name);
 

Minecraft.World/meson.build

@@ -25,6 +25,11 @@ world_sources = run_command(
   check : true,
 ).stdout().strip().split('\n')
 
+simdutf_dep = dependency('simdutf', 
+  fallback: ['simdutf', 'simdutf_dep'], 
+  default_options: ['utf8=true', 'utf16=true', 'utf32=true']
+)
+
 lib_world = static_library('Minecraft.World',
   world_sources,
   include_directories : include_directories('Platform', 'Platform/x64headers'),
@@ -34,17 +39,18 @@ lib_world = static_library('Minecraft.World',
     input_dep,
     profile_dep,
     storage_dep,
+    simdutf_dep,
   ],
   cpp_args : global_cpp_args + global_cpp_defs + [
     '-include', 'Platform/stdafx.h',
   ],
 )
 
-dep_zlib = dependency('zlib')
+zlib_dep = dependency('zlib')
-dep_crypto = dependency('libcrypto')  # for MD5 in Hasher.cpp on Linux
+crypto_dep = dependency('libcrypto')  # for MD5 in Hasher.cpp on Linux
 
 world_dep = declare_dependency(
   link_with : lib_world,
-  dependencies : [dep_crypto, dep_zlib],
+  dependencies : [crypto_dep, zlib_dep],
   include_directories : include_directories('Platform/x64headers'),
 )

subprojects/packagefiles/simdutf/meson.build

@@ -0,0 +1,61 @@
+project('simdutf',
+    'cpp',
+    default_options: ['cpp_std=c++23'],
+    version: '8.2.0',
+    license: ['MIT'],
+)
+
+pymod = import('python')
+python = pymod.find_installation('python3', required: true)
+
+simdutf_amalgamate_args = [
+    '--no-zip',
+    '--no-readme',
+    '--source-dir', meson.project_source_root() / 'src',
+    '--include-dir', meson.project_source_root() / 'include',
+    '--output-dir', meson.project_build_root(),
+]
+
+if get_option('utf8')
+    simdutf_amalgamate_args += ['--with-utf8']
+endif
+if get_option('utf16')
+    simdutf_amalgamate_args += ['--with-utf16']
+endif
+if get_option('utf32')
+    simdutf_amalgamate_args += ['--with-utf32']
+endif
+if get_option('ascii')
+    simdutf_amalgamate_args += ['--with-ascii']
+endif
+if get_option('latin1')
+    simdutf_amalgamate_args += ['--with-latin1']
+endif
+if get_option('base64')
+    simdutf_amalgamate_args += ['--with-base64']
+endif
+if get_option('detect-enc')
+    simdutf_amalgamate_args += ['--with-detect-enc']
+endif
+
+simdutf_amalgamate = custom_target(
+    'amalgamate',
+    input: [],
+    output: ['simdutf.h', 'simdutf.cpp'],
+    command: [
+        python,
+        files('singleheader' / 'amalgamate.py'),
+    ] + simdutf_amalgamate_args,
+)
+
+simdutf_lib = static_library(
+    'simdutf',
+    sources: simdutf_amalgamate,
+)
+
+simdutf_dep = declare_dependency(
+    sources: simdutf_amalgamate,
+    link_with: simdutf_lib,
+)
+
+meson.override_dependency('simdutf', simdutf_dep)

subprojects/packagefiles/simdutf/meson.options

@@ -0,0 +1,34 @@
+option('utf8',
+    type: 'boolean',
+    value: false,
+    description: 'Include UTF-8 support')
+
+option('utf16',
+    type: 'boolean',
+    value: false,
+    description: 'Include UTF-16 support')
+
+option('utf32',
+    type: 'boolean',
+    value: false,
+    description: 'Include UTF-32 support')
+
+option('base64',
+    type: 'boolean',
+    value: false,
+    description: 'Include Base64 support')
+
+option('detect-enc',
+    type: 'boolean',
+    value: false,
+    description: 'Include encoding detection support')
+
+option('ascii',
+    type: 'boolean',
+    value: false,
+    description: 'Include ASCII support')
+
+option('latin1',
+    type: 'boolean',
+    value: false,
+    description: 'Include Latin1 support')

subprojects/simdutf.wrap

@@ -0,0 +1,8 @@
+[wrap-git]
+url = https://github.com/simdutf/simdutf.git
+depth = 1
+revision = v8.2.0
+patch_directory = simdutf
+
+[provide]
+simdutf_dep = simdutf