如何正确处理大小 >= 2B 的渲染 utf-8 字符？

Question

我想渲染 utf-8 大小 >= 2 字节的字符。我已经把一切都搞定了。但是有一个问题。画一个字的时候，后面还有一个东西image

为了获取字形数据，我使用了 freetype。这是最小的实现，实际代码包含字距调整、SDF 等

我觉得需要说明的是图集。方法 "TextureAtlas::PackTexture(data, w, h)" 打包纹理数据和 returns 位置，原点 - 左上角 - 在图集 w 和 h 范围内。因此，第一个字符的原点 = [0, 0]，下一个宽度为 50 的字符的原点为 [50, 0]。简而言之。

    enum
    {
        DPI = 72,
        HIGHRES = 64
    };

    struct Glyph
    {
        uint32 codepoint = -1;
        uint32 width = 0; 
        uint32 height = 0;

        Vector2<int> bearing = 0;
        Vector2<float> advance = 0.0f;
        float s0, t0, s1, t1;
    };

    class TextureFont
    {
    public:
        TextureFont() = default;

        bool Initialize();
        void LoadFromFile(const std::string& filePath, float fontSize);

        Glyph* getGlyph(const char8_t* codepoint);
        Glyph* FindGlyph(const char8_t* codepoint);

        uint32 LoadGlyph(const char8_t* codepoint);

        int InitFreeType(float size);

        char* filename;

        vector<Glyph> glyphs;
        TextureAtlas atlas;

        FT_Library library;
        FT_Face face;

        float fontSize = 0.0f;
        float ascender = 0.0f;
        float descender = 0.0f;
        float height = 0.0f;
    };

int CharFromUtf8(unsigned int* out_char, const char* in_text, const char* in_text_end)
    {
        unsigned int c = (unsigned int)-1;
        const unsigned char* str = (const unsigned char*)in_text;
        if (!(*str & 0x80)) {
            c = (unsigned int)(*str++);
            *out_char = c;
            return 1;
        }
        if ((*str & 0xe0) == 0xc0) {
            *out_char = 0xFFFD;
            if (in_text_end && in_text_end - (const char*)str < 2) return 1;
            if (*str < 0xc2) return 2;
            c = (unsigned int)((*str++ & 0x1f) << 6);
            if ((*str & 0xc0) != 0x80) return 2;
            c += (*str++ & 0x3f);
            *out_char = c;
            return 2;
        }
        if ((*str & 0xf0) == 0xe0) {
            *out_char = 0xFFFD;
            if (in_text_end && in_text_end - (const char*)str < 3) return 1;
            if (*str == 0xe0 && (str[1] < 0xa0 || str[1] > 0xbf)) return 3;
            if (*str == 0xed && str[1] > 0x9f) return 3;
            c = (unsigned int)((*str++ & 0x0f) << 12);
            if ((*str & 0xc0) != 0x80) return 3;
            c += (unsigned int)((*str++ & 0x3f) << 6);
            if ((*str & 0xc0) != 0x80) return 3;
            c += (*str++ & 0x3f);
            *out_char = c;
            return 3;
        }
        if ((*str & 0xf8) == 0xf0) {
            *out_char = 0xFFFD;
            if (in_text_end && in_text_end - (const char*)str < 4) return 1;
            if (*str > 0xf4) return 4;
            if (*str == 0xf0 && (str[1] < 0x90 || str[1] > 0xbf)) return 4;
            if (*str == 0xf4 && str[1] > 0x8f) return 4; 
            c = (unsigned int)((*str++ & 0x07) << 18);
            if ((*str & 0xc0) != 0x80) return 4;
            c += (unsigned int)((*str++ & 0x3f) << 12);
            if ((*str & 0xc0) != 0x80) return 4;
            c += (unsigned int)((*str++ & 0x3f) << 6);
            if ((*str & 0xc0) != 0x80) return 4;
            c += (*str++ & 0x3f);
            if ((c & 0xFFFFF800) == 0xD800) return 4;
            *out_char = c;
            return 4;
        }
        *out_char = 0;
        return 0;
    }

    bool TextureFont::Initialize()
    {
        FT_Size_Metrics metrics;

        if (!InitFreeType(fontSize * 100.0f)) {
            return false;
        }

        metrics = face->size->metrics;
        ascender = (metrics.ascender >> 6) / 100.0f;
        descender = (metrics.descender >> 6) / 100.0f;
        height = (metrics.height >> 6) / 100.0f;

        FT_Done_Face(face);
        FT_Done_FreeType(library);

        return true;
    }

    int TextureFont::InitFreeType(float size)
    {
        FT_Matrix matrix = {
            static_cast<int>((1.0 / HIGHRES) * 0x10000L),
            static_cast<int>((0.0)           * 0x10000L),
            static_cast<int>((0.0)           * 0x10000L),
            static_cast<int>((1.0)           * 0x10000L)};
        FT_Error error;
        error = FT_Init_FreeType(&library);
        if (error) {
            EngineLogError("FREE_TYPE_ERROR: Could not Init FreeType!\n");
            FT_Done_FreeType(library);
            return 0;
        }

        error = FT_New_Face(library, filename, 0, &face);

        if (error) {
            EngineLogError("FREE_TYPE_ERROR: Could not create a new face!\n");
            FT_Done_FreeType(library);
            return 0;
        }

        error = FT_Select_Charmap(face, FT_ENCODING_UNICODE);
        if (error) {
            EngineLogError("FREE_TYPE_ERROR: Could not select charmap!\n");
            FT_Done_Face(face);
            return 0;
        }

        error = FT_Set_Char_Size(face, static_cast<ulong>(size * HIGHRES), 0, DPI * HIGHRES, DPI);
        if (error) {
            EngineLogError("FREE_TYPE_ERROR: Could not set char size!\n");
            FT_Done_Face(face);
            return 0;
        }

        FT_Set_Transform(face, &matrix, NULL);

        return 1;
    }

    void TextureFont::LoadFromFile(const std::string& filePath, float fontSize)
    {
        atlas.Create(512, 1);
        std::fill(atlas.buffer.begin(), atlas.buffer.end(), 0);
        this->fontSize = fontSize;  
        this->filename = strdup(filePath.c_str());

        Initialize();
    }

    Glyph* TextureFont::getGlyph(const char8_t* codepoint)
    {
        if (Glyph* glyph = FindGlyph(codepoint)) {
            return glyph;
        }

        if (LoadGlyph(codepoint)) {
            return FindGlyph(codepoint);
        }

        return nullptr;
    }

    Glyph* TextureFont::FindGlyph(const char8_t* codepoint)
    {
        Glyph* glyph = nullptr;
        uint32 ucodepoint;
        CharFromUtf8(&ucodepoint, (char*)codepoint, NULL);
        for (uint32 i = 0; i < glyphs.size(); ++i) {
            glyph = &glyphs[i];
            if (glyph->codepoint == ucodepoint) {
                return glyph;
            }
        }

        return nullptr;
    }

    uint32 TextureFont::LoadGlyph(const char8_t* codepoint)
    {
        FT_Error error = NULL;
        FT_Glyph ftGlyph = nullptr;
        FT_GlyphSlot slot = nullptr;
        FT_Bitmap bitmap;

        if (!InitFreeType(fontSize)) {
            return 0;
        }

        if (FindGlyph(codepoint)) {
            FT_Done_Face(face);
            FT_Done_FreeType(library);
            return 1;
        }

        unsigned int cp;
        CharFromUtf8(&cp, (char*)codepoint, NULL);
        uint32 glyphIndex = FT_Get_Char_Index(face, cp);

        int flag = 0;
        flag |= FT_LOAD_RENDER;
        flag |= FT_LOAD_FORCE_AUTOHINT;

        error = FT_Load_Glyph(face, glyphIndex, flag);
        if (error) {
            EngineLogError("FREE_TYPE_ERROR: Could not load the glyph (line {})!\n", __LINE__);
            FT_Done_Face(face);
            FT_Done_FreeType(library);
            return 0;
        }

        slot = face->glyph;
        bitmap = slot->bitmap;
        int glyphTop = slot->bitmap_top;
        int glyphLeft = slot->bitmap_left;

        uint32 srcWidth = bitmap.width / atlas.bytesPerPixel;
        uint32 srcHeight = bitmap.rows;

        uint32 tgtWidth = srcWidth;
        uint32 tgtHeight = srcHeight;

        auto buffer = std::make_unique<uchar[]>(tgtWidth * tgtHeight * atlas.bytesPerPixel);

        uchar* destPointer = buffer.get();
        uchar* srcPointer = bitmap.buffer;

        for (uint32 i = 0; i < srcHeight; ++i) {
            memcpy(destPointer, srcPointer, bitmap.width);
            destPointer += tgtWidth * atlas.bytesPerPixel;
            srcPointer += bitmap.pitch;
        }

        auto origin = atlas.PackTexture(buffer.get(), { tgtWidth, tgtHeight });

        float x = origin.x;
        float y = origin.y;

        Glyph current;
        current.codepoint = cp;
        current.width = tgtWidth;
        current.height = tgtHeight;
        current.bearing.x = glyphLeft;
        current.bearing.y = glyphTop;
        current.s0 = x / (float)atlas.textureSize.w;
        current.t0 = y / (float)atlas.textureSize.h;
        current.s1 = (x + tgtWidth) / (float)atlas.textureSize.w;
        current.t1 = (y + tgtHeight) / (float)atlas.textureSize.h;

        current.advance.x = slot->advance.x / (float)HIGHRES;
        current.advance.y = slot->advance.y / (float)HIGHRES;

        glyphs.push_back(current);

        FT_Done_Glyph(ftGlyph);
        FT_Done_Face(face);
        FT_Done_FreeType(library);

        return 1;
    }

为了呈现字符串（在本例中为单个字符），我遍历字符串大小、获取字形、更新图集并设置呈现数据。

文本是一个简单的四边形，带有带有适当 uv 的纹理。我认为没有必要解释里面的内容AddVertexData，因为它不会引起问题。

void DrawString(const std::u8string& string, float x, float y)
    {
        for (const auto& c : string) {
            auto glyph = textureFont.getGlyph(&c);

            auto& t = *(Texture2D*)texture.get();
            t.UpdateData(textureFont.atlas.buffer.data());

            float x0 = x + static_cast<float>(glyph->bearing.x);
            float y0 = y + (textureFont.ascender + textureFont.descender - static_cast<float>(glyph->bearing.y));
            float x1 = x0 + static_cast<float>(glyph->width);
            float y1 = y0 + static_cast<float>(glyph->height);

            float u0 = glyph->s0;
            float v0 = glyph->t0;
            float u1 = glyph->s1;
            float v1 = glyph->t1;

            //            position                uv                      color
            AddVertexData(Vector2<float>(x0, y0), Vector2<float>(u0, v0), 0xff0000ff);
            AddVertexData(Vector2<float>(x0, y1), Vector2<float>(u0, v1), 0xff0000ff);
            AddVertexData(Vector2<float>(x1, y1), Vector2<float>(u1, v1), 0xff0000ff);
            AddVertexData(Vector2<float>(x1, y0), Vector2<float>(u1, v0), 0xff0000ff);

            // indices for DrawElements() call
            // 0, 1, 2, 2, 3, 0
            AddRectElements();

            x += glyph->advance.x;
        }
    }

ę是utf-8 size == 2，所以循环跑了两次，但是只渲染了1个字符，不知道第二个字符（因为没有第二个字符），所以渲染空的四边形。

如何摆脱我要渲染的角色后面的四边形？

Answer 1

在你的DrawString函数中你有循环

for (const auto& c : string)

该循环将 逐字节 遍历字符串。因此，如果字符串包含双字节 "ę" 字符，则第一次迭代将获得第一个字节，第二次迭代将获得第二个字节。

您不能在此处使用基于范围的 for 循环，因为您需要跳过字符串中的字节。使用基于迭代器的循环或基于索引的循环。

例如

for (size_t i = 0; i < string.size(); /* nothing */) {
    // Here you need to get the number of bytes for the current character
    // Then you should increment the index by that amount
    i += byte_count_for_current_character;

    // ... rest of code
}

Answer 2

您的问题在 DrawString for (const auto& c : string)

您应该跳过用于编码先前字形的额外字符，那些与 0b10......:

匹配的字符

for (const auto& c : string) {
    if ((c & 0b1100'0000) == 0b1000'0000) {
        continue;
    }
// ...
}

或前进到最后一个字形读取的字节数。

Answer 3

对实际 UTF-8 解码函数的两次调用 CharFromUtf8 都会忽略其 return 值，即字符串指针应前进的字节数。而不是 for (const auto& c : string) 你应该有一个指针，你在每次迭代时将 return 值推进。

此外，由于您将在该循环内使用 CharFromUtf8 函数，因此您将知道 Unicode 代码点和要前进的字节数。然后，您可以重构 TextureFont 以将 unsigned int（即代码点）作为参数，而不是让它进行 UTF-8 解码。这将是更好的关注点分离。

Answer 4

其他答案已经确定了直接对 std::u8string 变量使用基于范围的 for 循环的问题。假设基于代码点的枚举是你想要的（它可能不是因为，一般来说，正确的字形选择取决于周围的代码点；你可能想迭代扩展的字形簇），你可以使用像 text_view 为代码点的迭代提供基于范围的支持。那个循环伤口看起来像：

auto tv = make_text_view<utf8_encoding>(string);
for (const auto& cp : tv) {
  ...
}

如何正确处理大小 >= 2B 的渲染 utf-8 字符？

How to handle rendering utf-8 characters with size >= 2B properly?

c++

text

utf-8

freetype

freetype2