utf8_sanitize.cpp

Go to the documentation of this file.

// SPDX-License-Identifier: Apache-2.0
#include <entropic/mcp/utf8_sanitize.h>
 
#include <cstdint>
 
namespace entropic::mcp {
 
namespace {
 
constexpr char kReplacement[] = "\xEF\xBF\xBD";  // U+FFFD
constexpr size_t kReplacementLen = 3;
 
inline bool is_cont(uint8_t b) { return (b & 0xC0) == 0x80; }
 
inline int follow_count(uint8_t b) {
    int n = -1;
    if (b < 0x80) { n = 0; }
    else if (b >= 0xC2 && b < 0xE0) { n = 1; }
    else if (b >= 0xE0 && b < 0xF0) { n = 2; }
    else if (b >= 0xF0 && b < 0xF5) { n = 3; }
    return n;
}
 
inline bool first_cont_in_range(uint8_t lead, uint8_t c1) {
    bool ok = is_cont(c1);  // default upper-bound 0xBF, lower-bound 0x80
    if (lead == 0xE0) { ok = ok && c1 >= 0xA0; }
    else if (lead == 0xED) { ok = ok && c1 <= 0x9F; }
    else if (lead == 0xF0) { ok = ok && c1 >= 0x90; }
    else if (lead == 0xF4) { ok = ok && c1 <= 0x8F; }
    return ok;
}
 
size_t valid_seq_len(const uint8_t* p, const uint8_t* end) {
    int n = follow_count(*p);
    bool ok = (n >= 0)
           && (n == 0
               || (p + n < end && first_cont_in_range(*p, p[1])));
    for (int i = 2; ok && i <= n; ++i) {
        ok = is_cont(p[i]);
    }
    return ok ? static_cast<size_t>(n + 1) : 0;
}
 
} // namespace
 
std::string sanitize_utf8(std::string_view input) {
    std::string out;
    out.reserve(input.size());
    const auto* p = reinterpret_cast<const uint8_t*>(input.data());
    const auto* end = p + input.size();
    while (p < end) {
        size_t len = valid_seq_len(p, end);
        if (len > 0) {
            out.append(reinterpret_cast<const char*>(p), len);
            p += len;
        } else {
            out.append(kReplacement, kReplacementLen);
            ++p;
        }
    }
    return out;
}
 
} // namespace entropic::mcp