Path: blob/main/contrib/llvm-project/libcxx/include/__format/unicode.h
35260 views
// -*- C++ -*-1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//89#ifndef _LIBCPP___FORMAT_UNICODE_H10#define _LIBCPP___FORMAT_UNICODE_H1112#include <__assert>13#include <__bit/countl.h>14#include <__concepts/same_as.h>15#include <__config>16#include <__format/extended_grapheme_cluster_table.h>17#include <__format/indic_conjunct_break_table.h>18#include <__iterator/concepts.h>19#include <__iterator/readable_traits.h> // iter_value_t20#include <__utility/unreachable.h>21#include <string_view>2223#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)24# pragma GCC system_header25#endif2627_LIBCPP_BEGIN_NAMESPACE_STD2829#if _LIBCPP_STD_VER >= 203031namespace __unicode {3233// Helper struct for the result of a consume operation.34//35// The status value for a correct code point is 0. This allows a valid value to36// be used without masking.37// When the decoding fails it know the number of code units affected. For the38// current use-cases that value is not needed, therefore it is not stored.39// The escape routine needs the number of code units for both a valid and40// invalid character and keeps track of it itself. Doing it in this result41// unconditionally would give some overhead when the value is unneeded.42struct __consume_result {43// When __status == __ok it contains the decoded code point.44// Else it contains the replacement character U+FFFD45char32_t __code_point : 31;4647enum : char32_t {48// Consumed a well-formed code point.49__ok = 0,50// Encountered invalid UTF-851__error = 152} __status : 1 {__ok};53};54static_assert(sizeof(__consume_result) == sizeof(char32_t));5556# ifndef _LIBCPP_HAS_NO_UNICODE5758/// Implements the grapheme cluster boundary rules59///60/// These rules are used to implement format's width estimation as stated in61/// [format.string.std]/1162///63/// The Standard refers to UAX \#29 for Unicode 12.0.064/// https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundary_Rules65///66/// The data tables used are67/// https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/GraphemeBreakProperty.txt68/// https://www.unicode.org/Public/UCD/latest/ucd/emoji/emoji-data.txt69/// https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/GraphemeBreakTest.txt (for testing only)7071inline constexpr char32_t __replacement_character = U'\ufffd';7273// The error of a consume operation.74//75// This sets the code point to the replacement character. This code point does76// not participate in the grapheme clustering, so grapheme clustering code can77// ignore the error status and always use the code point.78inline constexpr __consume_result __consume_result_error{__replacement_character, __consume_result::__error};7980[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool __is_high_surrogate(char32_t __value) {81return __value >= 0xd800 && __value <= 0xdbff;82}8384[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool __is_low_surrogate(char32_t __value) {85return __value >= 0xdc00 && __value <= 0xdfff;86}8788// https://www.unicode.org/glossary/#surrogate_code_point89[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool __is_surrogate(char32_t __value) {90return __value >= 0xd800 && __value <= 0xdfff;91}9293// https://www.unicode.org/glossary/#code_point94[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool __is_code_point(char32_t __value) {95return __value <= 0x10ffff;96}9798// https://www.unicode.org/glossary/#unicode_scalar_value99[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool __is_scalar_value(char32_t __value) {100return __unicode::__is_code_point(__value) && !__unicode::__is_surrogate(__value);101}102103template <contiguous_iterator _Iterator>104requires same_as<iter_value_t<_Iterator>, char>105_LIBCPP_HIDE_FROM_ABI constexpr bool __is_continuation(_Iterator __char, int __count) {106do {107if ((*__char & 0b1100'0000) != 0b1000'0000)108return false;109--__count;110++__char;111} while (__count);112return true;113}114115/// Helper class to extract a code unit from a Unicode character range.116///117/// The stored range is a view. There are multiple specialization for different118/// character types.119template <class _CharT>120class __code_point_view;121122/// UTF-8 specialization.123template <>124class __code_point_view<char> {125using _Iterator = basic_string_view<char>::const_iterator;126127public:128_LIBCPP_HIDE_FROM_ABI constexpr explicit __code_point_view(_Iterator __first, _Iterator __last)129: __first_(__first), __last_(__last) {}130131_LIBCPP_HIDE_FROM_ABI constexpr bool __at_end() const noexcept { return __first_ == __last_; }132_LIBCPP_HIDE_FROM_ABI constexpr _Iterator __position() const noexcept { return __first_; }133134// https://www.unicode.org/versions/latest/ch03.pdf#G7404135// Based on Table 3-7, Well-Formed UTF-8 Byte Sequences136//137// Code Points First Byte Second Byte Third Byte Fourth Byte Remarks138// U+0000..U+007F 00..7F U+0000..U+007F 1 code unit range139// C0..C1 80..BF invalid overlong encoding140// U+0080..U+07FF C2..DF 80..BF U+0080..U+07FF 2 code unit range141// E0 80..9F 80..BF invalid overlong encoding142// U+0800..U+0FFF E0 A0..BF 80..BF U+0800..U+FFFF 3 code unit range143// U+1000..U+CFFF E1..EC 80..BF 80..BF144// U+D000..U+D7FF ED 80..9F 80..BF145// U+D800..U+DFFF ED A0..BF 80..BF invalid encoding of surrogate code point146// U+E000..U+FFFF EE..EF 80..BF 80..BF147// F0 80..8F 80..BF 80..BF invalid overlong encoding148// U+10000..U+3FFFF F0 90..BF 80..BF 80..BF U+10000..U+10FFFF 4 code unit range149// U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF150// U+100000..U+10FFFF F4 80..8F 80..BF 80..BF151// F4 90..BF 80..BF 80..BF U+110000.. invalid code point range152//153// Unlike other parsers, these invalid entries are tested after decoding.154// - The parser always needs to consume these code units155// - The code is optimized for well-formed UTF-8156[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __consume_result __consume() noexcept {157_LIBCPP_ASSERT_INTERNAL(__first_ != __last_, "can't move beyond the end of input");158159// Based on the number of leading 1 bits the number of code units in the160// code point can be determined. See161// https://en.wikipedia.org/wiki/UTF-8#Encoding162switch (std::countl_one(static_cast<unsigned char>(*__first_))) {163case 0:164return {static_cast<unsigned char>(*__first_++)};165166case 2: {167if (__last_ - __first_ < 2 || !__unicode::__is_continuation(__first_ + 1, 1)) [[unlikely]]168break;169170char32_t __value = static_cast<unsigned char>(*__first_++) & 0x1f;171__value <<= 6;172__value |= static_cast<unsigned char>(*__first_++) & 0x3f;173174// These values should be encoded in 1 UTF-8 code unit.175if (__value < 0x0080) [[unlikely]]176return __consume_result_error;177178return {__value};179}180181case 3: {182if (__last_ - __first_ < 3 || !__unicode::__is_continuation(__first_ + 1, 2)) [[unlikely]]183break;184185char32_t __value = static_cast<unsigned char>(*__first_++) & 0x0f;186__value <<= 6;187__value |= static_cast<unsigned char>(*__first_++) & 0x3f;188__value <<= 6;189__value |= static_cast<unsigned char>(*__first_++) & 0x3f;190191// These values should be encoded in 1 or 2 UTF-8 code units.192if (__value < 0x0800) [[unlikely]]193return __consume_result_error;194195// A surrogate value is always encoded in 3 UTF-8 code units.196if (__unicode::__is_surrogate(__value)) [[unlikely]]197return __consume_result_error;198199return {__value};200}201202case 4: {203if (__last_ - __first_ < 4 || !__unicode::__is_continuation(__first_ + 1, 3)) [[unlikely]]204break;205206char32_t __value = static_cast<unsigned char>(*__first_++) & 0x07;207__value <<= 6;208__value |= static_cast<unsigned char>(*__first_++) & 0x3f;209__value <<= 6;210__value |= static_cast<unsigned char>(*__first_++) & 0x3f;211__value <<= 6;212__value |= static_cast<unsigned char>(*__first_++) & 0x3f;213214// These values should be encoded in 1, 2, or 3 UTF-8 code units.215if (__value < 0x10000) [[unlikely]]216return __consume_result_error;217218// A value too large is always encoded in 4 UTF-8 code units.219if (!__unicode::__is_code_point(__value)) [[unlikely]]220return __consume_result_error;221222return {__value};223}224}225// An invalid number of leading ones can be garbage or a code unit in the226// middle of a code point. By consuming one code unit the parser may get227// "in sync" after a few code units.228++__first_;229return __consume_result_error;230}231232private:233_Iterator __first_;234_Iterator __last_;235};236237# ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS238_LIBCPP_HIDE_FROM_ABI constexpr bool __is_surrogate_pair_high(wchar_t __value) {239return __value >= 0xd800 && __value <= 0xdbff;240}241242_LIBCPP_HIDE_FROM_ABI constexpr bool __is_surrogate_pair_low(wchar_t __value) {243return __value >= 0xdc00 && __value <= 0xdfff;244}245246/// This specialization depends on the size of wchar_t247/// - 2 UTF-16 (for example Windows and AIX)248/// - 4 UTF-32 (for example Linux)249template <>250class __code_point_view<wchar_t> {251using _Iterator = typename basic_string_view<wchar_t>::const_iterator;252253public:254static_assert(sizeof(wchar_t) == 2 || sizeof(wchar_t) == 4, "sizeof(wchar_t) has a not implemented value");255256_LIBCPP_HIDE_FROM_ABI constexpr explicit __code_point_view(_Iterator __first, _Iterator __last)257: __first_(__first), __last_(__last) {}258259_LIBCPP_HIDE_FROM_ABI constexpr _Iterator __position() const noexcept { return __first_; }260_LIBCPP_HIDE_FROM_ABI constexpr bool __at_end() const noexcept { return __first_ == __last_; }261262[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __consume_result __consume() noexcept {263_LIBCPP_ASSERT_INTERNAL(__first_ != __last_, "can't move beyond the end of input");264265char32_t __value = static_cast<char32_t>(*__first_++);266if constexpr (sizeof(wchar_t) == 2) {267if (__unicode::__is_low_surrogate(__value)) [[unlikely]]268return __consume_result_error;269270if (__unicode::__is_high_surrogate(__value)) {271if (__first_ == __last_ || !__unicode::__is_low_surrogate(static_cast<char32_t>(*__first_))) [[unlikely]]272return __consume_result_error;273274__value -= 0xd800;275__value <<= 10;276__value += static_cast<char32_t>(*__first_++) - 0xdc00;277__value += 0x10000;278279if (!__unicode::__is_code_point(__value)) [[unlikely]]280return __consume_result_error;281}282} else {283if (!__unicode::__is_scalar_value(__value)) [[unlikely]]284return __consume_result_error;285}286287return {__value};288}289290private:291_Iterator __first_;292_Iterator __last_;293};294# endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS295296// State machine to implement the Extended Grapheme Cluster Boundary297//298// The exact rules may change between Unicode versions.299// This implements the extended rules see300// https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries301class __extended_grapheme_cluster_break {302using __EGC_property = __extended_grapheme_custer_property_boundary::__property;303using __inCB_property = __indic_conjunct_break::__property;304305public:306_LIBCPP_HIDE_FROM_ABI constexpr explicit __extended_grapheme_cluster_break(char32_t __first_code_point)307: __prev_code_point_(__first_code_point),308__prev_property_(__extended_grapheme_custer_property_boundary::__get_property(__first_code_point)) {309// Initializes the active rule.310if (__prev_property_ == __EGC_property::__Extended_Pictographic)311__active_rule_ = __rule::__GB11_emoji;312else if (__prev_property_ == __EGC_property::__Regional_Indicator)313__active_rule_ = __rule::__GB12_GB13_regional_indicator;314else if (__indic_conjunct_break::__get_property(__first_code_point) == __inCB_property::__Consonant)315__active_rule_ = __rule::__GB9c_indic_conjunct_break;316}317318[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(char32_t __next_code_point) {319__EGC_property __next_property = __extended_grapheme_custer_property_boundary::__get_property(__next_code_point);320bool __result = __evaluate(__next_code_point, __next_property);321__prev_code_point_ = __next_code_point;322__prev_property_ = __next_property;323return __result;324}325326// The code point whose break propery are considered during the next327// evaluation cyle.328[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr char32_t __current_code_point() const { return __prev_code_point_; }329330private:331// The naming of the identifiers matches the Unicode standard.332// NOLINTBEGIN(readability-identifier-naming)333334[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool335__evaluate(char32_t __next_code_point, __EGC_property __next_property) {336switch (__active_rule_) {337case __rule::__none:338return __evaluate_none(__next_code_point, __next_property);339case __rule::__GB9c_indic_conjunct_break:340return __evaluate_GB9c_indic_conjunct_break(__next_code_point, __next_property);341case __rule::__GB11_emoji:342return __evaluate_GB11_emoji(__next_code_point, __next_property);343case __rule::__GB12_GB13_regional_indicator:344return __evaluate_GB12_GB13_regional_indicator(__next_code_point, __next_property);345}346__libcpp_unreachable();347}348349_LIBCPP_HIDE_FROM_ABI constexpr bool __evaluate_none(char32_t __next_code_point, __EGC_property __next_property) {350// *** Break at the start and end of text, unless the text is empty. ***351352_LIBCPP_ASSERT_INTERNAL(__prev_property_ != __EGC_property::__sot, "should be handled in the constructor"); // GB1353_LIBCPP_ASSERT_INTERNAL(__prev_property_ != __EGC_property::__eot, "should be handled by our caller"); // GB2354355// *** Do not break between a CR and LF. Otherwise, break before and after controls. ***356if (__prev_property_ == __EGC_property::__CR && __next_property == __EGC_property::__LF) // GB3357return false;358359if (__prev_property_ == __EGC_property::__Control || __prev_property_ == __EGC_property::__CR ||360__prev_property_ == __EGC_property::__LF) // GB4361return true;362363if (__next_property == __EGC_property::__Control || __next_property == __EGC_property::__CR ||364__next_property == __EGC_property::__LF) // GB5365return true;366367// *** Do not break Hangul syllable sequences. ***368if (__prev_property_ == __EGC_property::__L &&369(__next_property == __EGC_property::__L || __next_property == __EGC_property::__V ||370__next_property == __EGC_property::__LV || __next_property == __EGC_property::__LVT)) // GB6371return false;372373if ((__prev_property_ == __EGC_property::__LV || __prev_property_ == __EGC_property::__V) &&374(__next_property == __EGC_property::__V || __next_property == __EGC_property::__T)) // GB7375return false;376377if ((__prev_property_ == __EGC_property::__LVT || __prev_property_ == __EGC_property::__T) &&378__next_property == __EGC_property::__T) // GB8379return false;380381// *** Do not break before extending characters or ZWJ. ***382if (__next_property == __EGC_property::__Extend || __next_property == __EGC_property::__ZWJ)383return false; // GB9384385// *** Do not break before SpacingMarks, or after Prepend characters. ***386if (__next_property == __EGC_property::__SpacingMark) // GB9a387return false;388389if (__prev_property_ == __EGC_property::__Prepend) // GB9b390return false;391392// *** Do not break within certain combinations with Indic_Conjunct_Break (InCB)=Linker. ***393if (__indic_conjunct_break::__get_property(__next_code_point) == __inCB_property::__Consonant) {394__active_rule_ = __rule::__GB9c_indic_conjunct_break;395__GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Consonant;396return true;397}398399// *** Do not break within emoji modifier sequences or emoji zwj sequences. ***400if (__next_property == __EGC_property::__Extended_Pictographic) {401__active_rule_ = __rule::__GB11_emoji;402__GB11_emoji_state_ = __GB11_emoji_state::__Extended_Pictographic;403return true;404}405406// *** Do not break within emoji flag sequences ***407408// That is, do not break between regional indicator (RI) symbols if there409// is an odd number of RI characters before the break point.410if (__next_property == __EGC_property::__Regional_Indicator) { // GB12 + GB13411__active_rule_ = __rule::__GB12_GB13_regional_indicator;412return true;413}414415// *** Otherwise, break everywhere. ***416return true; // GB999417}418419[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool420__evaluate_GB9c_indic_conjunct_break(char32_t __next_code_point, __EGC_property __next_property) {421__inCB_property __break = __indic_conjunct_break::__get_property(__next_code_point);422if (__break == __inCB_property::__none) {423__active_rule_ = __rule::__none;424return __evaluate_none(__next_code_point, __next_property);425}426427switch (__GB9c_indic_conjunct_break_state_) {428case __GB9c_indic_conjunct_break_state::__Consonant:429if (__break == __inCB_property::__Extend) {430return false;431}432if (__break == __inCB_property::__Linker) {433__GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Linker;434return false;435}436__active_rule_ = __rule::__none;437return __evaluate_none(__next_code_point, __next_property);438439case __GB9c_indic_conjunct_break_state::__Linker:440if (__break == __inCB_property::__Extend) {441return false;442}443if (__break == __inCB_property::__Linker) {444return false;445}446if (__break == __inCB_property::__Consonant) {447__GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Consonant;448return false;449}450__active_rule_ = __rule::__none;451return __evaluate_none(__next_code_point, __next_property);452}453__libcpp_unreachable();454}455456[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool457__evaluate_GB11_emoji(char32_t __next_code_point, __EGC_property __next_property) {458switch (__GB11_emoji_state_) {459case __GB11_emoji_state::__Extended_Pictographic:460if (__next_property == __EGC_property::__Extend) {461__GB11_emoji_state_ = __GB11_emoji_state::__Extend;462return false;463}464[[fallthrough]];465case __GB11_emoji_state::__Extend:466if (__next_property == __EGC_property::__ZWJ) {467__GB11_emoji_state_ = __GB11_emoji_state::__ZWJ;468return false;469}470if (__next_property == __EGC_property::__Extend)471return false;472__active_rule_ = __rule::__none;473return __evaluate_none(__next_code_point, __next_property);474475case __GB11_emoji_state::__ZWJ:476if (__next_property == __EGC_property::__Extended_Pictographic) {477__GB11_emoji_state_ = __GB11_emoji_state::__Extended_Pictographic;478return false;479}480__active_rule_ = __rule::__none;481return __evaluate_none(__next_code_point, __next_property);482}483__libcpp_unreachable();484}485486[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool487__evaluate_GB12_GB13_regional_indicator(char32_t __next_code_point, __EGC_property __next_property) {488__active_rule_ = __rule::__none;489if (__next_property == __EGC_property::__Regional_Indicator)490return false;491return __evaluate_none(__next_code_point, __next_property);492}493494char32_t __prev_code_point_;495__EGC_property __prev_property_;496497enum class __rule {498__none,499__GB9c_indic_conjunct_break,500__GB11_emoji,501__GB12_GB13_regional_indicator,502};503__rule __active_rule_ = __rule::__none;504505enum class __GB11_emoji_state {506__Extended_Pictographic,507__Extend,508__ZWJ,509};510__GB11_emoji_state __GB11_emoji_state_ = __GB11_emoji_state::__Extended_Pictographic;511512enum class __GB9c_indic_conjunct_break_state {513__Consonant,514__Linker,515};516517__GB9c_indic_conjunct_break_state __GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Consonant;518519// NOLINTEND(readability-identifier-naming)520};521522/// Helper class to extract an extended grapheme cluster from a Unicode character range.523///524/// This function is used to determine the column width of an extended grapheme525/// cluster. In order to do that only the first code point is evaluated.526/// Therefore only this code point is extracted.527template <class _CharT>528class __extended_grapheme_cluster_view {529using _Iterator = typename basic_string_view<_CharT>::const_iterator;530531public:532_LIBCPP_HIDE_FROM_ABI constexpr explicit __extended_grapheme_cluster_view(_Iterator __first, _Iterator __last)533: __code_point_view_(__first, __last), __at_break_(__code_point_view_.__consume().__code_point) {}534535struct __cluster {536/// The first code point of the extended grapheme cluster.537///538/// The first code point is used to estimate the width of the extended539/// grapheme cluster.540char32_t __code_point_;541542/// Points one beyond the last code unit in the extended grapheme cluster.543///544/// It's expected the caller has the start position and thus can determine545/// the code unit range of the extended grapheme cluster.546_Iterator __last_;547};548549[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __cluster __consume() {550char32_t __code_point = __at_break_.__current_code_point();551_Iterator __position = __code_point_view_.__position();552while (!__code_point_view_.__at_end()) {553if (__at_break_(__code_point_view_.__consume().__code_point))554break;555__position = __code_point_view_.__position();556}557return {__code_point, __position};558}559560private:561__code_point_view<_CharT> __code_point_view_;562__extended_grapheme_cluster_break __at_break_;563};564565template <contiguous_iterator _Iterator>566__extended_grapheme_cluster_view(_Iterator, _Iterator) -> __extended_grapheme_cluster_view<iter_value_t<_Iterator>>;567568# else // _LIBCPP_HAS_NO_UNICODE569570// For ASCII every character is a "code point".571// This makes it easier to write code agnostic of the _LIBCPP_HAS_NO_UNICODE define.572template <class _CharT>573class __code_point_view {574using _Iterator = typename basic_string_view<_CharT>::const_iterator;575576public:577_LIBCPP_HIDE_FROM_ABI constexpr explicit __code_point_view(_Iterator __first, _Iterator __last)578: __first_(__first), __last_(__last) {}579580_LIBCPP_HIDE_FROM_ABI constexpr bool __at_end() const noexcept { return __first_ == __last_; }581_LIBCPP_HIDE_FROM_ABI constexpr _Iterator __position() const noexcept { return __first_; }582583[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __consume_result __consume() noexcept {584_LIBCPP_ASSERT_INTERNAL(__first_ != __last_, "can't move beyond the end of input");585return {static_cast<char32_t>(*__first_++)};586}587588private:589_Iterator __first_;590_Iterator __last_;591};592593# endif // _LIBCPP_HAS_NO_UNICODE594595} // namespace __unicode596597#endif //_LIBCPP_STD_VER >= 20598599_LIBCPP_END_NAMESPACE_STD600601#endif // _LIBCPP___FORMAT_UNICODE_H602603604