qchar.cpp

Go to the documentation of this file.

/****************************************************************************
**
** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies).
** Contact: http://www.qt-project.org/legal
**
** This file is part of the QtCore module of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Digia.  For licensing terms and
** conditions see http://qt.digia.com/licensing.  For further information
** use the contact form at http://qt.digia.com/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Digia gives you certain additional
** rights.  These rights are described in the Digia Qt LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
**
** $QT_END_LICENSE$
**
****************************************************************************/

// Don't define it while compiling this module, or USERS of Qt will
// not be able to link.
#ifdef QT_NO_CAST_FROM_ASCII
#  undef QT_NO_CAST_FROM_ASCII
#endif
#ifdef QT_NO_CAST_TO_ASCII
#  undef QT_NO_CAST_TO_ASCII
#endif
#include "qchar.h"

#include "qdatastream.h"
#include "qtextcodec.h"

#include "qunicodetables_p.h"
#include "qunicodetables.cpp"

QT_BEGIN_NAMESPACE

#ifndef QT_NO_CODEC_FOR_C_STRINGS
#  ifdef QT_NO_TEXTCODEC
#    define QT_NO_CODEC_FOR_C_STRINGS
#  endif
#endif

#define FLAG(x) (1 << (x))

QChar::QChar(char ch)
{
#ifndef QT_NO_CODEC_FOR_C_STRINGS
    if (QTextCodec::codecForCStrings())
        // #####
        ucs =  QTextCodec::codecForCStrings()->toUnicode(&ch, 1).at(0).unicode();
    else
#endif
        ucs = uchar(ch);
}

QChar::QChar(uchar ch)
{
#ifndef QT_NO_CODEC_FOR_C_STRINGS
    if (QTextCodec::codecForCStrings()) {
        // #####
        char c = char(ch);
        ucs =  QTextCodec::codecForCStrings()->toUnicode(&c, 1).at(0).unicode();
    } else
#endif
        ucs = ch;
}

bool QChar::isPrint() const
{
    const int test = FLAG(Other_Control) |
                     FLAG(Other_NotAssigned);
    return !(FLAG(qGetProp(ucs)->category) & test);
}

bool QChar::isSpace() const
{
    if(ucs >= 9 && ucs <=13)
        return true;
    const int test = FLAG(Separator_Space) |
                     FLAG(Separator_Line) |
                     FLAG(Separator_Paragraph);
    return FLAG(qGetProp(ucs)->category) & test;
}

bool QChar::isMark() const
{
    const int test = FLAG(Mark_NonSpacing) |
                     FLAG(Mark_SpacingCombining) |
                     FLAG(Mark_Enclosing);
    return FLAG(qGetProp(ucs)->category) & test;
}

bool QChar::isPunct() const
{
    const int test = FLAG(Punctuation_Connector) |
                     FLAG(Punctuation_Dash) |
                     FLAG(Punctuation_Open) |
                     FLAG(Punctuation_Close) |
                     FLAG(Punctuation_InitialQuote) |
                     FLAG(Punctuation_FinalQuote) |
                     FLAG(Punctuation_Other);
    return FLAG(qGetProp(ucs)->category) & test;
}

bool QChar::isLetter() const
{
    const int test = FLAG(Letter_Uppercase) |
                     FLAG(Letter_Lowercase) |
                     FLAG(Letter_Titlecase) |
                     FLAG(Letter_Modifier) |
                     FLAG(Letter_Other);
    return FLAG(qGetProp(ucs)->category) & test;
}

bool QChar::isNumber() const
{
    const int test = FLAG(Number_DecimalDigit) |
                     FLAG(Number_Letter) |
                     FLAG(Number_Other);
    return FLAG(qGetProp(ucs)->category) & test;
}

bool QChar::isLetterOrNumber() const
{
    const int test = FLAG(Letter_Uppercase) |
                     FLAG(Letter_Lowercase) |
                     FLAG(Letter_Titlecase) |
                     FLAG(Letter_Modifier) |
                     FLAG(Letter_Other) |
                     FLAG(Number_DecimalDigit) |
                     FLAG(Number_Letter) |
                     FLAG(Number_Other);
    return FLAG(qGetProp(ucs)->category) & test;
}


bool QChar::isDigit() const
{
    return (qGetProp(ucs)->category == Number_DecimalDigit);
}


bool QChar::isSymbol() const
{
    const int test = FLAG(Symbol_Math) |
                     FLAG(Symbol_Currency) |
                     FLAG(Symbol_Modifier) |
                     FLAG(Symbol_Other);
    return FLAG(qGetProp(ucs)->category) & test;
}

int QChar::digitValue() const
{
    return qGetProp(ucs)->digitValue;
}

int QChar::digitValue(ushort ucs2)
{
    return qGetProp(ucs2)->digitValue;
}

int QChar::digitValue(uint ucs4)
{
    if (ucs4 > UNICODE_LAST_CODEPOINT)
        return 0;
    return qGetProp(ucs4)->digitValue;
}

QChar::Category QChar::category() const
{
    return (QChar::Category) qGetProp(ucs)->category;
}

QChar::Category QChar::category(uint ucs4)
{
    if (ucs4 > UNICODE_LAST_CODEPOINT)
        return QChar::NoCategory;
    return (QChar::Category) qGetProp(ucs4)->category;
}

QChar::Category QChar::category(ushort ucs2)
{
    return (QChar::Category) qGetProp(ucs2)->category;
}


QChar::Direction QChar::direction() const
{
    return (QChar::Direction) qGetProp(ucs)->direction;
}

QChar::Direction QChar::direction(uint ucs4)
{
    if (ucs4 > UNICODE_LAST_CODEPOINT)
        return QChar::DirL;
    return (QChar::Direction) qGetProp(ucs4)->direction;
}

QChar::Direction QChar::direction(ushort ucs2)
{
    return (QChar::Direction) qGetProp(ucs2)->direction;
}

QChar::Joining QChar::joining() const
{
    return (QChar::Joining) qGetProp(ucs)->joining;
}

QChar::Joining QChar::joining(uint ucs4)
{
    if (ucs4 > UNICODE_LAST_CODEPOINT)
        return QChar::OtherJoining;
    return (QChar::Joining) qGetProp(ucs4)->joining;
}

QChar::Joining QChar::joining(ushort ucs2)
{
    return (QChar::Joining) qGetProp(ucs2)->joining;
}


bool QChar::hasMirrored() const
{
    return qGetProp(ucs)->mirrorDiff != 0;
}

QChar QChar::mirroredChar() const
{
    return ucs + qGetProp(ucs)->mirrorDiff;
}

uint QChar::mirroredChar(uint ucs4)
{
    if (ucs4 > UNICODE_LAST_CODEPOINT)
        return ucs4;
    return ucs4 + qGetProp(ucs4)->mirrorDiff;
}

ushort QChar::mirroredChar(ushort ucs2)
{
    return ucs2 + qGetProp(ucs2)->mirrorDiff;
}


enum {
    Hangul_SBase = 0xac00,
    Hangul_LBase = 0x1100,
    Hangul_VBase = 0x1161,
    Hangul_TBase = 0x11a7,
    Hangul_SCount = 11172,
    Hangul_LCount = 19,
    Hangul_VCount = 21,
    Hangul_TCount = 28,
    Hangul_NCount = 21*28
};

// buffer has to have a length of 3. It's needed for Hangul decomposition
static const unsigned short * QT_FASTCALL decompositionHelper
    (uint ucs4, int *length, int *tag, unsigned short *buffer)
{
    *length = 0;
    if (ucs4 > UNICODE_LAST_CODEPOINT)
        return 0;
    if (ucs4 >= Hangul_SBase && ucs4 < Hangul_SBase + Hangul_SCount) {
        int SIndex = ucs4 - Hangul_SBase;
        buffer[0] = Hangul_LBase + SIndex / Hangul_NCount; // L
        buffer[1] = Hangul_VBase + (SIndex % Hangul_NCount) / Hangul_TCount; // V
        buffer[2] = Hangul_TBase + SIndex % Hangul_TCount; // T
        *length = buffer[2] == Hangul_TBase ? 2 : 3;
        *tag = QChar::Canonical;
        return buffer;
    }

    const unsigned short index = GET_DECOMPOSITION_INDEX(ucs4);
    if (index == 0xffff)
        return 0;
    const unsigned short *decomposition = uc_decomposition_map+index;
    *tag = (*decomposition) & 0xff;
    *length = (*decomposition) >> 8;
    return decomposition+1;
}

QString QChar::decomposition() const
{
    return decomposition(ucs);
}

QString QChar::decomposition(uint ucs4)
{
    unsigned short buffer[3];
    int length;
    int tag;
    const unsigned short *d = decompositionHelper(ucs4, &length, &tag, buffer);
    return QString::fromUtf16(d, length);
}

QChar::Decomposition QChar::decompositionTag() const
{
    return decompositionTag(ucs);
}

QChar::Decomposition QChar::decompositionTag(uint ucs4)
{
    if (ucs4 > UNICODE_LAST_CODEPOINT)
        return QChar::NoDecomposition;
    const unsigned short index = GET_DECOMPOSITION_INDEX(ucs4);
    if (index == 0xffff)
        return QChar::NoDecomposition;
    return (QChar::Decomposition)(uc_decomposition_map[index] & 0xff);
}

unsigned char QChar::combiningClass() const
{
    return (unsigned char) qGetProp(ucs)->combiningClass;
}

unsigned char QChar::combiningClass(uint ucs4)
{
    if (ucs4 > UNICODE_LAST_CODEPOINT)
        return 0;
    return (unsigned char) qGetProp(ucs4)->combiningClass;
}

unsigned char QChar::combiningClass(ushort ucs2)
{
    return (unsigned char) qGetProp(ucs2)->combiningClass;
}

QChar::UnicodeVersion QChar::unicodeVersion() const
{
    return (QChar::UnicodeVersion) qGetProp(ucs)->unicodeVersion;
}

QChar::UnicodeVersion QChar::unicodeVersion(uint ucs4)
{
    if (ucs4 > UNICODE_LAST_CODEPOINT)
        return QChar::Unicode_Unassigned;
    return (QChar::UnicodeVersion) qGetProp(ucs4)->unicodeVersion;
}

QChar::UnicodeVersion QChar::unicodeVersion(ushort ucs2)
{
    return (QChar::UnicodeVersion) qGetProp(ucs2)->unicodeVersion;
}

QChar::UnicodeVersion QChar::currentUnicodeVersion()
{
    return UNICODE_DATA_VERSION;
}

QChar QChar::toLower() const
{
    const QUnicodeTables::Properties *p = qGetProp(ucs);
    if (!p->lowerCaseSpecial)
        return ucs + p->lowerCaseDiff;
    return ucs;
}

uint QChar::toLower(uint ucs4)
{
    if (ucs4 > UNICODE_LAST_CODEPOINT)
        return ucs4;
    const QUnicodeTables::Properties *p = qGetProp(ucs4);
    if (!p->lowerCaseSpecial)
        return ucs4 + p->lowerCaseDiff;
    return ucs4;
}

ushort QChar::toLower(ushort ucs2)
{
    const QUnicodeTables::Properties *p = qGetProp(ucs2);
    if (!p->lowerCaseSpecial)
        return ucs2 + p->lowerCaseDiff;
    return ucs2;
}

QChar QChar::toUpper() const
{
    const QUnicodeTables::Properties *p = qGetProp(ucs);
    if (!p->upperCaseSpecial)
        return ucs + p->upperCaseDiff;
    return ucs;
}

uint QChar::toUpper(uint ucs4)
{
    if (ucs4 > UNICODE_LAST_CODEPOINT)
        return ucs4;
    const QUnicodeTables::Properties *p = qGetProp(ucs4);
    if (!p->upperCaseSpecial)
        return ucs4 + p->upperCaseDiff;
    return ucs4;
}

ushort QChar::toUpper(ushort ucs2)
{
    const QUnicodeTables::Properties *p = qGetProp(ucs2);
    if (!p->upperCaseSpecial)
        return ucs2 + p->upperCaseDiff;
    return ucs2;
}

QChar QChar::toTitleCase() const
{
    const QUnicodeTables::Properties *p = qGetProp(ucs);
    if (!p->titleCaseSpecial)
        return ucs + p->titleCaseDiff;
    return ucs;
}

uint QChar::toTitleCase(uint ucs4)
{
    if (ucs4 > UNICODE_LAST_CODEPOINT)
        return ucs4;
    const QUnicodeTables::Properties *p = qGetProp(ucs4);
    if (!p->titleCaseSpecial)
        return ucs4 + p->titleCaseDiff;
    return ucs4;
}

ushort QChar::toTitleCase(ushort ucs2)
{
    const QUnicodeTables::Properties *p = qGetProp(ucs2);
    if (!p->titleCaseSpecial)
        return ucs2 + p->titleCaseDiff;
    return ucs2;
}


static inline uint foldCase(const ushort *ch, const ushort *start)
{
    uint c = *ch;
    if (QChar(c).isLowSurrogate() && ch > start && QChar(*(ch - 1)).isHighSurrogate())
        c = QChar::surrogateToUcs4(*(ch - 1), c);
    return *ch + qGetProp(c)->caseFoldDiff;
}

static inline uint foldCase(uint ch, uint &last)
{
    uint c = ch;
    if (QChar(c).isLowSurrogate() && QChar(last).isHighSurrogate())
        c = QChar::surrogateToUcs4(last, c);
    last = ch;
    return ch + qGetProp(c)->caseFoldDiff;
}

static inline ushort foldCase(ushort ch)
{
    return ch + qGetProp(ch)->caseFoldDiff;
}

QChar QChar::toCaseFolded() const
{
    return ucs + qGetProp(ucs)->caseFoldDiff;
}

uint QChar::toCaseFolded(uint ucs4)
{
    if (ucs4 > UNICODE_LAST_CODEPOINT)
        return ucs4;
    return ucs4 + qGetProp(ucs4)->caseFoldDiff;
}

ushort QChar::toCaseFolded(ushort ucs2)
{
    return ucs2 + qGetProp(ucs2)->caseFoldDiff;
}


#ifdef Q_COMPILER_MANGLES_RETURN_TYPE
const char QChar::toAscii() const
#else
char QChar::toAscii() const
#endif
{
#ifndef QT_NO_CODEC_FOR_C_STRINGS
    if (QTextCodec::codecForCStrings())
        // #####
        return QTextCodec::codecForCStrings()->fromUnicode(QString(*this)).at(0);
#endif
    return ucs > 0xff ? 0 : char(ucs);
}

QChar QChar::fromAscii(char c)
{
#ifndef QT_NO_CODEC_FOR_C_STRINGS
    if (QTextCodec::codecForCStrings())
        // #####
        return QTextCodec::codecForCStrings()->toUnicode(&c, 1).at(0).unicode();
#endif
    return QChar(ushort((uchar)c));
}

#ifndef QT_NO_DATASTREAM

QDataStream &operator<<(QDataStream &out, const QChar &chr)
{
    out << quint16(chr.unicode());
    return out;
}

QDataStream &operator>>(QDataStream &in, QChar &chr)
{
    quint16 u;
    in >> u;
    chr.unicode() = ushort(u);
    return in;
}
#endif // QT_NO_DATASTREAM

/*****************************************************************************
  Documentation of QChar related functions
 *****************************************************************************/

// ---------------------------------------------------------------------------


static void decomposeHelper(QString *str, bool canonical, QChar::UnicodeVersion version, int from)
{
    unsigned short buffer[3];

    QString &s = *str;

    const unsigned short *utf16 = reinterpret_cast<unsigned short *>(s.data());
    const unsigned short *uc = utf16 + s.length();
    while (uc != utf16 + from) {
        uint ucs4 = *(--uc);
        if (QChar(ucs4).isLowSurrogate() && uc != utf16) {
            ushort high = *(uc - 1);
            if (QChar(high).isHighSurrogate()) {
                --uc;
                ucs4 = QChar::surrogateToUcs4(high, ucs4);
            }
        }
        QChar::UnicodeVersion v = QChar::unicodeVersion(ucs4);
        if (v == QChar::Unicode_Unassigned || v > version)
            continue;
        int length;
        int tag;
        const unsigned short *d = decompositionHelper(ucs4, &length, &tag, buffer);
        if (!d || (canonical && tag != QChar::Canonical))
            continue;

        int pos = uc - utf16;
        s.replace(pos, QChar::requiresSurrogates(ucs4) ? 2 : 1, reinterpret_cast<const QChar *>(d), length);
        // since the insert invalidates the pointers and we do decomposition recursive
        utf16 = reinterpret_cast<unsigned short *>(s.data());
        uc = utf16 + pos + length;
    }
}


struct UCS2Pair {
    ushort u1;
    ushort u2;
};

inline bool operator<(ushort u1, const UCS2Pair &ligature)
{ return u1 < ligature.u1; }
inline bool operator<(const UCS2Pair &ligature, ushort u1)
{ return ligature.u1 < u1; }

static ushort ligatureHelper(ushort u1, ushort u2)
{
    // hangul L-V pair
    int LIndex = u1 - Hangul_LBase;
    if (0 <= LIndex && LIndex < Hangul_LCount) {
        int VIndex = u2 - Hangul_VBase;
        if (0 <= VIndex && VIndex < Hangul_VCount)
            return Hangul_SBase + (LIndex * Hangul_VCount + VIndex) * Hangul_TCount;
    }

    // hangul LV-T pair
    int SIndex = u1 - Hangul_SBase;
    if (0 <= SIndex && SIndex < Hangul_SCount && (SIndex % Hangul_TCount) == 0) {
        int TIndex = u2 - Hangul_TBase;
        if (0 <= TIndex && TIndex <= Hangul_TCount)
            return u1 + TIndex;
    }

    const unsigned short index = GET_LIGATURE_INDEX(u2);
    if (index == 0xffff)
        return 0;
    const unsigned short *ligatures = uc_ligature_map+index;
    ushort length = *ligatures++;
    {
        const UCS2Pair *data = reinterpret_cast<const UCS2Pair *>(ligatures);
        const UCS2Pair *r = qBinaryFind(data, data + length, u1);
        if (r != data + length)
            return r->u2;
    }

    return 0;
}

static void composeHelper(QString *str, QChar::UnicodeVersion version, int from)
{
    QString &s = *str;

    if (from < 0 || s.length() - from < 2)
        return;

    // the loop can partly ignore high Unicode as all ligatures are in the BMP
    int starter = -2; // to prevent starter == pos - 1
    int lastCombining = 255; // to prevent combining > lastCombining
    int pos = from;
    while (pos < s.length()) {
        uint uc = s.at(pos).unicode();
        if (QChar(uc).isHighSurrogate() && pos < s.length()-1) {
            ushort low = s.at(pos+1).unicode();
            if (QChar(low).isLowSurrogate()) {
                uc = QChar::surrogateToUcs4(uc, low);
                ++pos;
            }
        }
        const QUnicodeTables::Properties *p = qGetProp(uc);
        if (p->unicodeVersion == QChar::Unicode_Unassigned || p->unicodeVersion > version) {
            starter = -1; // to prevent starter == pos - 1
            lastCombining = 255; // to prevent combining > lastCombining
            ++pos;
            continue;
        }
        int combining = p->combiningClass;
        if ((starter == pos - 1 || combining > lastCombining) && starter >= from) {
            // allowed to form ligature with S
            QChar ligature = ligatureHelper(s.at(starter).unicode(), uc);
            if (ligature.unicode()) {
                s[starter] = ligature;
                s.remove(pos, 1);
                continue;
            }
        }
        if (!combining)
            starter = pos;
        lastCombining = combining;
        ++pos;
    }
}


static void canonicalOrderHelper(QString *str, QChar::UnicodeVersion version, int from)
{
    QString &s = *str;
    const int l = s.length()-1;
    int pos = from;
    while (pos < l) {
        int p2 = pos+1;
        uint u1 = s.at(pos).unicode();
        if (QChar(u1).isHighSurrogate()) {
            ushort low = s.at(p2).unicode();
            if (QChar(low).isLowSurrogate()) {
                u1 = QChar::surrogateToUcs4(u1, low);
                if (p2 >= l)
                    break;
                ++p2;
            }
        }
        uint u2 = s.at(p2).unicode();
        if (QChar(u2).isHighSurrogate() && p2 < l) {
            ushort low = s.at(p2+1).unicode();
            if (QChar(low).isLowSurrogate()) {
                u2 = QChar::surrogateToUcs4(u2, low);
                ++p2;
            }
        }

        ushort c2 = 0;
        {
            const QUnicodeTables::Properties *p = qGetProp(u2);
            if (p->unicodeVersion != QChar::Unicode_Unassigned && p->unicodeVersion <= version)
                c2 = p->combiningClass;
        }
        if (c2 == 0) {
            pos = p2+1;
            continue;
        }

        ushort c1 = 0;
        {
            const QUnicodeTables::Properties *p = qGetProp(u1);
            if (p->unicodeVersion != QChar::Unicode_Unassigned && p->unicodeVersion <= version)
                c1 = p->combiningClass;
        }

        if (c1 > c2) {
            QChar *uc = s.data();
            int p = pos;
            // exchange characters
            if (!QChar::requiresSurrogates(u2)) {
                uc[p++] = u2;
            } else {
                uc[p++] = QChar::highSurrogate(u2);
                uc[p++] = QChar::lowSurrogate(u2);
            }
            if (!QChar::requiresSurrogates(u1)) {
                uc[p++] = u1;
            } else {
                uc[p++] = QChar::highSurrogate(u1);
                uc[p++] = QChar::lowSurrogate(u1);
            }
            if (pos > 0)
                --pos;
            if (pos > 0 && s.at(pos).isLowSurrogate())
                --pos;
        } else {
            ++pos;
            if (QChar::requiresSurrogates(u1))
                ++pos;
        }
    }
}

QT_END_NAMESPACE