qxquerytokenizer.cpp

Go to the documentation of this file.

/****************************************************************************
**
** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies).
** Contact: http://www.qt-project.org/legal
**
** This file is part of the QtXmlPatterns module of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Digia.  For licensing terms and
** conditions see http://qt.digia.com/licensing.  For further information
** use the contact form at http://qt.digia.com/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Digia gives you certain additional
** rights.  These rights are described in the Digia Qt LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
**
** $QT_END_LICENSE$
**
****************************************************************************/

#include <QByteArray>

#include "qquerytransformparser_p.h"

#include "qxquerytokenizer_p.h"

#include "qtokenlookup.cpp"

QT_BEGIN_NAMESPACE

namespace QPatternist
{

#define handleWhitespace()                      \
{                                               \
    const TokenType t = consumeWhitespace();    \
    if(t != SUCCESS)                            \
        return Token(t);                        \
}

XQueryTokenizer::XQueryTokenizer(const QString &query,
                                 const QUrl &location,
                                 const State startingState) : Tokenizer(location)
                                                            , m_data(query)
                                                            , m_length(query.length())
                                                            , m_state(startingState)
                                                            , m_pos(0)
                                                            , m_line(1)
                                                            , m_columnOffset(0)
                                                            , m_scanOnly(false)
{
    Q_ASSERT(location.isValid() || location.isEmpty());
}

const QChar XQueryTokenizer::current() const
{
    if(m_pos < m_length)
        return m_data.at(m_pos);
    else
        return QChar();
}

char XQueryTokenizer::peekCurrent() const
{
    return current().toAscii();
}

int XQueryTokenizer::peekForColonColon() const
{
    /* Note, we don't modify m_pos in this function, so we need to do offset
     * calculations. */
    int pos = m_pos;

    while(pos < m_length)
    {
        switch(m_data.at(pos).toAscii())
        {
            /* Fallthrough these four. */
            case ' ':
            case '\t':
            case '\n':
            case '\r':
                break;
            case ':':
            {
                if(peekAhead((pos - m_pos) + 1) == ':')
                    return pos - m_pos;
                /* Fallthrough. */
            }
            default:
                return -1;
        }
        ++pos;
    }

    return -1;
}

Tokenizer::Token XQueryTokenizer::tokenAndChangeState(const TokenType code,
                                                      const State s,
                                                      const int advance)
{
    Q_ASSERT(advance >= 0);
    m_pos += advance;
    setState(s);
    return Token(code);
}

Tokenizer::Token XQueryTokenizer::tokenAndChangeState(const TokenType code,
                                                      const QString &value,
                                                      const State s)
{
    setState(s);
    return Token(code, value);
}

Tokenizer::Token XQueryTokenizer::tokenAndAdvance(const TokenType code,
                                                  const int advance)
{
    Q_ASSERT(advance >= 0);
    m_pos += advance;
    return Token(code);
}

QString XQueryTokenizer::normalizeEOL(const QString &input,
                                      const CharacterSkips &characterSkips)
{
    const int len = input.count();
    QString result;

    /* The likely hood is rather high it'll be the same content. */
    result.reserve(len);

    for(int i = 0; i < len; ++i)
    {
        const QChar &at = input.at(i);

        if(characterSkips.contains(i))
        {
            result.append(at);
            continue;
        }
        switch(input.at(i).unicode())
        {
            case '\r':
            {
                if(i + 1 < len && input.at(i + 1) == QLatin1Char('\n'))
                    ++i;

                /* Else, fallthrough. */
            }
            case '\n':
            {
                result.append(QLatin1Char('\n'));
                continue;
            }
            default:
            {
                result.append(at);
            }
        }
    }

    return result;
}

Tokenizer::TokenType XQueryTokenizer::consumeComment()
{
    /* Below, we return ERROR instead of END_OF_FILE such that the parser
     * sees an invalid comment. */
    while(m_pos < m_length)
    {
        switch(peekCurrent())
        {
            case ':':
            {
                ++m_pos; /* Consume ':' */
                if(atEnd())
                    return ERROR;

                if(peekCurrent() == ')')
                {
                    ++m_pos; /* Consume ')' */
                    return SUCCESS; /* The comment closed nicely. */
                }
                continue; /* We don't want to increment m_pos twice. */
            }
            case '(':
            { /* It looks like the start of a comment. */
                ++m_pos;

                if(atEnd())
                    return END_OF_FILE;
                else if(peekCurrent() == ':')
                {
                    /* And it is a nested comment -- parse it. */
                    const TokenType retval = consumeComment();
                    if(retval == SUCCESS)
                        continue; /* Continue with our "own" comment. */
                    else
                        return retval; /* Return the error in the nested comment. */
                }
                break;
            }
            case '\n':
            /* Fallthrough. */
            case '\r':
            {
                /* We want to count \r\n as a single line break. */
                if(peekAhead() == '\n')
                    ++m_pos;

                m_columnOffset = m_pos;
                ++m_line;

                break;
            }
        }
        ++m_pos;
    }

    return ERROR; /* Error: we reached the end while inside a comment. */
}

bool XQueryTokenizer::consumeRawWhitespace()
{
    while(m_pos < m_length)
    {
        switch(peekCurrent())
        {
            case ' ':
            case '\t':
                break;
            case '\n':
            case '\r':
            {
                if(peekAhead() == '\n')
                    ++m_pos;

                m_columnOffset = m_pos;
                ++m_line;

                break;
            }
            default:
                return false;
        }
        ++m_pos;
    }
    return true;
}

Tokenizer::TokenType XQueryTokenizer::consumeWhitespace()
{
    while(m_pos < m_length)
    {
        switch(peekCurrent())
        {
            case ' ':
            case '\t':
                break;
            case '\n':
            case '\r':
            {
                /* We want to count \r\n as a single line break. */
                if(peekAhead() == '\n')
                    ++m_pos;

                m_columnOffset = m_pos;
                ++m_line;

                break;
            }
            case '(':
            {
                if(peekAhead() == ':')
                {
                    m_pos += 2; /* Consume "(:" */

                    const TokenType comment = consumeComment();
                    if(comment == SUCCESS)
                        continue;
                    else
                        return comment;
                }
            }
            default:
                return SUCCESS;
        }
        ++m_pos;
    }

    return END_OF_FILE;
}

char XQueryTokenizer::peekAhead(const int length) const
{
    if(m_pos + length < m_length)
        return m_data.at(m_pos + length).toAscii();
    else
        return 0;
}

Tokenizer::Token XQueryTokenizer::error()
{
    return Token(ERROR);
}

bool XQueryTokenizer::isDigit(const char ch)
{
    return ch >= '0' && ch <= '9';
}

/* Replace with function in QXmlUtils. Write test cases for this. */
bool XQueryTokenizer::isNCNameStart(const QChar ch)
{
    if(ch == QLatin1Char('_'))
        return true;

    switch(ch.category())
    {
        case QChar::Letter_Lowercase:
        case QChar::Letter_Uppercase:
        case QChar::Letter_Other:
        case QChar::Letter_Titlecase:
        case QChar::Number_Letter:
            return true;
        default:
            return false;
    }
}

bool XQueryTokenizer::isNCNameBody(const QChar ch)
{
    switch(ch.unicode())
    {
        case '.':
        case '_':
        case '-':
            return true;
    }

    switch(ch.category())
    {
        case QChar::Letter_Lowercase:
        case QChar::Letter_Uppercase:
        case QChar::Letter_Other:
        case QChar::Letter_Titlecase:
        case QChar::Number_Letter:
        case QChar::Mark_SpacingCombining:
        case QChar::Mark_Enclosing:
        case QChar::Mark_NonSpacing:
        case QChar::Letter_Modifier:
        case QChar::Number_DecimalDigit:
            return true;
        default:
            return false;
    }
}

bool XQueryTokenizer::isPhraseKeyword(const TokenType code)
{
    switch(code)
    {
        /* Fallthrough all these. */
        case CASTABLE:
        case CAST:
        case COPY_NAMESPACES:
        case DECLARE:
        case EMPTY:
        case MODULE:
        case IMPORT:
        case INSTANCE:
        case ORDER:
        case ORDERING:
        case XQUERY:
        case STABLE:
        case TREAT:
            return true;
        default:
            return false;
    }
}

bool XQueryTokenizer::isOperatorKeyword(const TokenType code)
{
    switch(code)
    {
        /* Fallthrough all these. */
        case AS:
        case ASCENDING:
        case AT:
        case CASE:
        case CAST:
        case CASTABLE:
        case EQ:
        case EXTERNAL:
        case GE:
        case G_EQ:
        case G_GT:
        case G_LT:
        case G_NE:
        case GT:
        case IN:
        case INHERIT:
        case INSTANCE:
        case IS:
        case ITEM:
        case LE:
        case LT:
        case NE:
        case NO_INHERIT:
        case NO_PRESERVE:
        case OF:
        case PRESERVE:
        case RETURN:
        case STABLE:
        case TO:
        case TREAT:
            return true;
        default:
            return false;
    };
}

bool XQueryTokenizer::isTypeToken(const TokenType t)
{
    switch(t)
    {
        /* Fallthrough all these. */
        case ATTRIBUTE:
        case COMMENT:
        case DOCUMENT:
        case DOCUMENT_NODE:
        case ELEMENT:
        case ITEM:
        case NODE:
        case PROCESSING_INSTRUCTION:
        case SCHEMA_ATTRIBUTE:
        case SCHEMA_ELEMENT:
        case TEXT:
            return true;
        default:
            return false;
    }
}

Tokenizer::Token XQueryTokenizer::tokenizeNCNameOrQName()
{
    const int start = m_pos;

    const Token t1 = tokenizeNCName();
    if(t1.hasError())
        return t1;

    if(peekCurrent() != ':' || peekAhead() == '=')
        return t1;

    ++m_pos;

    const Token t2 = tokenizeNCName();
    if(t2.hasError())
        return t2;
    else
        return Token(QNAME, m_data.mid(start, m_pos - start));
}

Tokenizer::Token XQueryTokenizer::tokenizeNumberLiteral()
{
    setState(Operator);
    const int startPos = m_pos;
    bool hasDot = false;
    bool isXPath20 = false;

    for(; m_pos < m_length; ++m_pos)
    {
        QChar ch(current());

        char cell = ch.cell();

        if(cell == 'e' || cell == 'E')
        {
            isXPath20 = true;
            ++m_pos;
            ch = current();

            if(ch.row() != 0)
                break;

            cell = ch.cell();

            if(cell == '+' || cell == '-')
                continue;
        }

        if(isNCNameStart(ch))
            return error();

        if(cell < '0' || cell > '9')
        {
            if(cell == '.' && !hasDot)
                hasDot = true;
            else
                break;
        }
    }

    return Token(isXPath20 ? XPATH2_NUMBER : NUMBER, m_data.mid(startPos, m_pos - startPos));
}

QString XQueryTokenizer::tokenizeCharacterReference()
{
    Q_ASSERT(peekCurrent() == '&');

    const int theEnd = m_data.indexOf(QLatin1Char(';'), m_pos + 1);

    if(theEnd == -1) /* No ';' found, a syntax error. i18n. */
        return QString();

    QString content(m_data.mid(m_pos + 1, (theEnd - m_pos) - 1));
    m_pos = theEnd;

    const QChar charRef(charForReference(content));

    if(!charRef.isNull())
        return charRef;
    else if(content.startsWith(QLatin1Char('#')))
    {
        int base;

        /* It is only '#' or '#x'. */
        if(content.length() < 2)
            return QString();

        /* We got a hex number if it starts with 'x', otherwise it's a decimal. */
        if(content.at(1) == QLatin1Char('x'))
        {
            base = 16;
            content = content.mid(2); /* Remove "#x". */
        }
        else
        {
            base = 10;
            content = content.mid(1); /* Remove "#". */
        }

        bool conversionOK = false;
        const int codepoint = content.toInt(&conversionOK, base);

        if(conversionOK)
        {
            const QChar ch(codepoint);

            if(ch.isNull())
            {
                /* We likely have something which require surrogate pairs. */
                QString result;
                result += QChar(QChar::highSurrogate(codepoint));
                result += QChar(QChar::lowSurrogate(codepoint));
                return result;
            }
            else
                return ch;
        }
        else
            return QString();
    }
    else
        return QString();
}

int XQueryTokenizer::scanUntil(const char *const content)
{
    const int end = m_data.indexOf(QString::fromLatin1(content), m_pos);

    if(end == -1)
        return -1;
    else
    {
        const int len = end - m_pos;
        m_pos += len;
        return len;
    }
}

QChar XQueryTokenizer::charForReference(const QString &reference)
{
    if(m_charRefs.isEmpty())
    {
        /* Initialize. */
        m_charRefs.reserve(5);
        m_charRefs.insert(QLatin1String("lt"),     QLatin1Char('<'));
        m_charRefs.insert(QLatin1String("gt"),     QLatin1Char('>'));
        m_charRefs.insert(QLatin1String("amp"),    QLatin1Char('&'));
        m_charRefs.insert(QLatin1String("quot"),   QLatin1Char('"'));
        m_charRefs.insert(QLatin1String("apos"),   QLatin1Char('\''));
    }

    return m_charRefs.value(reference);
}

Tokenizer::Token XQueryTokenizer::tokenizeStringLiteral()
{
    const QChar delimiter(current());
    /* We cannot unfortunately just scan and then do mid(),
     * since we can encounter character references. */
    QString result;

    /* This is more likely than QString's default allocation. */
    result.reserve(8);

    CharacterSkips skipEOLNormalization;

    /* Advance over the initial quote character. */
    ++m_pos;

    for(; m_pos < m_length; ++m_pos)
    {
        const QChar c(current());

        if(c == QLatin1Char('&'))
        {
            const QString charRef(tokenizeCharacterReference());

            if(charRef.isNull())
                return error();
            else
            {
                skipEOLNormalization.insert(result.count());
                result.append(charRef);
            }

        }
        else if(c == delimiter)
        {
            /* Maybe the escaping mechanism is used. For instance, "s""s"
             * has the value `s"s'. */
            ++m_pos;

            if(current() == delimiter) /* Double quote. */
                result += delimiter;
            else
                return Token(STRING_LITERAL, normalizeEOL(result, skipEOLNormalization));
        }
        else
            result += c;
    }

    return error();
}

Tokenizer::Token XQueryTokenizer::tokenizeNCName()
{
    const int startPos = m_pos;

    if(m_pos < m_length && isNCNameStart(current()))
    {
        ++m_pos;

        for(; m_pos < m_length; ++m_pos)
        {
            if(!isNCNameBody(current()))
                break;
        }

        return Token(NCNAME, m_data.mid(startPos, m_pos - startPos));
    }
    else
        return error();
}

bool XQueryTokenizer::aheadEquals(const char *const chs,
                                  const int len,
                                  const int offset) const
{
    Q_ASSERT(len > 0);
    Q_ASSERT(qstrlen(chs) == uint(len));

    if(m_pos + len >= m_length)
        return false;

    for(int i = offset; i < (len + offset); ++i)
    {
        if(m_data.at(m_pos + i).toAscii() != chs[i - offset])
            return false;
    }

    return true;
}

const TokenMap *XQueryTokenizer::lookupKeyword(const QString &keyword)
{
    return TokenLookup::value(keyword.toAscii().constData(), keyword.length());
}

XQueryTokenizer::State XQueryTokenizer::state() const
{
    return m_state;
}

void XQueryTokenizer::setState(const State s)
{
    m_state = s;
}

void XQueryTokenizer::pushState(const State s)
{
    m_stateStack.push(s);
}

void XQueryTokenizer::pushState()
{
    m_stateStack.push(m_state);
}

void XQueryTokenizer::popState()
{
    /* QStack::pop() asserts if it's empty, so we need to check
     * it, since we might receive unbalanced curlies. */
    if(!m_stateStack.isEmpty())
        m_state = m_stateStack.pop();
}

Tokenizer::Token XQueryTokenizer::nextToken()
{
    switch(state())
    {
        /* We want to skip or do special whitespace handling for these
         * states. So fallthrough all of the following. */
        case AposAttributeContent:
        case Axis:
        case ElementContent:
        case EndTag:
        case Pragma:
        case PragmaContent:
        case ProcessingInstructionName:
        case QuotAttributeContent:
        case StartTag:
        case XMLComment:
            break;
        default:
            handleWhitespace();
    }

    switch(state())
    {
        case XMLSpaceDecl:
        /* Fallthrough. */
        case NamespaceKeyword:
        {
            switch(peekCurrent())
            {
                case ',':
                    return tokenAndAdvance(COMMA);
                case '"':
                /* Fallthrough. */
                case '\'':
                {
                    setState(NamespaceDecl);
                    return tokenizeStringLiteral();
                }
            }

            const Token id(tokenizeNCName());

            if(id.type != NCNAME)
                return id;

            const TokenMap *const keyword = lookupKeyword(id.value);
            if(keyword)
            {
                switch(keyword->token)
                {
                    case INHERIT:
                    /* Fallthrough. */
                    case NO_INHERIT:
                    {
                        setState(Default);
                        break;
                    }
                    case NAMESPACE:
                    {
                        setState(NamespaceDecl);
                        break;
                    }
                    case ORDERED:
                    /* Fallthrough. */
                    case UNORDERED:
                    /* Fallthrough. */
                    case STRIP:
                    {
                        setState(Default);
                        break;
                    }
                    case PRESERVE:
                    {
                        if(state() != NamespaceKeyword)
                            setState(Default);
                    }
                    default:
                        break;
                }

                return Token(keyword->token);
            }
            else
                return id;

            Q_ASSERT(false);
        }
        case NamespaceDecl:
        {
            switch(peekCurrent())
            {
                case '=':
                    return tokenAndAdvance(G_EQ);
                case ';':
                    return tokenAndChangeState(SEMI_COLON, Default);
                case '\'':
                /* Fallthrough. */
                case '\"':
                    return tokenizeStringLiteral();
            }

            const Token nc(tokenizeNCName());

            handleWhitespace();

            const char pc = peekCurrent();
            const TokenMap* const t = lookupKeyword(nc.value);

            if(pc == '\'' || (pc == '"' && t))
                return tokenAndChangeState(t->token, Default, 0);
            else
                return nc;

            Q_ASSERT(false);
        }
        case Axis:
        {
            if(peekCurrent() == ':')
            {
                Q_ASSERT(peekAhead() == ':');
                m_pos += 2;
                setState(AfterAxisSeparator);
                return Token(COLONCOLON);
            }
            /* Fallthrough. */
        }
        case AfterAxisSeparator:
        /* Fallthrough. */
        case Default:
           /* State Operator and state Default have a lot of tokens in common except
            * for minor differences. So we treat them the same way, and sprinkles logic
            * here and there to handle the small differences. */
        /* Fallthrough. */
        case Operator:
        {
            switch(peekCurrent())
            {
                case '=':
                    return tokenAndChangeState(G_EQ, Default);
                case '-':
                    return tokenAndChangeState(MINUS, Default);
                case '+':
                    return tokenAndChangeState(PLUS, Default);
                case '[':
                    return tokenAndChangeState(LBRACKET, Default);
                case ']':
                    return tokenAndChangeState(RBRACKET, Operator);
                case ',':
                    return tokenAndChangeState(COMMA, Default);
                case ';':
                    return tokenAndChangeState(SEMI_COLON, Default);
                case '$':
                    return tokenAndChangeState(DOLLAR, VarName);
                case '|':
                    return tokenAndChangeState(BAR, Default);
                case '?':
                    return tokenAndChangeState(QUESTION, Operator);
                case ')':
                    return tokenAndChangeState(RPAREN, Operator);
                case '@':
                    return tokenAndChangeState(AT_SIGN, Default);
                /* Fallthrough all these. */
                case '1':
                case '2':
                case '3':
                case '4':
                case '5':
                case '6':
                case '7':
                case '8':
                case '9':
                case '0':
                    return tokenizeNumberLiteral();
                case '.':
                {
                    const char next = peekAhead();
                    if(next == '.')
                        return tokenAndChangeState(DOTDOT, Operator, 2);
                    /* .5 is allowed, as short form for 0.5:
                     * <tt>[142]     DecimalLiteral     ::=     ("." Digits) | (Digits "." [0-9]*)</tt>
                     */
                    else if(isDigit(next))
                        return tokenizeNumberLiteral();
                    else
                        return tokenAndChangeState(DOT, Operator);
                }
                case '\'':
                /* Fallthrough. */
                case '"':
                {
                    setState(Operator);
                    return tokenizeStringLiteral();

                }
                case '(':
                {
                    if(peekAhead() == '#')
                        return tokenAndChangeState(PRAGMA_START, Pragma, 2);
                    else
                        return tokenAndChangeState(LPAREN, Default);
                }
                case '*':
                {
                    if(peekAhead() == ':')
                    {
                        m_pos += 2; /* Consume *:. */
                        const Token nc = tokenizeNCName();

                        if(nc.hasError())
                            return error();
                        else
                            return tokenAndChangeState(ANY_PREFIX, nc.value, Operator);
                    }
                    else
                        return tokenAndChangeState(STAR, state() == Default ? Operator : Default);
                }
                case ':':
                {
                    switch(peekAhead())
                    {
                        case '=':
                            return tokenAndChangeState(ASSIGN, Default, 2);
                        case ':':
                            return tokenAndChangeState(COLONCOLON, Default, 2);
                        default:
                            return error();
                    }
                }
                case '!':
                {
                    if(peekAhead() == '=')
                        return tokenAndChangeState(G_NE, Default, 2);
                    else
                        return error();
                }
                case '<':
                {
                    switch(peekAhead())
                    {
                        case '=':
                            return tokenAndChangeState(G_LE, Default, 2);
                        case '<':
                            return tokenAndChangeState(PRECEDES, Default, 2);
                        case '?':
                        {
                            pushState(Operator);
                            return tokenAndChangeState(PI_START, ProcessingInstructionName, 2);
                        }
                        case '!':
                        {
                            if(aheadEquals("!--", 3))
                            {
                                m_pos += 3; /* Consume "!--". */
                                pushState(Operator);
                                return tokenAndChangeState(COMMENT_START, XMLComment);
                            }
                            /* Fallthrough. It's a syntax error, and this is a good way to report it. */
                        }
                        default:
                        {
                            if((m_pos + 1) < m_length && isNCNameStart(m_data.at(m_pos + 1)))
                            {
                                /* We assume it's an element constructor. */
                                pushState(Operator);
                            }

                            return tokenAndChangeState(G_LT, state() == Operator ? Default : StartTag);
                        }
                    }
                }
                case '>':
                {
                    switch(peekAhead())
                    {
                        case '=':
                            return tokenAndChangeState(G_GE, Default, 2);
                        case '>':
                            return tokenAndChangeState(FOLLOWS, Default, 2);
                        default:
                            return tokenAndChangeState(G_GT, Default);
                    }
                }
                case '/':
                {
                    if(peekAhead() == '/')
                        return tokenAndChangeState(SLASHSLASH, Default, 2);
                    else
                        return tokenAndChangeState(SLASH, Default);
                }
                case '{':
                {
                    pushState(Operator);
                    return tokenAndChangeState(CURLY_LBRACE, Default);
                }
                case '}':
                {
                    popState();

                    return tokenAndAdvance(CURLY_RBRACE);
                }
            }

            /* Ok. We're in state Default or Operator, and it wasn't a simple
             * character. */

            const Token id(tokenizeNCName());

            if(id.type != NCNAME)
                return id;

            const TokenMap *const keyword = lookupKeyword(id.value);

            if(state() == Operator)
            {
                if(keyword)
                {
                    if(keyword->token == DEFAULT || keyword->token == ASCENDING || keyword->token == DESCENDING)
                        setState(Operator);
                    else if(keyword->token == RETURN)
                        setState(Default);
                    else if(isPhraseKeyword(keyword->token))
                    {
                        const TokenType ws = consumeWhitespace();
                        if(ws == ERROR)
                            return error();

                        const Token id2(tokenizeNCName());
                        const TokenMap *const keyword2 = lookupKeyword(id2.value);

                        if(keyword2)
                        {
                            if(keyword->token == TREAT && keyword2->token == AS)
                                setState(ItemType);
                            else if (keyword->token == CAST || (keyword->token == CASTABLE && keyword2->token == AS) || keyword2->token == BY)
                                setState(Default);

                            m_tokenStack.push(Token(keyword2->token));
                        }
                        else
                            m_tokenStack.push(id2);

                        return Token(keyword->token);
                    }
                    else
                    {
                        /* Such that we tokenize the second token in "empty greatest". */
                        if(keyword->token != EMPTY)
                            setState(Default);
                    }

                    if(keyword->token == AS || keyword->token == CASE)
                        setState(ItemType);

                    return Token(keyword->token);
                }
                else
                    return id;
            }

            Q_ASSERT(state() == Default || state() == Axis || state() == AfterAxisSeparator);

            /*
             * This is hard. Consider this:
             *
             * Valid:           child       ::nameTest
             * Valid:           child::     nameTest
             * Syntax Error:    child       :localName
             * Syntax Error:    child:      localName
             *
             * Consider "child ::name". Right now, we're here:
             *                ^
             * We don't know whether "child" is a prefix and hence the whitespace is invalid,
             * or whether it's an axis and hence skippable. */
            {
                const int wsLength = peekForColonColon();
                /* We cannot call handleWhitespace() because it returns on
                 * END_OF_FILE, and we have parsed up keyword, and we need to
                 * deal with that.
                 *
                 * If we have a colon colon, which means the whitespace is
                 * allowed, we skip it. */
                if(wsLength != -1)
                    m_pos += wsLength;
            }

            /* Handle name tests. */
            if(peekCurrent() == ':')
            {
                switch(peekAhead())
                {
                    case '=':
                        return id;
                    case '*':
                    {
                        m_pos += 2;
                        return tokenAndChangeState(ANY_LOCAL_NAME, id.value, Operator);
                    }
                    case ':':
                    {
                        /* We have an axis. */
                        setState(Axis);
                        return keyword ? Token(keyword->token) : id;
                    }
                    default:
                    {
                        /* It's a QName. */
                        ++m_pos; /* Consume the colon. */

                        const Token id2(tokenizeNCName());

                        if(id2.type != NCNAME)
                        {
                            --m_pos;
                            return id;
                        }

                        setState(Operator);
                        const int qNameLen = id.value.length() + id2.value.length() + 1;
                        return Token(QNAME, m_data.mid(m_pos - qNameLen, qNameLen));
                    }
                }
            }

            if(!keyword || isOperatorKeyword(keyword->token))
            {
                setState(Operator);
                return id;
            }

            const TokenType ws = consumeWhitespace();
            if(ws == ERROR) // TODO this should test for success. Write test.
                return Token(ERROR);

            if(atEnd())
            {
                setState(Operator);
                return id;
            }

            /* Let the if-body apply for constructors, and node type tests. */
            if(isTypeToken(keyword->token) ||
               keyword->token == TYPESWITCH ||
               keyword->token == ORDERED ||
               keyword->token == UNORDERED ||
               keyword->token == IF)
            {
                switch(peekCurrent())
                {
                    case '(':
                    {
                        // TODO See if we can remove DOCUMENT from isTypeToken.
                        if(isTypeToken(keyword->token) && keyword->token != DOCUMENT)
                        {
                            m_tokenStack.push(Token(LPAREN));
                            ++m_pos; /* Consume '('. */
                            pushState(Operator);

                            if(keyword->token == PROCESSING_INSTRUCTION)
                                setState(KindTestForPI);
                            else
                                setState(KindTest);

                            return Token(keyword->token);
                        }
                        else if(keyword->token == TYPESWITCH || keyword->token == IF)
                            return Token(keyword->token);
                        else /* It's a function call. */
                            return id;
                    }
                    case '{':
                    {
                        m_tokenStack.push(Token(CURLY_LBRACE));
                        ++m_pos; /* Consume '{'. */
                        pushState(Operator);
                        /* Stay in state Default. */
                        return Token(keyword->token);
                    }
                    default:
                    {
                        /* We have read in a token which is for instance
                         * "return", and now it can be an element
                         * test("element") a node kind test("element()"), or a
                         * computed element constructor("element name {...").
                         * We need to do a two-token lookahead here, because
                         * "element return" can be an element test followed by
                         * the return keyword, but it can also be an element
                         * constructor("element return {"). */
                        if(isNCNameStart(current()))
                        {
                            const int currentPos = m_pos;
                            const Token token2 = tokenizeNCNameOrQName();

                            if(token2.hasError())
                                return token2;

                            handleWhitespace();

                            if(peekCurrent() == '{')
                            {
                                /* An element constructor. */
                                m_tokenStack.push(token2);
                                return Token(keyword->token);
                            }

                            /* We jump back in the stream, we need to tokenize token2 according
                             * to the state. */
                            m_pos = currentPos;
                            setState(Operator);
                            return Token(NCNAME, QLatin1String(keyword->name));
                        }
                    }
                }
            }

            if(peekCurrent() == '$')
            {
                setState(VarName);
                return Token(keyword->token);
            }

            /* It's not a node type, it's not the typeswitch expression, but it is a function callsite. */
            if(peekCurrent() == '(')
                return id;
            else if(peekCurrent() == '{' && keyword->token == VALIDATE)
                return Token(keyword->token);

            if(!isNCNameStart(current()))
            {
                setState(Operator);
                return id;
            }

            const Token id2(tokenizeNCName());
            const TokenMap *const keyword2 = lookupKeyword(id2.value);

            if(!keyword2)
            {
                /* It's a syntax error. All cases of two subsequent ncnames are keywords(e.g, declarations). */
                setState(Operator);
                return id;
            }

            switch(keyword->token)
            {
                case DECLARE:
                {
                    switch(keyword2->token)
                    {
                        case VARIABLE:
                        /* Fallthrough. */
                        case FUNCTION:
                        {
                            m_tokenStack.push(Token(keyword2->token));
                            setState(Default);
                            return Token(keyword->token);
                        }
                        case OPTION:
                        {
                            m_tokenStack.push(Token(keyword2->token));
                            setState(Default);
                            return Token(keyword->token);
                        }
                        case COPY_NAMESPACES:
                        /* Fallthrough. */
                        case ORDERING:
                        {
                            m_tokenStack.push(Token(keyword2->token));
                            setState(NamespaceKeyword);
                            return Token(keyword->token);
                        }
                        case CONSTRUCTION:
                        {
                            // TODO identical to CONSTRUCTION?
                            m_tokenStack.push(Token(keyword2->token));
                            setState(Operator);
                            return Token(keyword->token);
                        }
                        case NAMESPACE:
                        /* Fallthrough. */
                        case BASEURI:
                        {
                            m_tokenStack.push(Token(keyword2->token));
                            setState(NamespaceDecl);
                            return Token(keyword->token);
                        }
                        case BOUNDARY_SPACE:
                        {
                            m_tokenStack.push(Token(keyword2->token));
                            setState(XMLSpaceDecl);
                            return Token(keyword->token);
                        }
                        case DEFAULT:
                        {
                            m_tokenStack.push(Token(keyword2->token));

                            const TokenType ws2 = consumeWhitespace();
                            if(ws2 != SUCCESS)
                            {
                                m_tokenStack.prepend(Token(ws2));
                                return Token(keyword->token);
                            }

                            const Token id3(tokenizeNCName());

                            if(id3.type != NCNAME)
                            {
                                m_tokenStack.prepend(id3);
                                return Token(keyword->token);
                            }

                            const TokenMap *const keyword3 = lookupKeyword(id3.value);
                            if(!keyword3)
                            {
                                m_tokenStack.prepend(id3);
                                return Token(keyword->token);
                            }
                            else
                            {
                                m_tokenStack.prepend(Token(keyword3->token));

                                if(keyword3->token == ORDER)
                                    setState(Operator);
                                else
                                    setState(NamespaceDecl);
                            }

                            return Token(keyword->token);
                        }
                        default:
                        {
                            m_tokenStack.push(Token(keyword2->token));
                            setState(Default);
                            return id;
                        }
                    }
                }
                case XQUERY:
                {
                    m_tokenStack.push(Token(keyword2->token));

                    if(keyword2->token == VERSION)
                    {
                        setState(NamespaceDecl);
                        return Token(keyword->token);
                    }
                    else
                    {
                        setState(Operator);
                        return id;
                    }
                }
                case IMPORT:
                {
                    m_tokenStack.push(Token(keyword2->token));

                    switch(keyword2->token)
                    {
                        case SCHEMA:
                        /* Fallthrough. */
                        case MODULE:
                        {
                            setState(NamespaceKeyword);
                            return Token(keyword->token);
                        }
                        default:
                        {
                            setState(Operator);
                            return id;
                        }
                    }
                }
                case VALIDATE:
                {
                    m_tokenStack.push(Token(keyword2->token));

                    switch(keyword2->token)
                    {
                        case LAX:
                        case STRICT:
                        {
                            pushState(Operator);
                            return Token(keyword->token);
                        }
                        default:
                        {
                            setState(Operator);
                            return id;
                        }
                    }
                }
                default:
                {
                    m_tokenStack.push(Token(keyword2->token));
                    setState(Operator);
                    return id;
                }
            }

            Q_ASSERT(false);

        }
        case VarName:
        {
            if(peekCurrent() == '$')
                return tokenAndAdvance(DOLLAR);

            setState(Operator);
            return tokenizeNCNameOrQName();
            Q_ASSERT(false);
        }
        case ItemType:
        {
            switch(peekCurrent())
            {
                case '(':
                    return tokenAndChangeState(LPAREN, KindTest);
                case '$':
                    return tokenAndChangeState(DOLLAR, VarName);
            }

            const Token name(tokenizeNCNameOrQName());

            if(name.hasError())
                return error();

            else if(name.type == QNAME)
            {
                setState(OccurrenceIndicator);
                return name;
            }
            else
            {
                const TokenMap *const keyword = lookupKeyword(name.value);

                if(keyword)
                {
                    pushState(OccurrenceIndicator);
                    return Token(keyword->token);
                }
                else
                {
                    setState(Default);
                    return name;
                }
            }
            Q_ASSERT(false);
        }
        case KindTest:
        {
            switch(peekCurrent())
            {
                case ')':
                {
                    popState();
                    return tokenAndAdvance(RPAREN);
                }
                case '(':
                    return tokenAndAdvance(LPAREN);
                case ',':
                    return tokenAndAdvance(COMMA);
                case '*':
                    return tokenAndAdvance(STAR);
                case '?':
                    return tokenAndAdvance(QUESTION);
                case '\'':
                /* Fallthrough. */
                case '"':
                    return tokenizeStringLiteral();
            }

            const Token nc(tokenizeNCNameOrQName());
            if(nc.hasError())
                return nc;

            const TokenType ws = consumeWhitespace();
            if(ws == ERROR)
                return error();

            if(peekCurrent() == '(')
            {
                const TokenMap *const keyword = lookupKeyword(nc.value);
                if(keyword)
                {
                    pushState(KindTest);
                    return Token(keyword->token);
                }
                else
                    return nc;
            }
            else
                return nc;
            Q_ASSERT(false);
        }
        case KindTestForPI:
        {
            switch(peekCurrent())
            {
                case ')':
                {
                    popState();
                    return tokenAndAdvance(RPAREN);
                }
                case '\'':
                /* Fallthrough. */
                case '"':
                    return tokenizeStringLiteral();
                default:
                    return tokenizeNCName();
            }
            Q_ASSERT(false);
        }
        case OccurrenceIndicator:
        {
            switch(peekCurrent())
            {
                case '?':
                    return tokenAndChangeState(QUESTION, Operator);
                case '*':
                    return tokenAndChangeState(STAR, Operator);
                case '+':
                    return tokenAndChangeState(PLUS, Operator);
                default:
                {
                    setState(Operator);
                    return nextToken();
                }
            }
            Q_ASSERT(false);
        }
        case XQueryVersion:
        {
            switch(peekCurrent())
            {
                case '\'':
                /* Fallthrough. */
                case '"':
                    return tokenizeStringLiteral();
                case ';':
                    return tokenAndChangeState(SEMI_COLON, Default);
            }

            const Token id(tokenizeNCName());

            if(id.type != NCNAME)
                return id;

            const TokenMap *const keyword = lookupKeyword(id.value);
            if(keyword)
                return tokenAndChangeState(keyword->token, Default);
            else
                return id;
            Q_ASSERT(false);
        }
        case StartTag:
        {
            if(peekAhead(-1) == '<')
            {
                if(current().isSpace())
                    return Token(ERROR);
            }
            else
            {
                if(consumeRawWhitespace())
                    return Token(END_OF_FILE);
            }

            switch(peekCurrent())
            {
                case '/':
                {
                    if(peekAhead() == '>')
                    {
                        m_pos += 2;

                        if(m_scanOnly)
                            return Token(POSITION_SET);
                        else
                        {
                            popState();
                            return Token(QUICK_TAG_END);
                        }
                    }
                    else
                        return error();
                }
                case '>':
                {
                    if(m_scanOnly)
                        return tokenAndChangeState(POSITION_SET, StartTag);
                    else
                        return tokenAndChangeState(G_GT, ElementContent);
                }
                case '=':
                    return tokenAndAdvance(G_EQ);
                case '\'':
                    return tokenAndChangeState(APOS, AposAttributeContent);
                case '"':
                    return tokenAndChangeState(QUOTE, QuotAttributeContent);
                default:
                    return tokenizeNCNameOrQName();
            }
            Q_ASSERT(false);
        }
        case AposAttributeContent:
        /* Fallthrough. */
        case QuotAttributeContent:
        {
            const QChar sep(state() == AposAttributeContent ? QLatin1Char('\'') : QLatin1Char('"'));
            QString result;
            result.reserve(20);

            if(m_scanOnly)
            {
                int stack = 0;
                return attributeAsRaw(sep, stack, m_pos, true, result);
            }

            Q_ASSERT(!m_scanOnly);
            while(true)
            {
                if(atEnd())
                {
                    /* In the case that the XSL-T tokenizer invokes us with
                     * default state QuotAttributeContent, we need to be able
                     * to return a single string, in case that is all we have
                     * accumulated. */
                    if(result.isEmpty())
                        return Token(END_OF_FILE);
                    else
                        return Token(STRING_LITERAL, result);
                }

                const QChar curr(current());

                if(curr == sep)
                {
                    if(m_pos + 1 == m_length)
                        return Token(END_OF_FILE);

                    if(m_data.at(m_pos + 1) == sep)
                    {
                        /* The quoting mechanism was used. */
                        m_pos += 2;
                        result.append(sep);
                        continue;
                    }

                    const QChar next(m_data.at(m_pos + 1));
                    if(!next.isSpace() && next != QLatin1Char('/') && next != QLatin1Char('>'))
                        return Token(ERROR); // i18n Space must separate attributes
                    else if(result.isEmpty())
                    {
                        return tokenAndChangeState(state() == AposAttributeContent ? APOS : QUOTE,
                                                   StartTag, 1);
                    }
                    else
                    {
                        /* Don't consume the sep, but leave it so we next time return a token for it. */
                        return Token(STRING_LITERAL, result);
                    }

                    ++m_pos;
                    continue;
                }
                else if(curr == QLatin1Char('{'))
                {
                    if(m_pos + 1 == m_length)
                        return Token(END_OF_FILE);
                    else if(peekAhead() == '{')
                    {
                        ++m_pos;
                        result.append(QLatin1Char('{'));
                    }
                    else
                    {
                        if(result.isEmpty())
                        {
                            /* The Attribute Value Template appeared directly in the attribute. */
                            pushState();
                            return tokenAndChangeState(CURLY_LBRACE, Default);
                        }
                        else
                        {
                            /* We don't advance, keep '{' as next token. */
                            return Token(STRING_LITERAL, result);
                        }
                    }
                }
                else if(curr == QLatin1Char('}'))
                {
                    if(m_pos + 1 == m_length)
                        return Token(END_OF_FILE);
                    else if(peekAhead() == '}')
                    {
                        ++m_pos;
                        result.append(QLatin1Char('}'));
                    }
                    else
                        return Token(ERROR);
                }
                else if(curr == QLatin1Char('&'))
                {
                    const QString ret(tokenizeCharacterReference());
                    if(ret.isNull())
                        return Token(ERROR);
                    else
                        result.append(ret);
                }
                else if(curr == QLatin1Char('<'))
                    return Token(STRING_LITERAL, result);
                else
                {
                    /* See Extensible Markup Language (XML) 1.0 (Fourth Edition),
                     * 3.3.3 Attribute-Value Normalization.
                     *
                     * However, it is complicated a bit by that AVN is defined on top of
                     * EOL normalization and we do those two in one go here. */
                    switch(curr.unicode())
                    {
                        case 0xD:
                        {
                            if(peekAhead() == '\n')
                            {
                                result.append(QLatin1Char(' '));
                                ++m_pos;
                                break;
                            }
                        }
                        case 0xA:
                        /* Fallthrough. */
                        case 0x9:
                        {
                            result.append(QLatin1Char(' '));
                            break;
                        }
                        default:
                            result.append(curr);
                    }
                }

                ++m_pos;
            }
            Q_ASSERT(false);
        }
        case ElementContent:
        {
            QString result;
            result.reserve(20);

            /* Whether the text node, result, may be whitespace only. Character references
             * and CDATA sections disables that. */
            bool mayBeWS = true;

            CharacterSkips skipEOLNormalization;

            while(true)
            {
                if(atEnd())
                    return Token(END_OF_FILE);

                switch(peekCurrent())
                {
                    case '<':
                    {
                        if(!result.isEmpty() && peekAhead(2) != '[')
                        {
                            /* We encountered the end, and it was not a CDATA section. */
                            /* We don't advance. Next time we'll handle the <... stuff. */
                            return Token(mayBeWS ? STRING_LITERAL : NON_BOUNDARY_WS, normalizeEOL(result, skipEOLNormalization));
                        }

                        ++m_pos;
                        if(atEnd())
                            return Token(END_OF_FILE);

                        const QChar ahead(current());
                        if(ahead.isSpace())
                            return error();
                        else if(ahead == QLatin1Char('/'))
                        {
                            if(m_pos + 1 == m_length)
                                return Token(END_OF_FILE);
                            else if(m_data.at(m_pos + 1).isSpace())
                                return error();
                            else
                                return tokenAndChangeState(BEGIN_END_TAG, EndTag);
                        }
                        else if(isNCNameStart(ahead))
                        {
                            pushState();
                            return tokenAndChangeState(G_LT, StartTag, 0);
                        }
                        else if(aheadEquals("!--", 3, 0))
                        {
                            pushState();
                            m_pos += 3;
                            return tokenAndChangeState(COMMENT_START, XMLComment, 0);
                        }
                        else if(aheadEquals("![CDATA[", 8, 0))
                        {
                            mayBeWS = false;
                            m_pos += 8;
                            const int start = m_pos;
                            const int len = scanUntil("]]>");

                            if(len == -1)
                                return Token(END_OF_FILE);

                            m_pos += 2; /* Consume "]]>". Note that m_pos is on '!'. */
                            result.append(m_data.mid(start, len));
                            break;
                        }
                        else if(ahead == QLatin1Char('?'))
                        {
                            pushState();
                            return tokenAndChangeState(PI_START, ProcessingInstructionName);
                        }
                        else
                            return Token(G_LT);
                    }
                    case '&':
                    {
                        const QString ret(tokenizeCharacterReference());
                        if(ret.isNull())
                            return Token(ERROR);
                        else
                        {
                            skipEOLNormalization.insert(result.count());
                            result.append(ret);
                            mayBeWS = false;
                            break;
                        }
                    }
                    case '{':
                    {
                        // TODO remove this check, also below.
                        if(m_pos + 1 == m_length)
                            return Token(END_OF_FILE);
                        else if(peekAhead() == '{')
                        {
                            ++m_pos;
                            result.append(QLatin1Char('{'));
                        }
                        else
                        {
                            if(result.isEmpty())
                            {
                                pushState();
                                return tokenAndChangeState(CURLY_LBRACE, Default);
                            }
                            else
                            {
                                /* We don't advance here. */
                                return Token(mayBeWS ? STRING_LITERAL : NON_BOUNDARY_WS, normalizeEOL(result, skipEOLNormalization));
                            }
                        }
                        break;
                    }
                    case '}':
                    {
                        if(m_pos + 1 == m_length)
                            return Token(END_OF_FILE);
                        else if(peekAhead() == '}')
                        {
                            ++m_pos;
                            result.append(QLatin1Char('}'));
                        }
                        else
                        {
                            /* This is a parse error, and the grammar won't be able
                             * to reduce this CURLY_RBRACE. */
                            return tokenAndChangeState(CURLY_RBRACE, Default);
                        }
                        break;
                    }
                    case '\n':
                    {
                        /* We want to translate \r\n into \n. */
                        if(peekAhead(-1) == '\r')
                            break;
                        /* else, fallthrough. */
                    }
                    case '\r':
                    {
                        result.append(QLatin1Char('\n'));
                        break;
                    }
                    default:
                    {
                        result.append(current());
                        break;
                    }
                }
                ++m_pos;
            }
            Q_ASSERT(false);
        }
        case ProcessingInstructionName:
        {
            const int start = m_pos;

            while(true)
            {
                ++m_pos;
                if(m_pos >= m_length)
                    return Token(END_OF_FILE);

                const QChar next(current());
                if(next.isSpace() || next == QLatin1Char('?'))
                {
                    return tokenAndChangeState(PI_TARGET, m_data.mid(start, m_pos - start),
                                               ProcessingInstructionContent);
                }
            }
            Q_ASSERT(false);
        }
        case ProcessingInstructionContent:
        {
            /* Consume whitespace between the name and the content. */
            if(consumeRawWhitespace())
                return Token(END_OF_FILE);

            const int start = m_pos;
            const int len = scanUntil("?>");

            if(len == -1)
                return Token(END_OF_FILE);
            else
            {
                m_pos += 2; /* Consume "?>" */
                popState();
                return Token(PI_CONTENT, normalizeEOL(m_data.mid(start, len), CharacterSkips()));
            }
            Q_ASSERT(false);
        }
        case EndTag:
        {
            if(consumeRawWhitespace())
                return END_OF_FILE;

            if(peekCurrent() == '>')
            {
                popState();
                return tokenAndAdvance(G_GT);
            }
            else
                return tokenizeNCNameOrQName();
            Q_ASSERT(false);
        }
        case XMLComment:
        {
            const int start = m_pos;
            const int len = scanUntil("--");

            if(len == -1)
                return END_OF_FILE;
            else
            {
                m_pos += 2; /* Consume "--". */
                popState();

                if(peekCurrent() == '>')
                {
                    ++m_pos;
                    return Token(COMMENT_CONTENT, normalizeEOL(m_data.mid(start, len), CharacterSkips()));
                }
                else
                    return error();
            }
            Q_ASSERT(false);
        }
        case Pragma:
        {
            /* Consume whitespace. */
            if(consumeRawWhitespace())
                return Token(END_OF_FILE);

            setState(PragmaContent);
            return tokenizeNCNameOrQName();
        }
        case PragmaContent:
        {
            QString result;
            result.reserve(20);

            const bool hasWS = m_pos < m_length && current().isSpace();

            /* Consume all whitespace up to the pragma content(if any). */
            if(consumeRawWhitespace())
                return Token(END_OF_FILE);

            if(peekCurrent() == '#' && peekAhead() == ')')
            {
                /* We reached the end, and there's no pragma content. */
                return tokenAndChangeState(PRAGMA_END, Default, 2);
            }
            else if(!hasWS)
            {
                /* A separating space is required if there's pragma content. */
                return error(); /* i18n */
            }

            const int start = m_pos;
            const int len = scanUntil("#)");
            if(len == -1)
                return Token(END_OF_FILE);

            return Token(STRING_LITERAL, m_data.mid(start, len));
            Q_ASSERT(false);
        }
    }

    Q_ASSERT(false);
    return error();
}

Tokenizer::Token XQueryTokenizer::attributeAsRaw(const QChar sep,
                                                 int &sepStack,
                                                 const int startPos,
                                                 const bool aInLiteral,
                                                 QString &result)
{
    bool inLiteral = aInLiteral;
    const char otherSep = (sep == QLatin1Char('"') ? '\'' : '"');

    while(true)
    {
        if(atEnd())
            return END_OF_FILE;

        if(peekCurrent() == sep.unicode())
        {
            if(inLiteral)
                inLiteral = false;
            else
                inLiteral = true;

            if(peekAhead() == sep.unicode())
            {
                /* The quoting mechanism was used. */
                result.append(current());
                m_pos += 2;
                continue;
            }
            else
            {
                /* Don't consume the separator, such that we
                 * return a token for it next time. */
                if(m_pos == startPos)
                {
                    ++m_pos;
                    setState(StartTag);
                    return Token(sep == QLatin1Char('"') ? QUOTE : APOS);
                }


                if(sepStack == 0)
                {
                    return Token(STRING_LITERAL, result);
                }
                else
                {
                    result.append(current());
                    ++m_pos;
                    continue;
                }
            }
        }
        else if(peekCurrent() == '&')
        {
            const QString ret(tokenizeCharacterReference());
            if(ret.isNull())
                return Token(ERROR);
            else
            {
                result.append(ret);
                ++m_pos;
                continue;
            }
        }
        else if(peekCurrent() == otherSep)
        {
            result.append(current());
            ++m_pos;

            if(peekCurrent() == otherSep)
                ++m_pos;

            if(inLiteral)
                inLiteral = false;
            else
                inLiteral = true;

            continue;
        }
        else if(peekCurrent() == '{')
        {
            result.append(current());

            if(peekAhead() == '{')
            {
                m_pos += 2;
                continue;
            }
            else
            {
                ++m_pos;
                ++sepStack;
                const Token t(attributeAsRaw(sep, sepStack, startPos, false, result));
                if(t.type != SUCCESS)
                    return t;
            }

        }
        else if(peekCurrent() == '}')
        {
            if(inLiteral && peekAhead() == '}')
            {
                result.append(current());
                m_pos += 2;
                continue;
            }
            else
            {
                ++m_pos;
                --sepStack;
                return Token(SUCCESS); /* The return value is arbitrary. */
            }
        }
        else
        {
            result.append(current());
            ++m_pos;
        }
    }
}

Tokenizer::Token XQueryTokenizer::nextToken(YYLTYPE *const sourceLocator)
{
    sourceLocator->first_line = m_line;
    sourceLocator->first_column = m_pos - m_columnOffset + 1; /* Plus 1, since m_pos is 0-based. */

    if(m_tokenStack.isEmpty())
        return nextToken();
    else
    {
        const Token retval(m_tokenStack.pop());

        switch(retval.type)
        {
            case MODULE:
            /* Fallthrough.*/
            case SCHEMA:
            /* Fallthrough.*/
            case COPY_NAMESPACES:
            {
                setState(NamespaceKeyword);
                break;
            }
            case VERSION:
            {
                setState(XQueryVersion);
                break;
            }
            case AS:
            /* Fallthrough. */
            case OF:
            {
                setState(ItemType);
                break;
            }
            default:
            {
                if(isOperatorKeyword(retval.type))
                    setState(Default);

                break;
            }
        };

        return retval;
    }
}

int XQueryTokenizer::commenceScanOnly()
{
    m_scanOnly = true;
    return m_pos;
}

void XQueryTokenizer::resumeTokenizationFrom(const int pos)
{
    m_scanOnly = false;
    m_pos = pos;
}

void XQueryTokenizer::setParserContext(const ParserContext::Ptr &)
{
}

#undef handleWhitespace

} // namespace QPatternist

QT_END_NAMESPACE