qtestcase.cpp

Go to the documentation of this file.

/****************************************************************************
**
** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies).
** Contact: http://www.qt-project.org/legal
**
** This file is part of the QtTest module of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Digia.  For licensing terms and
** conditions see http://qt.digia.com/licensing.  For further information
** use the contact form at http://qt.digia.com/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Digia gives you certain additional
** rights.  These rights are described in the Digia Qt LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
**
** $QT_END_LICENSE$
**
****************************************************************************/

#include "QtTest/qtestcase.h"
#include "QtTest/qtestassert.h"

#include <QtCore/qbytearray.h>
#include <QtCore/qmetaobject.h>
#include <QtCore/qobject.h>
#include <QtCore/qstringlist.h>
#include <QtCore/qvector.h>
#include <QtCore/qvarlengtharray.h>
#include <QtCore/qcoreapplication.h>
#include <QtCore/qfile.h>
#include <QtCore/qfileinfo.h>
#include <QtCore/qdir.h>
#include <QtCore/qprocess.h>
#include <QtCore/qdebug.h>

#include "QtTest/private/qtestlog_p.h"
#include "QtTest/private/qtesttable_p.h"
#include "QtTest/qtestdata.h"
#include "QtTest/private/qtestresult_p.h"
#include "QtTest/private/qsignaldumper_p.h"
#include "QtTest/private/qbenchmark_p.h"
#include "3rdparty/cycle_p.h"

#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>

#ifdef Q_OS_WIN
#include <windows.h> // for Sleep
#endif
#ifdef Q_OS_UNIX
#include <errno.h>
#include <signal.h>
#include <time.h>
#endif

#ifdef Q_WS_MAC
#include <Carbon/Carbon.h> // for SetFrontProcess
#ifdef QT_MAC_USE_COCOA
#include <IOKit/pwr_mgt/IOPMLib.h>
#else
#include <Security/AuthSession.h>
#endif
#undef verify
#endif

QT_BEGIN_NAMESPACE

namespace QTest
{
    static QObject *currentTestObject = 0;

    class TestFunction {
    public:
        TestFunction() : function_(-1), data_(0) {}
        void set(int function, char *data) { function_ = function; data_ = data; }
        char *data() const { return data_; }
        int function() const { return function_; }
        ~TestFunction() { delete[] data_; }
    private:
        int function_;
        char *data_;
    };
    static TestFunction * testFuncs = 0;
    static int testFuncCount = 0;

    static struct TestFuncCleanup
    {
        void cleanup()
        {
            delete[] testFuncs;
            testFuncCount = 0;
            testFuncs = 0;
        }

        ~TestFuncCleanup() { cleanup(); }
    } testFuncCleaner;

    static int keyDelay = -1;
    static int mouseDelay = -1;
    static int eventDelay = -1;
    static bool randomOrder = false;
    static int keyVerbose = -1;
    static unsigned int seed = 0;
    static bool seedSet = false;
#if defined(Q_OS_UNIX) && !defined(Q_OS_SYMBIAN)
    static bool noCrashHandler = false;
#endif

void filter_unprintable(char *str)
{
    char *idx = str;
    while (*idx) {
        if (((*idx < 0x20 && *idx != '\n' && *idx != '\t') || *idx > 0x7e))
            *idx = '?';
        ++idx;
    }
}

int qt_snprintf(char *str, int size, const char *format, ...)
{
    va_list ap;
    int res = 0;

    va_start(ap, format);
    qvsnprintf(str, size, format, ap);
    va_end(ap);
    str[size - 1] = '\0';

    filter_unprintable(str);

    return res;
}

static void invokeMethod(QObject *obj, const char *methodName)
{
    const QMetaObject *metaObject = obj->metaObject();
    int funcIndex = metaObject->indexOfMethod(methodName);
    if (funcIndex >= 0) {
        QMetaMethod method = metaObject->method(funcIndex);
        method.invoke(obj, Qt::DirectConnection);
    }
}

bool Q_TESTLIB_EXPORT defaultKeyVerbose()
{
    if (keyVerbose == -1) {
        keyVerbose = qgetenv("QTEST_KEYEVENT_VERBOSE").constData() ? 1 : 0;
    }
    return keyVerbose == 1;
}

int defaultEventDelay()
{
    if (eventDelay == -1) {
        if (qgetenv("QTEST_EVENT_DELAY").constData())
            eventDelay = atoi(qgetenv("QTEST_EVENT_DELAY"));
        else
            eventDelay = 0;
    }
    return eventDelay;
}

int Q_TESTLIB_EXPORT defaultMouseDelay()
{
    if (mouseDelay == -1) {
        if (qgetenv("QTEST_MOUSEEVENT_DELAY").constData())
            mouseDelay = atoi((qgetenv("QTEST_MOUSEEVENT_DELAY")));
        else
            mouseDelay = defaultEventDelay();
    }
    return mouseDelay;
}

int Q_TESTLIB_EXPORT defaultKeyDelay()
{
    if (keyDelay == -1) {
        if (qgetenv("QTEST_KEYEVENT_DELAY").constData())
            keyDelay = atoi(qgetenv("QTEST_KEYEVENT_DELAY").constData());
        else
            keyDelay = defaultEventDelay();
    }
    return keyDelay;
}

void seedRandom()
{
    static bool randomSeeded = false;
    if (!randomSeeded) {
        if (!QTest::seedSet) {
            QElapsedTimer timer;
            timer.start();
            QTest::seed = timer.msecsSinceReference();
        }
        qsrand(QTest::seed);
        randomSeeded = true;
    }
}

int qTestRandomSeed()
{
    Q_ASSERT(QTest::seedSet);
    return QTest::seed;
}

template<typename T>
void swap(T * array, int pos, int otherPos)
{
    T tmp = array[pos];
    array[pos] = array[otherPos];
    array[otherPos] = tmp;
}

template<typename T>
static void randomizeList(T * array, int size)
{
    for (int i = 0; i != size; i++) {
        int pos = qrand() % size;
        swap(array, pos, i);
    }
}

static bool isValidSlot(const QMetaMethod &sl)
{
    if (sl.access() != QMetaMethod::Private || !sl.parameterTypes().isEmpty()
        || qstrlen(sl.typeName()) || sl.methodType() != QMetaMethod::Slot)
        return false;
    const char *sig = sl.signature();
    int len = qstrlen(sig);
    if (len < 2)
        return false;
    if (sig[len - 2] != '(' || sig[len - 1] != ')')
        return false;
    if (len > 7 && strcmp(sig + (len - 7), "_data()") == 0)
        return false;
    if (strcmp(sig, "initTestCase()") == 0 || strcmp(sig, "cleanupTestCase()") == 0
        || strcmp(sig, "cleanup()") == 0 || strcmp(sig, "init()") == 0)
        return false;
    return true;
}

Q_TESTLIB_EXPORT bool printAvailableFunctions = false;
Q_TESTLIB_EXPORT bool printAvailableTags = false;
Q_TESTLIB_EXPORT QStringList testFunctions;
Q_TESTLIB_EXPORT QStringList testTags;

static void qPrintTestSlots()
{
    for (int i = 0; i < QTest::currentTestObject->metaObject()->methodCount(); ++i) {
        QMetaMethod sl = QTest::currentTestObject->metaObject()->method(i);
        if (isValidSlot(sl))
            printf("%s\n", sl.signature());
    }
}

static void qPrintDataTags()
{
    // Get global data tags:
    QTestTable::globalTestTable();
    invokeMethod(QTest::currentTestObject, "initTestCase_data()");
    const QTestTable *gTable = QTestTable::globalTestTable();

    const QMetaObject *currTestMetaObj = QTest::currentTestObject->metaObject();

    // Process test functions:
    for (int i = 0; i < currTestMetaObj->methodCount(); ++i) {
        QMetaMethod tf = currTestMetaObj->method(i);
        if (isValidSlot(tf)) {
            // Retrieve local tags:
            QStringList localTags;
            QTestTable table;
            char member[512];
            char *slot = qstrdup(tf.signature());
            slot[strlen(slot) - 2] = '\0';
            QTest::qt_snprintf(member, 512, "%s_data()", slot);
            invokeMethod(QTest::currentTestObject, member);
            for (int j = 0; j < table.dataCount(); ++j)
                localTags << QLatin1String(table.testData(j)->dataTag());

            // Print all tag combinations:
            if (gTable->dataCount() == 0) {
                if (localTags.count() == 0) {
                    // No tags at all, so just print the test function:
                    printf("%s %s\n", currTestMetaObj->className(), slot);
                } else {
                    // Only local tags, so print each of them:
                    for (int k = 0; k < localTags.size(); ++k)
                        printf(
                            "%s %s %s\n",
                            currTestMetaObj->className(), slot, localTags.at(k).toLatin1().data());
                }
            } else {
                for (int j = 0; j < gTable->dataCount(); ++j) {
                    if (localTags.count() == 0) {
                        // Only global tags, so print the current one:
                        printf(
                            "%s %s __global__ %s\n",
                            currTestMetaObj->className(), slot, gTable->testData(j)->dataTag());
                    } else {
                        // Local and global tags, so print each of the local ones and
                        // the current global one:
                        for (int k = 0; k < localTags.size(); ++k)
                            printf(
                                "%s %s %s __global__ %s\n", currTestMetaObj->className(), slot,
                                localTags.at(k).toLatin1().data(), gTable->testData(j)->dataTag());
                    }
                }
            }

            delete[] slot;
        }
    }
}

static int qToInt(char *str)
{
    char *pEnd;
    int l = (int)strtol(str, &pEnd, 10);
    if (*pEnd != 0) {
        printf("Invalid numeric parameter: '%s'\n", str);
        exit(1);
    }
    return l;
}

Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml)
{
    const char *testOptions =
         " options:\n"
         " -functions : Returns a list of current testfunctions\n"
         " -datatags  : Returns a list of current data tags.\n"
         "              A global data tag is preceded by ' __global__ '.\n"
         " -xunitxml  : Outputs results as XML XUnit document\n"
         " -xml       : Outputs results as XML document\n"
         " -lightxml  : Outputs results as stream of XML tags\n"
         " -flush     : Flushes the results\n"
         " -o filename: Writes all output into a file\n"
         " -silent    : Only outputs warnings and failures\n"
         " -v1        : Print enter messages for each testfunction\n"
         " -v2        : Also print out each QVERIFY/QCOMPARE/QTEST\n"
         " -vs        : Print every signal emitted\n"
         " -random    : Run testcases within each test in random order\n"
         " -seed n    : Positive integer to be used as seed for -random. If not specified,\n"
         "              the current time will be used as seed.\n"
         " -eventdelay ms    : Set default delay for mouse and keyboard simulation to ms milliseconds\n"
         " -keydelay ms      : Set default delay for keyboard simulation to ms milliseconds\n"
         " -mousedelay ms    : Set default delay for mouse simulation to ms milliseconds\n"
         " -keyevent-verbose : Turn on verbose messages for keyboard simulation\n"
         " -maxwarnings n    : Sets the maximum amount of messages to output.\n"
         "                     0 means unlimited, default: 2000\n"
#if defined(Q_OS_UNIX) && !defined(Q_OS_SYMBIAN)
         " -nocrashhandler   : Disables the crash handler\n"
#endif
         "\n"
         " Benchmark related options:\n"
#ifdef QTESTLIB_USE_VALGRIND
        " -callgrind      : Use callgrind to time benchmarks\n"
#endif
#ifdef HAVE_TICK_COUNTER
        " -tickcounter    : Use CPU tick counters to time benchmarks\n"
#endif
        " -eventcounter   : Counts events received during benchmarks\n"
        " -minimumvalue n : Sets the minimum acceptable measurement value\n"
        " -iterations  n  : Sets the number of accumulation iterations.\n"
        " -median  n      : Sets the number of median iterations.\n"
        " -vb             : Print out verbose benchmarking information.\n"
         "\n"
        " -help      : This help\n";

    for (int i = 1; i < argc; ++i) {
        if (strcmp(argv[i], "-help") == 0 || strcmp(argv[i], "--help") == 0
            || strcmp(argv[i], "/?") == 0) {
            printf(" Usage: %s [options] [testfunction[:testdata]]...\n"
                   "    By default, all testfunctions will be run.\n\n"
                   "%s", argv[0], testOptions);
            exit(0);
        } else if (strcmp(argv[i], "-functions") == 0) {
            if (qml) {
                QTest::printAvailableFunctions = true;
            } else {
                qPrintTestSlots();
                exit(0);
            }
        } else if (strcmp(argv[i], "-datatags") == 0) {
            QTest::printAvailableTags = true;
            if (!qml) {
                qPrintDataTags();
                exit(0);
            }
        } else if(strcmp(argv[i], "-xunitxml") == 0){
            QTestLog::setLogMode(QTestLog::XunitXML);
        } else if (strcmp(argv[i], "-xml") == 0) {
            QTestLog::setLogMode(QTestLog::XML);
        } else if (strcmp(argv[i], "-lightxml") == 0) {
            QTestLog::setLogMode(QTestLog::LightXML);
        }else if(strcmp(argv[i], "-flush") == 0){
            QTestLog::setFlushMode(QTestLog::FLushOn);
        } else if (strcmp(argv[i], "-silent") == 0) {
            QTestLog::setVerboseLevel(-1);
        } else if (strcmp(argv[i], "-v1") == 0) {
            QTestLog::setVerboseLevel(1);
        } else if (strcmp(argv[i], "-v2") == 0) {
            QTestLog::setVerboseLevel(2);
        } else if (strcmp(argv[i], "-vs") == 0) {
            QSignalDumper::startDump();
        } else if (strcmp(argv[i], "-o") == 0) {
            if (i + 1 >= argc) {
                printf("-o needs an extra parameter specifying the filename\n");
                exit(1);
            } else {
                QTestLog::redirectOutput(argv[++i]);
            }
        } else if (strcmp(argv[i], "-eventdelay") == 0) {
            if (i + 1 >= argc) {
                printf("-eventdelay needs an extra parameter to indicate the delay(ms)\n");
                exit(1);
            } else {
                QTest::eventDelay = qToInt(argv[++i]);
            }
        } else if (strcmp(argv[i], "-keydelay") == 0) {
            if (i + 1 >= argc) {
                printf("-keydelay needs an extra parameter to indicate the delay(ms)\n");
                exit(1);
            } else {
                QTest::keyDelay = qToInt(argv[++i]);
            }
        } else if (strcmp(argv[i], "-mousedelay") == 0) {
            if (i + 1 >= argc) {
                printf("-mousedelay needs an extra parameter to indicate the delay(ms)\n");
                exit(1);
            } else {
                QTest::mouseDelay = qToInt(argv[++i]);
            }
        } else if (strcmp(argv[i], "-maxwarnings") == 0) {
            if (i + 1 >= argc) {
                printf("-maxwarnings needs an extra parameter with the amount of warnings\n");
                exit(1);
            } else {
                QTestLog::setMaxWarnings(qToInt(argv[++i]));
            }
#if defined(Q_OS_UNIX) && !defined(Q_OS_SYMBIAN)
        } else if (strcmp(argv[i], "-nocrashhandler") == 0) {
            QTest::noCrashHandler = true;
#endif
        } else if (strcmp(argv[i], "-keyevent-verbose") == 0) {
            QTest::keyVerbose = 1;
#ifdef QTESTLIB_USE_VALGRIND
        } else if (strcmp(argv[i], "-callgrind") == 0) {
            if (QBenchmarkValgrindUtils::haveValgrind())
                if (QFileInfo(QDir::currentPath()).isWritable()) {
                    QBenchmarkGlobalData::current->setMode(QBenchmarkGlobalData::CallgrindParentProcess);
                } else {
                    printf("WARNING: Current directory not writable. Using the walltime measurer.\n");
                }
            else {
                printf("WARNING: Valgrind not found or too old. Make sure it is installed and in your path. "
                       "Using the walltime measurer.\n");
            }
        } else if (strcmp(argv[i], "-callgrindchild") == 0) { // "private" option
            QBenchmarkGlobalData::current->setMode(QBenchmarkGlobalData::CallgrindChildProcess);
            QBenchmarkGlobalData::current->callgrindOutFileBase =
                QBenchmarkValgrindUtils::outFileBase();
#endif
#ifdef HAVE_TICK_COUNTER
        } else if (strcmp(argv[i], "-tickcounter") == 0) {
            QBenchmarkGlobalData::current->setMode(QBenchmarkGlobalData::TickCounter);
#endif
        } else if (strcmp(argv[i], "-eventcounter") == 0) {
            QBenchmarkGlobalData::current->setMode(QBenchmarkGlobalData::EventCounter);
        } else if (strcmp(argv[i], "-random") == 0) {
            QTest::randomOrder = true;
        } else if (strcmp(argv[i], "-seed") == 0) {
            bool argumentOk = false;
            if (i + 1 < argc) {
                char * endpt = 0;
                long longSeed = strtol(argv[++i], &endpt, 10);
                argumentOk = (*endpt == '\0' && longSeed >= 0);
                QTest::seed = longSeed;
            }
            if (!argumentOk) {
                printf("-seed needs an extra positive integer parameter to specify the seed\n");
                exit(1);
            } else {
                QTest::seedSet = true;
            }
        } else if (strcmp(argv[i], "-minimumvalue") == 0) {
            if (i + 1 >= argc) {
                printf("-minimumvalue needs an extra parameter to indicate the minimum time(ms)\n");
                exit(1);
            } else {
                QBenchmarkGlobalData::current->walltimeMinimum = qToInt(argv[++i]);
            }
        } else if (strcmp(argv[i], "-iterations") == 0) {
            if (i + 1 >= argc) {
                printf("-iterations needs an extra parameter to indicate the number of iterations\n");
                exit(1);
            } else {
                QBenchmarkGlobalData::current->iterationCount = qToInt(argv[++i]);
            }
        } else if (strcmp(argv[i], "-median") == 0) {
            if (i + 1 >= argc) {
                printf("-median needs an extra parameter to indicate the number of median iterations\n");
                exit(1);
            } else {
                QBenchmarkGlobalData::current->medianIterationCount = qToInt(argv[++i]);
            }

        } else if (strcmp(argv[i], "-vb") == 0) {
            QBenchmarkGlobalData::current->verboseOutput = true;
        } else if (strcmp(argv[i], "-chart") == 0) {
            fprintf(stderr, "Warning: `-chart' option is not available\n");
        } else if (strcmp(argv[i], "-qws") == 0) {
            // do nothing
        } else if (strcmp(argv[i], "-graphicssystem") == 0) {
            // do nothing
            if (i + 1 >= argc) {
                printf("-graphicssystem needs an extra parameter specifying the graphics system\n");
                exit(1);
            } else {
                ++i;
            }
        } else if (argv[i][0] == '-') {
            printf("Unknown option: '%s'\n\n%s", argv[i], testOptions);
            exit(1);
        } else if (qml) {
            // We can't check the availability of test functions until
            // we load the QML files.  So just store the data for now.
            int colon = -1;
            int offset;
            for(offset = 0; *(argv[i]+offset); ++offset) {
                if (*(argv[i]+offset) == ':') {
                    if (*(argv[i]+offset+1) == ':') {
                        // "::" is used as a test name separator.
                        // e.g. "ClickTests::test_click:row1".
                        ++offset;
                    } else {
                        colon = offset;
                        break;
                    }
                }
            }
            if (colon == -1) {
                QTest::testFunctions += QString::fromLatin1(argv[i]);
                QTest::testTags += QString();
            } else {
                QTest::testFunctions +=
                    QString::fromLatin1(argv[i], colon);
                QTest::testTags +=
                    QString::fromLatin1(argv[i] + colon + 1);
            }
        } else {
            if (!QTest::testFuncs) {
                QTest::testFuncs = new QTest::TestFunction[512];
            }

            int colon = -1;
            char buf[512], *data=0;
            int off;
            for(off = 0; *(argv[i]+off); ++off) {
                if (*(argv[i]+off) == ':') {
                    colon = off;
                    break;
                }
            }
            if(colon != -1) {
                data = qstrdup(argv[i]+colon+1);
            }
            QTest::qt_snprintf(buf, qMin(512, off + 1), "%s", argv[i]); // copy text before the ':' into buf
            QTest::qt_snprintf(buf + off, qMin(512 - off, 3), "()");    // append "()"
            int idx = QTest::currentTestObject->metaObject()->indexOfMethod(buf);
            if (idx < 0 || !isValidSlot(QTest::currentTestObject->metaObject()->method(idx))) {
                printf("Unknown testfunction: '%s'\n", buf);
                printf("Available testfunctions:\n");
                qPrintTestSlots();
                exit(1);
            }
            testFuncs[testFuncCount].set(idx, data);
            testFuncCount++;
            QTEST_ASSERT(QTest::testFuncCount < 512);
        }
    }

    if (QTest::seedSet && !QTest::randomOrder) {
        printf("-seed requires -random\n");
        exit(1);
    }
}

QBenchmarkResult qMedian(const QList<QBenchmarkResult> &container)
{
    const int count = container.count();
    if (count == 0)
        return QBenchmarkResult();

    if (count == 1)
        return container.at(0);

    QList<QBenchmarkResult> containerCopy = container;
    qSort(containerCopy);

    const int middle = count / 2;

    // ### handle even-sized containers here by doing an aritmetic mean of the two middle items.
    return containerCopy.at(middle);
}

struct QTestDataSetter
{
    QTestDataSetter(QTestData *data)
    {
        QTestResult::setCurrentTestData(data);
    }
    ~QTestDataSetter()
    {
        QTestResult::setCurrentTestData(0);
    }
};

static void qInvokeTestMethodDataEntry(char *slot)
{
    /* Benchmarking: for each median iteration*/

    int i = (QBenchmarkGlobalData::current->measurer->needsWarmupIteration()) ? -1 : 0;

    QList<QBenchmarkResult> results;
    do {
        QBenchmarkTestMethodData::current->beginDataRun();

        /* Benchmarking: for each accumulation iteration*/
        bool invokeOk;
        do {
            QTestResult::setCurrentTestLocation(QTestResult::InitFunc);
            invokeMethod(QTest::currentTestObject, "init()");
            if (QTestResult::skipCurrentTest())
                break;

            QTestResult::setCurrentTestLocation(QTestResult::Func);

            QBenchmarkTestMethodData::current->result = QBenchmarkResult();
            QBenchmarkTestMethodData::current->resultAccepted = false;

            QBenchmarkGlobalData::current->context.tag =
                QLatin1String(
                    QTestResult::currentDataTag()
                    ? QTestResult::currentDataTag() : "");

            invokeOk = QMetaObject::invokeMethod(QTest::currentTestObject, slot,
                                                 Qt::DirectConnection);
            if (!invokeOk)
                QTestResult::addFailure("Unable to execute slot", __FILE__, __LINE__);

            QTestResult::setCurrentTestLocation(QTestResult::CleanupFunc);
            invokeMethod(QTest::currentTestObject, "cleanup()");
            QTestResult::setCurrentTestLocation(QTestResult::NoWhere);

            // If this test method has a benchmark, repeat until all measurements are
            // acceptable.
            // The QBENCHMARK macro increases the number of iterations for each run until
            // this happens.
        } while (invokeOk
                 && QBenchmarkTestMethodData::current->isBenchmark()
                 && QBenchmarkTestMethodData::current->resultsAccepted() == false);

        QBenchmarkTestMethodData::current->endDataRun();
        if (i > -1)  // iteration -1 is the warmup iteration.
            results.append(QBenchmarkTestMethodData::current->result);

        if (QBenchmarkTestMethodData::current->isBenchmark() &&
            QBenchmarkGlobalData::current->verboseOutput) {
                if (i == -1) {
                    qDebug() << "warmup stage result      :" << QBenchmarkTestMethodData::current->result.value;
                } else {
                    qDebug() << "accumulation stage result:" << QBenchmarkTestMethodData::current->result.value;
                }
            }
    } while (QBenchmarkTestMethodData::current->isBenchmark()
             && (++i < QBenchmarkGlobalData::current->adjustMedianIterationCount()));

    if (QBenchmarkTestMethodData::current->isBenchmark()
        && QBenchmarkTestMethodData::current->resultsAccepted())
        QTestLog::addBenchmarkResult(qMedian(results));
}

static bool qInvokeTestMethod(const char *slotName, const char *data=0)
{
    QTEST_ASSERT(slotName);

    QBenchmarkTestMethodData benchmarkData;
    QBenchmarkTestMethodData::current = &benchmarkData;

    QBenchmarkGlobalData::current->context.slotName = QLatin1String(slotName);

    char member[512];
    QTestTable table;

    char *slot = qstrdup(slotName);
    slot[strlen(slot) - 2] = '\0';
    QTestResult::setCurrentTestFunction(slot);

    const QTestTable *gTable = QTestTable::globalTestTable();
    const int globalDataCount = gTable->dataCount();
    int curGlobalDataIndex = 0;

    /* For each test function that has a *_data() table/function, do: */
    do {
        if (!gTable->isEmpty())
            QTestResult::setCurrentGlobalTestData(gTable->testData(curGlobalDataIndex));

        if (curGlobalDataIndex == 0) {
            QTestResult::setCurrentTestLocation(QTestResult::DataFunc);
            QTest::qt_snprintf(member, 512, "%s_data()", slot);
            invokeMethod(QTest::currentTestObject, member);

            // if we encounter a SkipAll in the _data slot, we skip the whole
            // testfunction, no matter how much global data exists
            if (QTestResult::skipCurrentTest()) {
                QTestResult::setCurrentGlobalTestData(0);
                break;
            }
        }

        bool foundFunction = false;
        if (!QTestResult::skipCurrentTest()) {
            int curDataIndex = 0;
            const int dataCount = table.dataCount();
            QTestResult::setSkipCurrentTest(false);

            // Data tag requested but none available?
            if (data && !dataCount) {
                // Let empty data tag through.
                if (!*data)
                    data = 0;
                else {
                    printf("Unknown testdata for function %s: '%s'\n", slotName, data);
                    printf("Function has no testdata.\n");
                    return false;
                }
            }

            /* For each entry in the data table, do: */
            do {
                if (!data || !qstrcmp(data, table.testData(curDataIndex)->dataTag())) {
                    foundFunction = true;
                    QTestDataSetter s(curDataIndex >= dataCount ? static_cast<QTestData *>(0)
                                                      : table.testData(curDataIndex));

                    qInvokeTestMethodDataEntry(slot);

                    if (QTestResult::skipCurrentTest())
                        // check whether SkipAll was requested
                        break;
                    if (data)
                        break;
                }
                ++curDataIndex;
            } while (curDataIndex < dataCount);
        }

        if (data && !foundFunction) {
            printf("Unknown testdata for function %s: '%s'\n", slotName, data);
            printf("Available testdata:\n");
            for(int i = 0; i < table.dataCount(); ++i)
                printf("%s\n", table.testData(i)->dataTag());
            return false;
        }

        QTestResult::setCurrentGlobalTestData(0);
        ++curGlobalDataIndex;
    } while (curGlobalDataIndex < globalDataCount);

    QTestResult::finishedCurrentTestFunction();
    QTestResult::setSkipCurrentTest(false);
    QTestResult::setCurrentTestData(0);
    delete[] slot;

    return true;
}

void *fetchData(QTestData *data, const char *tagName, int typeId)
{
    QTEST_ASSERT(typeId);
    QTEST_ASSERT_X(data, "QTest::fetchData()", "Test data requested, but no testdata available.");
    QTEST_ASSERT(data->parent());

    int idx = data->parent()->indexOf(tagName);

    if (idx == -1 || idx >= data->dataCount()) {
        qFatal("QFETCH: Requested testdata '%s' not available, check your _data function.",
                tagName);
    }

    if (typeId != data->parent()->elementTypeId(idx)) {
        qFatal("Requested type '%s' does not match available type '%s'.",
               QMetaType::typeName(typeId),
               QMetaType::typeName(data->parent()->elementTypeId(idx)));
    }

    return data->data(idx);
}

char *toHexRepresentation(const char *ba, int length)
{
    if(length == 0)
        return qstrdup("");

    /* We output at maximum about maxLen characters in order to avoid
     * running out of memory and flooding things when the byte array
     * is large.
     *
     * maxLen can't be for example 200 because QTestLib is sprinkled with fixed
     * size char arrays.
     * */
    const int maxLen = 50;
    const int len = qMin(maxLen, length);
    char *result = 0;

    if(length > maxLen) {
        const int size = len * 3 + 4;
        result = new char[size];

        char *const forElipsis = result + size - 5;
        forElipsis[0] = ' ';
        forElipsis[1] = '.';
        forElipsis[2] = '.';
        forElipsis[3] = '.';
        result[size - 1] = '\0';
    }
    else {
        const int size = len * 3;
        result = new char[size];
        result[size - 1] = '\0';
    }

    const char toHex[] = "0123456789ABCDEF";
    int i = 0;
    int o = 0;

    while(true) {
        const char at = ba[i];

        result[o] = toHex[(at >> 4) & 0x0F];
        ++o;
        result[o] = toHex[at & 0x0F];

        ++i;
        ++o;
        if(i == len)
            break;
        else {
            result[o] = ' ';
            ++o;
        }
    }

    return result;
}

static void qInvokeTestMethods(QObject *testObject)
{
    const QMetaObject *metaObject = testObject->metaObject();
    QTEST_ASSERT(metaObject);
    if (QTest::randomOrder) {
        QTestLog::startLogging(QTest::seed);
    } else {
        QTestLog::startLogging();
    }
    QTestResult::setCurrentTestFunction("initTestCase");
    QTestResult::setCurrentTestLocation(QTestResult::DataFunc);
    QTestTable::globalTestTable();
    invokeMethod(testObject, "initTestCase_data()");

    if (!QTestResult::skipCurrentTest() && !QTest::currentTestFailed()) {
        QTestResult::setCurrentTestLocation(QTestResult::InitFunc);
        invokeMethod(testObject, "initTestCase()");

        // finishedCurrentTestFunction() resets QTestResult::testFailed(), so use a local copy.
        const bool previousFailed = QTestResult::testFailed();
        QTestResult::finishedCurrentTestFunction();

        if(!QTestResult::skipCurrentTest() && !previousFailed) {

            if (QTest::testFuncs) {
                if (QTest::randomOrder)
                    randomizeList(QTest::testFuncs, QTest::testFuncCount);
                for (int i = 0; i != QTest::testFuncCount; i++) {
                    if (!qInvokeTestMethod(metaObject->method(QTest::testFuncs[i].function()).signature(),
                                                              QTest::testFuncs[i].data())) {
                        break;
                    }
                }
                testFuncCleaner.cleanup();
            } else {
                int methodCount = metaObject->methodCount();
                QMetaMethod *testMethods = new QMetaMethod[methodCount];
                for (int i = 0; i != methodCount; i++)
                    testMethods[i] = metaObject->method(i);
                if (QTest::randomOrder)
                    randomizeList(testMethods, methodCount);
                for (int i = 0; i != methodCount; i++) {
                    if (!isValidSlot(testMethods[i]))
                        continue;
                    if (!qInvokeTestMethod(testMethods[i].signature()))
                        break;
                }
                delete[] testMethods;
                testMethods = 0;
            }
        }

        QTestResult::setSkipCurrentTest(false);
        QTestResult::setCurrentTestFunction("cleanupTestCase");
        invokeMethod(testObject, "cleanupTestCase()");
    }
    QTestResult::finishedCurrentTestFunction();
    QTestResult::setCurrentTestFunction(0);
    QTestTable::clearGlobalTestTable();

    QTestLog::stopLogging();
}

#if defined(Q_OS_UNIX) && !defined(Q_OS_SYMBIAN)
class FatalSignalHandler
{
public:
    FatalSignalHandler();
    ~FatalSignalHandler();

private:
    static void signal(int);
    sigset_t handledSignals;
};

void FatalSignalHandler::signal(int signum)
{
    qFatal("Received signal %d", signum);
#if defined(Q_OS_INTEGRITY)
    {
        struct sigaction act;
        memset(&act, 0, sizeof(struct sigaction));
        act.sa_handler = SIG_DFL;
        sigaction(signum, &act, NULL);
    }
#endif
}

FatalSignalHandler::FatalSignalHandler()
{
    sigemptyset(&handledSignals);

    const int fatalSignals[] = {
         SIGHUP, SIGINT, SIGQUIT, SIGILL, SIGFPE, SIGSEGV, SIGPIPE, SIGTERM, 0 };

    struct sigaction act;
    memset(&act, 0, sizeof(act));
    act.sa_handler = FatalSignalHandler::signal;

    // Remove the handler after it is invoked.
#if !defined(Q_OS_INTEGRITY)
    act.sa_flags = SA_RESETHAND;
#endif
    // Block all fatal signals in our signal handler so we don't try to close
    // the testlog twice.
    sigemptyset(&act.sa_mask);
    for (int i = 0; fatalSignals[i]; ++i)
        sigaddset(&act.sa_mask, fatalSignals[i]);

    struct sigaction oldact;

    for (int i = 0; fatalSignals[i]; ++i) {
        sigaction(fatalSignals[i], &act, &oldact);
#ifndef Q_WS_QWS
        // Don't overwrite any non-default handlers
        // however, we need to replace the default QWS handlers
        if (
#ifdef SA_SIGINFO
            oldact.sa_flags & SA_SIGINFO ||
#endif
            oldact.sa_handler != SIG_DFL) {
            sigaction(fatalSignals[i], &oldact, 0);
        } else
#endif
        {
            sigaddset(&handledSignals, fatalSignals[i]);
        }
    }
}


FatalSignalHandler::~FatalSignalHandler()
{
    // Unregister any of our remaining signal handlers
    struct sigaction act;
    memset(&act, 0, sizeof(act));
    act.sa_handler = SIG_DFL;

    struct sigaction oldact;

    for (int i = 1; i < 32; ++i) {
        if (!sigismember(&handledSignals, i))
            continue;
        sigaction(i, &act, &oldact);

        // If someone overwrote it in the mean time, put it back
        if (oldact.sa_handler != FatalSignalHandler::signal)
            sigaction(i, &oldact, 0);
    }
}

#endif


} // namespace

int QTest::qExec(QObject *testObject, int argc, char **argv)
{
    QBenchmarkGlobalData benchmarkData;
    QBenchmarkGlobalData::current = &benchmarkData;

#ifdef QTESTLIB_USE_VALGRIND
    int callgrindChildExitCode = 0;
#endif

#ifdef Q_WS_MAC
     bool macNeedsActivate = qApp && (qstrcmp(qApp->metaObject()->className(), "QApplication") == 0);
#ifdef QT_MAC_USE_COCOA
     IOPMAssertionID powerID;
#endif
#endif
#ifndef QT_NO_EXCEPTIONS
    try {
#endif

 #if defined(Q_OS_WIN) && !defined(Q_OS_WINCE)
     SetErrorMode(SetErrorMode(0) | SEM_NOGPFAULTERRORBOX);
 #endif

#ifdef Q_WS_MAC
    // Starting with Qt 4.4, applications launched from the command line
    // no longer get focus automatically. Since some tests might depend
    // on this, call SetFrontProcess here to get the pre 4.4 behavior.
    if (macNeedsActivate) {
        ProcessSerialNumber psn = { 0, kCurrentProcess };
        SetFrontProcess(&psn);
#  ifdef QT_MAC_USE_COCOA
        IOReturn ok = IOPMAssertionCreate(kIOPMAssertionTypeNoDisplaySleep, kIOPMAssertionLevelOn, &powerID);
        if (ok != kIOReturnSuccess)
            macNeedsActivate = false; // no need to release the assertion on exit.
#  else
        UpdateSystemActivity(1); // Wake the display.
#  endif
    }
#endif

#if defined(Q_OS_SYMBIAN) && defined(Q_CC_NOKIAX86)
    // Delay execution of tests in Symbian emulator.
    // Needed to allow worst of other higher priority apps and services launched by emulator
    // to get out of the way before we run our test. Otherwise some of the timing sensitive tests
    // will not work properly.
    qSleep(3000);
#endif

    QTestResult::reset();

    QTEST_ASSERT(testObject);
    QTEST_ASSERT(!currentTestObject);
    currentTestObject = testObject;

    const QMetaObject *metaObject = testObject->metaObject();
    QTEST_ASSERT(metaObject);

    QTestResult::setCurrentTestObject(metaObject->className());
    if (argc > 0)
        QTestResult::setCurrentAppName(argv[0]);

    qtest_qParseArgs(argc, argv, false);
    if (QTest::randomOrder) {
        seedRandom();
    }
#ifdef QTESTLIB_USE_VALGRIND
    if (QBenchmarkGlobalData::current->mode() == QBenchmarkGlobalData::CallgrindParentProcess) {
        const QStringList origAppArgs(QCoreApplication::arguments());
        if (!QBenchmarkValgrindUtils::rerunThroughCallgrind(origAppArgs, callgrindChildExitCode))
            return -1;

        QBenchmarkValgrindUtils::cleanup();

    } else
#endif
    {
#if defined(Q_OS_UNIX) && !defined(Q_OS_SYMBIAN)
        QScopedPointer<FatalSignalHandler> handler;
        if (!noCrashHandler)
            handler.reset(new FatalSignalHandler);
#endif
        qInvokeTestMethods(testObject);
    }

#ifndef QT_NO_EXCEPTIONS
     } catch (...) {
         QTestResult::addFailure("Caught unhandled exception", __FILE__, __LINE__);
         if (QTestResult::currentTestFunction()) {
             QTestResult::finishedCurrentTestFunction();
             QTestResult::setCurrentTestFunction(0);
         }

        QTestLog::stopLogging();
#ifdef QT_MAC_USE_COCOA
         if (macNeedsActivate) {
             IOPMAssertionRelease(powerID);
         }
#endif
         currentTestObject = 0;

         // Rethrow exception to make debugging easier.
         throw;
         return 1;
     }
#  endif

    currentTestObject = 0;
#ifdef QT_MAC_USE_COCOA
     if (macNeedsActivate) {
         IOPMAssertionRelease(powerID);
     }
#endif

#if defined(QTEST_NOEXITCODE)
    return 0;
#else

#ifdef QTESTLIB_USE_VALGRIND
    if (QBenchmarkGlobalData::current->mode() == QBenchmarkGlobalData::CallgrindParentProcess)
        return callgrindChildExitCode;
#endif
    // make sure our exit code is never going above 127
    // since that could wrap and indicate 0 test fails
    return qMin(QTestResult::failCount(), 127);

#endif
}

int QTest::qExec(QObject *testObject, const QStringList &arguments)
{
    const int argc = arguments.count();
    QVarLengthArray<char *> argv(argc);

    QVector<QByteArray> args;
    args.reserve(argc);

    for(int i = 0; i < argc; ++i)
    {
        args.append(arguments.at(i).toLocal8Bit().constData());
        argv[i] = args.last().data();
    }

    return qExec(testObject, argc, argv.data());
}

void QTest::qFail(const char *statementStr, const char *file, int line)
{
    QTestResult::addFailure(statementStr, file, line);
}

bool QTest::qVerify(bool statement, const char *statementStr, const char *description,
                   const char *file, int line)
{
    return QTestResult::verify(statement, statementStr, description, file, line);
}

void QTest::qSkip(const char *message, QTest::SkipMode mode,
                 const char *file, int line)
{
    QTestResult::addSkip(message, mode, file, line);
    if (mode == QTest::SkipAll)
        QTestResult::setSkipCurrentTest(true);
}

bool QTest::qExpectFail(const char *dataIndex, const char *comment,
                       QTest::TestFailMode mode, const char *file, int line)
{
    return QTestResult::expectFail(dataIndex, qstrdup(comment), mode, file, line);
}

void QTest::qWarn(const char *message)
{
    QTestLog::warn(message);
}

void QTest::ignoreMessage(QtMsgType type, const char *message)
{
    QTestResult::ignoreMessage(type, message);
}

void *QTest::qData(const char *tagName, int typeId)
{
    return fetchData(QTestResult::currentTestData(), tagName, typeId);
}

void *QTest::qGlobalData(const char *tagName, int typeId)
{
    return fetchData(QTestResult::currentGlobalTestData(), tagName, typeId);
}

void *QTest::qElementData(const char *tagName, int metaTypeId)
{
    QTEST_ASSERT(tagName);
    QTestData *data = QTestResult::currentTestData();
    QTEST_ASSERT(data);
    QTEST_ASSERT(data->parent());

    int idx = data->parent()->indexOf(tagName);
    QTEST_ASSERT(idx != -1);
    QTEST_ASSERT(data->parent()->elementTypeId(idx) == metaTypeId);

    return data->data(data->parent()->indexOf(tagName));
}

void QTest::addColumnInternal(int id, const char *name)
{
    QTestTable *tbl = QTestTable::currentTestTable();
    QTEST_ASSERT_X(tbl, "QTest::addColumn()", "Cannot add testdata outside of a _data slot.");

    tbl->addColumn(id, name);
}

QTestData &QTest::newRow(const char *dataTag)
{
    QTestTable *tbl = QTestTable::currentTestTable();
    QTEST_ASSERT_X(tbl, "QTest::addColumn()", "Cannot add testdata outside of a _data slot.");

    return *tbl->newData(dataTag);
}

const char *QTest::currentAppName()
{
    return QTestResult::currentAppName();
}

const char *QTest::currentTestFunction()
{
    return QTestResult::currentTestFunction();
}

const char *QTest::currentDataTag()
{
    return QTestResult::currentDataTag();
}

bool QTest::currentTestFailed()
{
    return QTestResult::currentTestFailed();
}

void QTest::qSleep(int ms)
{
    QTEST_ASSERT(ms > 0);

#ifdef Q_OS_WIN
    Sleep(uint(ms));
#else
    struct timespec ts = { ms / 1000, (ms % 1000) * 1000 * 1000 };
    nanosleep(&ts, NULL);
#endif
}

QObject *QTest::testObject()
{
    return currentTestObject;
}

bool QTest::compare_helper(bool success, const char *msg, const char *file, int line)
{
    return QTestResult::compare(success, msg, file, line);
}

bool QTest::compare_helper(bool success, const char *msg, char *val1, char *val2,
                    const char *actual, const char *expected, const char *file, int line)
{
    return QTestResult::compare(success, msg, val1, val2, actual, expected, file, line);
}

template <>
Q_TESTLIB_EXPORT bool QTest::qCompare<float>(float const &t1, float const &t2, const char *actual, const char *expected,
                    const char *file, int line)
{
    return qFuzzyCompare(t1, t2)
            ? compare_helper(true, "COMPARE()", file, line)
            : compare_helper(false, "Compared floats are not the same (fuzzy compare)",
                             toString(t1), toString(t2), actual, expected, file, line);
}

template <>
Q_TESTLIB_EXPORT bool QTest::qCompare<double>(double const &t1, double const &t2, const char *actual, const char *expected,
                    const char *file, int line)
{
    return qFuzzyCompare(t1, t2)
            ? compare_helper(true, "COMPARE()", file, line)
            : compare_helper(false, "Compared doubles are not the same (fuzzy compare)",
                             toString(t1), toString(t2), actual, expected, file, line);
}

#define COMPARE_IMPL2(TYPE, FORMAT) \
template <> Q_TESTLIB_EXPORT char *QTest::toString<TYPE >(const TYPE &t) \
{ \
    char *msg = new char[128]; \
    qt_snprintf(msg, 128, #FORMAT, t); \
    return msg; \
}

COMPARE_IMPL2(short, %hd)
COMPARE_IMPL2(ushort, %hu)
COMPARE_IMPL2(int, %d)
COMPARE_IMPL2(uint, %u)
COMPARE_IMPL2(long, %ld)
COMPARE_IMPL2(ulong, %lu)
#if defined(Q_OS_WIN)
COMPARE_IMPL2(qint64, %I64d)
COMPARE_IMPL2(quint64, %I64u)
#else
COMPARE_IMPL2(qint64, %lld)
COMPARE_IMPL2(quint64, %llu)
#endif
COMPARE_IMPL2(bool, %d)
COMPARE_IMPL2(char, %c)
COMPARE_IMPL2(float, %g)
COMPARE_IMPL2(double, %lg)


char *QTest::toString(const char *str)
{
    if (!str)
        return 0;
    char *msg = new char[strlen(str) + 1];
    return qstrcpy(msg, str);
}

char *QTest::toString(const void *p)
{
    char *msg = new char[128];
    qt_snprintf(msg, 128, "%p", p);
    return msg;
}

bool QTest::compare_string_helper(const char *t1, const char *t2, const char *actual,
                                  const char *expected, const char *file, int line)
{
    return (qstrcmp(t1, t2) == 0)
            ? compare_helper(true, "COMPARE()", file, line)
            : compare_helper(false, "Compared strings are not the same",
                             toString(t1), toString(t2), actual, expected, file, line);
}

QT_END_NAMESPACE