GPUTestExpectationsParser.cpp

//
// Copyright 2019 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//

#include "GPUTestExpectationsParser.h"

#include <stddef.h>
#include <stdint.h>
#include <string.h>

#include "common/angleutils.h"
#include "common/debug.h"
#include "common/string_utils.h"

namespace angle
{

namespace
{

enum LineParserStage
{
    kLineParserBegin = 0,
    kLineParserBugID,
    kLineParserConfigs,
    kLineParserColon,
    kLineParserTestName,
    kLineParserEqual,
    kLineParserExpectations,
};

enum Token
{
    // os
    kConfigWinXP = 0,
    kConfigWinVista,
    kConfigWin7,
    kConfigWin8,
    kConfigWin10,
    kConfigWin,
    kConfigMacLeopard,
    kConfigMacSnowLeopard,
    kConfigMacLion,
    kConfigMacMountainLion,
    kConfigMacMavericks,
    kConfigMacYosemite,
    kConfigMacElCapitan,
    kConfigMacSierra,
    kConfigMacHighSierra,
    kConfigMacMojave,
    kConfigMac,
    kConfigIOS,
    kConfigLinux,
    kConfigChromeOS,
    kConfigAndroid,
    // gpu vendor
    kConfigNVIDIA,
    kConfigAMD,
    kConfigIntel,
    kConfigVMWare,
    // build type
    kConfigRelease,
    kConfigDebug,
    // ANGLE renderer
    kConfigD3D9,
    kConfigD3D11,
    kConfigGLDesktop,
    kConfigGLES,
    kConfigVulkan,
    kConfigSwiftShader,
    kConfigMetal,
    // Android devices
    kConfigNexus5X,
    kConfigPixel2,
    kConfigPixel4,
    // GPU devices
    kConfigNVIDIAQuadroP400,
    // PreRotation
    kConfigPreRotation,
    kConfigPreRotation90,
    kConfigPreRotation180,
    kConfigPreRotation270,
    // SPIR-V generation
    kConfigSPIRVGen,
    // expectation
    kExpectationPass,
    kExpectationFail,
    kExpectationFlaky,
    kExpectationTimeout,
    kExpectationSkip,
    // separator
    kSeparatorColon,
    kSeparatorEqual,

    kNumberOfExactMatchTokens,

    // others
    kTokenComment,
    kTokenWord,

    kNumberOfTokens,
};

enum ErrorType
{
    kErrorFileIO = 0,
    kErrorIllegalEntry,
    kErrorInvalidEntry,
    kErrorEntryWithExpectationConflicts,
    kErrorEntryWithDisallowedExpectation,
    kErrorEntriesOverlap,

    kNumberOfErrors,
};

struct TokenInfo
{
    constexpr TokenInfo()
        : name(nullptr),
          condition(GPUTestConfig::kConditionNone),
          expectation(GPUTestExpectationsParser::kGpuTestPass)
    {}

    constexpr TokenInfo(const char *nameIn,
                        GPUTestConfig::Condition conditionIn,
                        GPUTestExpectationsParser::GPUTestExpectation expectationIn)
        : name(nameIn), condition(conditionIn), expectation(expectationIn)
    {}

    constexpr TokenInfo(const char *nameIn, GPUTestConfig::Condition conditionIn)
        : TokenInfo(nameIn, conditionIn, GPUTestExpectationsParser::kGpuTestPass)
    {}

    const char *name;
    GPUTestConfig::Condition condition;
    GPUTestExpectationsParser::GPUTestExpectation expectation;
};

constexpr TokenInfo kTokenData[kNumberOfTokens] = {
    {"xp", GPUTestConfig::kConditionWinXP},
    {"vista", GPUTestConfig::kConditionWinVista},
    {"win7", GPUTestConfig::kConditionWin7},
    {"win8", GPUTestConfig::kConditionWin8},
    {"win10", GPUTestConfig::kConditionWin10},
    {"win", GPUTestConfig::kConditionWin},
    {"leopard", GPUTestConfig::kConditionMacLeopard},
    {"snowleopard", GPUTestConfig::kConditionMacSnowLeopard},
    {"lion", GPUTestConfig::kConditionMacLion},
    {"mountainlion", GPUTestConfig::kConditionMacMountainLion},
    {"mavericks", GPUTestConfig::kConditionMacMavericks},
    {"yosemite", GPUTestConfig::kConditionMacYosemite},
    {"elcapitan", GPUTestConfig::kConditionMacElCapitan},
    {"sierra", GPUTestConfig::kConditionMacSierra},
    {"highsierra", GPUTestConfig::kConditionMacHighSierra},
    {"mojave", GPUTestConfig::kConditionMacMojave},
    {"mac", GPUTestConfig::kConditionMac},
    {"ios", GPUTestConfig::kConditionIOS},
    {"linux", GPUTestConfig::kConditionLinux},
    {"chromeos", GPUTestConfig::kConditionNone},  // https://anglebug.com/3363 CrOS not supported
    {"android", GPUTestConfig::kConditionAndroid},
    {"nvidia", GPUTestConfig::kConditionNVIDIA},
    {"amd", GPUTestConfig::kConditionAMD},
    {"intel", GPUTestConfig::kConditionIntel},
    {"vmware", GPUTestConfig::kConditionVMWare},
    {"release", GPUTestConfig::kConditionRelease},
    {"debug", GPUTestConfig::kConditionDebug},
    {"d3d9", GPUTestConfig::kConditionD3D9},
    {"d3d11", GPUTestConfig::kConditionD3D11},
    {"opengl", GPUTestConfig::kConditionGLDesktop},
    {"gles", GPUTestConfig::kConditionGLES},
    {"vulkan", GPUTestConfig::kConditionVulkan},
    {"swiftshader", GPUTestConfig::kConditionSwiftShader},
    {"metal", GPUTestConfig::kConditionMetal},
    {"nexus5x", GPUTestConfig::kConditionNexus5X},
    {"pixel2orxl", GPUTestConfig::kConditionPixel2OrXL},
    {"pixel4orxl", GPUTestConfig::kConditionPixel4OrXL},
    {"quadrop400", GPUTestConfig::kConditionNVIDIAQuadroP400},
    {"prerotation", GPUTestConfig::kConditionPreRotation},
    {"prerotation90", GPUTestConfig::kConditionPreRotation90},
    {"prerotation180", GPUTestConfig::kConditionPreRotation180},
    {"prerotation270", GPUTestConfig::kConditionPreRotation270},
    {"spirvgen", GPUTestConfig::kConditionSPIRVGen},
    {"pass", GPUTestConfig::kConditionNone, GPUTestExpectationsParser::kGpuTestPass},
    {"fail", GPUTestConfig::kConditionNone, GPUTestExpectationsParser::kGpuTestFail},
    {"flaky", GPUTestConfig::kConditionNone, GPUTestExpectationsParser::kGpuTestFlaky},
    {"timeout", GPUTestConfig::kConditionNone, GPUTestExpectationsParser::kGpuTestTimeout},
    {"skip", GPUTestConfig::kConditionNone, GPUTestExpectationsParser::kGpuTestSkip},
    {":", GPUTestConfig::kConditionNone},  // kSeparatorColon
    {"=", GPUTestConfig::kConditionNone},  // kSeparatorEqual
    {},                                    // kNumberOfExactMatchTokens
    {},                                    // kTokenComment
    {},                                    // kTokenWord
};

const char *kErrorMessage[kNumberOfErrors] = {
    "file IO failed",
    "entry with wrong format",
    "entry invalid, likely unimplemented modifiers",
    "entry with expectation modifier conflicts",
    "entry with unsupported expectation",
    "two entries' configs overlap",
};

inline bool StartsWithASCII(const std::string &str, const std::string &search, bool caseSensitive)
{
    ASSERT(!caseSensitive);
    return str.compare(0, search.length(), search) == 0;
}

template <class Char>
inline Char ToLowerASCII(Char c)
{
    return (c >= 'A' && c <= 'Z') ? (c + ('a' - 'A')) : c;
}

template <typename Iter>
inline bool DoLowerCaseEqualsASCII(Iter a_begin, Iter a_end, const char *b)
{
    for (Iter it = a_begin; it != a_end; ++it, ++b)
    {
        if (!*b || ToLowerASCII(*it) != *b)
            return false;
    }
    return *b == 0;
}

inline bool LowerCaseEqualsASCII(const std::string &a, const char *b)
{
    return DoLowerCaseEqualsASCII(a.begin(), a.end(), b);
}

inline Token ParseToken(const std::string &word)
{
    if (StartsWithASCII(word, "//", false))
        return kTokenComment;

    for (int32_t i = 0; i < kNumberOfExactMatchTokens; ++i)
    {
        if (LowerCaseEqualsASCII(word, kTokenData[i].name))
            return static_cast<Token>(i);
    }
    return kTokenWord;
}

bool ConditionArrayIsSubset(const GPUTestConfig::ConditionArray &subset,
                            const GPUTestConfig::ConditionArray &superset)
{
    for (size_t subsetCondition : subset)
    {
        bool foundCondition = false;
        for (size_t supersetCondition : superset)
        {
            if (subsetCondition == supersetCondition)
            {
                foundCondition = true;
                break;
            }
        }

        if (!foundCondition)
        {
            return false;
        }
    }

    return true;
}

// If one array is completely contained within the other, then we say the conditions overlap.
bool ConditionsOverlap(const GPUTestConfig::ConditionArray &conditionsI,
                       const GPUTestConfig::ConditionArray &conditionsJ)
{
    return ConditionArrayIsSubset(conditionsI, conditionsJ) ||
           ConditionArrayIsSubset(conditionsJ, conditionsI);
}
}  // anonymous namespace

const char *GetConditionName(uint32_t condition)
{
    if (condition == GPUTestConfig::kConditionNone)
    {
        return nullptr;
    }

    for (const TokenInfo &info : kTokenData)
    {
        if (info.condition == condition)
        {
            // kConditionNone is used to tag tokens that aren't conditions, but this case has been
            // handled above.
            ASSERT(info.condition != GPUTestConfig::kConditionNone);
            return info.name;
        }
    }

    return nullptr;
}

GPUTestExpectationsParser::GPUTestExpectationsParser()
    : mExpectationsAllowMask(
          GPUTestExpectationsParser::kGpuTestPass | GPUTestExpectationsParser::kGpuTestFail |
          GPUTestExpectationsParser::kGpuTestFlaky | GPUTestExpectationsParser::kGpuTestTimeout |
          GPUTestExpectationsParser::kGpuTestSkip)
{
    // Some initial checks.
    ASSERT((static_cast<unsigned int>(kNumberOfTokens)) ==
           (sizeof(kTokenData) / sizeof(kTokenData[0])));
    ASSERT((static_cast<unsigned int>(kNumberOfErrors)) ==
           (sizeof(kErrorMessage) / sizeof(kErrorMessage[0])));
}

GPUTestExpectationsParser::~GPUTestExpectationsParser() = default;

bool GPUTestExpectationsParser::loadTestExpectationsImpl(const GPUTestConfig *config,
                                                         const std::string &data)
{
    mEntries.clear();
    mErrorMessages.clear();

    std::vector<std::string> lines = SplitString(data, "\n", TRIM_WHITESPACE, SPLIT_WANT_ALL);
    bool rt                        = true;
    for (size_t i = 0; i < lines.size(); ++i)
    {
        if (!parseLine(config, lines[i], i + 1))
            rt = false;
    }
    if (detectConflictsBetweenEntries())
    {
        mEntries.clear();
        rt = false;
    }

    return rt;
}

bool GPUTestExpectationsParser::loadTestExpectations(const GPUTestConfig &config,
                                                     const std::string &data)
{
    return loadTestExpectationsImpl(&config, data);
}

bool GPUTestExpectationsParser::loadAllTestExpectations(const std::string &data)
{
    return loadTestExpectationsImpl(nullptr, data);
}

bool GPUTestExpectationsParser::loadTestExpectationsFromFileImpl(const GPUTestConfig *config,
                                                                 const std::string &path)
{
    mEntries.clear();
    mErrorMessages.clear();

    std::string data;
    if (!ReadFileToString(path, &data))
    {
        mErrorMessages.push_back(kErrorMessage[kErrorFileIO]);
        return false;
    }
    return loadTestExpectationsImpl(config, data);
}

bool GPUTestExpectationsParser::loadTestExpectationsFromFile(const GPUTestConfig &config,
                                                             const std::string &path)
{
    return loadTestExpectationsFromFileImpl(&config, path);
}

bool GPUTestExpectationsParser::loadAllTestExpectationsFromFile(const std::string &path)
{
    return loadTestExpectationsFromFileImpl(nullptr, path);
}

int32_t GPUTestExpectationsParser::getTestExpectationImpl(const GPUTestConfig *config,
                                                          const std::string &testName)
{
    size_t maxExpectationLen            = 0;
    GPUTestExpectationEntry *foundEntry = nullptr;
    for (GPUTestExpectationEntry &entry : mEntries)
    {
        if (NamesMatchWithWildcard(entry.testName.c_str(), testName.c_str()))
        {
            size_t expectationLen = entry.testName.length();

            // Filter by condition first.
            bool satisfiesConditions = true;
            if (config)
            {
                for (size_t condition : entry.conditions)
                {
                    if (!config->getConditions()[condition])
                    {
                        satisfiesConditions = false;
                        break;
                    }
                }
            }

            // The longest/most specific matching expectation overrides any others.
            if (satisfiesConditions && expectationLen > maxExpectationLen)
            {
                maxExpectationLen = expectationLen;
                foundEntry        = &entry;
            }
        }
    }
    if (foundEntry != nullptr)
    {
        foundEntry->used = true;
        return foundEntry->testExpectation;
    }
    return kGpuTestPass;
}

int32_t GPUTestExpectationsParser::getTestExpectation(const std::string &testName)
{
    return getTestExpectationImpl(nullptr, testName);
}

int32_t GPUTestExpectationsParser::getTestExpectationWithConfig(const GPUTestConfig &config,
                                                                const std::string &testName)
{
    return getTestExpectationImpl(&config, testName);
}

const std::vector<std::string> &GPUTestExpectationsParser::getErrorMessages() const
{
    return mErrorMessages;
}

std::vector<std::string> GPUTestExpectationsParser::getUnusedExpectationsMessages() const
{
    std::vector<std::string> messages;
    std::vector<GPUTestExpectationsParser::GPUTestExpectationEntry> unusedExpectations =
        getUnusedExpectations();
    for (size_t i = 0; i < unusedExpectations.size(); ++i)
    {
        std::string message =
            "Line " + ToString(unusedExpectations[i].lineNumber) + ": expectation was unused.";
        messages.push_back(message);
    }
    return messages;
}

bool GPUTestExpectationsParser::parseLine(const GPUTestConfig *config,
                                          const std::string &lineData,
                                          size_t lineNumber)
{
    std::vector<std::string> tokens =
        SplitString(lineData, kWhitespaceASCII, KEEP_WHITESPACE, SPLIT_WANT_NONEMPTY);
    int32_t stage = kLineParserBegin;
    GPUTestExpectationEntry entry;
    entry.lineNumber = lineNumber;
    entry.used       = false;
    bool skipLine    = false;
    for (size_t i = 0; i < tokens.size() && !skipLine; ++i)
    {
        Token token = ParseToken(tokens[i]);
        switch (token)
        {
            case kTokenComment:
                skipLine = true;
                break;
            case kConfigWinXP:
            case kConfigWinVista:
            case kConfigWin7:
            case kConfigWin8:
            case kConfigWin10:
            case kConfigWin:
            case kConfigMacLeopard:
            case kConfigMacSnowLeopard:
            case kConfigMacLion:
            case kConfigMacMountainLion:
            case kConfigMacMavericks:
            case kConfigMacYosemite:
            case kConfigMacElCapitan:
            case kConfigMacSierra:
            case kConfigMacHighSierra:
            case kConfigMacMojave:
            case kConfigMac:
            case kConfigIOS:
            case kConfigLinux:
            case kConfigChromeOS:
            case kConfigAndroid:
            case kConfigNVIDIA:
            case kConfigAMD:
            case kConfigIntel:
            case kConfigVMWare:
            case kConfigRelease:
            case kConfigDebug:
            case kConfigD3D9:
            case kConfigD3D11:
            case kConfigGLDesktop:
            case kConfigGLES:
            case kConfigVulkan:
            case kConfigSwiftShader:
            case kConfigMetal:
            case kConfigNexus5X:
            case kConfigPixel2:
            case kConfigPixel4:
            case kConfigNVIDIAQuadroP400:
            case kConfigPreRotation:
            case kConfigPreRotation90:
            case kConfigPreRotation180:
            case kConfigPreRotation270:
            case kConfigSPIRVGen:
                // MODIFIERS, check each condition and add accordingly.
                if (stage != kLineParserConfigs && stage != kLineParserBugID)
                {
                    pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
                    return false;
                }
                {
                    bool err = false;
                    if (config)
                    {
                        if (!checkTokenCondition(*config, err, token, lineNumber))
                        {
                            skipLine = true;  // Move to the next line without adding this one.
                        }
                    }
                    else
                    {
                        // Store the conditions for later comparison if we don't have a config.
                        entry.conditions[kTokenData[token].condition] = true;
                    }
                    if (err)
                    {
                        return false;
                    }
                }
                if (stage == kLineParserBugID)
                {
                    stage++;
                }
                break;
            case kSeparatorColon:
                // :
                // If there are no modifiers, move straight to separator colon
                if (stage == kLineParserBugID)
                {
                    stage++;
                }
                if (stage != kLineParserConfigs)
                {
                    pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
                    return false;
                }
                stage++;
                break;
            case kSeparatorEqual:
                // =
                if (stage != kLineParserTestName)
                {
                    pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
                    return false;
                }
                stage++;
                break;
            case kTokenWord:
                // BUG_ID or TEST_NAME
                if (stage == kLineParserBegin)
                {
                    // Bug ID is not used for anything; ignore it.
                }
                else if (stage == kLineParserColon)
                {
                    entry.testName = tokens[i];
                }
                else
                {
                    pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
                    return false;
                }
                stage++;
                break;
            case kExpectationPass:
            case kExpectationFail:
            case kExpectationFlaky:
            case kExpectationTimeout:
            case kExpectationSkip:
                // TEST_EXPECTATIONS
                if (stage != kLineParserEqual && stage != kLineParserExpectations)
                {
                    pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
                    return false;
                }
                if (entry.testExpectation != 0)
                {
                    pushErrorMessage(kErrorMessage[kErrorEntryWithExpectationConflicts],
                                     lineNumber);
                    return false;
                }
                if ((mExpectationsAllowMask & kTokenData[token].expectation) == 0)
                {
                    pushErrorMessage(kErrorMessage[kErrorEntryWithDisallowedExpectation],
                                     lineNumber);
                    return false;
                }
                entry.testExpectation = kTokenData[token].expectation;
                if (stage == kLineParserEqual)
                    stage++;
                break;
            default:
                ASSERT(false);
                break;
        }
    }
    if (stage == kLineParserBegin || skipLine)
    {
        // The whole line is empty or all comments, or has been skipped to to a condition token.
        return true;
    }
    if (stage == kLineParserExpectations)
    {
        mEntries.push_back(entry);
        return true;
    }
    pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
    return false;
}

bool GPUTestExpectationsParser::checkTokenCondition(const GPUTestConfig &config,
                                                    bool &err,
                                                    int32_t token,
                                                    size_t lineNumber)
{
    if (token >= kNumberOfTokens)
    {
        pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
        err = true;
        return false;
    }

    if (kTokenData[token].condition == GPUTestConfig::kConditionNone ||
        kTokenData[token].condition >= GPUTestConfig::kNumberOfConditions)
    {
        pushErrorMessage(kErrorMessage[kErrorInvalidEntry], lineNumber);
        // error on any unsupported conditions
        err = true;
        return false;
    }
    err = false;
    return config.getConditions()[kTokenData[token].condition];
}

bool GPUTestExpectationsParser::detectConflictsBetweenEntries()
{
    bool rt = false;
    for (size_t i = 0; i < mEntries.size(); ++i)
    {
        for (size_t j = i + 1; j < mEntries.size(); ++j)
        {
            const GPUTestExpectationEntry &entryI = mEntries[i];
            const GPUTestExpectationEntry &entryJ = mEntries[j];
            if (entryI.testName == entryJ.testName &&
                ConditionsOverlap(entryI.conditions, entryJ.conditions))
            {
                pushErrorMessage(kErrorMessage[kErrorEntriesOverlap], entryI.lineNumber,
                                 entryJ.lineNumber);
                rt = true;
            }
        }
    }
    return rt;
}

std::vector<GPUTestExpectationsParser::GPUTestExpectationEntry>
GPUTestExpectationsParser::getUnusedExpectations() const
{
    std::vector<GPUTestExpectationsParser::GPUTestExpectationEntry> unusedExpectations;
    for (size_t i = 0; i < mEntries.size(); ++i)
    {
        if (!mEntries[i].used)
        {
            unusedExpectations.push_back(mEntries[i]);
        }
    }
    return unusedExpectations;
}

void GPUTestExpectationsParser::pushErrorMessage(const std::string &message, size_t lineNumber)
{
    mErrorMessages.push_back("Line " + ToString(lineNumber) + " : " + message.c_str());
}

void GPUTestExpectationsParser::pushErrorMessage(const std::string &message,
                                                 size_t entry1LineNumber,
                                                 size_t entry2LineNumber)
{
    mErrorMessages.push_back("Line " + ToString(entry1LineNumber) + " and " +
                             ToString(entry2LineNumber) + " : " + message.c_str());
}

GPUTestExpectationsParser::GPUTestExpectationEntry::GPUTestExpectationEntry()
    : testExpectation(0), lineNumber(0)
{}

}  // namespace angle