TestSuite.h

//
// Copyright 2019 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// TestSuite:
//   Basic implementation of a test harness in ANGLE.

#ifndef ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_
#define ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_

#include <map>
#include <memory>
#include <mutex>
#include <queue>
#include <string>
#include <thread>

#include "HistogramWriter.h"
#include "tests/test_expectations/GPUTestExpectationsParser.h"
#include "util/test_utils.h"

namespace angle
{
struct TestIdentifier
{
    TestIdentifier();
    TestIdentifier(const std::string &suiteNameIn, const std::string &nameIn);
    TestIdentifier(const TestIdentifier &other);
    ~TestIdentifier();

    TestIdentifier &operator=(const TestIdentifier &other);

    static bool ParseFromString(const std::string &str, TestIdentifier *idOut);

    bool valid() const { return !testName.empty(); }
    void sprintfName(char *outBuffer) const;

    std::string testSuiteName;
    std::string testName;
};

inline bool operator<(const TestIdentifier &a, const TestIdentifier &b)
{
    return std::tie(a.testSuiteName, a.testName) < std::tie(b.testSuiteName, b.testName);
}

inline bool operator==(const TestIdentifier &a, const TestIdentifier &b)
{
    return std::tie(a.testSuiteName, a.testName) == std::tie(b.testSuiteName, b.testName);
}

inline std::ostream &operator<<(std::ostream &os, const TestIdentifier &id)
{
    return os << id.testSuiteName << "." << id.testName;
}

enum class TestResultType
{
    Crash,
    Fail,
    NoResult,
    Pass,
    Skip,
    Timeout,
    Unknown,
};

const char *TestResultTypeToString(TestResultType type);

struct TestResult
{
    TestResultType type       = TestResultType::NoResult;
    double elapsedTimeSeconds = 0.0;
    uint32_t flakyFailures    = 0;
};

inline bool operator==(const TestResult &a, const TestResult &b)
{
    return a.type == b.type;
}

inline std::ostream &operator<<(std::ostream &os, const TestResult &result)
{
    return os << TestResultTypeToString(result.type);
}

struct TestResults
{
    TestResults();
    ~TestResults();

    std::map<TestIdentifier, TestResult> results;
    std::mutex currentTestMutex;
    TestIdentifier currentTest;
    Timer currentTestTimer;
    double currentTestTimeout = 0.0;
    bool allDone              = false;
    std::string testArtifactsFakeTestName;
    std::vector<std::string> testArtifactPaths;
};

struct FileLine
{
    const char *file;
    int line;
};

struct ProcessInfo : angle::NonCopyable
{
    ProcessInfo();
    ~ProcessInfo();
    ProcessInfo(ProcessInfo &&other);
    ProcessInfo &operator=(ProcessInfo &&rhs);

    ProcessHandle process;
    std::vector<TestIdentifier> testsInBatch;
    std::string resultsFileName;
    std::string filterFileName;
    std::string commandLine;
    std::string filterString;
};

using TestQueue = std::queue<std::vector<TestIdentifier>>;

class TestSuite
{
  public:
    TestSuite(int *argc, char **argv);
    ~TestSuite();

    int run();
    void onCrashOrTimeout(TestResultType crashOrTimeout);
    void addHistogramSample(const std::string &measurement,
                            const std::string &story,
                            double value,
                            const std::string &units);

    static TestSuite *GetInstance() { return mInstance; }

    // Returns the path to the artifact in the output directory.
    std::string addTestArtifact(const std::string &artifactName);

    int getShardIndex() const { return mShardIndex; }
    int getBatchId() const { return mBatchId; }

    // Test expectation processing.
    bool loadTestExpectationsFromFileWithConfig(const GPUTestConfig &config,
                                                const std::string &fileName);
    bool loadAllTestExpectationsFromFile(const std::string &fileName);
    int32_t getTestExpectation(const std::string &testName);
    void maybeUpdateTestTimeout(uint32_t testExpectation);
    int32_t getTestExpectationWithConfigAndUpdateTimeout(const GPUTestConfig &config,
                                                         const std::string &testName);
    bool logAnyUnusedTestExpectations();
    void setTestExpectationsAllowMask(uint32_t mask)
    {
        mTestExpectationsParser.setTestExpectationsAllowMask(mask);
    }

  private:
    bool parseSingleArg(const char *argument);
    bool launchChildTestProcess(uint32_t batchId, const std::vector<TestIdentifier> &testsInBatch);
    bool finishProcess(ProcessInfo *processInfo);
    int printFailuresAndReturnCount() const;
    void startWatchdog();
    void dumpTestExpectationsErrorMessages();
    int getSlowTestTimeout() const;

    static TestSuite *mInstance;

    std::string mTestExecutableName;
    std::string mTestSuiteName;
    TestQueue mTestQueue;
    std::string mFilterString;
    std::string mFilterFile;
    std::string mResultsDirectory;
    std::string mResultsFile;
    std::string mHistogramJsonFile;
    int mShardCount;
    int mShardIndex;
    angle::CrashCallback mCrashCallback;
    TestResults mTestResults;
    bool mBotMode;
    bool mDebugTestGroups;
    bool mGTestListTests;
    bool mListTests;
    bool mPrintTestStdout;
    bool mDisableCrashHandler;
    int mBatchSize;
    int mCurrentResultCount;
    int mTotalResultCount;
    int mMaxProcesses;
    int mTestTimeout;
    int mBatchTimeout;
    int mBatchId;
    int mFlakyRetries;
    int mMaxFailures;
    int mFailureCount;
    std::vector<std::string> mChildProcessArgs;
    std::map<TestIdentifier, FileLine> mTestFileLines;
    std::vector<ProcessInfo> mCurrentProcesses;
    std::thread mWatchdogThread;
    HistogramWriter mHistogramWriter;
    std::string mTestArtifactDirectory;
    GPUTestExpectationsParser mTestExpectationsParser;
};

bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut);
}  // namespace angle

#endif  // ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_