CoCalc provides the best real-time collaborative environment for Jupyter Notebooks, LaTeX documents, and SageMath, scalable from individual users to large groups and classes!
CoCalc provides the best real-time collaborative environment for Jupyter Notebooks, LaTeX documents, and SageMath, scalable from individual users to large groups and classes!
Path: blob/master/headless/Compare.cpp
Views: 1401
// Copyright (c) 2012- PPSSPP Project.12// This program is free software: you can redistribute it and/or modify3// it under the terms of the GNU General Public License as published by4// the Free Software Foundation, version 2.0 or later versions.56// This program is distributed in the hope that it will be useful,7// but WITHOUT ANY WARRANTY; without even the implied warranty of8// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the9// GNU General Public License 2.0 for more details.1011// A copy of the GPL 2.0 should have been included with the program.12// If not, see http://www.gnu.org/licenses/1314// Official git repository and contact information can be found at15// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.1617#include <algorithm>18#include <cmath>19#include <cstdarg>20#include <iostream>21#include <png.h>22#include <vector>2324#include "headless/Compare.h"25#include "Common/Data/Convert/ColorConv.h"26#include "Common/Data/Format/PNGLoad.h"27#include "Common/File/FileUtil.h"28#include "Common/StringUtils.h"29#include "Core/Loaders.h"3031#include "GPU/GPUState.h"32#include "GPU/Common/GPUDebugInterface.h"33#include "GPU/Common/TextureDecoder.h"343536bool teamCityMode = false;37std::string currentTestName = "";3839void TeamCityPrint(const char *fmt, ...)40{41if (!teamCityMode)42return;4344const int TEMP_BUFFER_SIZE = 32768;45char temp[TEMP_BUFFER_SIZE];4647va_list args;48va_start(args, fmt);49vsnprintf(temp, TEMP_BUFFER_SIZE - 1, fmt, args);50temp[TEMP_BUFFER_SIZE - 1] = '\0';51va_end(args);5253printf("##teamcity[%s]\n", temp);54}5556void GitHubActionsPrint(const char *type, const char *fmt, ...) {57if (!getenv("GITHUB_ACTIONS"))58return;5960const int TEMP_BUFFER_SIZE = 32768;61char temp[TEMP_BUFFER_SIZE];6263va_list args;64va_start(args, fmt);65vsnprintf(temp, TEMP_BUFFER_SIZE - 1, fmt, args);66temp[TEMP_BUFFER_SIZE - 1] = '\0';67va_end(args);6869printf("::%s file=%s::%s\n", type, currentTestName.c_str(), temp);70}7172struct BufferedLineReader {73const static int MAX_BUFFER = 5;74const static int TEMP_BUFFER_SIZE = 32768;7576BufferedLineReader(const std::string &data) : data_(data) {77}7879void Fill() {80while (valid_ < MAX_BUFFER && HasMoreLines()) {81buffer_[valid_++] = TrimNewlines(ReadLine());82}83}8485const std::string Peek(int pos) {86if (pos >= valid_) {87Fill();88}89if (pos >= valid_) {90return "";91}92return buffer_[pos];93}9495void Skip(int count) {96if (count > valid_) {97count = valid_;98}99valid_ -= count;100for (int i = 0; i < valid_; ++i) {101buffer_[i] = buffer_[i + count];102}103Fill();104}105106const std::string Consume() {107const std::string result = Peek(0);108Skip(1);109return result;110}111112bool HasLines() {113if (HasMoreLines()) {114return true;115}116// Don't say yes if it's a blank line.117for (int i = 0; i < valid_; ++i) {118if (!buffer_[i].empty()) {119return true;120}121}122return false;123}124125bool Compare(BufferedLineReader &other) {126if (Peek(0) != other.Peek(0)) {127return false;128}129130Skip(1);131other.Skip(1);132return true;133}134135protected:136BufferedLineReader() {137}138139bool HasMoreLines() {140return pos_ != data_.npos;141}142143std::string ReadLine() {144size_t next = data_.find('\n', pos_);145if (next == data_.npos) {146std::string result = data_.substr(pos_);147pos_ = next;148return result;149} else {150std::string result = data_.substr(pos_, next - pos_);151pos_ = next + 1;152return result;153}154}155156static std::string TrimNewlines(const std::string &s) {157size_t p = s.find_last_not_of("\r\n");158if (p == s.npos) {159return "";160}161return s.substr(0, p + 1);162}163164int valid_ = 0;165std::string buffer_[MAX_BUFFER];166const std::string data_;167size_t pos_ = 0;168};169170Path ExpectedScreenshotFromFilename(const Path &bootFilename) {171std::string extension = bootFilename.GetFileExtension();172if (extension.empty()) {173return bootFilename.WithExtraExtension(".bmp");174}175176// Let's use pngs as the default for ppdmp tests.177if (extension == ".ppdmp") {178return bootFilename.WithReplacedExtension(".png");179}180return bootFilename.WithReplacedExtension(".expected.bmp");181}182183static std::string ChopFront(std::string s, std::string front)184{185if (s.size() >= front.size())186{187if (s.substr(0, front.size()) == front)188return s.substr(front.size());189}190return s;191}192193static std::string ChopEnd(std::string s, std::string end)194{195if (s.size() >= end.size())196{197size_t endpos = s.size() - end.size();198if (s.substr(endpos) == end)199return s.substr(0, endpos);200}201return s;202}203204std::string GetTestName(const Path &bootFilename)205{206// Kinda ugly, trying to guesstimate the test name from filename...207return ChopEnd(ChopFront(ChopFront(bootFilename.ToString(), "tests/"), "pspautotests/tests/"), ".prx");208}209210bool CompareOutput(const Path &bootFilename, const std::string &output, bool verbose) {211Path expect_filename = bootFilename.GetFileExtension() == ".prx" ? bootFilename.WithReplacedExtension(".prx", ".expected") : bootFilename.WithExtraExtension(".expected");212std::unique_ptr<FileLoader> expect_loader(ConstructFileLoader(expect_filename));213214if (expect_loader->Exists()) {215std::string expect_results;216expect_results.resize(expect_loader->FileSize());217expect_results.resize(expect_loader->ReadAt(0, expect_loader->FileSize(), &expect_results[0]));218219BufferedLineReader expected(expect_results);220BufferedLineReader actual(output);221222bool failed = false;223while (expected.HasLines())224{225if (expected.Compare(actual))226continue;227228if (!failed)229{230TeamCityPrint("testFailed name='%s' message='Output different from expected file'", currentTestName.c_str());231GitHubActionsPrint("error", "Incorrect output for %s", currentTestName.c_str());232failed = true;233}234235// This is a really dirt simple comparing algorithm.236237// Perhaps it was an extra line?238if (expected.Peek(0) == actual.Peek(1) || !expected.HasLines())239printf("+ %s\n", actual.Consume().c_str());240// A single missing line?241else if (expected.Peek(1) == actual.Peek(0) || !actual.HasLines())242printf("- %s\n", expected.Consume().c_str());243else244{245printf("O %s\n", actual.Consume().c_str());246printf("E %s\n", expected.Consume().c_str());247}248}249250while (actual.HasLines())251{252// If it's a blank line, this will pass.253if (actual.Compare(expected))254continue;255256printf("+ %s\n", actual.Consume().c_str());257}258259if (verbose)260{261if (!failed)262{263printf("++++++++++++++ The Equal Output +++++++++++++\n");264printf("%s", output.c_str());265printf("+++++++++++++++++++++++++++++++++++++++++++++\n");266}267else268{269printf("============== output from failed %s:\n", GetTestName(bootFilename).c_str());270printf("%s", output.c_str());271printf("============== expected output:\n");272std::string fullExpected;273if (File::ReadTextFileToString(expect_filename, &fullExpected))274printf("%s", fullExpected.c_str());275printf("===============================\n");276}277}278279return !failed;280} else {281std::unique_ptr<FileLoader> screenshot(ConstructFileLoader(ExpectedScreenshotFromFilename(bootFilename)));282bool failed = true;283if (screenshot->Exists()) {284// Okay, just a screenshot then. Allow a pass with no output (i.e. screenshot match.)285failed = output.find_first_not_of(" \r\n\t") != output.npos;286if (failed) {287TeamCityPrint("testFailed name='%s' message='Output different from expected file'", currentTestName.c_str());288GitHubActionsPrint("error", "Incorrect output for %s", currentTestName.c_str());289}290} else {291fprintf(stderr, "Expectation file %s not found\n", expect_filename.c_str());292TeamCityPrint("testIgnored name='%s' message='Expects file missing'", currentTestName.c_str());293GitHubActionsPrint("error", "Expected file missing for %s", currentTestName.c_str());294}295296if (verbose || (screenshot->Exists() && failed)) {297BufferedLineReader actual(output);298while (actual.HasLines()) {299printf("+ %s\n", actual.Consume().c_str());300}301}302return !failed;303}304}305306static inline double CompareChannel(int pix1, int pix2) {307double diff = pix1 - pix2;308return diff * diff;309}310311static inline double ComparePixel(u32 pix1, u32 pix2) {312// Ignore alpha.313double r = CompareChannel(pix1 & 0xFF, pix2 & 0xFF);314double g = CompareChannel((pix1 >> 8) & 0xFF, (pix2 >> 8) & 0xFF);315double b = CompareChannel((pix1 >> 16) & 0xFF, (pix2 >> 16) & 0xFF);316317return r + g + b;318}319320std::vector<u32> TranslateDebugBufferToCompare(const GPUDebugBuffer *buffer, u32 stride, u32 h) {321// If the output was small, act like everything outside was 0.322// This can happen depending on viewport parameters.323u32 safeW = std::min(stride, buffer->GetStride());324u32 safeH = std::min(h, buffer->GetHeight());325326std::vector<u32> data;327data.resize(stride * h, 0);328329const u32 *pixels32 = (const u32 *)buffer->GetData();330const u16 *pixels16 = (const u16 *)buffer->GetData();331int outStride = buffer->GetStride();332if (!buffer->GetFlipped()) {333// Bitmaps are flipped, so we have to compare backwards in this case.334int toLastRow = outStride * (h > buffer->GetHeight() ? buffer->GetHeight() - 1 : h - 1);335pixels32 += toLastRow;336pixels16 += toLastRow;337outStride = -outStride;338}339340// Skip the bottom of the image in the buffer was smaller. Remember, we're flipped.341u32 *dst = &data[0];342if (safeH < h) {343dst += (h - safeH) * stride;344}345346for (u32 y = 0; y < safeH; ++y) {347switch (buffer->GetFormat()) {348case GPU_DBG_FORMAT_8888:349ConvertBGRA8888ToRGBA8888(&dst[y * stride], pixels32, safeW);350break;351case GPU_DBG_FORMAT_8888_BGRA:352memcpy(&dst[y * stride], pixels32, safeW * sizeof(u32));353break;354355case GPU_DBG_FORMAT_565:356ConvertRGB565ToBGRA8888(&dst[y * stride], pixels16, safeW);357break;358case GPU_DBG_FORMAT_5551:359ConvertRGBA5551ToBGRA8888(&dst[y * stride], pixels16, safeW);360break;361case GPU_DBG_FORMAT_4444:362ConvertRGBA4444ToBGRA8888(&dst[y * stride], pixels16, safeW);363break;364365default:366data.resize(0);367return data;368}369370pixels32 += outStride;371pixels16 += outStride;372}373374return data;375}376377378ScreenshotComparer::~ScreenshotComparer() {379if (reference_)380free(reference_);381}382383double ScreenshotComparer::Compare(const Path &screenshotFilename) {384if (pixels_.size() < stride_ * h_) {385error_ = "Buffer format conversion error";386return -1.0f;387}388389// We assume the bitmap is the specified size, not including whatever stride.390std::unique_ptr<FileLoader> loader(ConstructFileLoader(screenshotFilename));391if (loader->Exists()) {392uint8_t header[2];393if (loader->ReadAt(0, 2, header) != 2) {394error_ = "Unable to read screenshot data: " + screenshotFilename.ToVisualString();395return -1.0f;396}397398if (header[0] == 'B' && header[1] == 'M') {399reference_ = (u32 *)calloc(stride_ * h_, sizeof(u32));400referenceStride_ = stride_;401asBitmap_ = true;402// The bitmap header is 14 + 40 bytes. We could validate it but the test would fail either way.403if (reference_ && loader->ReadAt(14 + 40, sizeof(u32), stride_ * h_, reference_) != stride_ * h_) {404error_ = "Unable to read screenshot data: " + screenshotFilename.ToVisualString();405free(reference_);406reference_ = nullptr;407return -1.0f;408}409} else {410// For now, assume a PNG otherwise.411std::vector<uint8_t> compressed;412compressed.resize(loader->FileSize());413if (loader->ReadAt(0, compressed.size(), &compressed[0]) != compressed.size()) {414error_ = "Unable to read screenshot data: " + screenshotFilename.ToVisualString();415return -1.0f;416}417418int width, height;419if (!pngLoadPtr(&compressed[0], compressed.size(), &width, &height, (unsigned char **)&reference_)) {420error_ = "Unable to read screenshot data: " + screenshotFilename.ToVisualString();421if (reference_)422free(reference_);423reference_ = nullptr;424return -1.0f;425}426427referenceStride_ = width;428}429} else {430error_ = "Unable to read screenshot: " + screenshotFilename.ToVisualString();431return -1.0f;432}433434if (!reference_) {435error_ = "Unable to allocate screenshot data: " + screenshotFilename.ToVisualString();436return -1.0f;437}438439double errors = 0;440if (asBitmap_) {441// The reference is flipped and BGRA by default for the common BMP compare case.442for (u32 y = 0; y < h_; ++y) {443u32 yoff = y * referenceStride_;444for (u32 x = 0; x < w_; ++x)445errors += ComparePixel(pixels_[y * stride_ + x], reference_[yoff + x]);446}447} else {448// Just convert to BGRA for simplicity.449ConvertRGBA8888ToBGRA8888(reference_, reference_, h_ * referenceStride_);450for (u32 y = 0; y < h_; ++y) {451u32 yoff = (h_ - y - 1) * referenceStride_;452for (u32 x = 0; x < w_; ++x)453errors += ComparePixel(pixels_[y * stride_ + x], reference_[yoff + x]);454}455}456457// Convert to MSE, accounting for all three channels (RGB.)458return errors / (double)(w_ * h_ * 3);459}460461bool ScreenshotComparer::SaveActualBitmap(const Path &resultFilename) {462static const u8 header[14 + 40] = {4630x42, 0x4D, 0x38, 0x80, 0x08, 0x00, 0x00, 0x00,4640x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x28, 0x00,4650x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x10, 0x01,4660x00, 0x00, 0x01, 0x00, 0x20, 0x00, 0x00, 0x00,4670x00, 0x00, 0x02, 0x80, 0x08, 0x00, 0x12, 0x0B,4680x00, 0x00, 0x12, 0x0B, 0x00, 0x00, 0x00, 0x00,4690x00, 0x00, 0x00, 0x00, 0x00, 0x00,470};471472FILE *saved = File::OpenCFile(resultFilename, "wb");473if (saved) {474fwrite(&header, sizeof(header), 1, saved);475fwrite(pixels_.data(), sizeof(u32), stride_ * h_, saved);476fclose(saved);477478return true;479}480481return false;482}483484bool ScreenshotComparer::SaveVisualComparisonPNG(const Path &resultFilename) {485std::unique_ptr<u32[]> comparison(new u32[w_ * 2 * h_ * 2]);486487if (asBitmap_) {488// The reference is flipped and BGRA by default for the common BMP compare case.489for (u32 y = 0; y < h_; ++y) {490u32 yoff = y * referenceStride_;491u32 comparisonRow = (h_ - y - 1) * 2 * w_ * 2;492for (u32 x = 0; x < w_; ++x) {493PlotVisualComparison(comparison.get(), comparisonRow + x * 2, pixels_[y * stride_ + x], reference_[yoff + x]);494}495}496} else {497// Reference is already in BGRA either way.498for (u32 y = 0; y < h_; ++y) {499u32 yoff = (h_ - y - 1) * referenceStride_;500u32 comparisonRow = (h_ - y - 1) * 2 * w_ * 2;501for (u32 x = 0; x < w_; ++x) {502PlotVisualComparison(comparison.get(), comparisonRow + x * 2, pixels_[y * stride_ + x], reference_[yoff + x]);503}504}505}506507FILE *fp = File::OpenCFile(resultFilename, "wb");508if (!fp)509return false;510511png_image png;512memset(&png, 0, sizeof(png));513png.version = PNG_IMAGE_VERSION;514png.format = PNG_FORMAT_BGRA;515png.width = w_ * 2;516png.height = h_ * 2;517518bool success = png_image_write_to_stdio(&png, fp, 0, comparison.get(), w_ * 2 * 4, nullptr) != 0;519fclose(fp);520png_image_free(&png);521522return success && png.warning_or_error < 2;523}524525int ChannelDifference(u8 actual, u8 reference) {526int diff = actual > reference ? actual - reference : reference - actual;527if (diff == 0)528return 0;529if (diff < 4)530return 1;531if (diff < 8)532return 2;533if (diff < 16)534return 3;535if (diff < 32)536return 4;537return 5;538}539540int PixelDifference(u32 actual, u32 reference) {541int b = ChannelDifference((actual >> 0) & 0xFF, (reference >> 0) & 0xFF);542int g = ChannelDifference((actual >> 8) & 0xFF, (reference >> 8) & 0xFF);543int r = ChannelDifference((actual >> 16) & 0xFF, (reference >> 16) & 0xFF);544return std::max(b, std::max(g, r));545}546547void ScreenshotComparer::PlotVisualComparison(u32 *dst, u32 offset, u32 actual, u32 reference) {548int diff = PixelDifference(actual, reference);549dst[offset + 0] = actual | 0xFF000000;550dst[offset + 1] = actual | 0xFF000000;551dst[offset + w_ * 2 + 0] = reference | 0xFF000000;552553int alpha = 0x00000000;554switch (diff) {555case 0: alpha = 0xFF000000; break;556case 1: alpha = 0xEF000000; break;557case 2: alpha = 0xCF000000; break;558case 3: alpha = 0xAF000000; break;559case 4: alpha = 0x7F000000; break;560default: break;561}562563dst[offset + w_ * 2 + 1] = (reference & 0x00FFFFFF) | alpha;564}565566567