CoCalc provides the best real-time collaborative environment for Jupyter Notebooks, LaTeX documents, and SageMath, scalable from individual users to large groups and classes!
CoCalc provides the best real-time collaborative environment for Jupyter Notebooks, LaTeX documents, and SageMath, scalable from individual users to large groups and classes!
Path: blob/master/GPU/Common/TextureReplacer.cpp
Views: 1401
// Copyright (c) 2016- PPSSPP Project.12// This program is free software: you can redistribute it and/or modify3// it under the terms of the GNU General Public License as published by4// the Free Software Foundation, version 2.0 or later versions.56// This program is distributed in the hope that it will be useful,7// but WITHOUT ANY WARRANTY; without even the implied warranty of8// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the9// GNU General Public License 2.0 for more details.1011// A copy of the GPL 2.0 should have been included with the program.12// If not, see http://www.gnu.org/licenses/1314// Official git repository and contact information can be found at15// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.1617#include "ppsspp_config.h"1819#include <algorithm>20#include <cstring>21#include <memory>22#include <png.h>2324#include "ext/basis_universal/basisu_transcoder.h"25#include "ext/xxhash.h"2627#include "Common/Data/Convert/ColorConv.h"28#include "Common/Data/Format/IniFile.h"29#include "Common/Data/Format/ZIMLoad.h"30#include "Common/Data/Format/PNGLoad.h"31#include "Common/Data/Text/I18n.h"32#include "Common/Data/Text/Parsers.h"33#include "Common/File/VFS/DirectoryReader.h"34#include "Common/File/VFS/ZipFileReader.h"35#include "Common/File/FileUtil.h"36#include "Common/File/VFS/VFS.h"37#include "Common/LogReporting.h"38#include "Common/StringUtils.h"39#include "Common/System/OSD.h"40#include "Common/Thread/ParallelLoop.h"41#include "Common/Thread/Waitable.h"42#include "Common/Thread/ThreadManager.h"43#include "Common/TimeUtil.h"44#include "Core/Config.h"45#include "Core/System.h"46#include "Core/ThreadPools.h"47#include "Core/ELF/ParamSFO.h"48#include "GPU/Common/TextureReplacer.h"49#include "GPU/Common/TextureDecoder.h"5051static const std::string INI_FILENAME = "textures.ini";52static const std::string ZIP_FILENAME = "textures.zip";53static const std::string NEW_TEXTURE_DIR = "new/";54static const int VERSION = 1;55static const double MAX_CACHE_SIZE = 4.0;56static bool basisu_initialized = false;5758TextureReplacer::TextureReplacer(Draw::DrawContext *draw) {59if (!basisu_initialized) {60basist::basisu_transcoder_init();61basisu_initialized = true;62}63// We don't want to keep the draw object around, so extract the info we need.64if (draw->GetDataFormatSupport(Draw::DataFormat::BC3_UNORM_BLOCK)) formatSupport_.bc123 = true;65if (draw->GetDataFormatSupport(Draw::DataFormat::ASTC_4x4_UNORM_BLOCK)) formatSupport_.astc = true;66if (draw->GetDataFormatSupport(Draw::DataFormat::BC7_UNORM_BLOCK)) formatSupport_.bc7 = true;67if (draw->GetDataFormatSupport(Draw::DataFormat::ETC2_R8G8B8_UNORM_BLOCK)) formatSupport_.etc2 = true;68}6970TextureReplacer::~TextureReplacer() {71for (auto iter : levelCache_) {72delete iter.second;73}74delete vfs_;75}7677void TextureReplacer::NotifyConfigChanged() {78gameID_ = g_paramSFO.GetDiscID();7980bool wasReplaceEnabled = replaceEnabled_;81replaceEnabled_ = g_Config.bReplaceTextures;82saveEnabled_ = g_Config.bSaveNewTextures;83if (replaceEnabled_ || saveEnabled_) {84basePath_ = GetSysDirectory(DIRECTORY_TEXTURES) / gameID_;85replaceEnabled_ = replaceEnabled_ && File::IsDirectory(basePath_);86newTextureDir_ = basePath_ / NEW_TEXTURE_DIR;8788// If we're saving, auto-create the directory.89if (saveEnabled_ && !File::Exists(newTextureDir_)) {90INFO_LOG(Log::G3D, "Creating new texture directory: '%s'", newTextureDir_.ToVisualString().c_str());91File::CreateFullPath(newTextureDir_);92// We no longer create a nomedia file here, since we put one93// in the TEXTURES root.94}95}9697if (saveEnabled_) {98// Somewhat crude message, re-using translation strings.99auto d = GetI18NCategory(I18NCat::DEVELOPER);100auto di = GetI18NCategory(I18NCat::DIALOG);101g_OSD.Show(OSDType::MESSAGE_INFO, std::string(d->T("Save new textures")) + ": " + std::string(di->T("Enabled")), 2.0f);102}103104if (!replaceEnabled_ && wasReplaceEnabled) {105delete vfs_;106vfs_ = nullptr;107Decimate(ReplacerDecimateMode::ALL);108}109110if (replaceEnabled_) {111replaceEnabled_ = LoadIni();112}113}114115bool TextureReplacer::LoadIni() {116hash_ = ReplacedTextureHash::QUICK;117aliases_.clear();118hashranges_.clear();119filtering_.clear();120reducehashranges_.clear();121122allowVideo_ = false;123ignoreAddress_ = false;124reduceHash_ = false;125reduceHashGlobalValue = 0.5;126// Prevents dumping the mipmaps.127ignoreMipmap_ = false;128129delete vfs_;130vfs_ = nullptr;131132Path zipPath = basePath_ / ZIP_FILENAME;133134// First, check for textures.zip, which is used to reduce IO.135VFSBackend *dir = ZipFileReader::Create(zipPath, "", false);136if (!dir) {137INFO_LOG(Log::G3D, "%s wasn't a zip file - opening the directory %s instead.", zipPath.c_str(), basePath_.c_str());138vfsIsZip_ = false;139dir = new DirectoryReader(basePath_);140} else {141vfsIsZip_ = true;142}143144IniFile ini;145bool iniLoaded = ini.LoadFromVFS(*dir, INI_FILENAME);146147if (iniLoaded) {148if (!LoadIniValues(ini, dir)) {149delete dir;150return false;151}152153// Allow overriding settings per game id.154std::string overrideFilename;155if (ini.GetOrCreateSection("games")->Get(gameID_.c_str(), &overrideFilename, "")) {156if (!overrideFilename.empty() && overrideFilename != INI_FILENAME) {157IniFile overrideIni;158iniLoaded = overrideIni.LoadFromVFS(*dir, overrideFilename);159if (!iniLoaded) {160ERROR_LOG(Log::G3D, "Failed to load extra texture ini: %s", overrideFilename.c_str());161// Since this error is most likely to occure for texture pack creators, let's just bail here162// so that the creator is more likely to look in the logs for what happened.163delete dir;164return false;165}166167INFO_LOG(Log::G3D, "Loading extra texture ini: %s", overrideFilename.c_str());168if (!LoadIniValues(overrideIni, nullptr, true)) {169delete dir;170return false;171}172}173}174} else {175if (vfsIsZip_) {176// We don't accept zip files without inis.177ERROR_LOG(Log::G3D, "Texture pack lacking ini file: %s", basePath_.c_str());178delete dir;179return false;180} else {181WARN_LOG(Log::G3D, "Texture pack lacking ini file: %s", basePath_.c_str());182// Do what we can do anyway: Scan for textures and build the map.183std::map<ReplacementCacheKey, std::map<int, std::string>> filenameMap;184ScanForHashNamedFiles(dir, filenameMap);185186if (filenameMap.empty()) {187WARN_LOG(Log::G3D, "No replacement textures found.");188return false;189}190191ComputeAliasMap(filenameMap);192}193}194195auto gr = GetI18NCategory(I18NCat::GRAPHICS);196g_OSD.Show(OSDType::MESSAGE_SUCCESS, gr->T("Texture replacement pack activated"), 2.0f);197198vfs_ = dir;199200// If we have stuff loaded from before, need to update the vfs pointers to avoid201// crash on exit. The actual problem is that we tend to call LoadIni a little too much...202for (auto &repl : levelCache_) {203repl.second->vfs_ = vfs_;204}205206if (vfsIsZip_) {207INFO_LOG(Log::G3D, "Texture pack activated from '%s'", (basePath_ / ZIP_FILENAME).c_str());208} else {209INFO_LOG(Log::G3D, "Texture pack activated from '%s'", basePath_.c_str());210}211212// The ini doesn't have to exist for the texture directory or zip to be valid.213return true;214}215216void TextureReplacer::ScanForHashNamedFiles(VFSBackend *dir, std::map<ReplacementCacheKey, std::map<int, std::string>> &filenameMap) {217// Scan the root of the texture folder/zip and preinitialize the hash map.218std::vector<File::FileInfo> filesInRoot;219dir->GetFileListing("", &filesInRoot, nullptr);220for (auto file : filesInRoot) {221if (file.isDirectory)222continue;223if (file.name.empty() || file.name[0] == '.')224continue;225Path path(file.name);226std::string ext = path.GetFileExtension();227228std::string hash = file.name.substr(0, file.name.size() - ext.size());229if (!((hash.size() >= 26 && hash.size() <= 27 && hash[24] == '_') || hash.size() == 24)) {230continue;231}232// OK, it's hash-like enough to try to parse it into the map.233if (equalsNoCase(ext, ".ktx2") || equalsNoCase(ext, ".png") || equalsNoCase(ext, ".dds") || equalsNoCase(ext, ".zim")) {234ReplacementCacheKey key(0, 0);235int level = 0; // sscanf might fail to pluck the level, but that's ok, we default to 0. sscanf doesn't write to non-matched outputs.236if (sscanf(hash.c_str(), "%16llx%8x_%d", &key.cachekey, &key.hash, &level) >= 1) {237// INFO_LOG(Log::G3D, "hash-like file in root, adding: %s", file.name.c_str());238filenameMap[key][level] = file.name;239}240}241}242}243244void TextureReplacer::ComputeAliasMap(const std::map<ReplacementCacheKey, std::map<int, std::string>> &filenameMap) {245for (auto &pair : filenameMap) {246std::string alias;247int mipIndex = 0;248for (auto &level : pair.second) {249if (level.first == mipIndex) {250alias += level.second + "|";251mipIndex++;252} else {253WARN_LOG(Log::G3D, "Non-sequential mip index %d, breaking. filenames=%s", level.first, level.second.c_str());254break;255}256}257if (alias == "|") {258alias = ""; // marker for no replacement259}260// Replace any '\' with '/', to be safe and consistent. Since these are from the ini file, we do this on all platforms.261for (auto &c : alias) {262if (c == '\\') {263c = '/';264}265}266aliases_[pair.first] = alias;267}268}269270bool TextureReplacer::LoadIniValues(IniFile &ini, VFSBackend *dir, bool isOverride) {271auto options = ini.GetOrCreateSection("options");272std::string hash;273options->Get("hash", &hash, "");274if (strcasecmp(hash.c_str(), "quick") == 0) {275hash_ = ReplacedTextureHash::QUICK;276} else if (strcasecmp(hash.c_str(), "xxh32") == 0) {277hash_ = ReplacedTextureHash::XXH32;278} else if (strcasecmp(hash.c_str(), "xxh64") == 0) {279hash_ = ReplacedTextureHash::XXH64;280} else if (!isOverride || !hash.empty()) {281ERROR_LOG(Log::G3D, "Unsupported hash type: %s", hash.c_str());282return false;283}284285options->Get("video", &allowVideo_, allowVideo_);286options->Get("ignoreAddress", &ignoreAddress_, ignoreAddress_);287// Multiplies sizeInRAM/bytesPerLine in XXHASH by 0.5.288options->Get("reduceHash", &reduceHash_, reduceHash_);289options->Get("ignoreMipmap", &ignoreMipmap_, ignoreMipmap_);290if (reduceHash_ && hash_ == ReplacedTextureHash::QUICK) {291reduceHash_ = false;292ERROR_LOG(Log::G3D, "Texture Replacement: reduceHash option requires safer hash, use xxh32 or xxh64 instead.");293}294295if (ignoreAddress_ && hash_ == ReplacedTextureHash::QUICK) {296ignoreAddress_ = false;297ERROR_LOG(Log::G3D, "Texture Replacement: ignoreAddress option requires safer hash, use xxh32 or xxh64 instead.");298}299300int version = 0;301if (options->Get("version", &version, 0) && version > VERSION) {302ERROR_LOG(Log::G3D, "Unsupported texture replacement version %d, trying anyway", version);303}304305int badFileNameCount = 0;306307std::map<ReplacementCacheKey, std::map<int, std::string>> filenameMap;308309if (dir) {310ScanForHashNamedFiles(dir, filenameMap);311}312313std::string badFilenames;314315if (ini.HasSection("hashes")) {316auto hashes = ini.GetOrCreateSection("hashes")->ToMap();317// Format: hashname = filename.png318bool checkFilenames = saveEnabled_ && !g_Config.bIgnoreTextureFilenames && !vfsIsZip_;319320for (const auto &[k, v] : hashes) {321ReplacementCacheKey key(0, 0);322// sscanf might fail to pluck the level if omitted from the line, but that's ok, we default level to 0.323// sscanf doesn't write to non-matched outputs.324int level = 0;325if (sscanf(k.c_str(), "%16llx%8x_%d", &key.cachekey, &key.hash, &level) >= 1) {326// We allow empty filenames, to mark textures that we don't want to keep saving.327filenameMap[key][level] = v;328if (checkFilenames) {329// TODO: We should check for the union of these on all platforms, really.330#if PPSSPP_PLATFORM(WINDOWS)331bool bad = v.find_first_of("\\ABCDEFGHIJKLMNOPQRSTUVWXYZ:<>|?*") != std::string::npos;332// Uppercase probably means the filenames don't match.333// Avoiding an actual check of the filenames to avoid performance impact.334#else335bool bad = v.find_first_of("\\:<>|?*") != std::string::npos;336#endif337if (bad) {338badFileNameCount++;339if (badFileNameCount == 10) {340badFilenames.append("...");341} else if (badFileNameCount < 10) {342badFilenames.append(v);343badFilenames.push_back('\n');344}345}346}347} else if (k.empty()) {348INFO_LOG(Log::G3D, "Ignoring [hashes] line with empty key: '= %s'", v.c_str());349} else {350ERROR_LOG(Log::G3D, "Unsupported syntax under [hashes], ignoring: %s = ", k.c_str());351}352}353}354355// Now, translate the filenameMap to the final aliasMap.356ComputeAliasMap(filenameMap);357358if (badFileNameCount > 0) {359auto err = GetI18NCategory(I18NCat::ERRORS);360g_OSD.Show(OSDType::MESSAGE_WARNING, err->T("textures.ini filenames may not be cross - platform(banned characters)"), badFilenames, 6.0f);361WARN_LOG(Log::G3D, "Potentially bad filenames: %s", badFilenames.c_str());362}363364if (ini.HasSection("hashranges")) {365auto hashranges = ini.GetOrCreateSection("hashranges")->ToMap();366// Format: addr,w,h = newW,newH367for (const auto &[k, v] : hashranges) {368ParseHashRange(k, v);369}370}371372if (ini.HasSection("filtering")) {373auto filters = ini.GetOrCreateSection("filtering")->ToMap();374// Format: hashname = nearest or linear375for (const auto &[k, v] : filters) {376ParseFiltering(k, v);377}378}379380if (ini.HasSection("reducehashranges")) {381auto reducehashranges = ini.GetOrCreateSection("reducehashranges")->ToMap();382// Format: w,h = reducehashvalues383for (const auto &[k, v] : reducehashranges) {384ParseReduceHashRange(k, v);385}386}387388return true;389}390391void TextureReplacer::ParseHashRange(const std::string &key, const std::string &value) {392std::vector<std::string> keyParts;393SplitString(key, ',', keyParts);394std::vector<std::string> valueParts;395SplitString(value, ',', valueParts);396397if (keyParts.size() != 3 || valueParts.size() != 2) {398ERROR_LOG(Log::G3D, "Ignoring invalid hashrange %s = %s, expecting addr,w,h = w,h", key.c_str(), value.c_str());399return;400}401402// Allow addr not starting with 0x, for consistency. TryParse requires 0x to parse as hex.403if (!startsWith(keyParts[0], "0x") && !startsWith(keyParts[0], "0X")) {404keyParts[0] = "0x" + keyParts[0];405}406407u32 addr;408u32 fromW;409u32 fromH;410if (!TryParse(keyParts[0], &addr) || !TryParse(keyParts[1], &fromW) || !TryParse(keyParts[2], &fromH)) {411ERROR_LOG(Log::G3D, "Ignoring invalid hashrange %s = %s, key format is 0x12345678,512,512", key.c_str(), value.c_str());412return;413}414415u32 toW;416u32 toH;417if (!TryParse(valueParts[0], &toW) || !TryParse(valueParts[1], &toH)) {418ERROR_LOG(Log::G3D, "Ignoring invalid hashrange %s = %s, value format is 512,512", key.c_str(), value.c_str());419return;420}421422if (toW > fromW || toH > fromH) {423ERROR_LOG(Log::G3D, "Ignoring invalid hashrange %s = %s, range bigger than source", key.c_str(), value.c_str());424return;425}426427const u64 rangeKey = ((u64)addr << 32) | ((u64)fromW << 16) | fromH;428hashranges_[rangeKey] = WidthHeightPair(toW, toH);429}430431void TextureReplacer::ParseFiltering(const std::string &key, const std::string &value) {432ReplacementCacheKey itemKey(0, 0);433if (sscanf(key.c_str(), "%16llx%8x", &itemKey.cachekey, &itemKey.hash) >= 1) {434if (!strcasecmp(value.c_str(), "nearest")) {435filtering_[itemKey] = TEX_FILTER_FORCE_NEAREST;436} else if (!strcasecmp(value.c_str(), "linear")) {437filtering_[itemKey] = TEX_FILTER_FORCE_LINEAR;438} else if (!strcasecmp(value.c_str(), "auto")) {439filtering_[itemKey] = TEX_FILTER_AUTO;440} else {441ERROR_LOG(Log::G3D, "Unsupported syntax under [filtering]: %s", value.c_str());442}443} else {444ERROR_LOG(Log::G3D, "Unsupported syntax under [filtering]: %s", key.c_str());445}446}447448void TextureReplacer::ParseReduceHashRange(const std::string& key, const std::string& value) {449std::vector<std::string> keyParts;450SplitString(key, ',', keyParts);451std::vector<std::string> valueParts;452SplitString(value, ',', valueParts);453454if (keyParts.size() != 2 || valueParts.size() != 1) {455ERROR_LOG(Log::G3D, "Ignoring invalid reducehashrange %s = %s, expecting w,h = reducehashvalue", key.c_str(), value.c_str());456return;457}458459u32 forW;460u32 forH;461if (!TryParse(keyParts[0], &forW) || !TryParse(keyParts[1], &forH)) {462ERROR_LOG(Log::G3D, "Ignoring invalid reducehashrange %s = %s, key format is 512,512", key.c_str(), value.c_str());463return;464}465466float rhashvalue;467if (!TryParse(valueParts[0], &rhashvalue)) {468ERROR_LOG(Log::G3D, "Ignoring invalid reducehashrange %s = %s, value format is 0.5", key.c_str(), value.c_str());469return;470}471472if (rhashvalue == 0) {473ERROR_LOG(Log::G3D, "Ignoring invalid hashrange %s = %s, reducehashvalue can't be 0", key.c_str(), value.c_str());474return;475}476477const u64 reducerangeKey = ((u64)forW << 16) | forH;478reducehashranges_[reducerangeKey] = rhashvalue;479}480481u32 TextureReplacer::ComputeHash(u32 addr, int bufw, int w, int h, bool swizzled, GETextureFormat fmt, u16 maxSeenV) {482_dbg_assert_msg_(replaceEnabled_ || saveEnabled_, "Replacement not enabled");483484// TODO: Take swizzled into account, like in QuickTexHash().485// Note: Currently, only the MLB games are known to need this.486487if (!LookupHashRange(addr, w, h, &w, &h)) {488// There wasn't any hash range, let's fall back to maxSeenV logic.489if (h == 512 && maxSeenV < 512 && maxSeenV != 0) {490h = (int)maxSeenV;491}492}493494const u8 *checkp = Memory::GetPointerUnchecked(addr);495if (reduceHash_) {496reduceHashSize = LookupReduceHashRange(w, h);497// default to reduceHashGlobalValue which default is 0.5498}499if (bufw <= w) {500// We can assume the data is contiguous. These are the total used pixels.501const u32 totalPixels = bufw * h + (w - bufw);502const u32 sizeInRAM = (textureBitsPerPixel[fmt] * totalPixels) / 8 * reduceHashSize;503504switch (hash_) {505case ReplacedTextureHash::QUICK:506return StableQuickTexHash(checkp, sizeInRAM);507case ReplacedTextureHash::XXH32:508return XXH32(checkp, sizeInRAM, 0xBACD7814);509case ReplacedTextureHash::XXH64:510return XXH64(checkp, sizeInRAM, 0xBACD7814);511default:512return 0;513}514} else {515// We have gaps. Let's hash each row and sum.516const u32 bytesPerLine = (textureBitsPerPixel[fmt] * w) / 8 * reduceHashSize;517const u32 stride = (textureBitsPerPixel[fmt] * bufw) / 8;518519u32 result = 0;520switch (hash_) {521case ReplacedTextureHash::QUICK:522for (int y = 0; y < h; ++y) {523u32 rowHash = StableQuickTexHash(checkp, bytesPerLine);524result = (result * 11) ^ rowHash;525checkp += stride;526}527break;528529case ReplacedTextureHash::XXH32:530for (int y = 0; y < h; ++y) {531u32 rowHash = XXH32(checkp, bytesPerLine, 0xBACD7814);532result = (result * 11) ^ rowHash;533checkp += stride;534}535break;536537case ReplacedTextureHash::XXH64:538for (int y = 0; y < h; ++y) {539u32 rowHash = XXH64(checkp, bytesPerLine, 0xBACD7814);540result = (result * 11) ^ rowHash;541checkp += stride;542}543break;544545default:546break;547}548549return result;550}551}552553ReplacedTexture *TextureReplacer::FindReplacement(u64 cachekey, u32 hash, int w, int h) {554// Only actually replace if we're replacing. We might just be saving.555if (!Enabled() || !g_Config.bReplaceTextures) {556return nullptr;557}558559ReplacementCacheKey replacementKey(cachekey, hash);560auto it = cache_.find(replacementKey);561if (it != cache_.end()) {562return it->second.texture;563}564565ReplacementDesc desc;566desc.newW = w;567desc.newH = h;568desc.w = w;569desc.h = h;570desc.cachekey = cachekey;571desc.hash = hash;572LookupHashRange(cachekey >> 32, w, h, &desc.newW, &desc.newH);573574if (ignoreAddress_) {575cachekey = cachekey & 0xFFFFFFFFULL;576}577578bool foundAlias = false;579bool ignored = false;580std::string hashfiles = LookupHashFile(cachekey, hash, &foundAlias, &ignored);581582// Early-out for ignored textures, let's not bother even starting a thread task.583if (ignored) {584// WARN_LOG(Log::G3D, "Not found/ignored: %s (%d, %d)", hashfiles.c_str(), (int)foundReplacement, (int)ignored);585// Insert an entry into the cache for faster lookup next time.586ReplacedTextureRef ref{};587cache_.emplace(std::make_pair(replacementKey, ref));588return nullptr;589}590591desc.forceFiltering = (TextureFiltering)0; // invalid value592FindFiltering(cachekey, hash, &desc.forceFiltering);593594if (!foundAlias) {595// We'll just need to generate the names for each level.596// By default, we look for png since that's also what's dumped.597// For other file formats, use the ini to create aliases.598desc.filenames.resize(MAX_REPLACEMENT_MIP_LEVELS);599for (int level = 0; level < desc.filenames.size(); level++) {600desc.filenames[level] = TextureReplacer::HashName(cachekey, hash, level) + ".png";601}602desc.logId = desc.filenames[0];603desc.hashfiles = desc.filenames[0]; // The generated filename of the top level is used as the key in the data cache.604hashfiles.clear();605hashfiles.reserve(desc.filenames[0].size() * (desc.filenames.size() + 1));606for (int level = 0; level < desc.filenames.size(); level++) {607hashfiles += desc.filenames[level];608hashfiles.push_back('|');609}610} else {611desc.logId = hashfiles;612SplitString(hashfiles, '|', desc.filenames);613desc.hashfiles = hashfiles;614}615616_dbg_assert_(!hashfiles.empty());617// OK, we might already have a matching texture, we use hashfiles as a key. Look it up in the level cache.618auto iter = levelCache_.find(hashfiles);619if (iter != levelCache_.end()) {620// Insert an entry into the cache for faster lookup next time.621ReplacedTextureRef ref;622ref.hashfiles = hashfiles;623ref.texture = iter->second;624cache_.emplace(std::make_pair(replacementKey, ref));625return iter->second;626}627628// Final path - we actually need a new replacement texture, because we haven't seen "hashfiles" before.629desc.basePath = basePath_;630desc.formatSupport = formatSupport_;631632ReplacedTexture *texture = new ReplacedTexture(vfs_, desc);633634ReplacedTextureRef ref;635ref.hashfiles = hashfiles;636ref.texture = texture;637cache_.emplace(std::make_pair(replacementKey, ref));638639// Also, insert the level in the level cache so we can look up by desc_->hashfiles again.640levelCache_.emplace(std::make_pair(hashfiles, texture));641return texture;642}643644static bool WriteTextureToPNG(png_imagep image, const Path &filename, int convert_to_8bit, const void *buffer, png_int_32 row_stride, const void *colormap) {645FILE *fp = File::OpenCFile(filename, "wb");646if (!fp) {647ERROR_LOG(Log::IO, "Unable to open texture file '%s' for writing.", filename.c_str());648return false;649}650651if (png_image_write_to_stdio(image, fp, convert_to_8bit, buffer, row_stride, colormap)) {652fclose(fp);653return true;654} else {655ERROR_LOG(Log::System, "Texture PNG encode failed.");656fclose(fp);657remove(filename.c_str());658return false;659}660}661662// We save textures on threadpool tasks since it's a fire-and-forget task, and both I/O and png compression663// can be pretty slow.664class SaveTextureTask : public Task {665public:666std::vector<u8> rgbaData;667668int w = 0;669int h = 0;670int pitch = 0; // bytes671672Path filename;673Path saveFilename;674675u32 replacedInfoHash = 0;676677SaveTextureTask(std::vector<u8> &&_rgbaData) : rgbaData(std::move(_rgbaData)) {}678679// This must be set to I/O blocking because of Android storage (so we attach the thread to JNI), while being CPU heavy too.680TaskType Type() const override { return TaskType::IO_BLOCKING; }681682TaskPriority Priority() const override {683return TaskPriority::LOW;684}685686void Run() override {687// Should we skip writing if the newly saved data already exists?688if (File::Exists(saveFilename)) {689return;690}691692// And we always skip if the replace file already exists.693if (File::Exists(filename)) {694return;695}696697Path saveDirectory = saveFilename.NavigateUp();698if (!File::Exists(saveDirectory)) {699// Previously, we created a .nomedia file here. This is unnecessary as they have recursive behavior.700// When initializing (see NotifyConfigChange above) we create one in the "root" of the "new" folder.701File::CreateFullPath(saveDirectory);702}703704// Now that we've passed the checks, we change the file extension of the path we're actually705// going to write to to .png.706saveFilename = saveFilename.WithReplacedExtension(".png");707708png_image png{};709png.version = PNG_IMAGE_VERSION;710png.format = PNG_FORMAT_RGBA;711png.width = w;712png.height = h;713bool success = WriteTextureToPNG(&png, saveFilename, 0, rgbaData.data(), pitch, nullptr);714png_image_free(&png);715if (png.warning_or_error >= 2) {716ERROR_LOG(Log::G3D, "Saving texture to PNG produced errors.");717} else if (success) {718NOTICE_LOG(Log::G3D, "Saving texture for replacement: %08x / %dx%d in '%s'", replacedInfoHash, w, h, saveFilename.ToVisualString().c_str());719} else {720ERROR_LOG(Log::G3D, "Failed to write '%s'", saveFilename.c_str());721}722}723};724725bool TextureReplacer::WillSave(const ReplacedTextureDecodeInfo &replacedInfo) const {726if (!saveEnabled_)727return false;728// Don't save the PPGe texture.729if (replacedInfo.addr > 0x05000000 && replacedInfo.addr < PSP_GetKernelMemoryEnd())730return false;731if (replacedInfo.isVideo && !allowVideo_)732return false;733734return true;735}736737void TextureReplacer::NotifyTextureDecoded(ReplacedTexture *texture, const ReplacedTextureDecodeInfo &replacedInfo, const void *data, int pitch, int level, int origW, int origH, int scaledW, int scaledH) {738_assert_msg_(saveEnabled_, "Texture saving not enabled");739_assert_(pitch >= 0);740741if (!WillSave(replacedInfo)) {742// Ignore.743return;744}745746if (ignoreMipmap_ && level > 0) {747// Not saving higher mips.748return;749}750751u64 cachekey = replacedInfo.cachekey;752if (ignoreAddress_) {753cachekey = cachekey & 0xFFFFFFFFULL;754}755756bool foundAlias = false;757bool ignored = false;758std::string replacedLevelNames = LookupHashFile(cachekey, replacedInfo.hash, &foundAlias, &ignored);759if (ignored) {760// The ini file entry was set to empty string. We can early-out.761return;762}763764// Alright, get the specified filename for the level.765std::string hashfile;766if (!replacedLevelNames.empty()) {767// If the user has specified a name before, we get it here.768std::vector<std::string> names;769SplitString(replacedLevelNames, '|', names);770hashfile = names[std::min(level, (int)(names.size() - 1))];771} else {772// Generate a new PNG filename, complete with level.773hashfile = HashName(cachekey, replacedInfo.hash, level) + ".png";774}775776ReplacementCacheKey replacementKey(cachekey, replacedInfo.hash);777auto it = savedCache_.find(replacementKey);778if (it != savedCache_.end()) {779// We've already saved this texture. Ignore it.780// We don't really care about changing the scale factor during runtime, only confusing.781return;782}783double now = time_now_d();784785// Width/height of the image to save.786int w = scaledW;787int h = scaledH;788789// Only save the hashed portion of the PNG.790int lookupW;791int lookupH;792if (LookupHashRange(replacedInfo.addr, origW, origH, &lookupW, &lookupH)) {793w = lookupW * (scaledW / origW);794h = lookupH * (scaledH / origH);795}796797std::vector<u8> saveBuf;798799// Copy data to a buffer so we can send it to the thread. Might as well compact-away the pitch800// while we're at it.801saveBuf.resize(w * h * 4);802for (int y = 0; y < h; y++) {803memcpy((u8 *)saveBuf.data() + y * w * 4, (const u8 *)data + y * pitch, w * 4);804}805pitch = w * 4;806807SaveTextureTask *task = new SaveTextureTask(std::move(saveBuf));808809task->filename = basePath_ / hashfile;810task->saveFilename = newTextureDir_ / hashfile;811812task->w = w;813task->h = h;814task->pitch = pitch;815task->replacedInfoHash = replacedInfo.hash;816g_threadManager.EnqueueTask(task); // We don't care about waiting for the task. It'll be fine.817818// Remember that we've saved this for next time.819// Should be OK that the actual disk write may not be finished yet.820SavedTextureCacheData &saveData = savedCache_[replacementKey];821saveData.levelW[level] = w;822saveData.levelH[level] = h;823saveData.levelSaved[level] = true;824saveData.lastTimeSaved = now;825}826827void TextureReplacer::Decimate(ReplacerDecimateMode mode) {828// Allow replacements to be cached for a long time, although they're large.829double age = 1800.0;830if (mode == ReplacerDecimateMode::FORCE_PRESSURE) {831age = 90.0;832} else if (mode == ReplacerDecimateMode::ALL) {833age = 0.0;834} else if (lastTextureCacheSizeGB_ > 1.0) {835double pressure = std::min(MAX_CACHE_SIZE, lastTextureCacheSizeGB_) / MAX_CACHE_SIZE;836// Get more aggressive the closer we are to the max.837age = 90.0 + (1.0 - pressure) * 1710.0;838}839840const double threshold = time_now_d() - age;841size_t totalSize = 0;842for (auto &item : levelCache_) {843// During decimation, it's fine to try-lock here to avoid blocking the main thread while844// the level is being loaded - in that case we don't want to decimate anyway.845if (item.second->lock_.try_lock()) {846item.second->PurgeIfNotUsedSinceTime(threshold);847totalSize += item.second->GetTotalDataSize(); // TODO: Make something better.848item.second->lock_.unlock();849}850// don't actually delete the items here, just clean out the data.851}852853double totalSizeGB = totalSize / (1024.0 * 1024.0 * 1024.0);854if (totalSizeGB >= 1.0) {855WARN_LOG(Log::G3D, "Decimated replacements older than %fs, currently using %f GB of RAM", age, totalSizeGB);856}857lastTextureCacheSizeGB_ = totalSizeGB;858}859860template <typename Key, typename Value>861static typename std::unordered_map<Key, Value>::const_iterator LookupWildcard(const std::unordered_map<Key, Value> &map, Key &key, u64 cachekey, u32 hash, bool ignoreAddress) {862auto alias = map.find(key);863if (alias != map.end())864return alias;865866// Also check for a few more aliases with zeroed portions:867// Only clut hash (very dangerous in theory, in practice not more than missing "just" data hash)868key.cachekey = cachekey & 0xFFFFFFFFULL;869key.hash = 0;870alias = map.find(key);871if (alias != map.end())872return alias;873874if (!ignoreAddress) {875// No data hash.876key.cachekey = cachekey;877key.hash = 0;878alias = map.find(key);879if (alias != map.end())880return alias;881}882883// No address.884key.cachekey = cachekey & 0xFFFFFFFFULL;885key.hash = hash;886alias = map.find(key);887if (alias != map.end())888return alias;889890if (!ignoreAddress) {891// Address, but not clut hash (in case of garbage clut data.)892key.cachekey = cachekey & ~0xFFFFFFFFULL;893key.hash = hash;894alias = map.find(key);895if (alias != map.end())896return alias;897}898899// Anything with this data hash (a little dangerous.)900key.cachekey = 0;901key.hash = hash;902return map.find(key);903}904905bool TextureReplacer::FindFiltering(u64 cachekey, u32 hash, TextureFiltering *forceFiltering) {906if (!Enabled() || !g_Config.bReplaceTextures) {907return false;908}909910ReplacementCacheKey replacementKey(cachekey, hash);911auto filter = LookupWildcard(filtering_, replacementKey, cachekey, hash, ignoreAddress_);912if (filter == filtering_.end()) {913// Allow a global wildcard.914replacementKey.cachekey = 0;915replacementKey.hash = 0;916filter = filtering_.find(replacementKey);917}918if (filter != filtering_.end()) {919*forceFiltering = filter->second;920return true;921}922return false;923}924925std::string TextureReplacer::LookupHashFile(u64 cachekey, u32 hash, bool *foundAlias, bool *ignored) {926ReplacementCacheKey key(cachekey, hash);927auto alias = LookupWildcard(aliases_, key, cachekey, hash, ignoreAddress_);928if (alias != aliases_.end()) {929// Note: this will be blank if explicitly ignored.930*foundAlias = true;931*ignored = alias->second.empty();932return alias->second;933}934*foundAlias = false;935*ignored = false;936return "";937}938939std::string TextureReplacer::HashName(u64 cachekey, u32 hash, int level) {940char hashname[16 + 8 + 1 + 11 + 1] = {};941if (level > 0) {942snprintf(hashname, sizeof(hashname), "%016llx%08x_%d", cachekey, hash, level);943} else {944snprintf(hashname, sizeof(hashname), "%016llx%08x", cachekey, hash);945}946947return hashname;948}949950bool TextureReplacer::LookupHashRange(u32 addr, int w, int h, int *newW, int *newH) {951const u64 rangeKey = ((u64)addr << 32) | ((u64)w << 16) | h;952auto range = hashranges_.find(rangeKey);953if (range != hashranges_.end()) {954const WidthHeightPair &wh = range->second;955*newW = wh.first;956*newH = wh.second;957return true;958} else {959*newW = w;960*newH = h;961return false;962}963}964965float TextureReplacer::LookupReduceHashRange(int w, int h) {966const u64 reducerangeKey = ((u64)w << 16) | h;967auto range = reducehashranges_.find(reducerangeKey);968if (range != reducehashranges_.end()) {969float rhv = range->second;970return rhv;971}972else {973return reduceHashGlobalValue;974}975}976977bool TextureReplacer::IniExists(const std::string &gameID) {978if (gameID.empty())979return false;980981Path texturesDirectory = GetSysDirectory(DIRECTORY_TEXTURES) / gameID;982Path generatedFilename = texturesDirectory / INI_FILENAME;983return File::Exists(generatedFilename);984}985986bool TextureReplacer::GenerateIni(const std::string &gameID, Path &generatedFilename) {987if (gameID.empty())988return false;989990Path texturesDirectory = GetSysDirectory(DIRECTORY_TEXTURES) / gameID;991if (!File::Exists(texturesDirectory)) {992File::CreateFullPath(texturesDirectory);993}994995generatedFilename = texturesDirectory / INI_FILENAME;996if (File::Exists(generatedFilename))997return true;998999FILE *f = File::OpenCFile(generatedFilename, "wb");1000if (f) {1001// Unicode byte order mark1002fwrite("\xEF\xBB\xBF", 1, 3, f);10031004// Let's also write some defaults.1005fprintf(f, R"(# This describes your textures and set up options for texture replacement.1006# Documentation about the options and syntax is available here:1007# https://www.ppsspp.org/docs/reference/texture-replacement10081009[options]1010version = 11011hash = quick # options available: "quick", "xxh32" - more accurate, but slower, "xxh64" - more accurate and quite fast, but slower than xxh32 on 32 bit cpu's1012ignoreMipmap = true # Usually, can just generate them with basisu, no need to dump.1013reduceHash = false # Unsafe and can cause glitches in some cases, but allows to skip garbage data in some textures reducing endless duplicates as a side effect speeds up hashing as well, requires stronger hash like xxh32 or xxh641014ignoreAddress = false # Reduces duplicates at the cost of making hash less reliable, requires stronger hash like xxh32 or xxh64. Basically automatically sets the address to 0 in the dumped filenames.10151016[games]1017# Used to make it easier to install, and override settings for other regions.1018# Files still have to be copied to each TEXTURES folder.1019%s = %s10201021[hashes]1022# Use / for folders not \\, avoid special characters, and stick to lowercase.1023# See wiki for more info.10241025[hashranges]1026# This is useful for images that very clearly have smaller dimensions, like 480x272 image. They'll need to be redumped, since the hash will change. See the documentation.1027# Example: 08b31020,512,512 = 480,2721028# Example: 0x08b31020,512,512 = 480,27210291030[filtering]1031# You can enforce specific filtering modes with this. Available modes are linear, nearest, auto. See the docs.1032# Example: 08d3961000000909ba70b2af = nearest10331034[reducehashranges]1035# Lets you set texture sizes where the hash range is reduced by a factor. See the docs.1036# Example:1037512,512=0.510381039)", gameID.c_str(), INI_FILENAME.c_str());1040fclose(f);1041}1042return File::Exists(generatedFilename);1043}104410451046