CoCalc provides the best real-time collaborative environment for Jupyter Notebooks, LaTeX documents, and SageMath, scalable from individual users to large groups and classes!
CoCalc provides the best real-time collaborative environment for Jupyter Notebooks, LaTeX documents, and SageMath, scalable from individual users to large groups and classes!
Path: blob/master/Core/MIPS/x86/Jit.cpp
Views: 1401
// Copyright (c) 2012- PPSSPP Project.12// This program is free software: you can redistribute it and/or modify3// it under the terms of the GNU General Public License as published by4// the Free Software Foundation, version 2.0 or later versions.56// This program is distributed in the hope that it will be useful,7// but WITHOUT ANY WARRANTY; without even the implied warranty of8// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the9// GNU General Public License 2.0 for more details.1011// A copy of the GPL 2.0 should have been included with the program.12// If not, see http://www.gnu.org/licenses/1314// Official git repository and contact information can be found at15// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.1617#include "ppsspp_config.h"18#if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64)1920#include <algorithm>21#include <iterator>2223#include "Common/Math/math_util.h"24#include "Common/Profiler/Profiler.h"2526#include "Common/Serialize/Serializer.h"27#include "Common/Serialize/SerializeFuncs.h"28#include "Core/Core.h"29#include "Core/MemMap.h"30#include "Core/System.h"31#include "Core/CoreTiming.h"32#include "Core/Config.h"33#include "Core/Reporting.h"34#include "Core/Debugger/SymbolMap.h"35#include "Core/MIPS/MIPS.h"36#include "Core/MIPS/MIPSAnalyst.h"37#include "Core/MIPS/MIPSCodeUtils.h"38#include "Core/MIPS/MIPSInt.h"39#include "Core/MIPS/MIPSTables.h"40#include "Core/HLE/ReplaceTables.h"4142#include "RegCache.h"43#include "Jit.h"4445#include "Core/Debugger/Breakpoints.h"4647namespace MIPSComp48{49using namespace Gen;5051const bool USE_JIT_MISSMAP = false;52static std::map<std::string, u32> notJitOps;5354template<typename A, typename B>55std::pair<B,A> flip_pair(const std::pair<A,B> &p) {56return std::pair<B, A>(p.second, p.first);57}5859// This is called when Jit hits a breakpoint. Returns 1 when hit.60u32 JitBreakpoint(uint32_t addr)61{62// Should we skip this breakpoint?63if (CBreakPoints::CheckSkipFirst() == currentMIPS->pc || CBreakPoints::CheckSkipFirst() == addr)64return 0;6566BreakAction result = CBreakPoints::ExecBreakPoint(addr);67if ((result & BREAK_ACTION_PAUSE) == 0)68return 0;6970// There's probably a better place for this.71if (USE_JIT_MISSMAP) {72std::map<u32, std::string> notJitSorted;73std::transform(notJitOps.begin(), notJitOps.end(), std::inserter(notJitSorted, notJitSorted.begin()), flip_pair<std::string, u32>);7475std::string message;76char temp[256];77int remaining = 15;78for (auto it = notJitSorted.rbegin(), end = notJitSorted.rend(); it != end && --remaining >= 0; ++it)79{80snprintf(temp, 256, " (%d), ", it->first);81message += it->second + temp;82}8384if (message.size() > 2)85message.resize(message.size() - 2);8687NOTICE_LOG(Log::JIT, "Top ops compiled to interpreter: %s", message.c_str());88}8990return 1;91}9293static u32 JitMemCheck(u32 addr, u32 pc) {94// Should we skip this breakpoint?95if (CBreakPoints::CheckSkipFirst() == currentMIPS->pc)96return 0;9798// Did we already hit one?99if (coreState != CORE_RUNNING && coreState != CORE_NEXTFRAME)100return 1;101102// Note: pc may be the delay slot.103CBreakPoints::ExecOpMemCheck(addr, pc);104return coreState == CORE_RUNNING || coreState == CORE_NEXTFRAME ? 0 : 1;105}106107static void JitLogMiss(MIPSOpcode op)108{109if (USE_JIT_MISSMAP)110notJitOps[MIPSGetName(op)]++;111112MIPSInterpretFunc func = MIPSGetInterpretFunc(op);113func(op);114}115116#ifdef _MSC_VER117// JitBlockCache doesn't use this, just stores it.118#pragma warning(disable:4355)119#endif120Jit::Jit(MIPSState *mipsState)121: blocks(mipsState, this), mips_(mipsState) {122blocks.Init();123gpr.SetEmitter(this);124fpr.SetEmitter(this);125AllocCodeSpace(1024 * 1024 * 16);126GenerateFixedCode(jo);127128safeMemFuncs.Init(&thunks);129130js.startDefaultPrefix = mips_->HasDefaultPrefix();131132// The debugger sets this so that "go" on a breakpoint will actually... go.133// But if they reset, we can end up hitting it by mistake, since it's based on PC and ticks.134CBreakPoints::SetSkipFirst(0);135}136137Jit::~Jit() {138}139140void Jit::DoState(PointerWrap &p) {141auto s = p.Section("Jit", 1, 2);142if (!s)143return;144145Do(p, js.startDefaultPrefix);146if (p.mode == PointerWrap::MODE_READ && !js.startDefaultPrefix) {147WARN_LOG(Log::CPU, "Jit: An uneaten prefix was previously detected. Jitting in unknown-prefix mode.");148}149if (s >= 2) {150Do(p, js.hasSetRounding);151if (p.mode == PointerWrap::MODE_READ) {152js.lastSetRounding = 0;153}154} else {155js.hasSetRounding = 1;156}157158// The debugger sets this so that "go" on a breakpoint will actually... go.159// But if they load a state, we can end up hitting it by mistake, since it's based on PC and ticks.160CBreakPoints::SetSkipFirst(0);161}162163void Jit::UpdateFCR31() {164}165166void Jit::GetStateAndFlushAll(RegCacheState &state) {167gpr.GetState(state.gpr);168fpr.GetState(state.fpr);169FlushAll();170}171172void Jit::RestoreState(const RegCacheState& state) {173gpr.RestoreState(state.gpr);174fpr.RestoreState(state.fpr);175}176177void Jit::FlushAll() {178gpr.Flush();179fpr.Flush();180FlushPrefixV();181}182183void Jit::FlushPrefixV() {184if (js.startDefaultPrefix && !js.blockWrotePrefixes && js.HasNoPrefix()) {185// They started default, we never modified in memory, and they're default now.186// No reason to modify memory. This is common at end of blocks. Just clear dirty.187js.prefixSFlag = (JitState::PrefixState)(js.prefixSFlag & ~JitState::PREFIX_DIRTY);188js.prefixTFlag = (JitState::PrefixState)(js.prefixTFlag & ~JitState::PREFIX_DIRTY);189js.prefixDFlag = (JitState::PrefixState)(js.prefixDFlag & ~JitState::PREFIX_DIRTY);190return;191}192193if ((js.prefixSFlag & JitState::PREFIX_DIRTY) != 0) {194MOV(32, MIPSSTATE_VAR(vfpuCtrl[VFPU_CTRL_SPREFIX]), Imm32(js.prefixS));195js.prefixSFlag = (JitState::PrefixState) (js.prefixSFlag & ~JitState::PREFIX_DIRTY);196}197198if ((js.prefixTFlag & JitState::PREFIX_DIRTY) != 0) {199MOV(32, MIPSSTATE_VAR(vfpuCtrl[VFPU_CTRL_TPREFIX]), Imm32(js.prefixT));200js.prefixTFlag = (JitState::PrefixState) (js.prefixTFlag & ~JitState::PREFIX_DIRTY);201}202203if ((js.prefixDFlag & JitState::PREFIX_DIRTY) != 0) {204MOV(32, MIPSSTATE_VAR(vfpuCtrl[VFPU_CTRL_DPREFIX]), Imm32(js.prefixD));205js.prefixDFlag = (JitState::PrefixState) (js.prefixDFlag & ~JitState::PREFIX_DIRTY);206}207208// If we got here, we must've written prefixes to memory in this block.209js.blockWrotePrefixes = true;210}211212void Jit::WriteDowncount(int offset) {213const int downcount = js.downcountAmount + offset;214SUB(32, MIPSSTATE_VAR(downcount), downcount > 127 ? Imm32(downcount) : Imm8(downcount));215}216217void Jit::RestoreRoundingMode(bool force) {218// If the game has never set an interesting rounding mode, we can safely skip this.219if (force || js.hasSetRounding) {220CALL(restoreRoundingMode);221}222}223224void Jit::ApplyRoundingMode(bool force) {225// If the game has never set an interesting rounding mode, we can safely skip this.226if (force || js.hasSetRounding) {227CALL(applyRoundingMode);228}229}230231void Jit::UpdateRoundingMode(u32 fcr31) {232// We must set js.hasSetRounding at compile time, or this block will use the wrong rounding mode.233// The fcr31 parameter is -1 when not known at compile time, so we just assume it was changed.234if (fcr31 & 0x01000003) {235js.hasSetRounding = true;236}237}238239void Jit::ClearCache()240{241blocks.Clear();242ClearCodeSpace(0);243GenerateFixedCode(jo);244}245246void Jit::SaveFlags() {247PUSHF();248#if PPSSPP_ARCH(AMD64)249// On X64, the above misaligns the stack. However there might be a cheaper solution than this.250POP(64, R(EAX));251MOV(64, MIPSSTATE_VAR(saved_flags), R(EAX));252#endif253}254255void Jit::LoadFlags() {256#if PPSSPP_ARCH(AMD64)257MOV(64, R(EAX), MIPSSTATE_VAR(saved_flags));258PUSH(64, R(EAX));259#endif260POPF();261}262263void Jit::CompileDelaySlot(int flags, RegCacheState *state) {264// Need to offset the downcount which was already incremented for the branch + delay slot.265CheckJitBreakpoint(GetCompilerPC() + 4, -2);266267if (flags & DELAYSLOT_SAFE)268SaveFlags(); // preserve flag around the delay slot!269270js.inDelaySlot = true;271MIPSOpcode op = GetOffsetInstruction(1);272MIPSCompileOp(op, this);273js.inDelaySlot = false;274275if (flags & DELAYSLOT_FLUSH) {276if (state != NULL)277GetStateAndFlushAll(*state);278else279FlushAll();280}281if (flags & DELAYSLOT_SAFE)282LoadFlags(); // restore flag!283}284285void Jit::EatInstruction(MIPSOpcode op) {286MIPSInfo info = MIPSGetInfo(op);287if (info & DELAYSLOT) {288ERROR_LOG_REPORT_ONCE(ateDelaySlot, Log::JIT, "Ate a branch op.");289}290if (js.inDelaySlot) {291ERROR_LOG_REPORT_ONCE(ateInDelaySlot, Log::JIT, "Ate an instruction inside a delay slot.");292}293294CheckJitBreakpoint(GetCompilerPC() + 4, 0);295js.numInstructions++;296js.compilerPC += 4;297js.downcountAmount += MIPSGetInstructionCycleEstimate(op);298}299300void Jit::Compile(u32 em_address) {301PROFILE_THIS_SCOPE("jitc");302if (GetSpaceLeft() < 0x10000 || blocks.IsFull()) {303ClearCache();304}305306if (!Memory::IsValidAddress(em_address) || (em_address & 3) != 0) {307Core_ExecException(em_address, em_address, ExecExceptionType::JUMP);308return;309}310311// Sometimes we compile fairly large blocks, although it's uncommon.312BeginWrite(JitBlockCache::MAX_BLOCK_INSTRUCTIONS * 16);313314int block_num = blocks.AllocateBlock(em_address);315JitBlock *b = blocks.GetBlock(block_num);316DoJit(em_address, b);317_assert_msg_(b->originalAddress == em_address, "original %08x != em_address %08x (block %d)", b->originalAddress, em_address, b->blockNum);318blocks.FinalizeBlock(block_num, jo.enableBlocklink);319320EndWrite();321322bool cleanSlate = false;323324if (js.hasSetRounding && !js.lastSetRounding) {325WARN_LOG(Log::JIT, "Detected rounding mode usage, rebuilding jit with checks");326// Won't loop, since hasSetRounding is only ever set to 1.327js.lastSetRounding = js.hasSetRounding;328cleanSlate = true;329}330331// Drat. The VFPU hit an uneaten prefix at the end of a block.332if (js.startDefaultPrefix && js.MayHavePrefix()) {333WARN_LOG_REPORT(Log::JIT, "An uneaten prefix at end of block: %08x", GetCompilerPC() - 4);334js.LogPrefix();335336// Let's try that one more time. We won't get back here because we toggled the value.337js.startDefaultPrefix = false;338cleanSlate = true;339}340341if (cleanSlate) {342// Our assumptions are all wrong so it's clean-slate time.343ClearCache();344Compile(em_address);345}346}347348void Jit::RunLoopUntil(u64 globalticks) {349PROFILE_THIS_SCOPE("jit");350((void (*)())enterDispatcher)();351}352353u32 Jit::GetCompilerPC() {354return js.compilerPC;355}356357MIPSOpcode Jit::GetOffsetInstruction(int offset) {358return Memory::Read_Instruction(GetCompilerPC() + 4 * offset);359}360361const u8 *Jit::DoJit(u32 em_address, JitBlock *b) {362js.cancel = false;363js.blockStart = em_address;364js.compilerPC = em_address;365js.lastContinuedPC = 0;366js.initialBlockSize = 0;367js.nextExit = 0;368js.downcountAmount = 0;369js.curBlock = b;370js.compiling = true;371js.inDelaySlot = false;372js.blockWrotePrefixes = false;373js.afterOp = JitState::AFTER_NONE;374js.PrefixStart();375376// We add a check before the block, used when entering from a linked block.377b->checkedEntry = GetCodePtr();378// Downcount flag check. The last block decremented downcounter, and the flag should still be available.379FixupBranch skip = J_CC(CC_NS);380MOV(32, MIPSSTATE_VAR(pc), Imm32(js.blockStart));381JMP(outerLoop, true); // downcount hit zero - go advance.382SetJumpTarget(skip);383384b->normalEntry = GetCodePtr();385386MIPSAnalyst::AnalysisResults analysis = MIPSAnalyst::Analyze(em_address);387388gpr.Start(mips_, &js, &jo, analysis);389fpr.Start(mips_, &js, &jo, analysis, RipAccessible(&mips_->v[0]));390391js.numInstructions = 0;392while (js.compiling) {393// Jit breakpoints are quite fast, so let's do them in release too.394CheckJitBreakpoint(GetCompilerPC(), 0);395396MIPSOpcode inst = Memory::Read_Opcode_JIT(GetCompilerPC());397js.downcountAmount += MIPSGetInstructionCycleEstimate(inst);398399MIPSCompileOp(inst, this);400401if (js.afterOp & JitState::AFTER_CORE_STATE) {402// CORE_RUNNING is <= CORE_NEXTFRAME.403if (RipAccessible((const void *)&coreState)) {404CMP(32, M(&coreState), Imm32(CORE_NEXTFRAME)); // rip accessible405} else {406MOV(PTRBITS, R(RAX), ImmPtr((const void *)&coreState));407CMP(32, MatR(RAX), Imm32(CORE_NEXTFRAME));408}409FixupBranch skipCheck = J_CC(CC_LE, true);410// All cases of AFTER_CORE_STATE should update PC. We don't update here.411RegCacheState state;412GetStateAndFlushAll(state);413WriteSyscallExit();414415SetJumpTarget(skipCheck);416// If we didn't jump, we can keep our regs as they were.417RestoreState(state);418419js.afterOp = JitState::AFTER_NONE;420}421422js.compilerPC += 4;423js.numInstructions++;424425if (jo.Disabled(JitDisable::REGALLOC_GPR)) {426gpr.Flush();427}428if (jo.Disabled(JitDisable::REGALLOC_FPR)) {429fpr.Flush();430FlushPrefixV();431}432433// Safety check, in case we get a bunch of really large jit ops without a lot of branching.434if (GetSpaceLeft() < 0x800 || js.numInstructions >= JitBlockCache::MAX_BLOCK_INSTRUCTIONS) {435FlushAll();436WriteExit(GetCompilerPC(), js.nextExit++);437js.compiling = false;438}439}440441b->codeSize = (u32)(GetCodePtr() - b->normalEntry);442NOP();443AlignCode4();444if (js.lastContinuedPC == 0) {445b->originalSize = js.numInstructions;446} else {447// We continued at least once. Add the last proxy and set the originalSize correctly.448blocks.ProxyBlock(js.blockStart, js.lastContinuedPC, (GetCompilerPC() - js.lastContinuedPC) / sizeof(u32), GetCodePtr());449b->originalSize = js.initialBlockSize;450}451return b->normalEntry;452}453454void Jit::AddContinuedBlock(u32 dest) {455// The first block is the root block. When we continue, we create proxy blocks after that.456if (js.lastContinuedPC == 0)457js.initialBlockSize = js.numInstructions;458else459blocks.ProxyBlock(js.blockStart, js.lastContinuedPC, (GetCompilerPC() - js.lastContinuedPC) / sizeof(u32), GetCodePtr());460js.lastContinuedPC = dest;461}462463bool Jit::DescribeCodePtr(const u8 *ptr, std::string &name) {464if (ptr == applyRoundingMode)465name = "applyRoundingMode";466else if (ptr == dispatcher)467name = "dispatcher";468else if (ptr == dispatcherInEAXNoCheck)469name = "dispatcher (PC in EAX)";470else if (ptr == dispatcherNoCheck)471name = "dispatcherNoCheck";472else if (ptr == dispatcherCheckCoreState)473name = "dispatcherCheckCoreState";474else if (ptr == enterDispatcher)475name = "enterDispatcher";476else if (ptr == restoreRoundingMode)477name = "restoreRoundingMode";478else if (ptr == crashHandler)479name = "crashHandler";480else {481u32 jitAddr = blocks.GetAddressFromBlockPtr(ptr);482483// Returns 0 when it's valid, but unknown.484if (jitAddr == 0) {485name = "UnknownOrDeletedBlock";486} else if (jitAddr != (u32)-1) {487char temp[1024];488const std::string label = g_symbolMap ? g_symbolMap->GetDescription(jitAddr) : "";489if (!label.empty())490snprintf(temp, sizeof(temp), "%08x_%s", jitAddr, label.c_str());491else492snprintf(temp, sizeof(temp), "%08x", jitAddr);493name = temp;494} else if (IsInSpace(ptr)) {495if (ptr < endOfPregeneratedCode) {496name = "PreGenCode";497} else {498name = "Unknown";499}500} else if (thunks.IsInSpace(ptr)) {501name = "Thunk";502} else if (safeMemFuncs.IsInSpace(ptr)) {503name = "JitSafeMem";504} else {505// Not anywhere in jit, then.506return false;507}508}509// If we got here, one of the above cases matched.510return true;511}512513void Jit::Comp_RunBlock(MIPSOpcode op) {514// This shouldn't be necessary, the dispatcher should catch us before we get here.515ERROR_LOG(Log::JIT, "Comp_RunBlock");516}517518void Jit::LinkBlock(u8 *exitPoint, const u8 *checkedEntry) {519if (PlatformIsWXExclusive()) {520ProtectMemoryPages(exitPoint, 32, MEM_PROT_READ | MEM_PROT_WRITE);521}522XEmitter emit(exitPoint);523// Okay, this is a bit ugly, but we check here if it already has a JMP.524// That means it doesn't have a full exit to pad with INT 3.525bool prelinked = *emit.GetCodePointer() == 0xE9;526emit.JMP(checkedEntry, true);527if (!prelinked) {528ptrdiff_t actualSize = emit.GetWritableCodePtr() - exitPoint;529int pad = JitBlockCache::GetBlockExitSize() - (int)actualSize;530for (int i = 0; i < pad; ++i) {531emit.INT3();532}533}534if (PlatformIsWXExclusive()) {535ProtectMemoryPages(exitPoint, 32, MEM_PROT_READ | MEM_PROT_EXEC);536}537}538539void Jit::UnlinkBlock(u8 *checkedEntry, u32 originalAddress) {540if (PlatformIsWXExclusive()) {541ProtectMemoryPages(checkedEntry, 16, MEM_PROT_READ | MEM_PROT_WRITE);542}543// Send anyone who tries to run this block back to the dispatcher.544// Not entirely ideal, but .. pretty good.545// Spurious entrances from previously linked blocks can only come through checkedEntry546XEmitter emit(checkedEntry);547emit.MOV(32, MIPSSTATE_VAR(pc), Imm32(originalAddress));548emit.JMP(MIPSComp::jit->GetDispatcher(), true);549if (PlatformIsWXExclusive()) {550ProtectMemoryPages(checkedEntry, 16, MEM_PROT_READ | MEM_PROT_EXEC);551}552}553554bool Jit::ReplaceJalTo(u32 dest) {555const ReplacementTableEntry *entry = nullptr;556u32 funcSize = 0;557if (!CanReplaceJalTo(dest, &entry, &funcSize)) {558return false;559}560561// Warning - this might be bad if the code at the destination changes...562if (entry->flags & REPFLAG_ALLOWINLINE) {563// Jackpot! Just do it, no flushing. The code will be entirely inlined.564565// First, compile the delay slot. It's unconditional so no issues.566CompileDelaySlot(DELAYSLOT_NICE);567// Technically, we should write the unused return address to RA, but meh.568MIPSReplaceFunc repl = entry->jitReplaceFunc;569int cycles = (this->*repl)();570js.downcountAmount += cycles;571} else {572gpr.SetImm(MIPS_REG_RA, GetCompilerPC() + 8);573CompileDelaySlot(DELAYSLOT_NICE);574FlushAll();575MOV(32, MIPSSTATE_VAR(pc), Imm32(GetCompilerPC()));576RestoreRoundingMode();577ABI_CallFunction(entry->replaceFunc);578SUB(32, MIPSSTATE_VAR(downcount), R(EAX));579ApplyRoundingMode();580}581582js.compilerPC += 4;583// No writing exits, keep going!584585if (CBreakPoints::HasMemChecks()) {586// We could modify coreState, so we need to write PC and check.587// Otherwise, PC may end up on the jal. We add 4 to skip the delay slot.588MOV(32, MIPSSTATE_VAR(pc), Imm32(GetCompilerPC() + 4));589js.afterOp |= JitState::AFTER_CORE_STATE;590}591592// Add a trigger so that if the inlined code changes, we invalidate this block.593blocks.ProxyBlock(js.blockStart, dest, funcSize / sizeof(u32), GetCodePtr());594return true;595}596597void Jit::Comp_ReplacementFunc(MIPSOpcode op) {598// We get here if we execute the first instruction of a replaced function. This means599// that we do need to return to RA.600601// Inlined function calls (caught in jal) are handled differently.602603int index = op.encoding & MIPS_EMUHACK_VALUE_MASK;604605const ReplacementTableEntry *entry = GetReplacementFunc(index);606if (!entry) {607ERROR_LOG_REPORT_ONCE(replFunc, Log::HLE, "Invalid replacement op %08x at %08x", op.encoding, js.compilerPC);608return;609}610611u32 funcSize = g_symbolMap->GetFunctionSize(GetCompilerPC());612bool disabled = (entry->flags & REPFLAG_DISABLED) != 0;613if (!disabled && funcSize != SymbolMap::INVALID_ADDRESS && funcSize > sizeof(u32)) {614// We don't need to disable hooks, the code will still run.615if ((entry->flags & (REPFLAG_HOOKENTER | REPFLAG_HOOKEXIT)) == 0) {616// Any breakpoint at the func entry was already tripped, so we can still run the replacement.617// That's a common case - just to see how often the replacement hits.618disabled = CBreakPoints::RangeContainsBreakPoint(GetCompilerPC() + sizeof(u32), funcSize - sizeof(u32));619}620}621622// Hack for old savestates: Avoid stack overflow (MIPSCompileOp/CompReplacementFunc)623// Not sure about the cause.624Memory::Opcode origInstruction = Memory::Read_Instruction(GetCompilerPC(), true);625if (origInstruction.encoding == op.encoding) {626ERROR_LOG(Log::HLE, "Replacement broken (savestate problem?): %08x at %08x", op.encoding, GetCompilerPC());627return;628}629630if (disabled) {631MIPSCompileOp(origInstruction, this);632} else if (entry->jitReplaceFunc) {633MIPSReplaceFunc repl = entry->jitReplaceFunc;634int cycles = (this->*repl)();635636if (entry->flags & (REPFLAG_HOOKENTER | REPFLAG_HOOKEXIT)) {637// Compile the original instruction at this address. We ignore cycles for hooks.638MIPSCompileOp(origInstruction, this);639} else {640FlushAll();641MOV(32, R(ECX), MIPSSTATE_VAR(r[MIPS_REG_RA]));642js.downcountAmount += cycles;643WriteExitDestInReg(ECX);644js.compiling = false;645}646} else if (entry->replaceFunc) {647FlushAll();648649// Standard function call, nothing fancy.650// The function returns the number of cycles it took in EAX.651MOV(32, MIPSSTATE_VAR(pc), Imm32(GetCompilerPC()));652RestoreRoundingMode();653ABI_CallFunction(entry->replaceFunc);654655if (entry->flags & (REPFLAG_HOOKENTER | REPFLAG_HOOKEXIT)) {656// Compile the original instruction at this address. We ignore cycles for hooks.657ApplyRoundingMode();658MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC(), true), this);659} else {660CMP(32, R(EAX), Imm32(0));661FixupBranch positive = J_CC(CC_GE);662663MOV(32, R(ECX), MIPSSTATE_VAR(pc));664ADD(32, MIPSSTATE_VAR(downcount), R(EAX));665FixupBranch done = J();666667SetJumpTarget(positive);668MOV(32, R(ECX), MIPSSTATE_VAR(r[MIPS_REG_RA]));669SUB(32, MIPSSTATE_VAR(downcount), R(EAX));670671SetJumpTarget(done);672ApplyRoundingMode();673// Need to set flags again, ApplyRoundingMode destroyed them (and EAX.)674SUB(32, MIPSSTATE_VAR(downcount), Imm8(0));675WriteExitDestInReg(ECX);676js.compiling = false;677}678} else {679ERROR_LOG(Log::HLE, "Replacement function %s has neither jit nor regular impl", entry->name);680}681}682683void Jit::Comp_Generic(MIPSOpcode op) {684FlushAll();685MIPSInterpretFunc func = MIPSGetInterpretFunc(op);686_dbg_assert_msg_((MIPSGetInfo(op) & DELAYSLOT) == 0, "Cannot use interpreter for branch ops.");687688if (func)689{690// TODO: Maybe we'd be better off keeping the rounding mode within interp?691RestoreRoundingMode();692MOV(32, MIPSSTATE_VAR(pc), Imm32(GetCompilerPC()));693if (USE_JIT_MISSMAP)694ABI_CallFunctionC(&JitLogMiss, op.encoding);695else696ABI_CallFunctionC(func, op.encoding);697ApplyRoundingMode();698}699else700ERROR_LOG_REPORT(Log::JIT, "Trying to compile instruction %08x that can't be interpreted", op.encoding);701702const MIPSInfo info = MIPSGetInfo(op);703if ((info & IS_VFPU) != 0 && (info & VFPU_NO_PREFIX) == 0)704{705// If it does eat them, it'll happen in MIPSCompileOp().706if ((info & OUT_EAT_PREFIX) == 0)707js.PrefixUnknown();708709// Even if DISABLE'd, we want to set this flag so we overwrite.710if ((info & OUT_VFPU_PREFIX) != 0)711js.blockWrotePrefixes = true;712}713}714715static void HitInvalidBranch(uint32_t dest) {716Core_ExecException(dest, currentMIPS->pc, ExecExceptionType::JUMP);717}718719void Jit::WriteExit(u32 destination, int exit_num) {720_assert_msg_(exit_num < MAX_JIT_BLOCK_EXITS, "Expected a valid exit_num. dest=%08x", destination);721722if (!Memory::IsValidAddress(destination) || (destination & 3) != 0) {723ERROR_LOG_REPORT(Log::JIT, "Trying to write block exit to illegal destination %08x: pc = %08x", destination, currentMIPS->pc);724MOV(32, MIPSSTATE_VAR(pc), Imm32(GetCompilerPC()));725ABI_CallFunctionC(&HitInvalidBranch, destination);726js.afterOp |= JitState::AFTER_CORE_STATE;727}728// If we need to verify coreState, we may not jump yet.729if (js.afterOp & JitState::AFTER_CORE_STATE) {730// CORE_RUNNING is <= CORE_NEXTFRAME.731if (RipAccessible((const void *)&coreState)) {732CMP(32, M(&coreState), Imm32(CORE_NEXTFRAME)); // rip accessible733} else {734MOV(PTRBITS, R(RAX), ImmPtr((const void *)&coreState));735CMP(32, MatR(RAX), Imm32(CORE_NEXTFRAME));736}737FixupBranch skipCheck = J_CC(CC_LE);738// All cases of AFTER_CORE_STATE should update PC. We don't update here.739WriteSyscallExit();740SetJumpTarget(skipCheck);741}742743WriteDowncount();744745//If nobody has taken care of this yet (this can be removed when all branches are done)746JitBlock *b = js.curBlock;747b->exitAddress[exit_num] = destination;748b->exitPtrs[exit_num] = GetWritableCodePtr();749750// Link opportunity!751int block = blocks.GetBlockNumberFromStartAddress(destination);752if (block >= 0 && jo.enableBlocklink) {753// It exists! Joy of joy!754JMP(blocks.GetBlock(block)->checkedEntry, true);755b->linkStatus[exit_num] = true;756} else {757// No blocklinking.758MOV(32, MIPSSTATE_VAR(pc), Imm32(destination));759JMP(dispatcher, true);760761// Normally, exits are 15 bytes (MOV + &pc + dest + JMP + dest) on 64 or 32 bit.762// But just in case we somehow optimized, pad.763ptrdiff_t actualSize = GetWritableCodePtr() - b->exitPtrs[exit_num];764int pad = JitBlockCache::GetBlockExitSize() - (int)actualSize;765for (int i = 0; i < pad; ++i) {766INT3();767}768}769}770771static u32 IsValidJumpTarget(uint32_t addr) {772if (Memory::IsValidAddress(addr) && (addr & 3) == 0)773return 1;774return 0;775}776777static void HitInvalidJumpReg(uint32_t source) {778Core_ExecException(currentMIPS->pc, source, ExecExceptionType::JUMP);779currentMIPS->pc = source + 8;780}781782void Jit::WriteExitDestInReg(X64Reg reg) {783// If we need to verify coreState, we may not jump yet.784if (js.afterOp & JitState::AFTER_CORE_STATE) {785// CORE_RUNNING is <= CORE_NEXTFRAME.786if (RipAccessible((const void *)&coreState)) {787CMP(32, M(&coreState), Imm32(CORE_NEXTFRAME)); // rip accessible788} else {789X64Reg temp = reg == RAX ? RDX : RAX;790MOV(PTRBITS, R(temp), ImmPtr((const void *)&coreState));791CMP(32, MatR(temp), Imm32(CORE_NEXTFRAME));792}793FixupBranch skipCheck = J_CC(CC_LE);794// All cases of AFTER_CORE_STATE should update PC. We don't update here.795WriteSyscallExit();796SetJumpTarget(skipCheck);797}798799MOV(32, MIPSSTATE_VAR(pc), R(reg));800WriteDowncount();801802// Validate the jump to avoid a crash?803if (!g_Config.bFastMemory) {804CMP(32, R(reg), Imm32(PSP_GetKernelMemoryBase()));805FixupBranch tooLow = J_CC(CC_B);806CMP(32, R(reg), Imm32(PSP_GetUserMemoryEnd()));807FixupBranch tooHigh = J_CC(CC_AE);808809// Need to set neg flag again.810SUB(32, MIPSSTATE_VAR(downcount), Imm8(0));811if (reg == EAX)812J_CC(CC_NS, dispatcherInEAXNoCheck, true);813JMP(dispatcher, true);814815SetJumpTarget(tooLow);816SetJumpTarget(tooHigh);817818ABI_CallFunctionA((const void *)&IsValidJumpTarget, R(reg));819820// If we're ignoring, coreState didn't trip - so trip it now.821CMP(32, R(EAX), Imm32(0));822FixupBranch skip = J_CC(CC_NE);823ABI_CallFunctionC(&HitInvalidJumpReg, GetCompilerPC());824SetJumpTarget(skip);825826SUB(32, MIPSSTATE_VAR(downcount), Imm8(0));827JMP(dispatcherCheckCoreState, true);828} else if (reg == EAX) {829J_CC(CC_NS, dispatcherInEAXNoCheck, true);830JMP(dispatcher, true);831} else {832JMP(dispatcher, true);833}834}835836void Jit::WriteSyscallExit() {837WriteDowncount();838JMP(dispatcherCheckCoreState, true);839}840841bool Jit::CheckJitBreakpoint(u32 addr, int downcountOffset) {842if (CBreakPoints::IsAddressBreakPoint(addr)) {843SaveFlags();844FlushAll();845MOV(32, MIPSSTATE_VAR(pc), Imm32(GetCompilerPC()));846RestoreRoundingMode();847ABI_CallFunctionC(&JitBreakpoint, addr);848849// If 0, the conditional breakpoint wasn't taken.850CMP(32, R(EAX), Imm32(0));851FixupBranch skip = J_CC(CC_Z);852WriteDowncount(downcountOffset);853ApplyRoundingMode();854// Just to fix the stack.855LoadFlags();856JMP(dispatcherCheckCoreState, true);857SetJumpTarget(skip);858859ApplyRoundingMode();860LoadFlags();861return true;862}863864return false;865}866867void Jit::CheckMemoryBreakpoint(int instructionOffset, MIPSGPReg rs, int offset) {868if (!CBreakPoints::HasMemChecks())869return;870871int totalInstructionOffset = instructionOffset + (js.inDelaySlot ? 1 : 0);872uint32_t checkedPC = GetCompilerPC() + totalInstructionOffset * 4;873int size = MIPSAnalyst::OpMemoryAccessSize(checkedPC);874bool isWrite = MIPSAnalyst::IsOpMemoryWrite(checkedPC);875876// 0 because we normally execute before increasing.877int downcountOffset = js.inDelaySlot ? -2 : -1;878// TODO: In likely branches, downcount will be incorrect. This might make resume fail.879if (js.downcountAmount + downcountOffset < 0) {880downcountOffset = 0;881}882883if (gpr.IsImm(rs)) {884uint32_t iaddr = gpr.GetImm(rs) + offset;885MemCheck check;886if (CBreakPoints::GetMemCheckInRange(iaddr, size, &check)) {887if (!(check.cond & MEMCHECK_READ) && !isWrite)888return;889if (!(check.cond & MEMCHECK_WRITE) && isWrite)890return;891892// We need to flush, or conditions and log expressions will see old register values.893FlushAll();894895MOV(32, MIPSSTATE_VAR(pc), Imm32(GetCompilerPC()));896CallProtectedFunction(&JitMemCheck, iaddr, checkedPC);897898CMP(32, R(RAX), Imm32(0));899FixupBranch skipCheck = J_CC(CC_E);900WriteDowncount(downcountOffset);901JMP(dispatcherCheckCoreState, true);902903SetJumpTarget(skipCheck);904}905} else {906const auto memchecks = CBreakPoints::GetMemCheckRanges(isWrite);907bool possible = !memchecks.empty();908if (!possible)909return;910911gpr.Lock(rs);912gpr.MapReg(rs, true, false);913LEA(32, RAX, MDisp(gpr.RX(rs), offset));914gpr.UnlockAll();915916// We need to flush, or conditions and log expressions will see old register values.917FlushAll();918919std::vector<FixupBranch> hitChecks;920hitChecks.reserve(memchecks.size());921for (auto it = memchecks.begin(), end = memchecks.end(); it != end; ++it) {922if (it->end != 0) {923CMP(32, R(RAX), Imm32(it->start - size));924FixupBranch skipNext = J_CC(CC_BE);925926CMP(32, R(RAX), Imm32(it->end));927hitChecks.push_back(J_CC(CC_B, true));928929SetJumpTarget(skipNext);930} else {931CMP(32, R(RAX), Imm32(it->start));932hitChecks.push_back(J_CC(CC_E, true));933}934}935936FixupBranch noHits = J(true);937938// Okay, now land any hit here.939for (auto &fixup : hitChecks)940SetJumpTarget(fixup);941hitChecks.clear();942943MOV(32, MIPSSTATE_VAR(pc), Imm32(GetCompilerPC()));944CallProtectedFunction(&JitMemCheck, R(RAX), checkedPC);945946CMP(32, R(RAX), Imm32(0));947FixupBranch skipCheck = J_CC(CC_E);948WriteDowncount(downcountOffset);949JMP(dispatcherCheckCoreState, true);950951SetJumpTarget(skipCheck);952SetJumpTarget(noHits);953}954}955956void Jit::CallProtectedFunction(const void *func, const OpArg &arg1) {957// We don't regcache RCX, so the below is safe (and also faster, maybe branch prediction?)958ABI_CallFunctionA(thunks.ProtectFunction(func, 1), arg1);959}960961void Jit::CallProtectedFunction(const void *func, const OpArg &arg1, const OpArg &arg2) {962// We don't regcache RCX/RDX, so the below is safe (and also faster, maybe branch prediction?)963ABI_CallFunctionAA(thunks.ProtectFunction(func, 2), arg1, arg2);964}965966void Jit::CallProtectedFunction(const void *func, const u32 arg1, const u32 arg2) {967// We don't regcache RCX/RDX, so the below is safe (and also faster, maybe branch prediction?)968ABI_CallFunctionCC(thunks.ProtectFunction(func, 2), arg1, arg2);969}970971void Jit::CallProtectedFunction(const void *func, const OpArg &arg1, const u32 arg2) {972// We don't regcache RCX/RDX, so the below is safe (and also faster, maybe branch prediction?)973ABI_CallFunctionAC(thunks.ProtectFunction(func, 2), arg1, arg2);974}975976void Jit::Comp_DoNothing(MIPSOpcode op) { }977978MIPSOpcode Jit::GetOriginalOp(MIPSOpcode op) {979JitBlockCache *bc = GetBlockCache();980int block_num = bc->GetBlockNumberFromEmuHackOp(op, true);981if (block_num >= 0) {982return bc->GetOriginalFirstOp(block_num);983} else {984return op;985}986}987988} // namespace989990#endif // PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64)991992993