Hooking/Detouring d3d9 (Present/EndScene) - 似乎调用我的函数然后崩溃

Question

正如标题所说，我正在尝试hook DirectX 9 V-Table并在屏幕上显示一些信息，我一直在研究弯路和hook几天了，我认为我在一定程度上理解了它，但现在我不太确定如何调试这个问题。

我的钩子正在调用另一个函数 ShowMsg()，它最终将成为一个绘图函数，但现在它只显示一个 message-box。

我的 ShowMsg() 函数被调用，但是程序崩溃了。

我正在使用 DirectX Feb 2010 "SimpleSample" 应用程序。

这是我的代码，注释部分来自之前的测试，该测试从另一个测试应用程序挂接函数。

// dllmain.cpp : Defines the entry point for the DLL application.
#include "stdafx.h"
#include <iostream>
#include <Windows.h>
#include "VirtualTable.h"
#include <d3d9.h>
#pragma comment(lib, "d3d9.lib")
#include <d3dx9.h>
#pragma comment(lib, "d3dx9.lib")
#include <detours.h>
#pragma comment(lib, "detours.lib")

using namespace std;

typedef void(__thiscall* Present)(IDirect3DDevice9* device);
Present g_org_Present;

void ShowMsg()
{
    MessageBoxA(0, "This function was called.", "", 0);
}

void __fastcall hk_Present(IDirect3DDevice9* device)
{
    ShowMsg();
    //call the original function
    g_org_Present(device);
}

void InitiateHooks()
{
    HWND game_window = FindWindowA(NULL, "SimpleSample");

    auto d3dpp = D3DPRESENT_PARAMETERS{};

    auto d3d = Direct3DCreate9(D3D_SDK_VERSION);

    IDirect3DDevice9* device;

    d3dpp.BackBufferCount = 1;
    d3dpp.MultiSampleType = D3DMULTISAMPLE_NONE;
    d3dpp.SwapEffect = D3DSWAPEFFECT_DISCARD;
    d3dpp.hDeviceWindow = game_window;
    d3dpp.FullScreen_RefreshRateInHz = D3DPRESENT_RATE_DEFAULT;
    d3dpp.PresentationInterval = D3DPRESENT_INTERVAL_IMMEDIATE;
    d3dpp.BackBufferFormat = D3DFMT_R5G6B5;
    d3dpp.Windowed = TRUE;
    if (SUCCEEDED(d3d->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, game_window, D3DCREATE_HARDWARE_VERTEXPROCESSING, &d3dpp, &device)))
    {
        void** vmt = *(void***)device;
        DWORD oldProtection;
        VirtualProtect(&vmt[17], 4, PAGE_EXECUTE_READWRITE, &oldProtection);
        g_org_Present = (Present)vmt[17];
        vmt[17] = &hk_Present;
        VirtualProtect(&vmt[17], 4, oldProtection, 0);

        device->Present(NULL, NULL, NULL, NULL);
    }

//  VirtualTable* myTable = new VirtualTable();

    //get the pointer to the actual virtual method table from our pointer to our class instance
//  void** base = *(void***)myTable;

//  DWORD oldProtection;
    //one way to remove page protection(not the best but this is an example only)
//  VirtualProtect(&base[1], 4, PAGE_EXECUTE_READWRITE, &oldProtection);
    //save the original function
//  g_org_VirtualFunction01 = (VirtualFunction01_t)base[1];
    //overwrite
//  base[1] = &hk_VirtualFunction01;
    //restore page protection
//  VirtualProtect(&base[1], 4, oldProtection, 0);

    //call the virtual function (now hooked) from our class instance
//  myTable->VirtualFunction01();
}
#pragma endregion

BOOL APIENTRY DllMain( HMODULE hModule,
                       DWORD  ul_reason_for_call,
                       LPVOID lpReserved
                     )
{
    switch (ul_reason_for_call)
    {
    case DLL_PROCESS_ATTACH:
        CreateThread(0, 0x1000, (LPTHREAD_START_ROUTINE)InitiateHooks, 0, 0, NULL);
        break;
    case DLL_THREAD_ATTACH:
    case DLL_THREAD_DETACH:
    case DLL_PROCESS_DETACH:
        break;
    }
    return TRUE;
}

有人可以告诉我我在这里可能做错了什么，所以我可以解决这个问题，以供将来参考。

这是更新后的代码，我意识到我不应该直接调用函数所以我改变了它，也改变了它来尝试 hook/detour EndScene，也使用 MS Detours 而不是我认为的其他方法V-Table 修补，似乎正在调用我的 EndScene 挂钩，因为 MessageBox 被连续调用。但是，如果我注释掉 MessageBox，程序仍然会崩溃。

#include "stdafx.h"
#include <iostream>
#include <Windows.h>
#include <intrin.h>  
#include <tchar.h>
#include <tlhelp32.h>
#include <Psapi.h>
#include <winsock2.h>
#include <vector>
#include <ws2tcpip.h>
#pragma comment( lib, "Ws2_32.lib" )
#include <d3d9.h>
#pragma comment(lib, "d3d9.lib")
#include <d3dx9.h>
#pragma comment(lib, "d3dx9.lib")
#include <detours.h>
#pragma comment(lib, "detours.lib")

using namespace std;

D3DCOLOR RED = D3DCOLOR_ARGB(255, 255, 0, 0);

typedef HRESULT(__stdcall* EndScene) (IDirect3DDevice9*);
EndScene EndScene_orig;

HRESULT __stdcall EndScene_hook(IDirect3DDevice9* pDevice)
{
//  D3DRECT rec = { 100,100,200,200 };
//  pDevice->Clear(1, &rec, D3DCLEAR_TARGET, RED, 0, 0);
//  MessageBoxA(0, "We made it here...2", "", 0); //    <<<<----- This function is called over and over when not commented.
    return EndScene_orig(pDevice);
}

void InitHook()
{

    HWND game_window = FindWindow(NULL, _T("Skinned Mesh"));

    auto d3dpp = D3DPRESENT_PARAMETERS{};
    auto d3d = Direct3DCreate9(D3D_SDK_VERSION);
    if (d3d)
    {
        d3dpp.BackBufferCount = 1;
        d3dpp.MultiSampleType = D3DMULTISAMPLE_NONE;
        d3dpp.SwapEffect = D3DSWAPEFFECT_DISCARD;
        d3dpp.hDeviceWindow = game_window;
        d3dpp.FullScreen_RefreshRateInHz = D3DPRESENT_RATE_DEFAULT;
        d3dpp.PresentationInterval = D3DPRESENT_INTERVAL_IMMEDIATE;
        d3dpp.BackBufferFormat = D3DFMT_R5G6B5;
        d3dpp.Windowed = TRUE;
        IDirect3DDevice9* Device;
        if (SUCCEEDED(d3d->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, game_window, D3DCREATE_HARDWARE_VERTEXPROCESSING, &d3dpp, &Device)))
        {
//          MessageBoxA(0, "We made it here...", "", 0);

            DWORD* pVTable = *reinterpret_cast<DWORD**>(Device);

            DetourTransactionBegin();
            DetourUpdateThread(GetCurrentThread());

            EndScene_orig = (EndScene)pVTable[42];

            DetourAttach(&(LPVOID&)pVTable[42], (PBYTE)EndScene_hook);

            DetourTransactionCommit();
        }
    }

}


void SetupConsole()
{
    AllocConsole();
    freopen("CONOUT$", "wb", stdout);
    freopen("CONOUT$", "wb", stderr);
    freopen("CONIN$", "rb", stdin);
    SetConsoleTitle("CSGOHAX");
}


BOOL APIENTRY DllMain(HMODULE hModule,
    DWORD  ul_reason_for_call,
    LPVOID lpReserved
)
{
    switch (ul_reason_for_call)
    {
    case DLL_PROCESS_ATTACH:
        SetupConsole();
        CreateThread(0, 0, (LPTHREAD_START_ROUTINE)InitHook, 0, 0, NULL);
        break;
    case DLL_THREAD_ATTACH:
    case DLL_THREAD_DETACH:
    case DLL_PROCESS_DETACH:
        break;
    }
    return TRUE;
}

它必须是简单的我不认为是问题的东西..

Answer 1

我想通了，在IDA中逆向二进制，找到模块地址和EndScene函数及其地址，计算出偏移量。然后用ollydbg又找到了这个函数，用它做了一个签名，现在我可以使用签名扫描功能动态找到它了。

所以我可以得到这个签名的函数地址

DWORD dwEndScene = FindPattern("d3d9.dll",
    "\x6A\x18\xB8\x00\x00\x00\x00\xE8\x00\x00\x00\x00\x8B\x7D\x08\x8B\xDF\x8D\x47\x04\xF7\xDB\x1B\xDB\x23\xD8\x89\x5D\xE0\x33\xF6\x89\x75\xE4\x39\x73\x18\x75\x73",
    "xxx????x????xxxxxxxxxxxxxxxxxxxxxxxxxxx");

那我绕过函数

DetourTransactionBegin();
DetourUpdateThread(GetCurrentThread());
EndScene_orig = (oEndScene)(dwEndScene);
DetourAttach(&(LPVOID&)EndScene_orig, EndScene_hook);

这比我以前尝试使用虚拟设备在 V-Table 中查找函数要容易得多。

Hooking/Detouring d3d9 (Present/EndScene) - 似乎调用我的函数然后崩溃

Hooking/Detouring d3d9 (Present/EndScene) - Seems to call my function then crashes

c++

hook

directx-9

vtable

detours