1) Without symbols
a) The faulting Stack
The stack backtrace of the thread that AV'd would look like:
0:001> kvn
ChildEBP RetAddr Args to Child
WARNING: Stack unwind information not available. Following frames may be wrong.
00 00b9f70c 064712df 01c31eb0 00000001 01c31f7c badext!HttpExtensionProc+0x1e3
*** ERROR: Symbol file could not be found. Defaulted to export symbols for wam.dll -
01 00b9f728 65d82b06 01c31eb0 01c31f7c 01c31eb0 badext!HttpExtensionProc+0x5a
02 00b9f770 65d823a2 0209a338 01c32025 01c31eb0 wam!SE_TABLE::ReleaseExtension+0xbe8
*** ERROR: Symbol file could not be found. Defaulted to export symbols for RPCRT4.DLL -
03 00b9f7b4 77d339c0 006194c8 00000000 0000009e wam!SE_TABLE::ReleaseExtension+0x484
04 00b9f7dc 77d93570 65d8223f 00b9f7f8 00000005 RPCRT4!NdrServerInitialize+0x288
05 00b9fa54 77d949ac 021c44f0 002116c4 0018a648 RPCRT4!NdrStubCall2+0x570
*** ERROR: Symbol file could not be found. Defaulted to export symbols for ole32.dll -
06 00b9fab8 77b29e7f 021c44f0 0018a648 002116c4 RPCRT4!CStdStubBuffer_Invoke+0x9e
07 00b9fafc 77b29d6e 0018a648 021c3074 00224738 ole32!HDC_UserMarshal+0x78a
08 00b9fb44 77aa7ae7 0018a648 021c2ed8 021c44f0 ole32!HDC_UserMarshal+0x679
09 00b9fba8 77aa7a33 002116c4 00000000 021c44f0 ole32!CoGetCallContext+0x401
0a 00b9fbc4 77b29c89 0018a648 00000001 021c44f0 ole32!CoGetCallContext+0x34d
0b 00b9fbf4 77b29a89 0018a600 002116c4 021c44f0 ole32!HDC_UserMarshal+0x594
0c 00b9fcb4 77b2a12c 001a9ef0 00000000 021c3058 ole32!HDC_UserMarshal+0x394
0d 00b9fcf4 77d33721 0018a600 021c3058 021c7ef0 ole32!HDC_UserMarshal+0xa37
0e 00b9fd2c 77d33667 77b29f7b 021c7ef0 00b9fe0c RPCRT4!I_RpcFree+0x1ff
0f 00b9fd84 77d33579 00000000 00000000 00b9fe0c RPCRT4!I_RpcFree+0x145
10 00b9fda4 77d48e81 021c7ef0 00000000 00b9fe0c RPCRT4!I_RpcFree+0x57
11 00b9fdd4 77d34bc6 021c7ef0 021c7eb4 00000000 RPCRT4!RpcSsDestroyClientContext+0xa6
12 00b9fe10 77d346c5 021c7c88 001b6f68 80030001 RPCRT4!NdrCorrelationFree+0x1120
13 00b9fe28 77d422ff 021c7df8 00b9fe50 021c7c88 RPCRT4!NdrCorrelationFree+0xc1f
14 00b9ff74 77d420d9 77d42528 001b6f68 00070000 RPCRT4!NdrConformantArrayMemorySize+0x4c5
15 00b9ffa8 77d424da 001b7220 00b9ffec 77e887dd RPCRT4!NdrConformantArrayMemorySize+0x29f
*** ERROR: Symbol file could not be found. Defaulted to export symbols for KERNEL32.DLL -
16 00b9ffb4 77e887dd 001b7278 00070000 01000002 RPCRT4!NdrConformantArrayMemorySize+0x6a0
17 00b9ffec 00000000 77d424c2 001b7278 00000000 KERNEL32!GetModuleFileNameA+0x1d1 ]
In this stack backtrace, it is clear that the culprit is "badext.dll". So it is enough to tell the customer to contact the dll vendor to fix it. If it is an in-house component and the customer would like help
from Microsoft, they should open an advisory case to troubleshoot the issue further.
b) The regiters values
0:001> r eax=00000000 ebx=00000003 ecx=00000000 edx=014fb076 esi=78002041 edi=0209a7d8
eip=06471468 esp=00b9f70c ebp=00b9f70c iopl=0 nv up ei pl zr na po nc
cs=001b ss=0023 ds=0023 es=0023 fs=0038 gs=0000 efl=00000246
badext!HttpExtensionProc+1e3:
06471468 66c700d307 mov word ptr [eax],0x7d3 ds:0023:00000000=????
c) The disassmbly code
0:001> u eip
badext!HttpExtensionProc+1e3:
06471468 66c700d307 mov word ptr [eax],0x7d3
0647146d 6a01 push 0x1 0647146f 58 pop eax
06471470 5d pop ebp 06471471 c20800 ret 0x8
06471474 55 push ebp
06471475 8bec mov ebp,esp
06471477 837d0c01 cmp dword ptr [ebp+0xc],0x1
2) With symbols including public symbols of "badExt.dll"
First, we need to add the symbol path(company wide symbols) as well as symbols of our dll
0:001> .sympath+ \\symbols\symbols
Symbol search path is: \\symbols\symbols
0:001> .sympath+ c:\symbols\public (where public symbols of badExt.dll are located)
Symbol search path is: \\symbols\symbols;c:\symbols\public
The stack backtrace of the thread that AV'd would look like:
0:001> kvn50
# ChildEBP RetAddr Args to Child
00 00b9f70c 064712df 01c31eb0 00000001 01c31f7c badext!doAV1+0xb
01 00b9f728 65d82b06 01c31eb0 01c31f7c 01c31eb0 badext!HttpExtensionProc+0x5a
02 00b9f73c 65d82a97 01c31eb0 01c31f7c 01c31eb0 wam!HSE_APPDLL::ExecuteRequest+0x93 (FPO: [1,0,3]) (CONV: thiscall) [D:\nt\private\inet\iis\svcs\wam\object\isplocal.cxx @ 450]
03 00b9f770 65d823a2 0209a338 01c32025 01c31eb0 wam!WAM::InvokeExtension+0x35 (FPO: [Non-Fpo]) (CONV: thiscall) [D:\nt\private\inet\iis\svcs\wam\object\wamobj.cxx @ 913]
04 00b9f7b4 77d339c0 006194c8 00000000 0000009e wam!WAM::ProcessRequest+0x240 (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\inet\iis\svcs\wam\object\wamobj.cxx @ 643]
05 00b9f7dc 77d93570 65d8223f 00b9f7f8 00000005 RPCRT4!Invoke+0x30 [..\i386\stubless.asm @ 88]
06 00b9fa54 77d949ac 021c44f0 002116c4 0018a648 RPCRT4!NdrStubCall2+0x63d (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\rpc\ndr20\srvcall.c @ 1724]
07 00b9fab8 77b29e7f 021c44f0 0018a648 002116c4 RPCRT4!CStdStubBuffer_Invoke+0xec (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\rpc\ndr20\stub.c @ 1571]
08 00b9fafc 77b29d6e 0018a648 021c3074 00224738 ole32!SyncStubInvoke+0x61 (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\ole32\com\dcomrem\channelb.cxx @ 1136]
09 00b9fb44 77aa7ae7 0018a648 021c2ed8 021c44f0 ole32!StubInvoke+0xa8 (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\ole32\com\dcomrem\channelb.cxx @ 1334]
0a 00b9fba8 77aa7a33 002116c4 00000000 021c44f0 ole32!CCtxComChnl::ContextInvoke+0xbb (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\ole32\com\dcomrem\ctxchnl.cxx @ 1204]
0b 00b9fbc4 77b29c89 0018a648 00000001 021c44f0 ole32!MTAInvoke+0x18 (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\ole32\com\dcomrem\callctrl.cxx @ 1952]
0c 00b9fbf4 77b29a89 0018a600 002116c4 021c44f0 ole32!AppInvoke+0xb5 (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\ole32\com\dcomrem\channelb.cxx @ 1013]
0d 00b9fcb4 77b2a12c 001a9ef0 00000000 021c3058 ole32!ComInvokeWithLockAndIPID+0x29e (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\ole32\com\dcomrem\channelb.cxx @ 1578]
0e 00b9fcf4 77d33721 0018a600 021c3058 021c7ef0 ole32!ThreadInvoke+0x1b7 (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\ole32\com\dcomrem\channelb.cxx @ 3835]
0f 00b9fd2c 77d33667 77b29f7b 021c7ef0 00b9fe0c RPCRT4!DispatchToStubInC+0x84 (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\rpc\runtime\mtrt\dispatch.c @ 82]
10 00b9fd84 77d33579 00000000 00000000 00b9fe0c RPCRT4!RPC_INTERFACE::DispatchToStubWorker+0x100 (FPO: [Non-Fpo]) (CONV: thiscall) [D:\nt\private\rpc\runtime\mtrt\hndlsvr.cxx @ 543]
11 00b9fda4 77d48e81 021c7ef0 00000000 00b9fe0c RPCRT4!RPC_INTERFACE::DispatchToStub+0x5e (FPO: [Non-Fpo]) (CONV: thiscall) [D:\nt\private\rpc\runtime\mtrt\hndlsvr.cxx @ 407]
12 00b9fdd4 77d34bc6 021c7ef0 021c7eb4 00000000 RPCRT4!RPC_INTERFACE::DispatchToStubWithObject+0xa9 (FPO: [Non-Fpo]) (CONV: thiscall) [D:\nt\private\rpc\runtime\mtrt\hndlsvr.cxx @ 795]
13 00b9fe10 77d346c5 021c7c88 001b6f68 80030001 RPCRT4!LRPC_SCALL::DealWithRequestMessage+0x1c6 (FPO: [Non-Fpo]) (CONV: thiscall) [D:\nt\private\rpc\runtime\mtrt\lpcsvr.cxx @ 2665]
14 00b9fe28 77d422ff 021c7df8 00b9fe50 021c7c88 RPCRT4!LRPC_ADDRESS::DealWithLRPCRequest+0x10c (FPO: [Non-Fpo]) (CONV: thiscall) [D:\nt\private\rpc\runtime\mtrt\lpcsvr.cxx @ 867]
15 00b9ff74 77d420d9 77d42528 001b6f68 00070000 RPCRT4!LRPC_ADDRESS::ReceiveLotsaCalls+0x1eb (FPO: [Non-Fpo]) (CONV: thiscall) [D:\nt\private\rpc\runtime\mtrt\lpcsvr.cxx @ 1175]
16 00b9ff78 77d42528 001b6f68 00070000 01000002 RPCRT4!RecvLotsaCallsWrapper+0x9 (FPO: [1,0,0]) (CONV: stdcall) [D:\nt\private\rpc\runtime\mtrt\lpcsvr.cxx @ 37]
17 00b9ffa8 77d424da 001b7220 00b9ffec 77e887dd RPCRT4!BaseCachedThreadRoutine+0x4f (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\rpc\runtime\mtrt\hndlsvr.cxx @ 3586]
18 00b9ffb4 77e887dd 001b7278 00070000 01000002 RPCRT4!ThreadStartRoutine+0x18 (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\rpc\runtime\mtrt\threads.cxx @ 230]
19 00b9ffec 00000000 77d424c2 001b7278 00000000 KERNEL32!BaseThreadStart+0x52 (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\windows\base\client\support.c @ 460]
Now, with the public symbols line up, we see that the last call that AV'd was to the function "doAV1" that was called from the ISAPI extension
entry point "HttpExtensionProc".
Again, In this stack backtrace, it is clear that the culprit is "badext.dll". So it is enough to tell the customer to contact the dll vendor to fix it. If it is an in-house component and the customer would like help
from Microsoft, they should open an advisory case to troubleshoot the issue further.
Advanced Technique
Notice also that the call to our extension was invoked by "wam.dll" (Web Application Manager). It is the component responsible for handing
over any request that needs to be processed by an ISAPI extension.
One way to see the http request passed to our extension is to dump the structure of "pWamExecInfo".
0:006> .frame 3
03 00b9f770 65d823a2 wam!WAM::InvokeExtension+0x35 [D:\nt\private\inet\iis\svcs\wam\object\wamobj.cxx @ 913]
0:006> dv
this = 0
psExt = 0209a338
szISADllPath = 01c32025 "C:\Inetpub\wwwroot\Dev\badext.dll"
pWamExecInfo = 01c31eb0
ret = 0x6194c8
0:006> dt pWamExecInfo -r
Local var @ 0x136f780 Type WAM_EXEC_INFO*
0x01c31eb0
+0x000 ecb :
+0x000 cbSize : 0x90
+0x004 dwVersion : 0x40000
+0x008 ConnID : 0x01c31eb0
+0x00c dwHttpStatusCode : 0xc8
+0x010 lpszLogData : [80] ""
+0x060 lpszMethod : 0x01c32010 "GET"
+0x064 lpszQueryString : 0x01c32047 "s=av1_v1"
+0x068 lpszPathInfo : 0x01c31ffc ""
+0x06c lpszPathTranslated : 0x01c31ffd "c:\inetpub\wwwroot"
+0x070 cbTotalBytes : 0
+0x074 cbAvailable : 0
+0x078 lpbData : (null)
+0x07c lpszContentType : 0x01c32014 ""
+0x080 GetServerVariable : 0x65d815df wam!GetServerVariable+0
+0x084 WriteClient : 0x65d88484 wam!WriteClient+0
+0x088 ReadClient : 0x65d8861f wam!ReadClient+0
+0x08c ServerSupportFunction : 0x65d81d61 wam!ServerSupportFunction+0
+0x090 m_pIWamReqIIS : 0x001aa40c
.....
.....
.....
To verify that the symbols are lined up properly for our extension, issue this command.
0:006> lmv mbadext
start end module name 06470000 06475000
badext M (pdb symbols) c:\symbols\public\badext.pdb
Loaded symbol image file: C:\Inetpub\wwwroot\Dev\badext.dll Image path: C:\Inetpub\wwwroot\Dev\badext.dll
Timestamp: Wed Jan 29 18:07:14 2003 (3E385EA2)
Checksum: 0000EBCD
File version: 5.0.0.0
Product version: 5.0.0.0
File flags: 0 (Mask 3F)
File OS: 40004 NT Win32
File type: 2.0 Dll
File date: 00000000.00000000
CompanyName: Microsoft Corporation
ProductName: Internet Information Services
InternalName: badEXT.dll
OriginalFilename: badEXT.dll
ProductVersion: 5.00.0984
FileVersion: 5.00.0984
FileDescription: badEXT Test
LegalCopyright: Copyright (C) Microsoft Corp. 1981-1999
3) with Private Symbols "badExt.dll"
The stack backtrace of the thread that AV'd would look like:
0:001> kvn50
# ChildEBP RetAddr Args to Child
00 00b9f70c 064712df 01c31eb0 00000001 01c31f7c badext!doAV1+0xb (CONV: stdcall) [H:\nT\PRIVATE\inet\iis\svcs\w3\gateways\extDBG\badext.cpp @ 147]
01 00b9f728 65d82b06 01c31eb0 01c31f7c 01c31eb0 badext!HttpExtensionProc+0x5a (CONV: stdcall) [H:\nT\PRIVATE\inet\iis\svcs\w3\gateways\extDBG\badext.cpp @ 82]
02 00b9f73c 65d82a97 01c31eb0 01c31f7c 01c31eb0 wam!HSE_APPDLL::ExecuteRequest+0x93 (FPO: [1,0,3]) (CONV: thiscall) [D:\nt\private\inet\iis\svcs\wam\object\isplocal.cxx @ 450]
03 00b9f770 65d823a2 0209a338 01c32025 01c31eb0 wam!WAM::InvokeExtension+0x35 (FPO: [Non-Fpo]) (CONV: thiscall) [D:\nt\private\inet\iis\svcs\wam\object\wamobj.cxx @ 913]
04 00b9f7b4 77d339c0 006194c8 00000000 0000009e wam!WAM::ProcessRequest+0x240 (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\inet\iis\svcs\wam\object\wamobj.cxx @ 643]
05 00b9f7dc 77d93570 65d8223f 00b9f7f8 00000005 RPCRT4!Invoke+0x30 [..\i386\stubless.asm @ 88]
06 00b9fa54 77d949ac 021c44f0 002116c4 0018a648 RPCRT4!NdrStubCall2+0x63d (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\rpc\ndr20\srvcall.c @ 1724]
07 00b9fab8 77b29e7f 021c44f0 0018a648 002116c4 RPCRT4!CStdStubBuffer_Invoke+0xec (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\rpc\ndr20\stub.c @ 1571]
08 00b9fafc 77b29d6e 0018a648 021c3074 00224738 ole32!SyncStubInvoke+0x61 (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\ole32\com\dcomrem\channelb.cxx @ 1136]
09 00b9fb44 77aa7ae7 0018a648 021c2ed8 021c44f0 ole32!StubInvoke+0xa8 (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\ole32\com\dcomrem\channelb.cxx @ 1334]
0a 00b9fba8 77aa7a33 002116c4 00000000 021c44f0 ole32!CCtxComChnl::ContextInvoke+0xbb (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\ole32\com\dcomrem\ctxchnl.cxx @ 1204]
0b 00b9fbc4 77b29c89 0018a648 00000001 021c44f0 ole32!MTAInvoke+0x18 (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\ole32\com\dcomrem\callctrl.cxx @ 1952]
0c 00b9fbf4 77b29a89 0018a600 002116c4 021c44f0 ole32!AppInvoke+0xb5 (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\ole32\com\dcomrem\channelb.cxx @ 1013]
0d 00b9fcb4 77b2a12c 001a9ef0 00000000 021c3058 ole32!ComInvokeWithLockAndIPID+0x29e (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\ole32\com\dcomrem\channelb.cxx @ 1578]
0e 00b9fcf4 77d33721 0018a600 021c3058 021c7ef0 ole32!ThreadInvoke+0x1b7 (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\ole32\com\dcomrem\channelb.cxx @ 3835]
0f 00b9fd2c 77d33667 77b29f7b 021c7ef0 00b9fe0c RPCRT4!DispatchToStubInC+0x84 (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\rpc\runtime\mtrt\dispatch.c @ 82]
10 00b9fd84 77d33579 00000000 00000000 00b9fe0c RPCRT4!RPC_INTERFACE::DispatchToStubWorker+0x100 (FPO: [Non-Fpo]) (CONV: thiscall) [D:\nt\private\rpc\runtime\mtrt\hndlsvr.cxx @ 543]
11 00b9fda4 77d48e81 021c7ef0 00000000 00b9fe0c RPCRT4!RPC_INTERFACE::DispatchToStub+0x5e (FPO: [Non-Fpo]) (CONV: thiscall) [D:\nt\private\rpc\runtime\mtrt\hndlsvr.cxx @ 407]
12 00b9fdd4 77d34bc6 021c7ef0 021c7eb4 00000000 RPCRT4!RPC_INTERFACE::DispatchToStubWithObject+0xa9 (FPO: [Non-Fpo]) (CONV: thiscall) [D:\nt\private\rpc\runtime\mtrt\hndlsvr.cxx @ 795]
13 00b9fe10 77d346c5 021c7c88 001b6f68 80030001 RPCRT4!LRPC_SCALL::DealWithRequestMessage+0x1c6 (FPO: [Non-Fpo]) (CONV: thiscall) [D:\nt\private\rpc\runtime\mtrt\lpcsvr.cxx @ 2665]
14 00b9fe28 77d422ff 021c7df8 00b9fe50 021c7c88 RPCRT4!LRPC_ADDRESS::DealWithLRPCRequest+0x10c (FPO: [Non-Fpo]) (CONV: thiscall) [D:\nt\private\rpc\runtime\mtrt\lpcsvr.cxx @ 867]
15 00b9ff74 77d420d9 77d42528 001b6f68 00070000 RPCRT4!LRPC_ADDRESS::ReceiveLotsaCalls+0x1eb (FPO: [Non-Fpo]) (CONV: thiscall) [D:\nt\private\rpc\runtime\mtrt\lpcsvr.cxx @ 1175]
16 00b9ff78 77d42528 001b6f68 00070000 01000002 RPCRT4!RecvLotsaCallsWrapper+0x9 (FPO: [1,0,0]) (CONV: stdcall) [D:\nt\private\rpc\runtime\mtrt\lpcsvr.cxx @ 37]
17 00b9ffa8 77d424da 001b7220 00b9ffec 77e887dd RPCRT4!BaseCachedThreadRoutine+0x4f (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\rpc\runtime\mtrt\hndlsvr.cxx @ 3586]
18 00b9ffb4 77e887dd 001b7278 00070000 01000002 RPCRT4!ThreadStartRoutine+0x18 (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\rpc\runtime\mtrt\threads.cxx @ 230]
19 00b9ffec 00000000 77d424c2 001b7278 00000000 KERNEL32!BaseThreadStart+0x52 (FPO: [Non-Fpo]) (CONV: stdcall) [D:\nt\private\windows\base\client\support.c @ 460]
Notice in frame OO and 01 the line of the code as well as the name and location of the source code. Going back to the source code in badext.cpp:
Line 82 looks like
DWORD WINAPI HttpExtensionProc(EXTENSION_CONTROL_BLOCK *lpEcb)
{
.....
.....
if (!strncmp(szS, "av1", 3))
{
return doAV1(lpEcb, 1); //Line 82
}
.....
.....
}
Line 147 looks like
DWORD doAV1(EXTENSION_CONTROL_BLOCK *lpEcb, int Variation)
{
if (1 == Variation)
{
SYSTEMTIME* pSysTime = NULL;
pSysTime->wYear = 2003; // Line 147: av right here.
}
return HSE_STATUS_SUCCESS;
}