in Python/Product/PyDebugAttach/PyDebugAttach.cpp [722:1185]
bool DoAttach(HMODULE module, ConnectionInfo& connInfo, bool isDebug) {
// Python DLL?
auto isInit = (Py_IsInitialized*)GetProcAddress(module, "Py_IsInitialized");
if (isInit != nullptr && isInit()) {
DWORD interpreterId = INFINITE;
for (DWORD curInterp = 0; curInterp < MAX_INTERPRETERS; curInterp++) {
if (_interpreterInfo[curInterp] != nullptr &&
_interpreterInfo[curInterp]->Interpreter == module) {
interpreterId = curInterp;
break;
}
}
if (interpreterId == INFINITE) {
connInfo.ReportError(ConnError_UnknownVersion);
return FALSE;
}
auto version = GetPythonVersion(module);
// found initialized Python runtime, gather and check the APIs we need for a successful attach...
auto addPendingCall = (Py_AddPendingCall*)GetProcAddress(module, "Py_AddPendingCall");
auto interpHead = (PyInterpreterState_Head*)GetProcAddress(module, "PyInterpreterState_Head");
auto gilEnsure = (PyGILState_Ensure*)GetProcAddress(module, "PyGILState_Ensure");
auto gilRelease = (PyGILState_Release*)GetProcAddress(module, "PyGILState_Release");
auto threadHead = (PyInterpreterState_ThreadHead*)GetProcAddress(module, "PyInterpreterState_ThreadHead");
auto initThreads = (PyEval_Lock*)GetProcAddress(module, "PyEval_InitThreads");
auto releaseLock = (PyEval_Lock*)GetProcAddress(module, "PyEval_ReleaseLock");
auto threadsInited = (PyEval_ThreadsInitialized*)GetProcAddress(module, "PyEval_ThreadsInitialized");
auto threadNext = (PyThreadState_Next*)GetProcAddress(module, "PyThreadState_Next");
auto threadSwap = (PyThreadState_Swap*)GetProcAddress(module, "PyThreadState_Swap");
auto pyDictNew = (PyDict_New*)GetProcAddress(module, "PyDict_New");
auto pyCompileString = (Py_CompileString*)GetProcAddress(module, "Py_CompileString");
auto pyEvalCode = (PyEval_EvalCode*)GetProcAddress(module, "PyEval_EvalCode");
auto getDictItem = (PyDict_GetItemString*)GetProcAddress(module, "PyDict_GetItemString");
auto call = (PyObject_CallFunctionObjArgs*)GetProcAddress(module, "PyObject_CallFunctionObjArgs");
auto getBuiltins = (PyEval_GetBuiltins*)GetProcAddress(module, "PyEval_GetBuiltins");
auto dictSetItem = (PyDict_SetItemString*)GetProcAddress(module, "PyDict_SetItemString");
PyInt_FromLong* intFromLong;
PyString_FromString* strFromString;
PyInt_FromSize_t* intFromSizeT;
if (version >= PythonVersion_30) {
intFromLong = (PyInt_FromLong*)GetProcAddress(module, "PyLong_FromLong");
intFromSizeT = (PyInt_FromSize_t*)GetProcAddress(module, "PyLong_FromSize_t");
if (version >= PythonVersion_33) {
strFromString = (PyString_FromString*)GetProcAddress(module, "PyUnicode_FromString");
} else {
strFromString = (PyString_FromString*)GetProcAddress(module, "PyUnicodeUCS2_FromString");
}
} else {
intFromLong = (PyInt_FromLong*)GetProcAddress(module, "PyInt_FromLong");
strFromString = (PyString_FromString*)GetProcAddress(module, "PyString_FromString");
intFromSizeT = (PyInt_FromSize_t*)GetProcAddress(module, "PyInt_FromSize_t");
}
auto errOccurred = (PyErr_Occurred*)GetProcAddress(module, "PyErr_Occurred");
auto pyErrFetch = (PyErr_Fetch*)GetProcAddress(module, "PyErr_Fetch");
auto pyErrRestore = (PyErr_Restore*)GetProcAddress(module, "PyErr_Restore");
auto pyErrPrint = (PyErr_Print*)GetProcAddress(module, "PyErr_Print");
auto pyImportMod = (PyImport_ImportModule*) GetProcAddress(module, "PyImport_ImportModule");
auto pyGetAttr = (PyObject_GetAttrString*)GetProcAddress(module, "PyObject_GetAttrString");
auto pySetAttr = (PyObject_SetAttrString*)GetProcAddress(module, "PyObject_SetAttrString");
auto pyNone = (PyObject*)GetProcAddress(module, "_Py_NoneStruct");
auto boolFromLong = (PyBool_FromLong*)GetProcAddress(module, "PyBool_FromLong");
auto getThreadTls = (PyThread_get_key_value*)GetProcAddress(module, "PyThread_get_key_value");
auto setThreadTls = (PyThread_set_key_value*)GetProcAddress(module, "PyThread_set_key_value");
auto delThreadTls = (PyThread_delete_key_value*)GetProcAddress(module, "PyThread_delete_key_value");
auto PyCFrame_Type = (PyTypeObject*)GetProcAddress(module, "PyCFrame_Type");
auto pyObjectRepr = (PyObject_Repr*)GetProcAddress(module, "PyObject_Repr");
auto pyUnicodeAsWideChar = (PyUnicode_AsWideChar*)GetProcAddress(module,
version < PythonVersion_33 ? "PyUnicodeUCS2_AsWideChar" : "PyUnicode_AsWideChar");
// Either _PyThreadState_Current or _PyThreadState_UncheckedGet are required
auto curPythonThread = (PyThreadState**)(void*)GetProcAddress(module, "_PyThreadState_Current");
auto getPythonThread = (_PyThreadState_UncheckedGet*)GetProcAddress(module, "_PyThreadState_UncheckedGet");
// Either _Py_CheckInterval or _PyEval_[GS]etSwitchInterval are useful, but not required
auto intervalCheck = (int*)GetProcAddress(module, "_Py_CheckInterval");
auto getSwitchInterval = (_PyEval_GetSwitchInterval*)GetProcAddress(module, "_PyEval_GetSwitchInterval");
auto setSwitchInterval = (_PyEval_SetSwitchInterval*)GetProcAddress(module, "_PyEval_SetSwitchInterval");
if (addPendingCall == nullptr || interpHead == nullptr || gilEnsure == nullptr || gilRelease == nullptr || threadHead == nullptr ||
initThreads == nullptr || releaseLock == nullptr || threadsInited == nullptr || threadNext == nullptr || threadSwap == nullptr ||
pyDictNew == nullptr || pyCompileString == nullptr || pyEvalCode == nullptr || getDictItem == nullptr || call == nullptr ||
getBuiltins == nullptr || dictSetItem == nullptr || intFromLong == nullptr || pyErrRestore == nullptr || pyErrFetch == nullptr ||
errOccurred == nullptr || pyImportMod == nullptr || pyGetAttr == nullptr || pyNone == nullptr || pySetAttr == nullptr || boolFromLong == nullptr ||
getThreadTls == nullptr || setThreadTls == nullptr || delThreadTls == nullptr || pyObjectRepr == nullptr || pyUnicodeAsWideChar == nullptr ||
(curPythonThread == nullptr && getPythonThread == nullptr)) {
// we're missing some APIs, we cannot attach.
connInfo.ReportError(ConnError_PythonNotFound);
return false;
}
auto head = interpHead();
if (head == nullptr) {
// this interpreter is loaded but not initialized.
connInfo.ReportError(ConnError_InterpreterNotInitialized);
return false;
}
bool threadSafeAddPendingCall = false;
// check that we're a supported version
if (version == PythonVersion_Unknown) {
connInfo.ReportError(ConnError_UnknownVersion);
return false;
} else if (version < PythonVersion_26) {
connInfo.ReportError(ConnError_UnsupportedVersion);
return false;
} else if (version >= PythonVersion_27 && version != PythonVersion_30) {
threadSafeAddPendingCall = true;
}
connInfo.SetVersion(version);
// we know everything we need for VS to continue the attach.
connInfo.ReportError(ConnError_None);
SetEvent(connInfo.Buffer->AttachStartingEvent);
if (!threadsInited()) {
int saveIntervalCheck;
unsigned long saveLongIntervalCheck;
if (intervalCheck != nullptr) {
// not available on 3.2
saveIntervalCheck = *intervalCheck;
*intervalCheck = -1; // lower the interval check so pending calls are processed faster
saveLongIntervalCheck = 0; // prevent compiler warning
} else if (getSwitchInterval != nullptr && setSwitchInterval != nullptr) {
saveLongIntervalCheck = getSwitchInterval();
setSwitchInterval(0);
saveIntervalCheck = 0; // prevent compiler warning
}
else {
saveIntervalCheck = 0; // prevent compiler warning
saveLongIntervalCheck = 0; // prevent compiler warning
}
//
// Multiple thread support has not been initialized in the interpreter. We need multi threading support
// to block any actively running threads and setup the debugger attach state.
//
// We need to initialize multiple threading support but we need to do so safely. One option is to call
// Py_AddPendingCall and have our callback then initialize multi threading. This is completely safe on 2.7
// and up. Unfortunately that doesn't work if we're not actively running code on the main thread (blocked on a lock
// or reading input). It's also not thread safe pre-2.7 so we need to make sure it's safe to call on down-level
// interpreters.
//
// Another option is to make sure no code is running - if there is no active thread then we can safely call
// PyEval_InitThreads and we're in business. But to know this is safe we need to first suspend all the other
// threads in the process and then inspect if any code is running.
//
// Finally if code is running after we've suspended the threads then we can go ahead and do Py_AddPendingCall
// on down-level interpreters as long as we're sure no one else is making a call to Py_AddPendingCall at the same
// time.
//
// Therefore our strategy becomes: Make the Py_AddPendingCall on interpreters where it's thread safe. Then suspend
// all threads - if a threads IP is in Py_AddPendingCall resume and try again. Once we've got all of the threads
// stopped and not in Py_AddPendingCall (which calls no functions its self, you can see this and it's size in the
// debugger) then see if we have a current thread. If not go ahead and initialize multiple threading (it's now safe,
// no Python code is running). Otherwise add the pending call and repeat. If at any point during this process
// threading becomes initialized (due to our pending call or the Python code creating a new thread) then we're done
// and we just resume all of the presently suspended threads.
ThreadMap suspendedThreads;
g_initedEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
HandleHolder holder(g_initedEvent);
bool addedPendingCall = false;
if (addPendingCall != nullptr && threadSafeAddPendingCall) {
// we're on a thread safe Python version, go ahead and pend our call to initialize threading.
addPendingCall(&AttachCallback, initThreads);
addedPendingCall = true;
}
#define TICKS_DIFF(prev, cur) ((cur) >= (prev)) ? ((cur)-(prev)) : ((0xFFFFFFFF-(prev))+(cur))
const DWORD ticksPerSecond = 1000;
ULONGLONG startTickCount = GetTickCount64();
do {
SuspendThreads(suspendedThreads, addPendingCall, threadsInited);
if (!threadsInited()) {
auto curPyThread = getPythonThread ? getPythonThread() : *curPythonThread;
if (curPyThread == nullptr) {
// no threads are currently running, it is safe to initialize multi threading.
PyGILState_STATE gilState;
if (version >= PythonVersion_34) {
// in 3.4 due to http://bugs.python.org/issue20891,
// we need to create our thread state manually
// before we can call PyGILState_Ensure() before we
// can call PyEval_InitThreads().
// Don't require this function unless we need it.
auto threadNew = (PyThreadState_NewFunc*)GetProcAddress(module, "PyThreadState_New");
if (threadNew != nullptr) {
threadNew(head);
}
}
if (version >= PythonVersion_32) {
// in 3.2 due to the new GIL and later we can't call Py_InitThreads
// without a thread being initialized.
// So we use PyGilState_Ensure here to first
// initialize the current thread, and then we use
// Py_InitThreads to bring up multi-threading.
// Some context here: http://bugs.python.org/issue11329
// http://pytools.codeplex.com/workitem/834
gilState = gilEnsure();
}
else {
gilState = PyGILState_LOCKED; // prevent compiler warning
}
initThreads();
if (version >= PythonVersion_32) {
// we will release the GIL here
gilRelease(gilState);
} else {
releaseLock();
}
} else if (!addedPendingCall) {
// someone holds the GIL but no one is actively adding any pending calls. We can pend our call
// and initialize threads.
addPendingCall(&AttachCallback, initThreads);
addedPendingCall = true;
}
}
ResumeThreads(suspendedThreads);
} while (!threadsInited() &&
(TICKS_DIFF(startTickCount, GetTickCount64())) < (ticksPerSecond * 20) &&
!addedPendingCall);
if (!threadsInited()) {
if (addedPendingCall) {
// we've added our call to initialize multi-threading, we can now wait
// until Python code actually starts running.
SetEvent(connInfo.Buffer->AttachDoneEvent);
::WaitForSingleObject(g_initedEvent, INFINITE);
} else {
connInfo.ReportError(ConnError_TimeOut);
return false;
}
} else {
SetEvent(connInfo.Buffer->AttachDoneEvent);
}
if (intervalCheck != nullptr) {
*intervalCheck = saveIntervalCheck;
} else if (setSwitchInterval != nullptr) {
setSwitchInterval(saveLongIntervalCheck);
}
} else {
SetEvent(connInfo.Buffer->AttachDoneEvent);
}
if (g_heap != nullptr) {
HeapDestroy(g_heap);
g_heap = nullptr;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// go ahead and bring in the debugger module and initialize all threads in the process...
GilHolder gilLock(gilEnsure, gilRelease); // acquire and hold the GIL until done...
auto pyTrue = boolFromLong(1);
auto pyFalse = boolFromLong(0);
auto filename = GetCurrentModuleFilename();
if (filename.length() == 0) {
return nullptr;
}
wchar_t drive[_MAX_DRIVE], dir[_MAX_DIR], file[_MAX_FNAME], ext[_MAX_EXT];
_wsplitpath_s(filename.c_str(), drive, _MAX_DRIVE, dir, _MAX_DIR, file, _MAX_FNAME, ext, _MAX_EXT);
wchar_t ptvsdLoaderPath[MAX_PATH];
_wmakepath_s(ptvsdLoaderPath, drive, dir, L"ptvsd_loader", L".py");
auto globalsDict = PyObjectHolder(isDebug, pyDictNew());
if (!LoadAndEvaluateCode(ptvsdLoaderPath, "ptvsd_loader.py", connInfo, isDebug, globalsDict.ToPython(),
pyCompileString, dictSetItem, pyEvalCode, strFromString, getBuiltins, pyErrPrint)) {
return false;
}
// now initialize debugger process wide state
auto attach_process = PyObjectHolder(isDebug, getDictItem(globalsDict.ToPython(), "attach_process"), true);
auto new_thread = PyObjectHolder(isDebug, getDictItem(globalsDict.ToPython(), "new_thread"), true);
auto set_debugger_dll_handle = PyObjectHolder(isDebug, getDictItem(globalsDict.ToPython(), "set_debugger_dll_handle"), true);
_interpreterInfo[interpreterId]->NewThreadFunction = new PyObjectHolder(
isDebug,
getDictItem(globalsDict.ToPython(), "new_external_thread"),
true);
if (*attach_process == nullptr || *new_thread == nullptr || *set_debugger_dll_handle == nullptr) {
connInfo.ReportErrorAfterAttachDone(ConnError_LoadDebuggerBadDebugger);
return false;
}
auto pyPortNum = PyObjectHolder(isDebug, intFromLong(connInfo.Buffer->PortNumber));
auto debugId = PyObjectHolder(isDebug, strFromString(connInfo.Buffer->DebugId));
auto debugOptions = PyObjectHolder(isDebug, strFromString(connInfo.Buffer->DebugOptions));
DecRef(call(attach_process.ToPython(), pyPortNum.ToPython(), debugId.ToPython(), debugOptions.ToPython(), pyTrue, pyFalse, NULL), isDebug);
if (auto err = errOccurred()) {
PyObject *type, *value, *traceback;
pyErrFetch(&type, &value, &traceback);
auto repr = PyObjectHolder(isDebug, pyObjectRepr(value));
wchar_t reprText[0x1000] = {};
pyUnicodeAsWideChar(repr.ToPython(), reprText, sizeof(reprText) / sizeof(reprText[0]) - 1);
fputws(reprText, stderr);
connInfo.ReportErrorAfterAttachDone(ConnError_LoadDebuggerFailed);
return false;
}
auto sysMod = PyObjectHolder(isDebug, pyImportMod("sys"));
if (*sysMod == nullptr) {
connInfo.ReportErrorAfterAttachDone(ConnError_SysNotFound);
return false;
}
auto settrace = PyObjectHolder(isDebug, pyGetAttr(sysMod.ToPython(), "settrace"));
if (*settrace == nullptr) {
connInfo.ReportErrorAfterAttachDone(ConnError_SysSetTraceNotFound);
return false;
}
auto gettrace = PyObjectHolder(isDebug, pyGetAttr(sysMod.ToPython(), "gettrace"));
// we need to walk the thread list each time after we've initialized a thread so that we are always
// dealing w/ a valid thread list (threads can exit when we run code and therefore the current thread
// could be corrupt). We also don't care about newly created threads as our start_new_thread wrapper
// will handle those. So we collect the initial set of threads first here so that we don't keep iterating
// if the program is spawning large numbers of threads.
unordered_set<PyThreadState*> initialThreads;
for (auto curThread = threadHead(head); curThread != nullptr; curThread = threadNext(curThread)) {
initialThreads.insert(curThread);
}
unordered_set<PyThreadState*> seenThreads;
{
// find what index is holding onto the thread state...
auto curPyThread = getPythonThread ? getPythonThread() : *curPythonThread;
int threadStateIndex = -1;
for (int i = 0; i < 100000; i++) {
void* value = getThreadTls(i);
if (value == curPyThread) {
threadStateIndex = i;
break;
}
}
bool foundThread;
int processedThreads = 0;
do {
foundThread = false;
for (auto curThread = threadHead(head); curThread != nullptr; curThread = threadNext(curThread)) {
if (initialThreads.find(curThread) == initialThreads.end() ||
seenThreads.insert(curThread).second == false) {
continue;
}
foundThread = true;
processedThreads++;
DWORD threadId = GetPythonThreadId(version, curThread);
// skip this thread - it doesn't really have any Python code on it...
if (threadId != GetCurrentThreadId()) {
// create new debugger Thread object on our injected thread
auto pyThreadId = PyObjectHolder(isDebug, intFromLong(threadId));
PyFrameObject* frame;
// update all of the frames so they have our trace func
if (PyThreadState_25_27::IsFor(version)) {
frame = ((PyThreadState_25_27*)curThread)->frame;
} else if (PyThreadState_30_33::IsFor(version)) {
frame = ((PyThreadState_30_33*)curThread)->frame;
} else if (PyThreadState_34_36::IsFor(version)) {
frame = ((PyThreadState_34_36*)curThread)->frame;
} else if (PyThreadState_37::IsFor(version)) {
frame = ((PyThreadState_37*)curThread)->frame;
} else {
_ASSERTE(false);
frame = nullptr; // prevent compiler warning
}
auto threadObj = PyObjectHolder(isDebug, call(new_thread.ToPython(), pyThreadId.ToPython(), pyTrue, frame, NULL));
if (threadObj.ToPython() == pyNone || *threadObj == nullptr) {
break;
}
// switch to our new thread so we can call sys.settrace on it...
// all of the work here needs to be minimal - in particular we shouldn't
// ever evaluate user defined code as we could end up switching to this
// thread on the main thread and corrupting state.
delThreadTls(threadStateIndex);
setThreadTls(threadStateIndex, curThread);
auto prevThread = threadSwap(curThread);
// save and restore the error in case something funky happens...
auto errOccured = errOccurred();
PyObject* type = nullptr;
PyObject* value = nullptr;
PyObject* traceback = nullptr;
if (errOccured) {
pyErrFetch(&type, &value, &traceback);
}
auto traceFunc = PyObjectHolder(isDebug, pyGetAttr(threadObj.ToPython(), "trace_func"));
if (*gettrace == NULL) {
DecRef(call(settrace.ToPython(), traceFunc.ToPython(), NULL), isDebug);
} else {
auto existingTraceFunc = PyObjectHolder(isDebug, call(gettrace.ToPython(), NULL));
DecRef(call(settrace.ToPython(), traceFunc.ToPython(), NULL), isDebug);
if (existingTraceFunc.ToPython() != pyNone) {
pySetAttr(threadObj.ToPython(), "prev_trace_func", existingTraceFunc.ToPython());
}
}
if (errOccured) {
pyErrRestore(type, value, traceback);
}
// update all of the frames so they have our trace func
auto curFrame = (PyFrameObject*)GetPyObjectPointerNoDebugInfo(isDebug, frame);
while (curFrame != nullptr) {
// Special case for CFrame objects
// Stackless CFrame does not have a trace function
// This will just prevent a crash on attach.
if (((PyObject*)curFrame)->ob_type != PyCFrame_Type) {
DecRef(curFrame->f_trace, isDebug);
IncRef(*traceFunc);
curFrame->f_trace = traceFunc.ToPython();
}
curFrame = (PyFrameObject*)GetPyObjectPointerNoDebugInfo(isDebug, curFrame->f_back);
}
delThreadTls(threadStateIndex);
setThreadTls(threadStateIndex, prevThread);
threadSwap(prevThread);
}
break;
}
} while (foundThread);
}
HMODULE hModule = NULL;
if (GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, (LPCTSTR)GetCurrentModuleFilename, &hModule) != 0) {
// set our handle so we can be unloaded on detach...
DecRef(call(set_debugger_dll_handle.ToPython(), intFromSizeT((size_t)hModule), nullptr), isDebug);
}
return true;
}
connInfo.ReportError(ConnError_PythonNotFound);
return false;
}