![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to encryptedioqueue.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [truecrypt] / truecrypt / driver|
Return to encryptedioqueue.c CVS log | Up to [truecrypt] / truecrypt / driver |
1.1 root 1: /* 1.1.1.8 root 2: Copyright (c) 2008-2009 TrueCrypt Foundation. All rights reserved. 1.1 root 3: 1.1.1.9 ! root 4: Governed by the TrueCrypt License 2.7 the full text of which is contained 1.1 root 5: in the file License.txt included in TrueCrypt binary and source code 6: distribution packages. 7: */ 8: 9: #include "TCdefs.h" 10: #include "Apidrvr.h" 11: #include "Ntdriver.h" 1.1.1.6 root 12: #include "DriveFilter.h" 1.1 root 13: #include "EncryptedIoQueue.h" 1.1.1.5 root 14: #include "EncryptionThreadPool.h" 15: #include "Volumes.h" 1.1 root 16: 17: 1.1.1.8 root 18: static void AcquireBufferPoolMutex (EncryptedIoQueue *queue) 19: { 20: NTSTATUS status; 21: 22: status = KeWaitForMutexObject (&queue->BufferPoolMutex, Executive, KernelMode, FALSE, NULL); 23: if (!NT_SUCCESS (status)) 24: TC_BUG_CHECK (status); 25: } 26: 27: 28: static void ReleaseBufferPoolMutex (EncryptedIoQueue *queue) 29: { 30: KeReleaseMutex (&queue->BufferPoolMutex, FALSE); 31: } 32: 33: 34: static void *GetPoolBuffer (EncryptedIoQueue *queue, ULONG requestedSize) 35: { 36: EncryptedIoQueueBuffer *buffer; 37: void *bufferAddress = NULL; 38: BOOL requestedSizePresentInPool = FALSE; 39: 40: while (TRUE) 41: { 42: AcquireBufferPoolMutex (queue); 43: 44: for (buffer = queue->FirstPoolBuffer; ; buffer = buffer->NextBuffer) 45: { 46: if (buffer && buffer->Size == requestedSize) 47: { 48: requestedSizePresentInPool = TRUE; 49: 50: if (!buffer->InUse) 51: { 52: // Reuse a free buffer 53: buffer->InUse = TRUE; 54: bufferAddress = buffer->Address; 55: break; 56: } 57: } 58: 59: if (!buffer || !buffer->NextBuffer) 60: { 1.1.1.9 ! root 61: EncryptedIoQueueBuffer *newBuffer; ! 62: ! 63: if (requestedSizePresentInPool && !queue->StartPending) ! 64: break; ! 65: 1.1.1.8 root 66: // Allocate a new buffer 1.1.1.9 ! root 67: newBuffer = TCalloc (sizeof (EncryptedIoQueueBuffer)); 1.1.1.8 root 68: if (!newBuffer) 69: { 70: bufferAddress = NULL; 71: break; 72: } 73: 74: bufferAddress = TCalloc (requestedSize); 75: if (bufferAddress) 76: { 77: newBuffer->NextBuffer = NULL; 78: newBuffer->Address = bufferAddress; 79: newBuffer->Size = requestedSize; 80: newBuffer->InUse = TRUE; 81: 82: if (!buffer) 83: queue->FirstPoolBuffer = newBuffer; 84: else 85: buffer->NextBuffer = newBuffer; 86: } 87: else 88: TCfree (newBuffer); 89: 90: break; 91: } 92: } 93: 94: ReleaseBufferPoolMutex (queue); 95: 96: if (bufferAddress || !requestedSizePresentInPool || queue->StartPending) 97: break; 98: 1.1.1.9 ! root 99: KeWaitForSingleObject (&queue->PoolBufferFreeEvent, Executive, KernelMode, FALSE, NULL); 1.1.1.8 root 100: } 101: 102: return bufferAddress; 103: } 104: 105: 106: static void ReleasePoolBuffer (EncryptedIoQueue *queue, void *address) 107: { 108: EncryptedIoQueueBuffer *buffer; 109: AcquireBufferPoolMutex (queue); 110: 111: for (buffer = queue->FirstPoolBuffer; buffer != NULL; buffer = buffer->NextBuffer) 112: { 113: if (buffer->Address == address) 114: { 115: ASSERT (buffer->InUse); 116: 117: buffer->InUse = FALSE; 118: break; 119: } 120: } 121: 122: ReleaseBufferPoolMutex (queue); 1.1.1.9 ! root 123: KeSetEvent (&queue->PoolBufferFreeEvent, IO_DISK_INCREMENT, FALSE); 1.1.1.8 root 124: } 125: 126: 127: static void FreePoolBuffers (EncryptedIoQueue *queue) 128: { 129: EncryptedIoQueueBuffer *buffer; 130: AcquireBufferPoolMutex (queue); 131: 132: for (buffer = queue->FirstPoolBuffer; buffer != NULL; ) 133: { 134: EncryptedIoQueueBuffer *nextBuffer = buffer->NextBuffer; 135: 136: ASSERT (!buffer->InUse); 137: 138: TCfree (buffer->Address); 139: TCfree (buffer); 140: 141: buffer = nextBuffer; 142: } 143: 144: queue->FirstPoolBuffer = NULL; 145: ReleaseBufferPoolMutex (queue); 146: } 147: 148: 1.1 root 149: static void DecrementOutstandingIoCount (EncryptedIoQueue *queue) 150: { 151: if (InterlockedDecrement (&queue->OutstandingIoCount) == 0 && (queue->SuspendPending || queue->StopPending)) 1.1.1.3 root 152: KeSetEvent (&queue->NoOutstandingIoEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 153: } 154: 155: 1.1.1.5 root 156: static void OnItemCompleted (EncryptedIoQueueItem *item, BOOL freeItem) 1.1 root 157: { 158: DecrementOutstandingIoCount (item->Queue); 159: 160: if (item->Queue->IsFilterDevice) 161: IoReleaseRemoveLock (&item->Queue->RemoveLock, item->OriginalIrp); 162: 163: if (NT_SUCCESS (item->Status)) 164: { 165: if (item->Write) 166: item->Queue->TotalBytesWritten += item->OriginalLength; 167: else 168: item->Queue->TotalBytesRead += item->OriginalLength; 169: } 170: 1.1.1.5 root 171: if (freeItem) 1.1.1.8 root 172: ReleasePoolBuffer (item->Queue, item); 1.1 root 173: } 174: 175: 176: static NTSTATUS CompleteOriginalIrp (EncryptedIoQueueItem *item, NTSTATUS status, ULONG_PTR information) 177: { 1.1.1.8 root 178: #ifdef TC_TRACE_IO_QUEUE 179: Dump ("< %I64d [%I64d] %c status=%x info=%I64d\n", item->OriginalIrpOffset, GetElapsedTime (&item->Queue->LastPerformanceCounter), item->Write ? 'W' : 'R', status, (int64) information); 180: #endif 181: 1.1 root 182: TCCompleteDiskIrp (item->OriginalIrp, status, information); 1.1.1.5 root 183: 184: item->Status = status; 185: OnItemCompleted (item, TRUE); 186: 1.1 root 187: return status; 188: } 189: 190: 191: static void AcquireFragmentBuffer (EncryptedIoQueue *queue, byte *buffer) 192: { 193: NTSTATUS status = STATUS_INVALID_PARAMETER; 194: 195: if (buffer == queue->FragmentBufferA) 196: { 197: status = KeWaitForSingleObject (&queue->FragmentBufferAFreeEvent, Executive, KernelMode, FALSE, NULL); 198: } 199: else if (buffer == queue->FragmentBufferB) 200: { 201: status = KeWaitForSingleObject (&queue->FragmentBufferBFreeEvent, Executive, KernelMode, FALSE, NULL); 202: } 203: 204: if (!NT_SUCCESS (status)) 205: TC_BUG_CHECK (status); 206: } 207: 208: 209: static void ReleaseFragmentBuffer (EncryptedIoQueue *queue, byte *buffer) 210: { 211: if (buffer == queue->FragmentBufferA) 212: { 1.1.1.3 root 213: KeSetEvent (&queue->FragmentBufferAFreeEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 214: } 215: else if (buffer == queue->FragmentBufferB) 216: { 1.1.1.3 root 217: KeSetEvent (&queue->FragmentBufferBFreeEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 218: } 219: else 220: { 221: TC_BUG_CHECK (STATUS_INVALID_PARAMETER); 222: } 223: } 224: 225: 226: static VOID CompletionThreadProc (PVOID threadArg) 227: { 228: EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg; 229: PLIST_ENTRY listEntry; 230: EncryptedIoRequest *request; 231: UINT64_STRUCT dataUnit; 232: 1.1.1.5 root 233: if (IsEncryptionThreadPoolRunning()) 234: KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY); 235: 1.1 root 236: while (!queue->ThreadExitRequested) 237: { 238: if (!NT_SUCCESS (KeWaitForSingleObject (&queue->CompletionThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL))) 239: continue; 240: 241: if (queue->ThreadExitRequested) 242: break; 243: 244: while ((listEntry = ExInterlockedRemoveHeadList (&queue->CompletionThreadQueue, &queue->CompletionThreadQueueLock))) 245: { 246: request = CONTAINING_RECORD (listEntry, EncryptedIoRequest, CompletionListEntry); 247: 1.1.1.3 root 248: if (request->EncryptedLength > 0 && NT_SUCCESS (request->Item->Status)) 1.1 root 249: { 250: ASSERT (request->EncryptedOffset + request->EncryptedLength <= request->Offset.QuadPart + request->Length); 251: dataUnit.Value = (request->Offset.QuadPart + request->EncryptedOffset) / ENCRYPTION_DATA_UNIT_SIZE; 1.1.1.3 root 252: 253: if (queue->CryptoInfo->bPartitionInInactiveSysEncScope) 254: dataUnit.Value += queue->CryptoInfo->FirstDataUnitNo.Value; 1.1.1.5 root 255: else if (queue->RemapEncryptedArea) 256: dataUnit.Value += queue->RemappedAreaDataUnitOffset; 1.1.1.3 root 257: 1.1 root 258: DecryptDataUnits (request->Data + request->EncryptedOffset, &dataUnit, request->EncryptedLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo); 259: } 260: 261: if (request->CompleteOriginalIrp) 262: { 263: CompleteOriginalIrp (request->Item, request->Item->Status, 264: NT_SUCCESS (request->Item->Status) ? request->Item->OriginalLength : 0); 265: } 1.1.1.3 root 266: else 267: { 268: InterlockedDecrement (&request->Item->OutstandingRequestCount); 269: KeSetEvent (&queue->RequestCompletedEvent, IO_DISK_INCREMENT, FALSE); 270: } 1.1 root 271: 1.1.1.8 root 272: ReleasePoolBuffer (queue, request); 1.1 root 273: } 274: } 275: 276: PsTerminateSystemThread (STATUS_SUCCESS); 277: } 278: 279: 1.1.1.8 root 280: static NTSTATUS TCCachedRead (EncryptedIoQueue *queue, IO_STATUS_BLOCK *ioStatus, PVOID buffer, LARGE_INTEGER offset, ULONG length) 281: { 282: queue->LastReadOffset = offset; 283: queue->LastReadLength = length; 284: 285: if (queue->ReadAheadBufferValid && queue->ReadAheadOffset.QuadPart == offset.QuadPart && queue->ReadAheadLength >= length) 286: { 287: memcpy (buffer, queue->ReadAheadBuffer, length); 288: 289: if (!queue->IsFilterDevice) 290: { 291: ioStatus->Information = length; 292: ioStatus->Status = STATUS_SUCCESS; 293: } 294: 295: return STATUS_SUCCESS; 296: } 297: 298: if (queue->IsFilterDevice) 299: return TCReadDevice (queue->LowerDeviceObject, buffer, offset, length); 300: 301: return ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, ioStatus, buffer, length, &offset, NULL); 302: } 303: 304: 1.1 root 305: static VOID IoThreadProc (PVOID threadArg) 306: { 307: EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg; 308: PLIST_ENTRY listEntry; 309: EncryptedIoRequest *request; 310: 1.1.1.4 root 311: KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY); 312: 1.1.1.6 root 313: if (!queue->IsFilterDevice && queue->SecurityClientContext) 314: { 315: #ifdef DEBUG 316: NTSTATUS status = 317: #endif 318: SeImpersonateClientEx (queue->SecurityClientContext, NULL); 319: ASSERT (NT_SUCCESS (status)); 320: } 321: 1.1 root 322: while (!queue->ThreadExitRequested) 323: { 324: if (!NT_SUCCESS (KeWaitForSingleObject (&queue->IoThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL))) 325: continue; 326: 327: if (queue->ThreadExitRequested) 328: break; 329: 330: while ((listEntry = ExInterlockedRemoveHeadList (&queue->IoThreadQueue, &queue->IoThreadQueueLock))) 331: { 1.1.1.8 root 332: InterlockedDecrement (&queue->IoThreadPendingRequestCount); 1.1 root 333: request = CONTAINING_RECORD (listEntry, EncryptedIoRequest, ListEntry); 1.1.1.3 root 334: 1.1.1.8 root 335: #ifdef TC_TRACE_IO_QUEUE 336: Dump ("%c %I64d [%I64d] roff=%I64d rlen=%d\n", request->Item->Write ? 'W' : 'R', request->Item->OriginalIrpOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), request->Offset.QuadPart, request->Length); 337: #endif 338: 1.1.1.3 root 339: // Perform IO request if no preceding request of the item failed 340: if (NT_SUCCESS (request->Item->Status)) 1.1 root 341: { 1.1.1.3 root 342: if (queue->IsFilterDevice) 343: { 1.1.1.5 root 344: if (queue->RemapEncryptedArea && request->EncryptedLength > 0) 345: { 346: if (request->EncryptedLength != request->Length) 347: { 348: // Up to three subfragments may be required to handle a partially remapped fragment 349: int subFragment; 350: byte *subFragmentData = request->Data; 351: 352: for (subFragment = 0 ; subFragment < 3; ++subFragment) 353: { 354: LARGE_INTEGER subFragmentOffset; 355: ULONG subFragmentLength; 356: subFragmentOffset.QuadPart = request->Offset.QuadPart; 357: 358: switch (subFragment) 359: { 360: case 0: 361: subFragmentLength = (ULONG) request->EncryptedOffset; 362: break; 363: 364: case 1: 365: subFragmentOffset.QuadPart += request->EncryptedOffset + queue->RemappedAreaOffset; 366: subFragmentLength = request->EncryptedLength; 367: break; 368: 369: case 2: 370: subFragmentOffset.QuadPart += request->EncryptedOffset + request->EncryptedLength; 371: subFragmentLength = (ULONG) (request->Length - (request->EncryptedOffset + request->EncryptedLength)); 372: break; 373: } 374: 375: if (subFragmentLength > 0) 376: { 377: if (request->Item->Write) 378: request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, subFragmentData, subFragmentOffset, subFragmentLength); 379: else 1.1.1.8 root 380: request->Item->Status = TCCachedRead (queue, NULL, subFragmentData, subFragmentOffset, subFragmentLength); 1.1.1.5 root 381: 382: subFragmentData += subFragmentLength; 383: } 384: } 385: } 386: else 387: { 388: // Remap the fragment 389: LARGE_INTEGER remappedOffset; 390: remappedOffset.QuadPart = request->Offset.QuadPart + queue->RemappedAreaOffset; 391: 392: if (request->Item->Write) 393: request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, remappedOffset, request->Length); 394: else 1.1.1.8 root 395: request->Item->Status = TCCachedRead (queue, NULL, request->Data, remappedOffset, request->Length); 1.1.1.5 root 396: } 397: } 1.1.1.3 root 398: else 1.1.1.5 root 399: { 400: if (request->Item->Write) 401: request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, request->Offset, request->Length); 402: else 1.1.1.8 root 403: request->Item->Status = TCCachedRead (queue, NULL, request->Data, request->Offset, request->Length); 1.1.1.5 root 404: } 1.1.1.3 root 405: } 1.1 root 406: else 1.1.1.3 root 407: { 408: IO_STATUS_BLOCK ioStatus; 1.1 root 409: 1.1.1.3 root 410: if (request->Item->Write) 411: request->Item->Status = ZwWriteFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, request->Data, request->Length, &request->Offset, NULL); 412: else 1.1.1.8 root 413: request->Item->Status = TCCachedRead (queue, &ioStatus, request->Data, request->Offset, request->Length); 1.1.1.5 root 414: 415: if (NT_SUCCESS (request->Item->Status) && ioStatus.Information != request->Length) 416: request->Item->Status = STATUS_END_OF_FILE; 1.1.1.3 root 417: } 1.1 root 418: } 419: 1.1.1.3 root 420: if (request->Item->Write) 1.1 root 421: { 1.1.1.8 root 422: queue->ReadAheadBufferValid = FALSE; 423: 1.1 root 424: ReleaseFragmentBuffer (queue, request->Data); 425: 426: if (request->CompleteOriginalIrp) 427: { 428: CompleteOriginalIrp (request->Item, request->Item->Status, 429: NT_SUCCESS (request->Item->Status) ? request->Item->OriginalLength : 0); 430: } 431: else 432: { 1.1.1.3 root 433: InterlockedDecrement (&request->Item->OutstandingRequestCount); 434: KeSetEvent (&queue->RequestCompletedEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 435: } 436: 1.1.1.8 root 437: ReleasePoolBuffer (queue, request); 1.1 root 438: } 1.1.1.3 root 439: else 440: { 1.1.1.8 root 441: BOOL readAhead = FALSE; 442: 1.1.1.3 root 443: if (NT_SUCCESS (request->Item->Status)) 444: memcpy (request->OrigDataBufferFragment, request->Data, request->Length); 445: 446: ReleaseFragmentBuffer (queue, request->Data); 447: request->Data = request->OrigDataBufferFragment; 448: 1.1.1.8 root 449: if (request->CompleteOriginalIrp 450: && queue->LastReadLength > 0 451: && NT_SUCCESS (request->Item->Status) 452: && InterlockedExchangeAdd (&queue->IoThreadPendingRequestCount, 0) == 0) 453: { 454: readAhead = TRUE; 455: InterlockedIncrement (&queue->OutstandingIoCount); 456: } 457: 1.1.1.3 root 458: ExInterlockedInsertTailList (&queue->CompletionThreadQueue, &request->CompletionListEntry, &queue->CompletionThreadQueueLock); 459: KeSetEvent (&queue->CompletionThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE); 1.1.1.8 root 460: 461: if (readAhead) 462: { 463: queue->ReadAheadBufferValid = FALSE; 464: queue->ReadAheadOffset.QuadPart = queue->LastReadOffset.QuadPart + queue->LastReadLength; 465: queue->ReadAheadLength = queue->LastReadLength; 466: 467: #ifdef TC_TRACE_IO_QUEUE 468: Dump ("A %I64d [%I64d] roff=%I64d rlen=%d\n", request->Item->OriginalIrpOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), queue->ReadAheadOffset, queue->ReadAheadLength); 469: #endif 470: 471: if (queue->ReadAheadOffset.QuadPart + queue->ReadAheadLength <= queue->MaxReadAheadOffset.QuadPart) 472: { 473: if (queue->IsFilterDevice) 474: { 475: queue->ReadAheadBufferValid = NT_SUCCESS (TCReadDevice (queue->LowerDeviceObject, queue->ReadAheadBuffer, queue->ReadAheadOffset, queue->ReadAheadLength)); 476: } 477: else 478: { 479: IO_STATUS_BLOCK ioStatus; 480: queue->ReadAheadBufferValid = NT_SUCCESS (ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, queue->ReadAheadBuffer, queue->ReadAheadLength, &queue->ReadAheadOffset, NULL)); 481: queue->ReadAheadLength = (ULONG) ioStatus.Information; 482: } 483: } 484: 485: DecrementOutstandingIoCount (queue); 486: } 1.1.1.3 root 487: } 1.1 root 488: } 489: } 490: 491: PsTerminateSystemThread (STATUS_SUCCESS); 492: } 493: 494: 495: static VOID MainThreadProc (PVOID threadArg) 496: { 497: EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg; 498: PLIST_ENTRY listEntry; 499: EncryptedIoQueueItem *item; 500: 501: LARGE_INTEGER fragmentOffset; 502: ULONG dataRemaining; 503: PUCHAR activeFragmentBuffer = queue->FragmentBufferA; 504: PUCHAR dataBuffer; 505: EncryptedIoRequest *request; 506: uint64 intersectStart; 507: uint32 intersectLength; 508: 1.1.1.8 root 509: int64 mdlWaitTime; 510: LARGE_INTEGER mdlWaitPerfCounter; 1.1.1.5 root 511: 512: if (IsEncryptionThreadPoolRunning()) 513: KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY); 514: 1.1 root 515: while (!queue->ThreadExitRequested) 516: { 517: if (!NT_SUCCESS (KeWaitForSingleObject (&queue->MainThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL))) 518: continue; 519: 520: while ((listEntry = ExInterlockedRemoveHeadList (&queue->MainThreadQueue, &queue->MainThreadQueueLock))) 521: { 1.1.1.5 root 522: PIRP irp = CONTAINING_RECORD (listEntry, IRP, Tail.Overlay.ListEntry); 523: PIO_STACK_LOCATION irpSp = IoGetCurrentIrpStackLocation (irp); 524: 1.1 root 525: if (queue->Suspended) 526: KeWaitForSingleObject (&queue->QueueResumedEvent, Executive, KernelMode, FALSE, NULL); 1.1.1.5 root 527: 1.1.1.8 root 528: item = GetPoolBuffer (queue, sizeof (EncryptedIoQueueItem)); 1.1.1.5 root 529: if (!item) 530: { 531: EncryptedIoQueueItem stackItem; 532: stackItem.Queue = queue; 533: stackItem.OriginalIrp = irp; 534: stackItem.Status = STATUS_INSUFFICIENT_RESOURCES; 535: 536: TCCompleteDiskIrp (irp, STATUS_INSUFFICIENT_RESOURCES, 0); 537: OnItemCompleted (&stackItem, FALSE); 538: continue; 1.1 root 539: } 1.1.1.5 root 540: 541: item->Queue = queue; 542: item->OriginalIrp = irp; 543: item->OutstandingRequestCount = 0; 544: item->Status = STATUS_SUCCESS; 545: 546: IoSetCancelRoutine (irp, NULL); 547: if (irp->Cancel) 1.1 root 548: { 549: CompleteOriginalIrp (item, STATUS_CANCELLED, 0); 550: continue; 551: } 552: 1.1.1.5 root 553: switch (irpSp->MajorFunction) 554: { 555: case IRP_MJ_READ: 556: item->Write = FALSE; 557: item->OriginalOffset = irpSp->Parameters.Read.ByteOffset; 558: item->OriginalLength = irpSp->Parameters.Read.Length; 559: break; 560: 561: case IRP_MJ_WRITE: 562: item->Write = TRUE; 563: item->OriginalOffset = irpSp->Parameters.Write.ByteOffset; 564: item->OriginalLength = irpSp->Parameters.Write.Length; 565: break; 566: 567: default: 568: CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); 569: continue; 570: } 571: 1.1.1.8 root 572: #ifdef TC_TRACE_IO_QUEUE 573: item->OriginalIrpOffset = item->OriginalOffset; 574: #endif 1.1 root 575: 1.1.1.8 root 576: // Handle misaligned read operations to work around a bug in Windows System Assessment Tool which does not follow FILE_FLAG_NO_BUFFERING requirements when benchmarking disk devices 1.1.1.7 root 577: if (queue->IsFilterDevice 578: && !item->Write 579: && item->OriginalLength > 0 580: && (item->OriginalLength & (ENCRYPTION_DATA_UNIT_SIZE - 1)) == 0 581: && (item->OriginalOffset.QuadPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0) 582: { 583: byte *buffer; 584: ULONG alignedLength = item->OriginalLength + ENCRYPTION_DATA_UNIT_SIZE; 585: LARGE_INTEGER alignedOffset; 586: alignedOffset.QuadPart = item->OriginalOffset.QuadPart & ~((LONGLONG) ENCRYPTION_DATA_UNIT_SIZE - 1); 587: 588: buffer = TCalloc (alignedLength); 589: if (!buffer) 590: { 591: CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); 592: continue; 593: } 594: 595: item->Status = TCReadDevice (queue->LowerDeviceObject, buffer, alignedOffset, alignedLength); 596: 597: if (NT_SUCCESS (item->Status)) 598: { 599: UINT64_STRUCT dataUnit; 600: 601: dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, HighPagePriority); 602: if (!dataBuffer) 603: { 604: TCfree (buffer); 605: CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); 606: continue; 607: } 608: 1.1.1.8 root 609: if (queue->EncryptedAreaStart != -1 && queue->EncryptedAreaEnd != -1) 1.1.1.7 root 610: { 1.1.1.8 root 611: GetIntersection (alignedOffset.QuadPart, alignedLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength); 612: if (intersectLength > 0) 613: { 614: dataUnit.Value = intersectStart / ENCRYPTION_DATA_UNIT_SIZE; 615: DecryptDataUnits (buffer + (intersectStart - alignedOffset.QuadPart), &dataUnit, intersectLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo); 616: } 1.1.1.7 root 617: } 618: 619: memcpy (dataBuffer, buffer + (item->OriginalOffset.LowPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)), item->OriginalLength); 620: } 621: 622: TCfree (buffer); 623: CompleteOriginalIrp (item, item->Status, NT_SUCCESS (item->Status) ? item->OriginalLength : 0); 624: continue; 625: } 626: 1.1 root 627: // Validate offset and length 1.1.1.3 root 628: if (item->OriginalLength == 0 629: || (item->OriginalLength & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0 630: || (item->OriginalOffset.QuadPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0 1.1 root 631: || (!queue->IsFilterDevice && item->OriginalOffset.QuadPart + item->OriginalLength > queue->VirtualDeviceLength)) 632: { 633: CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); 634: continue; 635: } 636: 1.1.1.8 root 637: #ifdef TC_TRACE_IO_QUEUE 638: Dump ("Q %I64d [%I64d] %c len=%d\n", item->OriginalOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), item->Write ? 'W' : 'R', item->OriginalLength); 639: #endif 1.1.1.5 root 640: 1.1 root 641: if (!queue->IsFilterDevice) 642: { 1.1.1.3 root 643: // Adjust the offset for host file or device 644: if (queue->CryptoInfo->hiddenVolume) 645: item->OriginalOffset.QuadPart += queue->CryptoInfo->hiddenVolumeOffset; 646: else 647: item->OriginalOffset.QuadPart += queue->CryptoInfo->volDataAreaOffset; 648: 1.1 root 649: // Hidden volume protection 650: if (item->Write && queue->CryptoInfo->bProtectHiddenVolume) 651: { 652: // If there has already been a write operation denied in order to protect the 653: // hidden volume (since the volume mount time) 654: if (queue->CryptoInfo->bHiddenVolProtectionAction) 655: { 656: // Do not allow writing to this volume anymore. This is to fake a complete volume 657: // or system failure (otherwise certain kinds of inconsistency within the file 658: // system could indicate that this volume has used hidden volume protection). 659: CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); 660: continue; 661: } 662: 663: // Verify that no byte is going to be written to the hidden volume area 1.1.1.3 root 664: if (RegionsOverlap ((unsigned __int64) item->OriginalOffset.QuadPart, 665: (unsigned __int64) item->OriginalOffset.QuadPart + item->OriginalLength - 1, 1.1 root 666: queue->CryptoInfo->hiddenVolumeOffset, 1.1.1.5 root 667: (unsigned __int64) queue->CryptoInfo->hiddenVolumeOffset + queue->CryptoInfo->hiddenVolumeProtectedSize - 1)) 1.1 root 668: { 1.1.1.5 root 669: Dump ("Hidden volume protection triggered: write %I64d-%I64d (protected %I64d-%I64d)\n", item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, queue->CryptoInfo->hiddenVolumeOffset, queue->CryptoInfo->hiddenVolumeOffset + queue->CryptoInfo->hiddenVolumeProtectedSize - 1); 1.1 root 670: queue->CryptoInfo->bHiddenVolProtectionAction = TRUE; 671: 672: // Deny this write operation to prevent the hidden volume from being overwritten 673: CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); 674: continue; 675: } 676: } 677: } 1.1.1.6 root 678: else if (item->Write && IsHiddenSystemRunning() 679: && (RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, SECTOR_SIZE, TC_BOOT_LOADER_AREA_SECTOR_COUNT * SECTOR_SIZE - 1) 680: || RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, GetBootDriveLength(), _I64_MAX))) 681: { 682: Dump ("Preventing write to boot loader or host protected area\n"); 683: CompleteOriginalIrp (item, STATUS_MEDIA_WRITE_PROTECTED, 0); 684: continue; 685: } 1.1 root 686: 687: // Original IRP data buffer 1.1.1.5 root 688: mdlWaitTime = 0; 1.1.1.8 root 689: mdlWaitPerfCounter.QuadPart = 0; 1.1.1.5 root 690: while (TRUE) 691: { 692: dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, HighPagePriority); 693: 694: if (dataBuffer || mdlWaitTime >= TC_ENC_IO_QUEUE_MEM_ALLOC_TIMEOUT) 695: break; 696: 1.1.1.8 root 697: if (mdlWaitPerfCounter.QuadPart == 0) 698: GetElapsedTimeInit (&mdlWaitPerfCounter); 699: 700: TCSleep (TC_ENC_IO_QUEUE_MEM_ALLOC_RETRY_DELAY); 701: mdlWaitTime += GetElapsedTime (&mdlWaitPerfCounter) / 1000; 1.1.1.5 root 702: } 703: 1.1 root 704: if (dataBuffer == NULL) 705: { 706: CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); 707: continue; 708: } 709: 710: // Divide data block to fragments to enable efficient overlapping of encryption and IO operations 711: 712: dataRemaining = item->OriginalLength; 713: fragmentOffset = item->OriginalOffset; 714: 715: while (dataRemaining > 0) 716: { 717: BOOL isLastFragment = dataRemaining <= TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; 718: 719: ULONG dataFragmentLength = isLastFragment ? dataRemaining : TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; 720: activeFragmentBuffer = (activeFragmentBuffer == queue->FragmentBufferA ? queue->FragmentBufferB : queue->FragmentBufferA); 721: 722: // Create IO request 1.1.1.8 root 723: request = GetPoolBuffer (queue, sizeof (EncryptedIoRequest)); 1.1 root 724: if (!request) 725: { 1.1.1.3 root 726: while (InterlockedExchangeAdd (&item->OutstandingRequestCount, 0) > 0) 727: KeWaitForSingleObject (&queue->RequestCompletedEvent, Executive, KernelMode, FALSE, NULL); 728: 1.1 root 729: CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); 730: break; 731: } 732: 1.1.1.8 root 733: InterlockedIncrement (&queue->IoThreadPendingRequestCount); 734: 1.1 root 735: request->Item = item; 736: request->CompleteOriginalIrp = isLastFragment; 737: request->Offset = fragmentOffset; 738: request->Data = activeFragmentBuffer; 739: request->OrigDataBufferFragment = dataBuffer; 740: request->Length = dataFragmentLength; 741: 742: if (queue->IsFilterDevice) 743: { 1.1.1.8 root 744: if (queue->EncryptedAreaStart == -1 || queue->EncryptedAreaEnd == -1) 745: { 746: request->EncryptedLength = 0; 747: } 748: else 749: { 750: // Get intersection of data fragment with encrypted area 751: GetIntersection (fragmentOffset.QuadPart, dataFragmentLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength); 1.1 root 752: 1.1.1.8 root 753: request->EncryptedOffset = intersectStart - fragmentOffset.QuadPart; 754: request->EncryptedLength = intersectLength; 755: } 1.1 root 756: } 757: else 758: { 759: request->EncryptedOffset = 0; 760: request->EncryptedLength = dataFragmentLength; 761: } 762: 763: AcquireFragmentBuffer (queue, activeFragmentBuffer); 764: 765: if (item->Write) 766: { 767: // Encrypt data 768: memcpy (activeFragmentBuffer, dataBuffer, dataFragmentLength); 769: 770: if (request->EncryptedLength > 0) 771: { 772: UINT64_STRUCT dataUnit; 773: ASSERT (request->EncryptedOffset + request->EncryptedLength <= request->Offset.QuadPart + request->Length); 774: 775: dataUnit.Value = (request->Offset.QuadPart + request->EncryptedOffset) / ENCRYPTION_DATA_UNIT_SIZE; 776: 1.1.1.3 root 777: if (queue->CryptoInfo->bPartitionInInactiveSysEncScope) 778: dataUnit.Value += queue->CryptoInfo->FirstDataUnitNo.Value; 1.1.1.5 root 779: else if (queue->RemapEncryptedArea) 780: dataUnit.Value += queue->RemappedAreaDataUnitOffset; 781: 1.1.1.3 root 782: EncryptDataUnits (activeFragmentBuffer + request->EncryptedOffset, &dataUnit, request->EncryptedLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo); 1.1 root 783: } 784: } 785: 786: // Queue IO request 1.1.1.3 root 787: InterlockedIncrement (&item->OutstandingRequestCount); 788: 1.1 root 789: ExInterlockedInsertTailList (&queue->IoThreadQueue, &request->ListEntry, &queue->IoThreadQueueLock); 1.1.1.3 root 790: KeSetEvent (&queue->IoThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 791: 792: if (isLastFragment) 793: break; 794: 795: dataRemaining -= TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; 796: dataBuffer += TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; 797: fragmentOffset.QuadPart += TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; 798: } 799: } 800: } 801: 802: PsTerminateSystemThread (STATUS_SUCCESS); 803: } 804: 805: 806: NTSTATUS EncryptedIoQueueAddIrp (EncryptedIoQueue *queue, PIRP irp) 807: { 808: NTSTATUS status; 809: 810: InterlockedIncrement (&queue->OutstandingIoCount); 811: if (queue->StopPending) 812: { 813: Dump ("STATUS_DEVICE_NOT_READY out=%d\n", queue->OutstandingIoCount); 814: status = STATUS_DEVICE_NOT_READY; 815: goto err; 816: } 817: 818: if (queue->IsFilterDevice) 819: { 820: status = IoAcquireRemoveLock (&queue->RemoveLock, irp); 821: if (!NT_SUCCESS (status)) 822: goto err; 823: } 824: 1.1.1.8 root 825: #ifdef TC_TRACE_IO_QUEUE 826: { 827: PIO_STACK_LOCATION irpSp = IoGetCurrentIrpStackLocation (irp); 828: Dump ("* %I64d [%I64d] %c len=%d out=%d\n", irpSp->MajorFunction == IRP_MJ_WRITE ? irpSp->Parameters.Write.ByteOffset : irpSp->Parameters.Read.ByteOffset, GetElapsedTime (&queue->LastPerformanceCounter), irpSp->MajorFunction == IRP_MJ_WRITE ? 'W' : 'R', irpSp->MajorFunction == IRP_MJ_WRITE ? irpSp->Parameters.Write.Length : irpSp->Parameters.Read.Length, queue->OutstandingIoCount); 829: } 830: #endif 831: 1.1 root 832: IoMarkIrpPending (irp); 833: 1.1.1.5 root 834: ExInterlockedInsertTailList (&queue->MainThreadQueue, &irp->Tail.Overlay.ListEntry, &queue->MainThreadQueueLock); 1.1.1.3 root 835: KeSetEvent (&queue->MainThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 836: 837: return STATUS_PENDING; 838: 839: err: 840: DecrementOutstandingIoCount (queue); 841: return status; 842: } 843: 844: 845: NTSTATUS EncryptedIoQueueHoldWhenIdle (EncryptedIoQueue *queue, int64 timeout) 846: { 847: NTSTATUS status; 848: ASSERT (!queue->Suspended); 849: 850: queue->SuspendPending = TRUE; 851: 852: while (TRUE) 853: { 854: while (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0) 855: { 856: LARGE_INTEGER waitTimeout; 857: 858: waitTimeout.QuadPart = timeout * -10000; 859: status = KeWaitForSingleObject (&queue->NoOutstandingIoEvent, Executive, KernelMode, FALSE, timeout != 0 ? &waitTimeout : NULL); 860: 861: if (status == STATUS_TIMEOUT) 862: status = STATUS_UNSUCCESSFUL; 863: 864: if (!NT_SUCCESS (status)) 1.1.1.8 root 865: { 866: queue->SuspendPending = FALSE; 1.1 root 867: return status; 1.1.1.8 root 868: } 1.1.1.6 root 869: 1.1.1.7 root 870: TCSleep (1); 1.1.1.6 root 871: if (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0) 1.1.1.8 root 872: { 873: queue->SuspendPending = FALSE; 1.1.1.6 root 874: return STATUS_UNSUCCESSFUL; 1.1.1.8 root 875: } 1.1 root 876: } 877: 878: KeClearEvent (&queue->QueueResumedEvent); 879: queue->Suspended = TRUE; 880: 881: if (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) == 0) 882: break; 883: 884: queue->Suspended = FALSE; 1.1.1.3 root 885: KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 886: } 887: 1.1.1.8 root 888: queue->ReadAheadBufferValid = FALSE; 1.1 root 889: 1.1.1.8 root 890: queue->SuspendPending = FALSE; 1.1 root 891: return STATUS_SUCCESS; 892: } 893: 894: 895: BOOL EncryptedIoQueueIsSuspended (EncryptedIoQueue *queue) 896: { 897: return queue->Suspended; 898: } 899: 900: 901: BOOL EncryptedIoQueueIsRunning (EncryptedIoQueue *queue) 902: { 903: return !queue->StopPending; 904: } 905: 906: 907: NTSTATUS EncryptedIoQueueResumeFromHold (EncryptedIoQueue *queue) 908: { 909: ASSERT (queue->Suspended); 910: 911: queue->Suspended = FALSE; 1.1.1.3 root 912: KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 913: 914: return STATUS_SUCCESS; 915: } 916: 917: 1.1.1.6 root 918: NTSTATUS EncryptedIoQueueStart (EncryptedIoQueue *queue) 1.1 root 919: { 920: NTSTATUS status; 1.1.1.8 root 921: EncryptedIoQueueBuffer *buffer; 922: int i; 923: 924: queue->StartPending = TRUE; 1.1 root 925: queue->ThreadExitRequested = FALSE; 926: 1.1.1.8 root 927: queue->OutstandingIoCount = 0; 928: queue->IoThreadPendingRequestCount = 0; 929: 930: queue->FirstPoolBuffer = NULL; 931: KeInitializeMutex (&queue->BufferPoolMutex, 0); 932: 1.1 root 933: KeInitializeEvent (&queue->NoOutstandingIoEvent, SynchronizationEvent, FALSE); 1.1.1.3 root 934: KeInitializeEvent (&queue->RequestCompletedEvent, SynchronizationEvent, FALSE); 1.1.1.9 ! root 935: KeInitializeEvent (&queue->PoolBufferFreeEvent, SynchronizationEvent, FALSE); 1.1 root 936: KeInitializeEvent (&queue->QueueResumedEvent, SynchronizationEvent, FALSE); 937: 938: queue->FragmentBufferA = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE); 939: if (!queue->FragmentBufferA) 940: goto noMemory; 941: 942: queue->FragmentBufferB = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE); 943: if (!queue->FragmentBufferB) 944: goto noMemory; 945: 946: KeInitializeEvent (&queue->FragmentBufferAFreeEvent, SynchronizationEvent, TRUE); 947: KeInitializeEvent (&queue->FragmentBufferBFreeEvent, SynchronizationEvent, TRUE); 948: 1.1.1.8 root 949: queue->ReadAheadBufferValid = FALSE; 950: queue->ReadAheadBuffer = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE); 951: if (!queue->ReadAheadBuffer) 952: goto noMemory; 953: 954: // Preallocate buffers 955: for (i = 0; i < TC_ENC_IO_QUEUE_PREALLOCATED_IO_REQUEST_COUNT; ++i) 956: { 957: if (i < TC_ENC_IO_QUEUE_PREALLOCATED_ITEM_COUNT && !GetPoolBuffer (queue, sizeof (EncryptedIoQueueItem))) 958: goto noMemory; 959: 960: if (!GetPoolBuffer (queue, sizeof (EncryptedIoRequest))) 961: goto noMemory; 962: } 963: 964: for (buffer = queue->FirstPoolBuffer; buffer != NULL; buffer = buffer->NextBuffer) 965: { 966: buffer->InUse = FALSE; 967: } 968: 1.1 root 969: // Main thread 970: InitializeListHead (&queue->MainThreadQueue); 971: KeInitializeSpinLock (&queue->MainThreadQueueLock); 972: KeInitializeEvent (&queue->MainThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE); 973: 974: status = TCStartThread (MainThreadProc, queue, &queue->MainThread); 975: if (!NT_SUCCESS (status)) 976: goto err; 977: 978: // IO thread 979: InitializeListHead (&queue->IoThreadQueue); 980: KeInitializeSpinLock (&queue->IoThreadQueueLock); 981: KeInitializeEvent (&queue->IoThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE); 982: 1.1.1.6 root 983: status = TCStartThread (IoThreadProc, queue, &queue->IoThread); 1.1 root 984: if (!NT_SUCCESS (status)) 985: { 986: queue->ThreadExitRequested = TRUE; 987: TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent); 988: goto err; 989: } 990: 991: // Completion thread 992: InitializeListHead (&queue->CompletionThreadQueue); 993: KeInitializeSpinLock (&queue->CompletionThreadQueueLock); 994: KeInitializeEvent (&queue->CompletionThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE); 995: 996: status = TCStartThread (CompletionThreadProc, queue, &queue->CompletionThread); 997: if (!NT_SUCCESS (status)) 998: { 999: queue->ThreadExitRequested = TRUE; 1000: TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent); 1001: TCStopThread (queue->IoThread, &queue->IoThreadQueueNotEmptyEvent); 1002: goto err; 1003: } 1004: 1.1.1.8 root 1005: #ifdef TC_TRACE_IO_QUEUE 1006: GetElapsedTimeInit (&queue->LastPerformanceCounter); 1007: #endif 1008: 1.1 root 1009: queue->StopPending = FALSE; 1.1.1.8 root 1010: queue->StartPending = FALSE; 1011: 1.1 root 1012: Dump ("Queue started\n"); 1013: return STATUS_SUCCESS; 1014: 1015: noMemory: 1016: status = STATUS_INSUFFICIENT_RESOURCES; 1017: 1018: err: 1019: if (queue->FragmentBufferA) 1020: TCfree (queue->FragmentBufferA); 1021: if (queue->FragmentBufferB) 1022: TCfree (queue->FragmentBufferB); 1023: 1.1.1.8 root 1024: FreePoolBuffers (queue); 1025: 1026: queue->StartPending = FALSE; 1.1 root 1027: return status; 1028: } 1029: 1030: 1031: NTSTATUS EncryptedIoQueueStop (EncryptedIoQueue *queue) 1032: { 1033: ASSERT (!queue->StopPending); 1034: queue->StopPending = TRUE; 1035: 1036: while (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0) 1037: { 1038: KeWaitForSingleObject (&queue->NoOutstandingIoEvent, Executive, KernelMode, FALSE, NULL); 1039: } 1040: 1041: Dump ("Queue stopping out=%d\n", queue->OutstandingIoCount); 1042: 1043: queue->ThreadExitRequested = TRUE; 1044: 1045: TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent); 1046: TCStopThread (queue->IoThread, &queue->IoThreadQueueNotEmptyEvent); 1047: TCStopThread (queue->CompletionThread, &queue->CompletionThreadQueueNotEmptyEvent); 1048: 1049: TCfree (queue->FragmentBufferA); 1050: TCfree (queue->FragmentBufferB); 1.1.1.8 root 1051: TCfree (queue->ReadAheadBuffer); 1052: 1053: FreePoolBuffers (queue); 1.1 root 1054: 1055: Dump ("Queue stopped out=%d\n", queue->OutstandingIoCount); 1056: return STATUS_SUCCESS; 1057: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.