![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to encryptedioqueue.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [truecrypt] / truecrypt / driver|
Return to encryptedioqueue.c CVS log | Up to [truecrypt] / truecrypt / driver |
1.1 root 1: /* 1.1.1.11! root 2: Copyright (c) 2008-2009 TrueCrypt Developers Association. All rights reserved. 1.1 root 3: 1.1.1.11! root 4: Governed by the TrueCrypt License 2.8 the full text of which is contained in ! 5: the file License.txt included in TrueCrypt binary and source code distribution ! 6: packages. 1.1 root 7: */ 8: 9: #include "TCdefs.h" 10: #include "Apidrvr.h" 11: #include "Ntdriver.h" 1.1.1.6 root 12: #include "DriveFilter.h" 1.1 root 13: #include "EncryptedIoQueue.h" 1.1.1.5 root 14: #include "EncryptionThreadPool.h" 15: #include "Volumes.h" 1.1 root 16: 17: 1.1.1.8 root 18: static void AcquireBufferPoolMutex (EncryptedIoQueue *queue) 19: { 20: NTSTATUS status; 21: 22: status = KeWaitForMutexObject (&queue->BufferPoolMutex, Executive, KernelMode, FALSE, NULL); 23: if (!NT_SUCCESS (status)) 24: TC_BUG_CHECK (status); 25: } 26: 27: 28: static void ReleaseBufferPoolMutex (EncryptedIoQueue *queue) 29: { 30: KeReleaseMutex (&queue->BufferPoolMutex, FALSE); 31: } 32: 33: 34: static void *GetPoolBuffer (EncryptedIoQueue *queue, ULONG requestedSize) 35: { 36: EncryptedIoQueueBuffer *buffer; 37: void *bufferAddress = NULL; 38: BOOL requestedSizePresentInPool = FALSE; 39: 40: while (TRUE) 41: { 42: AcquireBufferPoolMutex (queue); 43: 44: for (buffer = queue->FirstPoolBuffer; ; buffer = buffer->NextBuffer) 45: { 46: if (buffer && buffer->Size == requestedSize) 47: { 48: requestedSizePresentInPool = TRUE; 49: 50: if (!buffer->InUse) 51: { 52: // Reuse a free buffer 53: buffer->InUse = TRUE; 54: bufferAddress = buffer->Address; 55: break; 56: } 57: } 58: 59: if (!buffer || !buffer->NextBuffer) 60: { 1.1.1.9 root 61: EncryptedIoQueueBuffer *newBuffer; 62: 63: if (requestedSizePresentInPool && !queue->StartPending) 64: break; 65: 1.1.1.8 root 66: // Allocate a new buffer 1.1.1.9 root 67: newBuffer = TCalloc (sizeof (EncryptedIoQueueBuffer)); 1.1.1.8 root 68: if (!newBuffer) 69: { 70: bufferAddress = NULL; 71: break; 72: } 73: 74: bufferAddress = TCalloc (requestedSize); 75: if (bufferAddress) 76: { 77: newBuffer->NextBuffer = NULL; 78: newBuffer->Address = bufferAddress; 79: newBuffer->Size = requestedSize; 80: newBuffer->InUse = TRUE; 81: 82: if (!buffer) 83: queue->FirstPoolBuffer = newBuffer; 84: else 85: buffer->NextBuffer = newBuffer; 86: } 87: else 88: TCfree (newBuffer); 89: 90: break; 91: } 92: } 93: 94: ReleaseBufferPoolMutex (queue); 95: 96: if (bufferAddress || !requestedSizePresentInPool || queue->StartPending) 97: break; 98: 1.1.1.9 root 99: KeWaitForSingleObject (&queue->PoolBufferFreeEvent, Executive, KernelMode, FALSE, NULL); 1.1.1.8 root 100: } 101: 102: return bufferAddress; 103: } 104: 105: 106: static void ReleasePoolBuffer (EncryptedIoQueue *queue, void *address) 107: { 108: EncryptedIoQueueBuffer *buffer; 109: AcquireBufferPoolMutex (queue); 110: 111: for (buffer = queue->FirstPoolBuffer; buffer != NULL; buffer = buffer->NextBuffer) 112: { 113: if (buffer->Address == address) 114: { 115: ASSERT (buffer->InUse); 116: 117: buffer->InUse = FALSE; 118: break; 119: } 120: } 121: 122: ReleaseBufferPoolMutex (queue); 1.1.1.9 root 123: KeSetEvent (&queue->PoolBufferFreeEvent, IO_DISK_INCREMENT, FALSE); 1.1.1.8 root 124: } 125: 126: 127: static void FreePoolBuffers (EncryptedIoQueue *queue) 128: { 129: EncryptedIoQueueBuffer *buffer; 130: AcquireBufferPoolMutex (queue); 131: 132: for (buffer = queue->FirstPoolBuffer; buffer != NULL; ) 133: { 134: EncryptedIoQueueBuffer *nextBuffer = buffer->NextBuffer; 135: 136: ASSERT (!buffer->InUse); 137: 138: TCfree (buffer->Address); 139: TCfree (buffer); 140: 141: buffer = nextBuffer; 142: } 143: 144: queue->FirstPoolBuffer = NULL; 145: ReleaseBufferPoolMutex (queue); 146: } 147: 148: 1.1 root 149: static void DecrementOutstandingIoCount (EncryptedIoQueue *queue) 150: { 151: if (InterlockedDecrement (&queue->OutstandingIoCount) == 0 && (queue->SuspendPending || queue->StopPending)) 1.1.1.3 root 152: KeSetEvent (&queue->NoOutstandingIoEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 153: } 154: 155: 1.1.1.5 root 156: static void OnItemCompleted (EncryptedIoQueueItem *item, BOOL freeItem) 1.1 root 157: { 158: DecrementOutstandingIoCount (item->Queue); 1.1.1.10 root 159: IoReleaseRemoveLock (&item->Queue->RemoveLock, item->OriginalIrp); 1.1 root 160: 161: if (NT_SUCCESS (item->Status)) 162: { 163: if (item->Write) 164: item->Queue->TotalBytesWritten += item->OriginalLength; 165: else 166: item->Queue->TotalBytesRead += item->OriginalLength; 167: } 168: 1.1.1.5 root 169: if (freeItem) 1.1.1.8 root 170: ReleasePoolBuffer (item->Queue, item); 1.1 root 171: } 172: 173: 174: static NTSTATUS CompleteOriginalIrp (EncryptedIoQueueItem *item, NTSTATUS status, ULONG_PTR information) 175: { 1.1.1.8 root 176: #ifdef TC_TRACE_IO_QUEUE 177: Dump ("< %I64d [%I64d] %c status=%x info=%I64d\n", item->OriginalIrpOffset, GetElapsedTime (&item->Queue->LastPerformanceCounter), item->Write ? 'W' : 'R', status, (int64) information); 178: #endif 179: 1.1 root 180: TCCompleteDiskIrp (item->OriginalIrp, status, information); 1.1.1.5 root 181: 182: item->Status = status; 183: OnItemCompleted (item, TRUE); 184: 1.1 root 185: return status; 186: } 187: 188: 189: static void AcquireFragmentBuffer (EncryptedIoQueue *queue, byte *buffer) 190: { 191: NTSTATUS status = STATUS_INVALID_PARAMETER; 192: 193: if (buffer == queue->FragmentBufferA) 194: { 195: status = KeWaitForSingleObject (&queue->FragmentBufferAFreeEvent, Executive, KernelMode, FALSE, NULL); 196: } 197: else if (buffer == queue->FragmentBufferB) 198: { 199: status = KeWaitForSingleObject (&queue->FragmentBufferBFreeEvent, Executive, KernelMode, FALSE, NULL); 200: } 201: 202: if (!NT_SUCCESS (status)) 203: TC_BUG_CHECK (status); 204: } 205: 206: 207: static void ReleaseFragmentBuffer (EncryptedIoQueue *queue, byte *buffer) 208: { 209: if (buffer == queue->FragmentBufferA) 210: { 1.1.1.3 root 211: KeSetEvent (&queue->FragmentBufferAFreeEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 212: } 213: else if (buffer == queue->FragmentBufferB) 214: { 1.1.1.3 root 215: KeSetEvent (&queue->FragmentBufferBFreeEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 216: } 217: else 218: { 219: TC_BUG_CHECK (STATUS_INVALID_PARAMETER); 220: } 221: } 222: 223: 224: static VOID CompletionThreadProc (PVOID threadArg) 225: { 226: EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg; 227: PLIST_ENTRY listEntry; 228: EncryptedIoRequest *request; 229: UINT64_STRUCT dataUnit; 230: 1.1.1.5 root 231: if (IsEncryptionThreadPoolRunning()) 232: KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY); 233: 1.1 root 234: while (!queue->ThreadExitRequested) 235: { 236: if (!NT_SUCCESS (KeWaitForSingleObject (&queue->CompletionThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL))) 237: continue; 238: 239: if (queue->ThreadExitRequested) 240: break; 241: 242: while ((listEntry = ExInterlockedRemoveHeadList (&queue->CompletionThreadQueue, &queue->CompletionThreadQueueLock))) 243: { 244: request = CONTAINING_RECORD (listEntry, EncryptedIoRequest, CompletionListEntry); 245: 1.1.1.3 root 246: if (request->EncryptedLength > 0 && NT_SUCCESS (request->Item->Status)) 1.1 root 247: { 248: ASSERT (request->EncryptedOffset + request->EncryptedLength <= request->Offset.QuadPart + request->Length); 249: dataUnit.Value = (request->Offset.QuadPart + request->EncryptedOffset) / ENCRYPTION_DATA_UNIT_SIZE; 1.1.1.3 root 250: 251: if (queue->CryptoInfo->bPartitionInInactiveSysEncScope) 252: dataUnit.Value += queue->CryptoInfo->FirstDataUnitNo.Value; 1.1.1.5 root 253: else if (queue->RemapEncryptedArea) 254: dataUnit.Value += queue->RemappedAreaDataUnitOffset; 1.1.1.3 root 255: 1.1 root 256: DecryptDataUnits (request->Data + request->EncryptedOffset, &dataUnit, request->EncryptedLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo); 257: } 258: 259: if (request->CompleteOriginalIrp) 260: { 261: CompleteOriginalIrp (request->Item, request->Item->Status, 262: NT_SUCCESS (request->Item->Status) ? request->Item->OriginalLength : 0); 263: } 264: 1.1.1.8 root 265: ReleasePoolBuffer (queue, request); 1.1 root 266: } 267: } 268: 269: PsTerminateSystemThread (STATUS_SUCCESS); 270: } 271: 272: 1.1.1.8 root 273: static NTSTATUS TCCachedRead (EncryptedIoQueue *queue, IO_STATUS_BLOCK *ioStatus, PVOID buffer, LARGE_INTEGER offset, ULONG length) 274: { 275: queue->LastReadOffset = offset; 276: queue->LastReadLength = length; 277: 278: if (queue->ReadAheadBufferValid && queue->ReadAheadOffset.QuadPart == offset.QuadPart && queue->ReadAheadLength >= length) 279: { 280: memcpy (buffer, queue->ReadAheadBuffer, length); 281: 282: if (!queue->IsFilterDevice) 283: { 284: ioStatus->Information = length; 285: ioStatus->Status = STATUS_SUCCESS; 286: } 287: 288: return STATUS_SUCCESS; 289: } 290: 291: if (queue->IsFilterDevice) 292: return TCReadDevice (queue->LowerDeviceObject, buffer, offset, length); 293: 294: return ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, ioStatus, buffer, length, &offset, NULL); 295: } 296: 297: 1.1 root 298: static VOID IoThreadProc (PVOID threadArg) 299: { 300: EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg; 301: PLIST_ENTRY listEntry; 302: EncryptedIoRequest *request; 303: 1.1.1.4 root 304: KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY); 305: 1.1.1.6 root 306: if (!queue->IsFilterDevice && queue->SecurityClientContext) 307: { 308: #ifdef DEBUG 309: NTSTATUS status = 310: #endif 311: SeImpersonateClientEx (queue->SecurityClientContext, NULL); 312: ASSERT (NT_SUCCESS (status)); 313: } 314: 1.1 root 315: while (!queue->ThreadExitRequested) 316: { 317: if (!NT_SUCCESS (KeWaitForSingleObject (&queue->IoThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL))) 318: continue; 319: 320: if (queue->ThreadExitRequested) 321: break; 322: 323: while ((listEntry = ExInterlockedRemoveHeadList (&queue->IoThreadQueue, &queue->IoThreadQueueLock))) 324: { 1.1.1.8 root 325: InterlockedDecrement (&queue->IoThreadPendingRequestCount); 1.1 root 326: request = CONTAINING_RECORD (listEntry, EncryptedIoRequest, ListEntry); 1.1.1.3 root 327: 1.1.1.8 root 328: #ifdef TC_TRACE_IO_QUEUE 329: Dump ("%c %I64d [%I64d] roff=%I64d rlen=%d\n", request->Item->Write ? 'W' : 'R', request->Item->OriginalIrpOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), request->Offset.QuadPart, request->Length); 330: #endif 331: 1.1.1.3 root 332: // Perform IO request if no preceding request of the item failed 333: if (NT_SUCCESS (request->Item->Status)) 1.1 root 334: { 1.1.1.3 root 335: if (queue->IsFilterDevice) 336: { 1.1.1.5 root 337: if (queue->RemapEncryptedArea && request->EncryptedLength > 0) 338: { 339: if (request->EncryptedLength != request->Length) 340: { 341: // Up to three subfragments may be required to handle a partially remapped fragment 342: int subFragment; 343: byte *subFragmentData = request->Data; 344: 345: for (subFragment = 0 ; subFragment < 3; ++subFragment) 346: { 347: LARGE_INTEGER subFragmentOffset; 348: ULONG subFragmentLength; 349: subFragmentOffset.QuadPart = request->Offset.QuadPart; 350: 351: switch (subFragment) 352: { 353: case 0: 354: subFragmentLength = (ULONG) request->EncryptedOffset; 355: break; 356: 357: case 1: 358: subFragmentOffset.QuadPart += request->EncryptedOffset + queue->RemappedAreaOffset; 359: subFragmentLength = request->EncryptedLength; 360: break; 361: 362: case 2: 363: subFragmentOffset.QuadPart += request->EncryptedOffset + request->EncryptedLength; 364: subFragmentLength = (ULONG) (request->Length - (request->EncryptedOffset + request->EncryptedLength)); 365: break; 366: } 367: 368: if (subFragmentLength > 0) 369: { 370: if (request->Item->Write) 371: request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, subFragmentData, subFragmentOffset, subFragmentLength); 372: else 1.1.1.8 root 373: request->Item->Status = TCCachedRead (queue, NULL, subFragmentData, subFragmentOffset, subFragmentLength); 1.1.1.5 root 374: 375: subFragmentData += subFragmentLength; 376: } 377: } 378: } 379: else 380: { 381: // Remap the fragment 382: LARGE_INTEGER remappedOffset; 383: remappedOffset.QuadPart = request->Offset.QuadPart + queue->RemappedAreaOffset; 384: 385: if (request->Item->Write) 386: request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, remappedOffset, request->Length); 387: else 1.1.1.8 root 388: request->Item->Status = TCCachedRead (queue, NULL, request->Data, remappedOffset, request->Length); 1.1.1.5 root 389: } 390: } 1.1.1.3 root 391: else 1.1.1.5 root 392: { 393: if (request->Item->Write) 394: request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, request->Offset, request->Length); 395: else 1.1.1.8 root 396: request->Item->Status = TCCachedRead (queue, NULL, request->Data, request->Offset, request->Length); 1.1.1.5 root 397: } 1.1.1.3 root 398: } 1.1 root 399: else 1.1.1.3 root 400: { 401: IO_STATUS_BLOCK ioStatus; 1.1 root 402: 1.1.1.3 root 403: if (request->Item->Write) 404: request->Item->Status = ZwWriteFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, request->Data, request->Length, &request->Offset, NULL); 405: else 1.1.1.8 root 406: request->Item->Status = TCCachedRead (queue, &ioStatus, request->Data, request->Offset, request->Length); 1.1.1.5 root 407: 408: if (NT_SUCCESS (request->Item->Status) && ioStatus.Information != request->Length) 409: request->Item->Status = STATUS_END_OF_FILE; 1.1.1.3 root 410: } 1.1 root 411: } 412: 1.1.1.3 root 413: if (request->Item->Write) 1.1 root 414: { 1.1.1.8 root 415: queue->ReadAheadBufferValid = FALSE; 416: 1.1 root 417: ReleaseFragmentBuffer (queue, request->Data); 418: 419: if (request->CompleteOriginalIrp) 420: { 421: CompleteOriginalIrp (request->Item, request->Item->Status, 422: NT_SUCCESS (request->Item->Status) ? request->Item->OriginalLength : 0); 423: } 424: 1.1.1.8 root 425: ReleasePoolBuffer (queue, request); 1.1 root 426: } 1.1.1.3 root 427: else 428: { 1.1.1.8 root 429: BOOL readAhead = FALSE; 430: 1.1.1.3 root 431: if (NT_SUCCESS (request->Item->Status)) 432: memcpy (request->OrigDataBufferFragment, request->Data, request->Length); 433: 434: ReleaseFragmentBuffer (queue, request->Data); 435: request->Data = request->OrigDataBufferFragment; 436: 1.1.1.8 root 437: if (request->CompleteOriginalIrp 438: && queue->LastReadLength > 0 439: && NT_SUCCESS (request->Item->Status) 440: && InterlockedExchangeAdd (&queue->IoThreadPendingRequestCount, 0) == 0) 441: { 442: readAhead = TRUE; 443: InterlockedIncrement (&queue->OutstandingIoCount); 444: } 445: 1.1.1.3 root 446: ExInterlockedInsertTailList (&queue->CompletionThreadQueue, &request->CompletionListEntry, &queue->CompletionThreadQueueLock); 447: KeSetEvent (&queue->CompletionThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE); 1.1.1.8 root 448: 449: if (readAhead) 450: { 451: queue->ReadAheadBufferValid = FALSE; 452: queue->ReadAheadOffset.QuadPart = queue->LastReadOffset.QuadPart + queue->LastReadLength; 453: queue->ReadAheadLength = queue->LastReadLength; 454: 455: if (queue->ReadAheadOffset.QuadPart + queue->ReadAheadLength <= queue->MaxReadAheadOffset.QuadPart) 456: { 1.1.1.11! root 457: #ifdef TC_TRACE_IO_QUEUE ! 458: Dump ("A %I64d [%I64d] roff=%I64d rlen=%d\n", request->Item->OriginalIrpOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), queue->ReadAheadOffset, queue->ReadAheadLength); ! 459: #endif 1.1.1.8 root 460: if (queue->IsFilterDevice) 461: { 462: queue->ReadAheadBufferValid = NT_SUCCESS (TCReadDevice (queue->LowerDeviceObject, queue->ReadAheadBuffer, queue->ReadAheadOffset, queue->ReadAheadLength)); 463: } 464: else 465: { 466: IO_STATUS_BLOCK ioStatus; 467: queue->ReadAheadBufferValid = NT_SUCCESS (ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, queue->ReadAheadBuffer, queue->ReadAheadLength, &queue->ReadAheadOffset, NULL)); 468: queue->ReadAheadLength = (ULONG) ioStatus.Information; 469: } 470: } 471: 472: DecrementOutstandingIoCount (queue); 473: } 1.1.1.3 root 474: } 1.1 root 475: } 476: } 477: 478: PsTerminateSystemThread (STATUS_SUCCESS); 479: } 480: 481: 482: static VOID MainThreadProc (PVOID threadArg) 483: { 484: EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg; 485: PLIST_ENTRY listEntry; 486: EncryptedIoQueueItem *item; 487: 488: LARGE_INTEGER fragmentOffset; 489: ULONG dataRemaining; 490: PUCHAR activeFragmentBuffer = queue->FragmentBufferA; 491: PUCHAR dataBuffer; 492: EncryptedIoRequest *request; 493: uint64 intersectStart; 494: uint32 intersectLength; 495: 1.1.1.5 root 496: if (IsEncryptionThreadPoolRunning()) 497: KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY); 498: 1.1 root 499: while (!queue->ThreadExitRequested) 500: { 501: if (!NT_SUCCESS (KeWaitForSingleObject (&queue->MainThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL))) 502: continue; 503: 504: while ((listEntry = ExInterlockedRemoveHeadList (&queue->MainThreadQueue, &queue->MainThreadQueueLock))) 505: { 1.1.1.5 root 506: PIRP irp = CONTAINING_RECORD (listEntry, IRP, Tail.Overlay.ListEntry); 507: PIO_STACK_LOCATION irpSp = IoGetCurrentIrpStackLocation (irp); 508: 1.1 root 509: if (queue->Suspended) 510: KeWaitForSingleObject (&queue->QueueResumedEvent, Executive, KernelMode, FALSE, NULL); 1.1.1.5 root 511: 1.1.1.8 root 512: item = GetPoolBuffer (queue, sizeof (EncryptedIoQueueItem)); 1.1.1.5 root 513: item->Queue = queue; 514: item->OriginalIrp = irp; 515: item->Status = STATUS_SUCCESS; 516: 517: IoSetCancelRoutine (irp, NULL); 518: if (irp->Cancel) 1.1 root 519: { 520: CompleteOriginalIrp (item, STATUS_CANCELLED, 0); 521: continue; 522: } 523: 1.1.1.5 root 524: switch (irpSp->MajorFunction) 525: { 526: case IRP_MJ_READ: 527: item->Write = FALSE; 528: item->OriginalOffset = irpSp->Parameters.Read.ByteOffset; 529: item->OriginalLength = irpSp->Parameters.Read.Length; 530: break; 531: 532: case IRP_MJ_WRITE: 533: item->Write = TRUE; 534: item->OriginalOffset = irpSp->Parameters.Write.ByteOffset; 535: item->OriginalLength = irpSp->Parameters.Write.Length; 536: break; 537: 538: default: 539: CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); 540: continue; 541: } 542: 1.1.1.8 root 543: #ifdef TC_TRACE_IO_QUEUE 544: item->OriginalIrpOffset = item->OriginalOffset; 545: #endif 1.1 root 546: 1.1.1.8 root 547: // Handle misaligned read operations to work around a bug in Windows System Assessment Tool which does not follow FILE_FLAG_NO_BUFFERING requirements when benchmarking disk devices 1.1.1.7 root 548: if (queue->IsFilterDevice 549: && !item->Write 550: && item->OriginalLength > 0 551: && (item->OriginalLength & (ENCRYPTION_DATA_UNIT_SIZE - 1)) == 0 552: && (item->OriginalOffset.QuadPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0) 553: { 554: byte *buffer; 555: ULONG alignedLength = item->OriginalLength + ENCRYPTION_DATA_UNIT_SIZE; 556: LARGE_INTEGER alignedOffset; 557: alignedOffset.QuadPart = item->OriginalOffset.QuadPart & ~((LONGLONG) ENCRYPTION_DATA_UNIT_SIZE - 1); 558: 559: buffer = TCalloc (alignedLength); 560: if (!buffer) 561: { 562: CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); 563: continue; 564: } 565: 566: item->Status = TCReadDevice (queue->LowerDeviceObject, buffer, alignedOffset, alignedLength); 567: 568: if (NT_SUCCESS (item->Status)) 569: { 570: UINT64_STRUCT dataUnit; 571: 572: dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, HighPagePriority); 573: if (!dataBuffer) 574: { 575: TCfree (buffer); 576: CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); 577: continue; 578: } 579: 1.1.1.8 root 580: if (queue->EncryptedAreaStart != -1 && queue->EncryptedAreaEnd != -1) 1.1.1.7 root 581: { 1.1.1.8 root 582: GetIntersection (alignedOffset.QuadPart, alignedLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength); 583: if (intersectLength > 0) 584: { 585: dataUnit.Value = intersectStart / ENCRYPTION_DATA_UNIT_SIZE; 586: DecryptDataUnits (buffer + (intersectStart - alignedOffset.QuadPart), &dataUnit, intersectLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo); 587: } 1.1.1.7 root 588: } 589: 590: memcpy (dataBuffer, buffer + (item->OriginalOffset.LowPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)), item->OriginalLength); 591: } 592: 593: TCfree (buffer); 594: CompleteOriginalIrp (item, item->Status, NT_SUCCESS (item->Status) ? item->OriginalLength : 0); 595: continue; 596: } 597: 1.1 root 598: // Validate offset and length 1.1.1.3 root 599: if (item->OriginalLength == 0 600: || (item->OriginalLength & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0 601: || (item->OriginalOffset.QuadPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0 1.1 root 602: || (!queue->IsFilterDevice && item->OriginalOffset.QuadPart + item->OriginalLength > queue->VirtualDeviceLength)) 603: { 604: CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); 605: continue; 606: } 607: 1.1.1.8 root 608: #ifdef TC_TRACE_IO_QUEUE 609: Dump ("Q %I64d [%I64d] %c len=%d\n", item->OriginalOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), item->Write ? 'W' : 'R', item->OriginalLength); 610: #endif 1.1.1.5 root 611: 1.1 root 612: if (!queue->IsFilterDevice) 613: { 1.1.1.3 root 614: // Adjust the offset for host file or device 615: if (queue->CryptoInfo->hiddenVolume) 616: item->OriginalOffset.QuadPart += queue->CryptoInfo->hiddenVolumeOffset; 617: else 618: item->OriginalOffset.QuadPart += queue->CryptoInfo->volDataAreaOffset; 619: 1.1 root 620: // Hidden volume protection 621: if (item->Write && queue->CryptoInfo->bProtectHiddenVolume) 622: { 623: // If there has already been a write operation denied in order to protect the 624: // hidden volume (since the volume mount time) 625: if (queue->CryptoInfo->bHiddenVolProtectionAction) 626: { 627: // Do not allow writing to this volume anymore. This is to fake a complete volume 628: // or system failure (otherwise certain kinds of inconsistency within the file 629: // system could indicate that this volume has used hidden volume protection). 630: CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); 631: continue; 632: } 633: 634: // Verify that no byte is going to be written to the hidden volume area 1.1.1.3 root 635: if (RegionsOverlap ((unsigned __int64) item->OriginalOffset.QuadPart, 636: (unsigned __int64) item->OriginalOffset.QuadPart + item->OriginalLength - 1, 1.1 root 637: queue->CryptoInfo->hiddenVolumeOffset, 1.1.1.5 root 638: (unsigned __int64) queue->CryptoInfo->hiddenVolumeOffset + queue->CryptoInfo->hiddenVolumeProtectedSize - 1)) 1.1 root 639: { 1.1.1.5 root 640: Dump ("Hidden volume protection triggered: write %I64d-%I64d (protected %I64d-%I64d)\n", item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, queue->CryptoInfo->hiddenVolumeOffset, queue->CryptoInfo->hiddenVolumeOffset + queue->CryptoInfo->hiddenVolumeProtectedSize - 1); 1.1 root 641: queue->CryptoInfo->bHiddenVolProtectionAction = TRUE; 642: 643: // Deny this write operation to prevent the hidden volume from being overwritten 644: CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); 645: continue; 646: } 647: } 648: } 1.1.1.10 root 649: else if (item->Write 650: && RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, TC_BOOT_VOLUME_HEADER_SECTOR_OFFSET, TC_BOOT_VOLUME_HEADER_SECTOR_OFFSET + TC_BOOT_ENCRYPTION_VOLUME_HEADER_SIZE - 1)) 651: { 652: // Prevent inappropriately designed software from damaging important data that may be out of sync with the backup on the Rescue Disk (such as the end of the encrypted area). 653: Dump ("Preventing write to the system encryption key data area\n"); 654: CompleteOriginalIrp (item, STATUS_MEDIA_WRITE_PROTECTED, 0); 655: continue; 656: } 1.1.1.6 root 657: else if (item->Write && IsHiddenSystemRunning() 658: && (RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, SECTOR_SIZE, TC_BOOT_LOADER_AREA_SECTOR_COUNT * SECTOR_SIZE - 1) 659: || RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, GetBootDriveLength(), _I64_MAX))) 660: { 661: Dump ("Preventing write to boot loader or host protected area\n"); 662: CompleteOriginalIrp (item, STATUS_MEDIA_WRITE_PROTECTED, 0); 663: continue; 664: } 1.1 root 665: 1.1.1.10 root 666: dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, HighPagePriority); 1.1.1.5 root 667: 1.1 root 668: if (dataBuffer == NULL) 669: { 670: CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); 671: continue; 672: } 673: 674: // Divide data block to fragments to enable efficient overlapping of encryption and IO operations 675: 676: dataRemaining = item->OriginalLength; 677: fragmentOffset = item->OriginalOffset; 678: 679: while (dataRemaining > 0) 680: { 681: BOOL isLastFragment = dataRemaining <= TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; 682: 683: ULONG dataFragmentLength = isLastFragment ? dataRemaining : TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; 684: activeFragmentBuffer = (activeFragmentBuffer == queue->FragmentBufferA ? queue->FragmentBufferB : queue->FragmentBufferA); 685: 1.1.1.8 root 686: InterlockedIncrement (&queue->IoThreadPendingRequestCount); 687: 1.1.1.10 root 688: // Create IO request 689: request = GetPoolBuffer (queue, sizeof (EncryptedIoRequest)); 1.1 root 690: request->Item = item; 691: request->CompleteOriginalIrp = isLastFragment; 692: request->Offset = fragmentOffset; 693: request->Data = activeFragmentBuffer; 694: request->OrigDataBufferFragment = dataBuffer; 695: request->Length = dataFragmentLength; 696: 697: if (queue->IsFilterDevice) 698: { 1.1.1.8 root 699: if (queue->EncryptedAreaStart == -1 || queue->EncryptedAreaEnd == -1) 700: { 701: request->EncryptedLength = 0; 702: } 703: else 704: { 705: // Get intersection of data fragment with encrypted area 706: GetIntersection (fragmentOffset.QuadPart, dataFragmentLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength); 1.1 root 707: 1.1.1.8 root 708: request->EncryptedOffset = intersectStart - fragmentOffset.QuadPart; 709: request->EncryptedLength = intersectLength; 710: } 1.1 root 711: } 712: else 713: { 714: request->EncryptedOffset = 0; 715: request->EncryptedLength = dataFragmentLength; 716: } 717: 718: AcquireFragmentBuffer (queue, activeFragmentBuffer); 719: 720: if (item->Write) 721: { 722: // Encrypt data 723: memcpy (activeFragmentBuffer, dataBuffer, dataFragmentLength); 724: 725: if (request->EncryptedLength > 0) 726: { 727: UINT64_STRUCT dataUnit; 728: ASSERT (request->EncryptedOffset + request->EncryptedLength <= request->Offset.QuadPart + request->Length); 729: 730: dataUnit.Value = (request->Offset.QuadPart + request->EncryptedOffset) / ENCRYPTION_DATA_UNIT_SIZE; 731: 1.1.1.3 root 732: if (queue->CryptoInfo->bPartitionInInactiveSysEncScope) 733: dataUnit.Value += queue->CryptoInfo->FirstDataUnitNo.Value; 1.1.1.5 root 734: else if (queue->RemapEncryptedArea) 735: dataUnit.Value += queue->RemappedAreaDataUnitOffset; 736: 1.1.1.3 root 737: EncryptDataUnits (activeFragmentBuffer + request->EncryptedOffset, &dataUnit, request->EncryptedLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo); 1.1 root 738: } 739: } 740: 741: // Queue IO request 742: ExInterlockedInsertTailList (&queue->IoThreadQueue, &request->ListEntry, &queue->IoThreadQueueLock); 1.1.1.3 root 743: KeSetEvent (&queue->IoThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 744: 745: if (isLastFragment) 746: break; 747: 748: dataRemaining -= TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; 749: dataBuffer += TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; 750: fragmentOffset.QuadPart += TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; 751: } 752: } 753: } 754: 755: PsTerminateSystemThread (STATUS_SUCCESS); 756: } 757: 758: 759: NTSTATUS EncryptedIoQueueAddIrp (EncryptedIoQueue *queue, PIRP irp) 760: { 761: NTSTATUS status; 762: 763: InterlockedIncrement (&queue->OutstandingIoCount); 764: if (queue->StopPending) 765: { 766: Dump ("STATUS_DEVICE_NOT_READY out=%d\n", queue->OutstandingIoCount); 767: status = STATUS_DEVICE_NOT_READY; 768: goto err; 769: } 770: 1.1.1.10 root 771: status = IoAcquireRemoveLock (&queue->RemoveLock, irp); 772: if (!NT_SUCCESS (status)) 773: goto err; 1.1 root 774: 1.1.1.8 root 775: #ifdef TC_TRACE_IO_QUEUE 776: { 777: PIO_STACK_LOCATION irpSp = IoGetCurrentIrpStackLocation (irp); 778: Dump ("* %I64d [%I64d] %c len=%d out=%d\n", irpSp->MajorFunction == IRP_MJ_WRITE ? irpSp->Parameters.Write.ByteOffset : irpSp->Parameters.Read.ByteOffset, GetElapsedTime (&queue->LastPerformanceCounter), irpSp->MajorFunction == IRP_MJ_WRITE ? 'W' : 'R', irpSp->MajorFunction == IRP_MJ_WRITE ? irpSp->Parameters.Write.Length : irpSp->Parameters.Read.Length, queue->OutstandingIoCount); 779: } 780: #endif 781: 1.1 root 782: IoMarkIrpPending (irp); 783: 1.1.1.5 root 784: ExInterlockedInsertTailList (&queue->MainThreadQueue, &irp->Tail.Overlay.ListEntry, &queue->MainThreadQueueLock); 1.1.1.3 root 785: KeSetEvent (&queue->MainThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 786: 787: return STATUS_PENDING; 788: 789: err: 790: DecrementOutstandingIoCount (queue); 791: return status; 792: } 793: 794: 795: NTSTATUS EncryptedIoQueueHoldWhenIdle (EncryptedIoQueue *queue, int64 timeout) 796: { 797: NTSTATUS status; 798: ASSERT (!queue->Suspended); 799: 800: queue->SuspendPending = TRUE; 801: 802: while (TRUE) 803: { 804: while (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0) 805: { 806: LARGE_INTEGER waitTimeout; 807: 808: waitTimeout.QuadPart = timeout * -10000; 809: status = KeWaitForSingleObject (&queue->NoOutstandingIoEvent, Executive, KernelMode, FALSE, timeout != 0 ? &waitTimeout : NULL); 810: 811: if (status == STATUS_TIMEOUT) 812: status = STATUS_UNSUCCESSFUL; 813: 814: if (!NT_SUCCESS (status)) 1.1.1.8 root 815: { 816: queue->SuspendPending = FALSE; 1.1 root 817: return status; 1.1.1.8 root 818: } 1.1.1.6 root 819: 1.1.1.7 root 820: TCSleep (1); 1.1.1.6 root 821: if (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0) 1.1.1.8 root 822: { 823: queue->SuspendPending = FALSE; 1.1.1.6 root 824: return STATUS_UNSUCCESSFUL; 1.1.1.8 root 825: } 1.1 root 826: } 827: 828: KeClearEvent (&queue->QueueResumedEvent); 829: queue->Suspended = TRUE; 830: 831: if (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) == 0) 832: break; 833: 834: queue->Suspended = FALSE; 1.1.1.3 root 835: KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 836: } 837: 1.1.1.8 root 838: queue->ReadAheadBufferValid = FALSE; 1.1 root 839: 1.1.1.8 root 840: queue->SuspendPending = FALSE; 1.1 root 841: return STATUS_SUCCESS; 842: } 843: 844: 845: BOOL EncryptedIoQueueIsSuspended (EncryptedIoQueue *queue) 846: { 847: return queue->Suspended; 848: } 849: 850: 851: BOOL EncryptedIoQueueIsRunning (EncryptedIoQueue *queue) 852: { 853: return !queue->StopPending; 854: } 855: 856: 857: NTSTATUS EncryptedIoQueueResumeFromHold (EncryptedIoQueue *queue) 858: { 859: ASSERT (queue->Suspended); 860: 861: queue->Suspended = FALSE; 1.1.1.3 root 862: KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 863: 864: return STATUS_SUCCESS; 865: } 866: 867: 1.1.1.6 root 868: NTSTATUS EncryptedIoQueueStart (EncryptedIoQueue *queue) 1.1 root 869: { 870: NTSTATUS status; 1.1.1.8 root 871: EncryptedIoQueueBuffer *buffer; 872: int i; 873: 874: queue->StartPending = TRUE; 1.1 root 875: queue->ThreadExitRequested = FALSE; 876: 1.1.1.8 root 877: queue->OutstandingIoCount = 0; 878: queue->IoThreadPendingRequestCount = 0; 879: 880: queue->FirstPoolBuffer = NULL; 881: KeInitializeMutex (&queue->BufferPoolMutex, 0); 882: 1.1 root 883: KeInitializeEvent (&queue->NoOutstandingIoEvent, SynchronizationEvent, FALSE); 1.1.1.9 root 884: KeInitializeEvent (&queue->PoolBufferFreeEvent, SynchronizationEvent, FALSE); 1.1 root 885: KeInitializeEvent (&queue->QueueResumedEvent, SynchronizationEvent, FALSE); 886: 887: queue->FragmentBufferA = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE); 888: if (!queue->FragmentBufferA) 889: goto noMemory; 890: 891: queue->FragmentBufferB = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE); 892: if (!queue->FragmentBufferB) 893: goto noMemory; 894: 895: KeInitializeEvent (&queue->FragmentBufferAFreeEvent, SynchronizationEvent, TRUE); 896: KeInitializeEvent (&queue->FragmentBufferBFreeEvent, SynchronizationEvent, TRUE); 897: 1.1.1.8 root 898: queue->ReadAheadBufferValid = FALSE; 899: queue->ReadAheadBuffer = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE); 900: if (!queue->ReadAheadBuffer) 901: goto noMemory; 902: 903: // Preallocate buffers 904: for (i = 0; i < TC_ENC_IO_QUEUE_PREALLOCATED_IO_REQUEST_COUNT; ++i) 905: { 906: if (i < TC_ENC_IO_QUEUE_PREALLOCATED_ITEM_COUNT && !GetPoolBuffer (queue, sizeof (EncryptedIoQueueItem))) 907: goto noMemory; 908: 909: if (!GetPoolBuffer (queue, sizeof (EncryptedIoRequest))) 910: goto noMemory; 911: } 912: 913: for (buffer = queue->FirstPoolBuffer; buffer != NULL; buffer = buffer->NextBuffer) 914: { 915: buffer->InUse = FALSE; 916: } 917: 1.1 root 918: // Main thread 919: InitializeListHead (&queue->MainThreadQueue); 920: KeInitializeSpinLock (&queue->MainThreadQueueLock); 921: KeInitializeEvent (&queue->MainThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE); 922: 923: status = TCStartThread (MainThreadProc, queue, &queue->MainThread); 924: if (!NT_SUCCESS (status)) 925: goto err; 926: 927: // IO thread 928: InitializeListHead (&queue->IoThreadQueue); 929: KeInitializeSpinLock (&queue->IoThreadQueueLock); 930: KeInitializeEvent (&queue->IoThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE); 931: 1.1.1.6 root 932: status = TCStartThread (IoThreadProc, queue, &queue->IoThread); 1.1 root 933: if (!NT_SUCCESS (status)) 934: { 935: queue->ThreadExitRequested = TRUE; 936: TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent); 937: goto err; 938: } 939: 940: // Completion thread 941: InitializeListHead (&queue->CompletionThreadQueue); 942: KeInitializeSpinLock (&queue->CompletionThreadQueueLock); 943: KeInitializeEvent (&queue->CompletionThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE); 944: 945: status = TCStartThread (CompletionThreadProc, queue, &queue->CompletionThread); 946: if (!NT_SUCCESS (status)) 947: { 948: queue->ThreadExitRequested = TRUE; 949: TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent); 950: TCStopThread (queue->IoThread, &queue->IoThreadQueueNotEmptyEvent); 951: goto err; 952: } 953: 1.1.1.8 root 954: #ifdef TC_TRACE_IO_QUEUE 955: GetElapsedTimeInit (&queue->LastPerformanceCounter); 956: #endif 957: 1.1 root 958: queue->StopPending = FALSE; 1.1.1.8 root 959: queue->StartPending = FALSE; 960: 1.1 root 961: Dump ("Queue started\n"); 962: return STATUS_SUCCESS; 963: 964: noMemory: 965: status = STATUS_INSUFFICIENT_RESOURCES; 966: 967: err: 968: if (queue->FragmentBufferA) 969: TCfree (queue->FragmentBufferA); 970: if (queue->FragmentBufferB) 971: TCfree (queue->FragmentBufferB); 972: 1.1.1.8 root 973: FreePoolBuffers (queue); 974: 975: queue->StartPending = FALSE; 1.1 root 976: return status; 977: } 978: 979: 980: NTSTATUS EncryptedIoQueueStop (EncryptedIoQueue *queue) 981: { 982: ASSERT (!queue->StopPending); 983: queue->StopPending = TRUE; 984: 985: while (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0) 986: { 987: KeWaitForSingleObject (&queue->NoOutstandingIoEvent, Executive, KernelMode, FALSE, NULL); 988: } 989: 990: Dump ("Queue stopping out=%d\n", queue->OutstandingIoCount); 991: 992: queue->ThreadExitRequested = TRUE; 993: 994: TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent); 995: TCStopThread (queue->IoThread, &queue->IoThreadQueueNotEmptyEvent); 996: TCStopThread (queue->CompletionThread, &queue->CompletionThreadQueueNotEmptyEvent); 997: 998: TCfree (queue->FragmentBufferA); 999: TCfree (queue->FragmentBufferB); 1.1.1.8 root 1000: TCfree (queue->ReadAheadBuffer); 1001: 1002: FreePoolBuffers (queue); 1.1 root 1003: 1004: Dump ("Queue stopped out=%d\n", queue->OutstandingIoCount); 1005: return STATUS_SUCCESS; 1006: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.