![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to encryptedioqueue.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [truecrypt] / truecrypt / driver|
Return to encryptedioqueue.c CVS log | Up to [truecrypt] / truecrypt / driver |
1.1 root 1: /* 1.1.1.8 root 2: Copyright (c) 2008-2009 TrueCrypt Foundation. All rights reserved. 1.1 root 3: 1.1.1.10! root 4: Governed by the TrueCrypt License 2.8 the full text of which is contained 1.1 root 5: in the file License.txt included in TrueCrypt binary and source code 6: distribution packages. 7: */ 8: 9: #include "TCdefs.h" 10: #include "Apidrvr.h" 11: #include "Ntdriver.h" 1.1.1.6 root 12: #include "DriveFilter.h" 1.1 root 13: #include "EncryptedIoQueue.h" 1.1.1.5 root 14: #include "EncryptionThreadPool.h" 15: #include "Volumes.h" 1.1 root 16: 17: 1.1.1.8 root 18: static void AcquireBufferPoolMutex (EncryptedIoQueue *queue) 19: { 20: NTSTATUS status; 21: 22: status = KeWaitForMutexObject (&queue->BufferPoolMutex, Executive, KernelMode, FALSE, NULL); 23: if (!NT_SUCCESS (status)) 24: TC_BUG_CHECK (status); 25: } 26: 27: 28: static void ReleaseBufferPoolMutex (EncryptedIoQueue *queue) 29: { 30: KeReleaseMutex (&queue->BufferPoolMutex, FALSE); 31: } 32: 33: 34: static void *GetPoolBuffer (EncryptedIoQueue *queue, ULONG requestedSize) 35: { 36: EncryptedIoQueueBuffer *buffer; 37: void *bufferAddress = NULL; 38: BOOL requestedSizePresentInPool = FALSE; 39: 40: while (TRUE) 41: { 42: AcquireBufferPoolMutex (queue); 43: 44: for (buffer = queue->FirstPoolBuffer; ; buffer = buffer->NextBuffer) 45: { 46: if (buffer && buffer->Size == requestedSize) 47: { 48: requestedSizePresentInPool = TRUE; 49: 50: if (!buffer->InUse) 51: { 52: // Reuse a free buffer 53: buffer->InUse = TRUE; 54: bufferAddress = buffer->Address; 55: break; 56: } 57: } 58: 59: if (!buffer || !buffer->NextBuffer) 60: { 1.1.1.9 root 61: EncryptedIoQueueBuffer *newBuffer; 62: 63: if (requestedSizePresentInPool && !queue->StartPending) 64: break; 65: 1.1.1.8 root 66: // Allocate a new buffer 1.1.1.9 root 67: newBuffer = TCalloc (sizeof (EncryptedIoQueueBuffer)); 1.1.1.8 root 68: if (!newBuffer) 69: { 70: bufferAddress = NULL; 71: break; 72: } 73: 74: bufferAddress = TCalloc (requestedSize); 75: if (bufferAddress) 76: { 77: newBuffer->NextBuffer = NULL; 78: newBuffer->Address = bufferAddress; 79: newBuffer->Size = requestedSize; 80: newBuffer->InUse = TRUE; 81: 82: if (!buffer) 83: queue->FirstPoolBuffer = newBuffer; 84: else 85: buffer->NextBuffer = newBuffer; 86: } 87: else 88: TCfree (newBuffer); 89: 90: break; 91: } 92: } 93: 94: ReleaseBufferPoolMutex (queue); 95: 96: if (bufferAddress || !requestedSizePresentInPool || queue->StartPending) 97: break; 98: 1.1.1.9 root 99: KeWaitForSingleObject (&queue->PoolBufferFreeEvent, Executive, KernelMode, FALSE, NULL); 1.1.1.8 root 100: } 101: 102: return bufferAddress; 103: } 104: 105: 106: static void ReleasePoolBuffer (EncryptedIoQueue *queue, void *address) 107: { 108: EncryptedIoQueueBuffer *buffer; 109: AcquireBufferPoolMutex (queue); 110: 111: for (buffer = queue->FirstPoolBuffer; buffer != NULL; buffer = buffer->NextBuffer) 112: { 113: if (buffer->Address == address) 114: { 115: ASSERT (buffer->InUse); 116: 117: buffer->InUse = FALSE; 118: break; 119: } 120: } 121: 122: ReleaseBufferPoolMutex (queue); 1.1.1.9 root 123: KeSetEvent (&queue->PoolBufferFreeEvent, IO_DISK_INCREMENT, FALSE); 1.1.1.8 root 124: } 125: 126: 127: static void FreePoolBuffers (EncryptedIoQueue *queue) 128: { 129: EncryptedIoQueueBuffer *buffer; 130: AcquireBufferPoolMutex (queue); 131: 132: for (buffer = queue->FirstPoolBuffer; buffer != NULL; ) 133: { 134: EncryptedIoQueueBuffer *nextBuffer = buffer->NextBuffer; 135: 136: ASSERT (!buffer->InUse); 137: 138: TCfree (buffer->Address); 139: TCfree (buffer); 140: 141: buffer = nextBuffer; 142: } 143: 144: queue->FirstPoolBuffer = NULL; 145: ReleaseBufferPoolMutex (queue); 146: } 147: 148: 1.1 root 149: static void DecrementOutstandingIoCount (EncryptedIoQueue *queue) 150: { 151: if (InterlockedDecrement (&queue->OutstandingIoCount) == 0 && (queue->SuspendPending || queue->StopPending)) 1.1.1.3 root 152: KeSetEvent (&queue->NoOutstandingIoEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 153: } 154: 155: 1.1.1.5 root 156: static void OnItemCompleted (EncryptedIoQueueItem *item, BOOL freeItem) 1.1 root 157: { 158: DecrementOutstandingIoCount (item->Queue); 1.1.1.10! root 159: IoReleaseRemoveLock (&item->Queue->RemoveLock, item->OriginalIrp); 1.1 root 160: 161: if (NT_SUCCESS (item->Status)) 162: { 163: if (item->Write) 164: item->Queue->TotalBytesWritten += item->OriginalLength; 165: else 166: item->Queue->TotalBytesRead += item->OriginalLength; 167: } 168: 1.1.1.5 root 169: if (freeItem) 1.1.1.8 root 170: ReleasePoolBuffer (item->Queue, item); 1.1 root 171: } 172: 173: 174: static NTSTATUS CompleteOriginalIrp (EncryptedIoQueueItem *item, NTSTATUS status, ULONG_PTR information) 175: { 1.1.1.8 root 176: #ifdef TC_TRACE_IO_QUEUE 177: Dump ("< %I64d [%I64d] %c status=%x info=%I64d\n", item->OriginalIrpOffset, GetElapsedTime (&item->Queue->LastPerformanceCounter), item->Write ? 'W' : 'R', status, (int64) information); 178: #endif 179: 1.1 root 180: TCCompleteDiskIrp (item->OriginalIrp, status, information); 1.1.1.5 root 181: 182: item->Status = status; 183: OnItemCompleted (item, TRUE); 184: 1.1 root 185: return status; 186: } 187: 188: 189: static void AcquireFragmentBuffer (EncryptedIoQueue *queue, byte *buffer) 190: { 191: NTSTATUS status = STATUS_INVALID_PARAMETER; 192: 193: if (buffer == queue->FragmentBufferA) 194: { 195: status = KeWaitForSingleObject (&queue->FragmentBufferAFreeEvent, Executive, KernelMode, FALSE, NULL); 196: } 197: else if (buffer == queue->FragmentBufferB) 198: { 199: status = KeWaitForSingleObject (&queue->FragmentBufferBFreeEvent, Executive, KernelMode, FALSE, NULL); 200: } 201: 202: if (!NT_SUCCESS (status)) 203: TC_BUG_CHECK (status); 204: } 205: 206: 207: static void ReleaseFragmentBuffer (EncryptedIoQueue *queue, byte *buffer) 208: { 209: if (buffer == queue->FragmentBufferA) 210: { 1.1.1.3 root 211: KeSetEvent (&queue->FragmentBufferAFreeEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 212: } 213: else if (buffer == queue->FragmentBufferB) 214: { 1.1.1.3 root 215: KeSetEvent (&queue->FragmentBufferBFreeEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 216: } 217: else 218: { 219: TC_BUG_CHECK (STATUS_INVALID_PARAMETER); 220: } 221: } 222: 223: 224: static VOID CompletionThreadProc (PVOID threadArg) 225: { 226: EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg; 227: PLIST_ENTRY listEntry; 228: EncryptedIoRequest *request; 229: UINT64_STRUCT dataUnit; 230: 1.1.1.5 root 231: if (IsEncryptionThreadPoolRunning()) 232: KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY); 233: 1.1 root 234: while (!queue->ThreadExitRequested) 235: { 236: if (!NT_SUCCESS (KeWaitForSingleObject (&queue->CompletionThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL))) 237: continue; 238: 239: if (queue->ThreadExitRequested) 240: break; 241: 242: while ((listEntry = ExInterlockedRemoveHeadList (&queue->CompletionThreadQueue, &queue->CompletionThreadQueueLock))) 243: { 244: request = CONTAINING_RECORD (listEntry, EncryptedIoRequest, CompletionListEntry); 245: 1.1.1.3 root 246: if (request->EncryptedLength > 0 && NT_SUCCESS (request->Item->Status)) 1.1 root 247: { 248: ASSERT (request->EncryptedOffset + request->EncryptedLength <= request->Offset.QuadPart + request->Length); 249: dataUnit.Value = (request->Offset.QuadPart + request->EncryptedOffset) / ENCRYPTION_DATA_UNIT_SIZE; 1.1.1.3 root 250: 251: if (queue->CryptoInfo->bPartitionInInactiveSysEncScope) 252: dataUnit.Value += queue->CryptoInfo->FirstDataUnitNo.Value; 1.1.1.5 root 253: else if (queue->RemapEncryptedArea) 254: dataUnit.Value += queue->RemappedAreaDataUnitOffset; 1.1.1.3 root 255: 1.1 root 256: DecryptDataUnits (request->Data + request->EncryptedOffset, &dataUnit, request->EncryptedLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo); 257: } 258: 259: if (request->CompleteOriginalIrp) 260: { 261: CompleteOriginalIrp (request->Item, request->Item->Status, 262: NT_SUCCESS (request->Item->Status) ? request->Item->OriginalLength : 0); 263: } 264: 1.1.1.8 root 265: ReleasePoolBuffer (queue, request); 1.1 root 266: } 267: } 268: 269: PsTerminateSystemThread (STATUS_SUCCESS); 270: } 271: 272: 1.1.1.8 root 273: static NTSTATUS TCCachedRead (EncryptedIoQueue *queue, IO_STATUS_BLOCK *ioStatus, PVOID buffer, LARGE_INTEGER offset, ULONG length) 274: { 275: queue->LastReadOffset = offset; 276: queue->LastReadLength = length; 277: 278: if (queue->ReadAheadBufferValid && queue->ReadAheadOffset.QuadPart == offset.QuadPart && queue->ReadAheadLength >= length) 279: { 280: memcpy (buffer, queue->ReadAheadBuffer, length); 281: 282: if (!queue->IsFilterDevice) 283: { 284: ioStatus->Information = length; 285: ioStatus->Status = STATUS_SUCCESS; 286: } 287: 288: return STATUS_SUCCESS; 289: } 290: 291: if (queue->IsFilterDevice) 292: return TCReadDevice (queue->LowerDeviceObject, buffer, offset, length); 293: 294: return ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, ioStatus, buffer, length, &offset, NULL); 295: } 296: 297: 1.1 root 298: static VOID IoThreadProc (PVOID threadArg) 299: { 300: EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg; 301: PLIST_ENTRY listEntry; 302: EncryptedIoRequest *request; 303: 1.1.1.4 root 304: KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY); 305: 1.1.1.6 root 306: if (!queue->IsFilterDevice && queue->SecurityClientContext) 307: { 308: #ifdef DEBUG 309: NTSTATUS status = 310: #endif 311: SeImpersonateClientEx (queue->SecurityClientContext, NULL); 312: ASSERT (NT_SUCCESS (status)); 313: } 314: 1.1 root 315: while (!queue->ThreadExitRequested) 316: { 317: if (!NT_SUCCESS (KeWaitForSingleObject (&queue->IoThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL))) 318: continue; 319: 320: if (queue->ThreadExitRequested) 321: break; 322: 323: while ((listEntry = ExInterlockedRemoveHeadList (&queue->IoThreadQueue, &queue->IoThreadQueueLock))) 324: { 1.1.1.8 root 325: InterlockedDecrement (&queue->IoThreadPendingRequestCount); 1.1 root 326: request = CONTAINING_RECORD (listEntry, EncryptedIoRequest, ListEntry); 1.1.1.3 root 327: 1.1.1.8 root 328: #ifdef TC_TRACE_IO_QUEUE 329: Dump ("%c %I64d [%I64d] roff=%I64d rlen=%d\n", request->Item->Write ? 'W' : 'R', request->Item->OriginalIrpOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), request->Offset.QuadPart, request->Length); 330: #endif 331: 1.1.1.3 root 332: // Perform IO request if no preceding request of the item failed 333: if (NT_SUCCESS (request->Item->Status)) 1.1 root 334: { 1.1.1.3 root 335: if (queue->IsFilterDevice) 336: { 1.1.1.5 root 337: if (queue->RemapEncryptedArea && request->EncryptedLength > 0) 338: { 339: if (request->EncryptedLength != request->Length) 340: { 341: // Up to three subfragments may be required to handle a partially remapped fragment 342: int subFragment; 343: byte *subFragmentData = request->Data; 344: 345: for (subFragment = 0 ; subFragment < 3; ++subFragment) 346: { 347: LARGE_INTEGER subFragmentOffset; 348: ULONG subFragmentLength; 349: subFragmentOffset.QuadPart = request->Offset.QuadPart; 350: 351: switch (subFragment) 352: { 353: case 0: 354: subFragmentLength = (ULONG) request->EncryptedOffset; 355: break; 356: 357: case 1: 358: subFragmentOffset.QuadPart += request->EncryptedOffset + queue->RemappedAreaOffset; 359: subFragmentLength = request->EncryptedLength; 360: break; 361: 362: case 2: 363: subFragmentOffset.QuadPart += request->EncryptedOffset + request->EncryptedLength; 364: subFragmentLength = (ULONG) (request->Length - (request->EncryptedOffset + request->EncryptedLength)); 365: break; 366: } 367: 368: if (subFragmentLength > 0) 369: { 370: if (request->Item->Write) 371: request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, subFragmentData, subFragmentOffset, subFragmentLength); 372: else 1.1.1.8 root 373: request->Item->Status = TCCachedRead (queue, NULL, subFragmentData, subFragmentOffset, subFragmentLength); 1.1.1.5 root 374: 375: subFragmentData += subFragmentLength; 376: } 377: } 378: } 379: else 380: { 381: // Remap the fragment 382: LARGE_INTEGER remappedOffset; 383: remappedOffset.QuadPart = request->Offset.QuadPart + queue->RemappedAreaOffset; 384: 385: if (request->Item->Write) 386: request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, remappedOffset, request->Length); 387: else 1.1.1.8 root 388: request->Item->Status = TCCachedRead (queue, NULL, request->Data, remappedOffset, request->Length); 1.1.1.5 root 389: } 390: } 1.1.1.3 root 391: else 1.1.1.5 root 392: { 393: if (request->Item->Write) 394: request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, request->Offset, request->Length); 395: else 1.1.1.8 root 396: request->Item->Status = TCCachedRead (queue, NULL, request->Data, request->Offset, request->Length); 1.1.1.5 root 397: } 1.1.1.3 root 398: } 1.1 root 399: else 1.1.1.3 root 400: { 401: IO_STATUS_BLOCK ioStatus; 1.1 root 402: 1.1.1.3 root 403: if (request->Item->Write) 404: request->Item->Status = ZwWriteFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, request->Data, request->Length, &request->Offset, NULL); 405: else 1.1.1.8 root 406: request->Item->Status = TCCachedRead (queue, &ioStatus, request->Data, request->Offset, request->Length); 1.1.1.5 root 407: 408: if (NT_SUCCESS (request->Item->Status) && ioStatus.Information != request->Length) 409: request->Item->Status = STATUS_END_OF_FILE; 1.1.1.3 root 410: } 1.1 root 411: } 412: 1.1.1.3 root 413: if (request->Item->Write) 1.1 root 414: { 1.1.1.8 root 415: queue->ReadAheadBufferValid = FALSE; 416: 1.1 root 417: ReleaseFragmentBuffer (queue, request->Data); 418: 419: if (request->CompleteOriginalIrp) 420: { 421: CompleteOriginalIrp (request->Item, request->Item->Status, 422: NT_SUCCESS (request->Item->Status) ? request->Item->OriginalLength : 0); 423: } 424: 1.1.1.8 root 425: ReleasePoolBuffer (queue, request); 1.1 root 426: } 1.1.1.3 root 427: else 428: { 1.1.1.8 root 429: BOOL readAhead = FALSE; 430: 1.1.1.3 root 431: if (NT_SUCCESS (request->Item->Status)) 432: memcpy (request->OrigDataBufferFragment, request->Data, request->Length); 433: 434: ReleaseFragmentBuffer (queue, request->Data); 435: request->Data = request->OrigDataBufferFragment; 436: 1.1.1.8 root 437: if (request->CompleteOriginalIrp 438: && queue->LastReadLength > 0 439: && NT_SUCCESS (request->Item->Status) 440: && InterlockedExchangeAdd (&queue->IoThreadPendingRequestCount, 0) == 0) 441: { 442: readAhead = TRUE; 443: InterlockedIncrement (&queue->OutstandingIoCount); 444: } 445: 1.1.1.3 root 446: ExInterlockedInsertTailList (&queue->CompletionThreadQueue, &request->CompletionListEntry, &queue->CompletionThreadQueueLock); 447: KeSetEvent (&queue->CompletionThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE); 1.1.1.8 root 448: 449: if (readAhead) 450: { 451: queue->ReadAheadBufferValid = FALSE; 452: queue->ReadAheadOffset.QuadPart = queue->LastReadOffset.QuadPart + queue->LastReadLength; 453: queue->ReadAheadLength = queue->LastReadLength; 454: 455: #ifdef TC_TRACE_IO_QUEUE 456: Dump ("A %I64d [%I64d] roff=%I64d rlen=%d\n", request->Item->OriginalIrpOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), queue->ReadAheadOffset, queue->ReadAheadLength); 457: #endif 458: 459: if (queue->ReadAheadOffset.QuadPart + queue->ReadAheadLength <= queue->MaxReadAheadOffset.QuadPart) 460: { 461: if (queue->IsFilterDevice) 462: { 463: queue->ReadAheadBufferValid = NT_SUCCESS (TCReadDevice (queue->LowerDeviceObject, queue->ReadAheadBuffer, queue->ReadAheadOffset, queue->ReadAheadLength)); 464: } 465: else 466: { 467: IO_STATUS_BLOCK ioStatus; 468: queue->ReadAheadBufferValid = NT_SUCCESS (ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, queue->ReadAheadBuffer, queue->ReadAheadLength, &queue->ReadAheadOffset, NULL)); 469: queue->ReadAheadLength = (ULONG) ioStatus.Information; 470: } 471: } 472: 473: DecrementOutstandingIoCount (queue); 474: } 1.1.1.3 root 475: } 1.1 root 476: } 477: } 478: 479: PsTerminateSystemThread (STATUS_SUCCESS); 480: } 481: 482: 483: static VOID MainThreadProc (PVOID threadArg) 484: { 485: EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg; 486: PLIST_ENTRY listEntry; 487: EncryptedIoQueueItem *item; 488: 489: LARGE_INTEGER fragmentOffset; 490: ULONG dataRemaining; 491: PUCHAR activeFragmentBuffer = queue->FragmentBufferA; 492: PUCHAR dataBuffer; 493: EncryptedIoRequest *request; 494: uint64 intersectStart; 495: uint32 intersectLength; 496: 1.1.1.5 root 497: if (IsEncryptionThreadPoolRunning()) 498: KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY); 499: 1.1 root 500: while (!queue->ThreadExitRequested) 501: { 502: if (!NT_SUCCESS (KeWaitForSingleObject (&queue->MainThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL))) 503: continue; 504: 505: while ((listEntry = ExInterlockedRemoveHeadList (&queue->MainThreadQueue, &queue->MainThreadQueueLock))) 506: { 1.1.1.5 root 507: PIRP irp = CONTAINING_RECORD (listEntry, IRP, Tail.Overlay.ListEntry); 508: PIO_STACK_LOCATION irpSp = IoGetCurrentIrpStackLocation (irp); 509: 1.1 root 510: if (queue->Suspended) 511: KeWaitForSingleObject (&queue->QueueResumedEvent, Executive, KernelMode, FALSE, NULL); 1.1.1.5 root 512: 1.1.1.8 root 513: item = GetPoolBuffer (queue, sizeof (EncryptedIoQueueItem)); 1.1.1.5 root 514: item->Queue = queue; 515: item->OriginalIrp = irp; 516: item->Status = STATUS_SUCCESS; 517: 518: IoSetCancelRoutine (irp, NULL); 519: if (irp->Cancel) 1.1 root 520: { 521: CompleteOriginalIrp (item, STATUS_CANCELLED, 0); 522: continue; 523: } 524: 1.1.1.5 root 525: switch (irpSp->MajorFunction) 526: { 527: case IRP_MJ_READ: 528: item->Write = FALSE; 529: item->OriginalOffset = irpSp->Parameters.Read.ByteOffset; 530: item->OriginalLength = irpSp->Parameters.Read.Length; 531: break; 532: 533: case IRP_MJ_WRITE: 534: item->Write = TRUE; 535: item->OriginalOffset = irpSp->Parameters.Write.ByteOffset; 536: item->OriginalLength = irpSp->Parameters.Write.Length; 537: break; 538: 539: default: 540: CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); 541: continue; 542: } 543: 1.1.1.8 root 544: #ifdef TC_TRACE_IO_QUEUE 545: item->OriginalIrpOffset = item->OriginalOffset; 546: #endif 1.1 root 547: 1.1.1.8 root 548: // Handle misaligned read operations to work around a bug in Windows System Assessment Tool which does not follow FILE_FLAG_NO_BUFFERING requirements when benchmarking disk devices 1.1.1.7 root 549: if (queue->IsFilterDevice 550: && !item->Write 551: && item->OriginalLength > 0 552: && (item->OriginalLength & (ENCRYPTION_DATA_UNIT_SIZE - 1)) == 0 553: && (item->OriginalOffset.QuadPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0) 554: { 555: byte *buffer; 556: ULONG alignedLength = item->OriginalLength + ENCRYPTION_DATA_UNIT_SIZE; 557: LARGE_INTEGER alignedOffset; 558: alignedOffset.QuadPart = item->OriginalOffset.QuadPart & ~((LONGLONG) ENCRYPTION_DATA_UNIT_SIZE - 1); 559: 560: buffer = TCalloc (alignedLength); 561: if (!buffer) 562: { 563: CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); 564: continue; 565: } 566: 567: item->Status = TCReadDevice (queue->LowerDeviceObject, buffer, alignedOffset, alignedLength); 568: 569: if (NT_SUCCESS (item->Status)) 570: { 571: UINT64_STRUCT dataUnit; 572: 573: dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, HighPagePriority); 574: if (!dataBuffer) 575: { 576: TCfree (buffer); 577: CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); 578: continue; 579: } 580: 1.1.1.8 root 581: if (queue->EncryptedAreaStart != -1 && queue->EncryptedAreaEnd != -1) 1.1.1.7 root 582: { 1.1.1.8 root 583: GetIntersection (alignedOffset.QuadPart, alignedLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength); 584: if (intersectLength > 0) 585: { 586: dataUnit.Value = intersectStart / ENCRYPTION_DATA_UNIT_SIZE; 587: DecryptDataUnits (buffer + (intersectStart - alignedOffset.QuadPart), &dataUnit, intersectLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo); 588: } 1.1.1.7 root 589: } 590: 591: memcpy (dataBuffer, buffer + (item->OriginalOffset.LowPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)), item->OriginalLength); 592: } 593: 594: TCfree (buffer); 595: CompleteOriginalIrp (item, item->Status, NT_SUCCESS (item->Status) ? item->OriginalLength : 0); 596: continue; 597: } 598: 1.1 root 599: // Validate offset and length 1.1.1.3 root 600: if (item->OriginalLength == 0 601: || (item->OriginalLength & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0 602: || (item->OriginalOffset.QuadPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0 1.1 root 603: || (!queue->IsFilterDevice && item->OriginalOffset.QuadPart + item->OriginalLength > queue->VirtualDeviceLength)) 604: { 605: CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); 606: continue; 607: } 608: 1.1.1.8 root 609: #ifdef TC_TRACE_IO_QUEUE 610: Dump ("Q %I64d [%I64d] %c len=%d\n", item->OriginalOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), item->Write ? 'W' : 'R', item->OriginalLength); 611: #endif 1.1.1.5 root 612: 1.1 root 613: if (!queue->IsFilterDevice) 614: { 1.1.1.3 root 615: // Adjust the offset for host file or device 616: if (queue->CryptoInfo->hiddenVolume) 617: item->OriginalOffset.QuadPart += queue->CryptoInfo->hiddenVolumeOffset; 618: else 619: item->OriginalOffset.QuadPart += queue->CryptoInfo->volDataAreaOffset; 620: 1.1 root 621: // Hidden volume protection 622: if (item->Write && queue->CryptoInfo->bProtectHiddenVolume) 623: { 624: // If there has already been a write operation denied in order to protect the 625: // hidden volume (since the volume mount time) 626: if (queue->CryptoInfo->bHiddenVolProtectionAction) 627: { 628: // Do not allow writing to this volume anymore. This is to fake a complete volume 629: // or system failure (otherwise certain kinds of inconsistency within the file 630: // system could indicate that this volume has used hidden volume protection). 631: CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); 632: continue; 633: } 634: 635: // Verify that no byte is going to be written to the hidden volume area 1.1.1.3 root 636: if (RegionsOverlap ((unsigned __int64) item->OriginalOffset.QuadPart, 637: (unsigned __int64) item->OriginalOffset.QuadPart + item->OriginalLength - 1, 1.1 root 638: queue->CryptoInfo->hiddenVolumeOffset, 1.1.1.5 root 639: (unsigned __int64) queue->CryptoInfo->hiddenVolumeOffset + queue->CryptoInfo->hiddenVolumeProtectedSize - 1)) 1.1 root 640: { 1.1.1.5 root 641: Dump ("Hidden volume protection triggered: write %I64d-%I64d (protected %I64d-%I64d)\n", item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, queue->CryptoInfo->hiddenVolumeOffset, queue->CryptoInfo->hiddenVolumeOffset + queue->CryptoInfo->hiddenVolumeProtectedSize - 1); 1.1 root 642: queue->CryptoInfo->bHiddenVolProtectionAction = TRUE; 643: 644: // Deny this write operation to prevent the hidden volume from being overwritten 645: CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); 646: continue; 647: } 648: } 649: } 1.1.1.10! root 650: else if (item->Write ! 651: && RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, TC_BOOT_VOLUME_HEADER_SECTOR_OFFSET, TC_BOOT_VOLUME_HEADER_SECTOR_OFFSET + TC_BOOT_ENCRYPTION_VOLUME_HEADER_SIZE - 1)) ! 652: { ! 653: // Prevent inappropriately designed software from damaging important data that may be out of sync with the backup on the Rescue Disk (such as the end of the encrypted area). ! 654: Dump ("Preventing write to the system encryption key data area\n"); ! 655: CompleteOriginalIrp (item, STATUS_MEDIA_WRITE_PROTECTED, 0); ! 656: continue; ! 657: } 1.1.1.6 root 658: else if (item->Write && IsHiddenSystemRunning() 659: && (RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, SECTOR_SIZE, TC_BOOT_LOADER_AREA_SECTOR_COUNT * SECTOR_SIZE - 1) 660: || RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, GetBootDriveLength(), _I64_MAX))) 661: { 662: Dump ("Preventing write to boot loader or host protected area\n"); 663: CompleteOriginalIrp (item, STATUS_MEDIA_WRITE_PROTECTED, 0); 664: continue; 665: } 1.1 root 666: 1.1.1.10! root 667: dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, HighPagePriority); 1.1.1.5 root 668: 1.1 root 669: if (dataBuffer == NULL) 670: { 671: CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); 672: continue; 673: } 674: 675: // Divide data block to fragments to enable efficient overlapping of encryption and IO operations 676: 677: dataRemaining = item->OriginalLength; 678: fragmentOffset = item->OriginalOffset; 679: 680: while (dataRemaining > 0) 681: { 682: BOOL isLastFragment = dataRemaining <= TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; 683: 684: ULONG dataFragmentLength = isLastFragment ? dataRemaining : TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; 685: activeFragmentBuffer = (activeFragmentBuffer == queue->FragmentBufferA ? queue->FragmentBufferB : queue->FragmentBufferA); 686: 1.1.1.8 root 687: InterlockedIncrement (&queue->IoThreadPendingRequestCount); 688: 1.1.1.10! root 689: // Create IO request ! 690: request = GetPoolBuffer (queue, sizeof (EncryptedIoRequest)); 1.1 root 691: request->Item = item; 692: request->CompleteOriginalIrp = isLastFragment; 693: request->Offset = fragmentOffset; 694: request->Data = activeFragmentBuffer; 695: request->OrigDataBufferFragment = dataBuffer; 696: request->Length = dataFragmentLength; 697: 698: if (queue->IsFilterDevice) 699: { 1.1.1.8 root 700: if (queue->EncryptedAreaStart == -1 || queue->EncryptedAreaEnd == -1) 701: { 702: request->EncryptedLength = 0; 703: } 704: else 705: { 706: // Get intersection of data fragment with encrypted area 707: GetIntersection (fragmentOffset.QuadPart, dataFragmentLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength); 1.1 root 708: 1.1.1.8 root 709: request->EncryptedOffset = intersectStart - fragmentOffset.QuadPart; 710: request->EncryptedLength = intersectLength; 711: } 1.1 root 712: } 713: else 714: { 715: request->EncryptedOffset = 0; 716: request->EncryptedLength = dataFragmentLength; 717: } 718: 719: AcquireFragmentBuffer (queue, activeFragmentBuffer); 720: 721: if (item->Write) 722: { 723: // Encrypt data 724: memcpy (activeFragmentBuffer, dataBuffer, dataFragmentLength); 725: 726: if (request->EncryptedLength > 0) 727: { 728: UINT64_STRUCT dataUnit; 729: ASSERT (request->EncryptedOffset + request->EncryptedLength <= request->Offset.QuadPart + request->Length); 730: 731: dataUnit.Value = (request->Offset.QuadPart + request->EncryptedOffset) / ENCRYPTION_DATA_UNIT_SIZE; 732: 1.1.1.3 root 733: if (queue->CryptoInfo->bPartitionInInactiveSysEncScope) 734: dataUnit.Value += queue->CryptoInfo->FirstDataUnitNo.Value; 1.1.1.5 root 735: else if (queue->RemapEncryptedArea) 736: dataUnit.Value += queue->RemappedAreaDataUnitOffset; 737: 1.1.1.3 root 738: EncryptDataUnits (activeFragmentBuffer + request->EncryptedOffset, &dataUnit, request->EncryptedLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo); 1.1 root 739: } 740: } 741: 742: // Queue IO request 743: ExInterlockedInsertTailList (&queue->IoThreadQueue, &request->ListEntry, &queue->IoThreadQueueLock); 1.1.1.3 root 744: KeSetEvent (&queue->IoThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 745: 746: if (isLastFragment) 747: break; 748: 749: dataRemaining -= TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; 750: dataBuffer += TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; 751: fragmentOffset.QuadPart += TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; 752: } 753: } 754: } 755: 756: PsTerminateSystemThread (STATUS_SUCCESS); 757: } 758: 759: 760: NTSTATUS EncryptedIoQueueAddIrp (EncryptedIoQueue *queue, PIRP irp) 761: { 762: NTSTATUS status; 763: 764: InterlockedIncrement (&queue->OutstandingIoCount); 765: if (queue->StopPending) 766: { 767: Dump ("STATUS_DEVICE_NOT_READY out=%d\n", queue->OutstandingIoCount); 768: status = STATUS_DEVICE_NOT_READY; 769: goto err; 770: } 771: 1.1.1.10! root 772: status = IoAcquireRemoveLock (&queue->RemoveLock, irp); ! 773: if (!NT_SUCCESS (status)) ! 774: goto err; 1.1 root 775: 1.1.1.8 root 776: #ifdef TC_TRACE_IO_QUEUE 777: { 778: PIO_STACK_LOCATION irpSp = IoGetCurrentIrpStackLocation (irp); 779: Dump ("* %I64d [%I64d] %c len=%d out=%d\n", irpSp->MajorFunction == IRP_MJ_WRITE ? irpSp->Parameters.Write.ByteOffset : irpSp->Parameters.Read.ByteOffset, GetElapsedTime (&queue->LastPerformanceCounter), irpSp->MajorFunction == IRP_MJ_WRITE ? 'W' : 'R', irpSp->MajorFunction == IRP_MJ_WRITE ? irpSp->Parameters.Write.Length : irpSp->Parameters.Read.Length, queue->OutstandingIoCount); 780: } 781: #endif 782: 1.1 root 783: IoMarkIrpPending (irp); 784: 1.1.1.5 root 785: ExInterlockedInsertTailList (&queue->MainThreadQueue, &irp->Tail.Overlay.ListEntry, &queue->MainThreadQueueLock); 1.1.1.3 root 786: KeSetEvent (&queue->MainThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 787: 788: return STATUS_PENDING; 789: 790: err: 791: DecrementOutstandingIoCount (queue); 792: return status; 793: } 794: 795: 796: NTSTATUS EncryptedIoQueueHoldWhenIdle (EncryptedIoQueue *queue, int64 timeout) 797: { 798: NTSTATUS status; 799: ASSERT (!queue->Suspended); 800: 801: queue->SuspendPending = TRUE; 802: 803: while (TRUE) 804: { 805: while (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0) 806: { 807: LARGE_INTEGER waitTimeout; 808: 809: waitTimeout.QuadPart = timeout * -10000; 810: status = KeWaitForSingleObject (&queue->NoOutstandingIoEvent, Executive, KernelMode, FALSE, timeout != 0 ? &waitTimeout : NULL); 811: 812: if (status == STATUS_TIMEOUT) 813: status = STATUS_UNSUCCESSFUL; 814: 815: if (!NT_SUCCESS (status)) 1.1.1.8 root 816: { 817: queue->SuspendPending = FALSE; 1.1 root 818: return status; 1.1.1.8 root 819: } 1.1.1.6 root 820: 1.1.1.7 root 821: TCSleep (1); 1.1.1.6 root 822: if (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0) 1.1.1.8 root 823: { 824: queue->SuspendPending = FALSE; 1.1.1.6 root 825: return STATUS_UNSUCCESSFUL; 1.1.1.8 root 826: } 1.1 root 827: } 828: 829: KeClearEvent (&queue->QueueResumedEvent); 830: queue->Suspended = TRUE; 831: 832: if (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) == 0) 833: break; 834: 835: queue->Suspended = FALSE; 1.1.1.3 root 836: KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 837: } 838: 1.1.1.8 root 839: queue->ReadAheadBufferValid = FALSE; 1.1 root 840: 1.1.1.8 root 841: queue->SuspendPending = FALSE; 1.1 root 842: return STATUS_SUCCESS; 843: } 844: 845: 846: BOOL EncryptedIoQueueIsSuspended (EncryptedIoQueue *queue) 847: { 848: return queue->Suspended; 849: } 850: 851: 852: BOOL EncryptedIoQueueIsRunning (EncryptedIoQueue *queue) 853: { 854: return !queue->StopPending; 855: } 856: 857: 858: NTSTATUS EncryptedIoQueueResumeFromHold (EncryptedIoQueue *queue) 859: { 860: ASSERT (queue->Suspended); 861: 862: queue->Suspended = FALSE; 1.1.1.3 root 863: KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE); 1.1 root 864: 865: return STATUS_SUCCESS; 866: } 867: 868: 1.1.1.6 root 869: NTSTATUS EncryptedIoQueueStart (EncryptedIoQueue *queue) 1.1 root 870: { 871: NTSTATUS status; 1.1.1.8 root 872: EncryptedIoQueueBuffer *buffer; 873: int i; 874: 875: queue->StartPending = TRUE; 1.1 root 876: queue->ThreadExitRequested = FALSE; 877: 1.1.1.8 root 878: queue->OutstandingIoCount = 0; 879: queue->IoThreadPendingRequestCount = 0; 880: 881: queue->FirstPoolBuffer = NULL; 882: KeInitializeMutex (&queue->BufferPoolMutex, 0); 883: 1.1 root 884: KeInitializeEvent (&queue->NoOutstandingIoEvent, SynchronizationEvent, FALSE); 1.1.1.9 root 885: KeInitializeEvent (&queue->PoolBufferFreeEvent, SynchronizationEvent, FALSE); 1.1 root 886: KeInitializeEvent (&queue->QueueResumedEvent, SynchronizationEvent, FALSE); 887: 888: queue->FragmentBufferA = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE); 889: if (!queue->FragmentBufferA) 890: goto noMemory; 891: 892: queue->FragmentBufferB = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE); 893: if (!queue->FragmentBufferB) 894: goto noMemory; 895: 896: KeInitializeEvent (&queue->FragmentBufferAFreeEvent, SynchronizationEvent, TRUE); 897: KeInitializeEvent (&queue->FragmentBufferBFreeEvent, SynchronizationEvent, TRUE); 898: 1.1.1.8 root 899: queue->ReadAheadBufferValid = FALSE; 900: queue->ReadAheadBuffer = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE); 901: if (!queue->ReadAheadBuffer) 902: goto noMemory; 903: 904: // Preallocate buffers 905: for (i = 0; i < TC_ENC_IO_QUEUE_PREALLOCATED_IO_REQUEST_COUNT; ++i) 906: { 907: if (i < TC_ENC_IO_QUEUE_PREALLOCATED_ITEM_COUNT && !GetPoolBuffer (queue, sizeof (EncryptedIoQueueItem))) 908: goto noMemory; 909: 910: if (!GetPoolBuffer (queue, sizeof (EncryptedIoRequest))) 911: goto noMemory; 912: } 913: 914: for (buffer = queue->FirstPoolBuffer; buffer != NULL; buffer = buffer->NextBuffer) 915: { 916: buffer->InUse = FALSE; 917: } 918: 1.1 root 919: // Main thread 920: InitializeListHead (&queue->MainThreadQueue); 921: KeInitializeSpinLock (&queue->MainThreadQueueLock); 922: KeInitializeEvent (&queue->MainThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE); 923: 924: status = TCStartThread (MainThreadProc, queue, &queue->MainThread); 925: if (!NT_SUCCESS (status)) 926: goto err; 927: 928: // IO thread 929: InitializeListHead (&queue->IoThreadQueue); 930: KeInitializeSpinLock (&queue->IoThreadQueueLock); 931: KeInitializeEvent (&queue->IoThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE); 932: 1.1.1.6 root 933: status = TCStartThread (IoThreadProc, queue, &queue->IoThread); 1.1 root 934: if (!NT_SUCCESS (status)) 935: { 936: queue->ThreadExitRequested = TRUE; 937: TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent); 938: goto err; 939: } 940: 941: // Completion thread 942: InitializeListHead (&queue->CompletionThreadQueue); 943: KeInitializeSpinLock (&queue->CompletionThreadQueueLock); 944: KeInitializeEvent (&queue->CompletionThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE); 945: 946: status = TCStartThread (CompletionThreadProc, queue, &queue->CompletionThread); 947: if (!NT_SUCCESS (status)) 948: { 949: queue->ThreadExitRequested = TRUE; 950: TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent); 951: TCStopThread (queue->IoThread, &queue->IoThreadQueueNotEmptyEvent); 952: goto err; 953: } 954: 1.1.1.8 root 955: #ifdef TC_TRACE_IO_QUEUE 956: GetElapsedTimeInit (&queue->LastPerformanceCounter); 957: #endif 958: 1.1 root 959: queue->StopPending = FALSE; 1.1.1.8 root 960: queue->StartPending = FALSE; 961: 1.1 root 962: Dump ("Queue started\n"); 963: return STATUS_SUCCESS; 964: 965: noMemory: 966: status = STATUS_INSUFFICIENT_RESOURCES; 967: 968: err: 969: if (queue->FragmentBufferA) 970: TCfree (queue->FragmentBufferA); 971: if (queue->FragmentBufferB) 972: TCfree (queue->FragmentBufferB); 973: 1.1.1.8 root 974: FreePoolBuffers (queue); 975: 976: queue->StartPending = FALSE; 1.1 root 977: return status; 978: } 979: 980: 981: NTSTATUS EncryptedIoQueueStop (EncryptedIoQueue *queue) 982: { 983: ASSERT (!queue->StopPending); 984: queue->StopPending = TRUE; 985: 986: while (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0) 987: { 988: KeWaitForSingleObject (&queue->NoOutstandingIoEvent, Executive, KernelMode, FALSE, NULL); 989: } 990: 991: Dump ("Queue stopping out=%d\n", queue->OutstandingIoCount); 992: 993: queue->ThreadExitRequested = TRUE; 994: 995: TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent); 996: TCStopThread (queue->IoThread, &queue->IoThreadQueueNotEmptyEvent); 997: TCStopThread (queue->CompletionThread, &queue->CompletionThreadQueueNotEmptyEvent); 998: 999: TCfree (queue->FragmentBufferA); 1000: TCfree (queue->FragmentBufferB); 1.1.1.8 root 1001: TCfree (queue->ReadAheadBuffer); 1002: 1003: FreePoolBuffers (queue); 1.1 root 1004: 1005: Dump ("Queue stopped out=%d\n", queue->OutstandingIoCount); 1006: return STATUS_SUCCESS; 1007: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.