--- truecrypt/driver/encryptedioqueue.c 2018/04/24 16:48:47 1.1.1.1 +++ truecrypt/driver/encryptedioqueue.c 2018/04/24 16:59:51 1.1.1.7 @@ -1,7 +1,7 @@ /* Copyright (c) 2008 TrueCrypt Foundation. All rights reserved. - Governed by the TrueCrypt License 2.4 the full text of which is contained + Governed by the TrueCrypt License 2.6 the full text of which is contained in the file License.txt included in TrueCrypt binary and source code distribution packages. */ @@ -9,17 +9,20 @@ #include "TCdefs.h" #include "Apidrvr.h" #include "Ntdriver.h" +#include "DriveFilter.h" #include "EncryptedIoQueue.h" +#include "EncryptionThreadPool.h" +#include "Volumes.h" static void DecrementOutstandingIoCount (EncryptedIoQueue *queue) { if (InterlockedDecrement (&queue->OutstandingIoCount) == 0 && (queue->SuspendPending || queue->StopPending)) - KeSetEvent (&queue->NoOutstandingIoEvent, 0, FALSE); + KeSetEvent (&queue->NoOutstandingIoEvent, IO_DISK_INCREMENT, FALSE); } -static void OnItemCompleted (EncryptedIoQueueItem *item) +static void OnItemCompleted (EncryptedIoQueueItem *item, BOOL freeItem) { DecrementOutstandingIoCount (item->Queue); @@ -34,15 +37,20 @@ static void OnItemCompleted (EncryptedIo item->Queue->TotalBytesRead += item->OriginalLength; } - TCfree (item); + if (freeItem) + TCfree (item); } static NTSTATUS CompleteOriginalIrp (EncryptedIoQueueItem *item, NTSTATUS status, ULONG_PTR information) { //Dump ("Queue comp offset=%I64d status=%x info=%p out=%d\n", item->OriginalOffset, status, information, item->Queue->OutstandingIoCount - 1); + TCCompleteDiskIrp (item->OriginalIrp, status, information); - OnItemCompleted (item); + + item->Status = status; + OnItemCompleted (item, TRUE); + return status; } @@ -69,11 +77,11 @@ static void ReleaseFragmentBuffer (Encry { if (buffer == queue->FragmentBufferA) { - KeSetEvent (&queue->FragmentBufferAFreeEvent, 0, FALSE); + KeSetEvent (&queue->FragmentBufferAFreeEvent, IO_DISK_INCREMENT, FALSE); } else if (buffer == queue->FragmentBufferB) { - KeSetEvent (&queue->FragmentBufferBFreeEvent, 0, FALSE); + KeSetEvent (&queue->FragmentBufferBFreeEvent, IO_DISK_INCREMENT, FALSE); } else { @@ -89,6 +97,9 @@ static VOID CompletionThreadProc (PVOID EncryptedIoRequest *request; UINT64_STRUCT dataUnit; + if (IsEncryptionThreadPoolRunning()) + KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY); + while (!queue->ThreadExitRequested) { if (!NT_SUCCESS (KeWaitForSingleObject (&queue->CompletionThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL))) @@ -101,10 +112,16 @@ static VOID CompletionThreadProc (PVOID { request = CONTAINING_RECORD (listEntry, EncryptedIoRequest, CompletionListEntry); - if (request->EncryptedLength > 0) + if (request->EncryptedLength > 0 && NT_SUCCESS (request->Item->Status)) { ASSERT (request->EncryptedOffset + request->EncryptedLength <= request->Offset.QuadPart + request->Length); dataUnit.Value = (request->Offset.QuadPart + request->EncryptedOffset) / ENCRYPTION_DATA_UNIT_SIZE; + + if (queue->CryptoInfo->bPartitionInInactiveSysEncScope) + dataUnit.Value += queue->CryptoInfo->FirstDataUnitNo.Value; + else if (queue->RemapEncryptedArea) + dataUnit.Value += queue->RemappedAreaDataUnitOffset; + DecryptDataUnits (request->Data + request->EncryptedOffset, &dataUnit, request->EncryptedLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo); } @@ -113,6 +130,11 @@ static VOID CompletionThreadProc (PVOID CompleteOriginalIrp (request->Item, request->Item->Status, NT_SUCCESS (request->Item->Status) ? request->Item->OriginalLength : 0); } + else + { + InterlockedDecrement (&request->Item->OutstandingRequestCount); + KeSetEvent (&queue->RequestCompletedEvent, IO_DISK_INCREMENT, FALSE); + } TCfree (request); } @@ -128,6 +150,17 @@ static VOID IoThreadProc (PVOID threadAr PLIST_ENTRY listEntry; EncryptedIoRequest *request; + KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY); + + if (!queue->IsFilterDevice && queue->SecurityClientContext) + { +#ifdef DEBUG + NTSTATUS status = +#endif + SeImpersonateClientEx (queue->SecurityClientContext, NULL); + ASSERT (NT_SUCCESS (status)); + } + while (!queue->ThreadExitRequested) { if (!NT_SUCCESS (KeWaitForSingleObject (&queue->IoThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL))) @@ -139,41 +172,89 @@ static VOID IoThreadProc (PVOID threadAr while ((listEntry = ExInterlockedRemoveHeadList (&queue->IoThreadQueue, &queue->IoThreadQueueLock))) { request = CONTAINING_RECORD (listEntry, EncryptedIoRequest, ListEntry); - - // IO request - if (queue->IsFilterDevice) - { - if (request->Item->Write) - request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, request->Offset, request->Length); - else - request->Item->Status = TCReadDevice (queue->LowerDeviceObject, request->Data, request->Offset, request->Length); - } - else + + // Perform IO request if no preceding request of the item failed + if (NT_SUCCESS (request->Item->Status)) { - IO_STATUS_BLOCK ioStatus; - - if (request->Item->Write) - request->Item->Status = ZwWriteFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, request->Data, request->Length, &request->Offset, NULL); + if (queue->IsFilterDevice) + { + if (queue->RemapEncryptedArea && request->EncryptedLength > 0) + { + if (request->EncryptedLength != request->Length) + { + // Up to three subfragments may be required to handle a partially remapped fragment + int subFragment; + byte *subFragmentData = request->Data; + + for (subFragment = 0 ; subFragment < 3; ++subFragment) + { + LARGE_INTEGER subFragmentOffset; + ULONG subFragmentLength; + subFragmentOffset.QuadPart = request->Offset.QuadPart; + + switch (subFragment) + { + case 0: + subFragmentLength = (ULONG) request->EncryptedOffset; + break; + + case 1: + subFragmentOffset.QuadPart += request->EncryptedOffset + queue->RemappedAreaOffset; + subFragmentLength = request->EncryptedLength; + break; + + case 2: + subFragmentOffset.QuadPart += request->EncryptedOffset + request->EncryptedLength; + subFragmentLength = (ULONG) (request->Length - (request->EncryptedOffset + request->EncryptedLength)); + break; + } + + if (subFragmentLength > 0) + { + if (request->Item->Write) + request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, subFragmentData, subFragmentOffset, subFragmentLength); + else + request->Item->Status = TCReadDevice (queue->LowerDeviceObject, subFragmentData, subFragmentOffset, subFragmentLength); + + subFragmentData += subFragmentLength; + } + } + } + else + { + // Remap the fragment + LARGE_INTEGER remappedOffset; + remappedOffset.QuadPart = request->Offset.QuadPart + queue->RemappedAreaOffset; + + if (request->Item->Write) + request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, remappedOffset, request->Length); + else + request->Item->Status = TCReadDevice (queue->LowerDeviceObject, request->Data, remappedOffset, request->Length); + } + } + else + { + if (request->Item->Write) + request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, request->Offset, request->Length); + else + request->Item->Status = TCReadDevice (queue->LowerDeviceObject, request->Data, request->Offset, request->Length); + } + } else - request->Item->Status = ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, request->Data, request->Length, &request->Offset, NULL); - } - - if (!request->Item->Write && NT_SUCCESS (request->Item->Status)) - { - // Successful read completed - if (!request->CompleteOriginalIrp) - KeSetEvent (&request->Item->IoRequestCompletedEvent, 0, FALSE); + { + IO_STATUS_BLOCK ioStatus; - // Copy fragment to original IRP buffer - memcpy (request->OrigDataBufferFragment, request->Data, request->Length); - ReleaseFragmentBuffer (queue, request->Data); - request->Data = request->OrigDataBufferFragment; + if (request->Item->Write) + request->Item->Status = ZwWriteFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, request->Data, request->Length, &request->Offset, NULL); + else + request->Item->Status = ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, request->Data, request->Length, &request->Offset, NULL); - // Queue decryption to completion thread - ExInterlockedInsertTailList (&queue->CompletionThreadQueue, &request->CompletionListEntry, &queue->CompletionThreadQueueLock); - KeSetEvent (&queue->CompletionThreadQueueNotEmptyEvent, 0, FALSE); + if (NT_SUCCESS (request->Item->Status) && ioStatus.Information != request->Length) + request->Item->Status = STATUS_END_OF_FILE; + } } - else + + if (request->Item->Write) { ReleaseFragmentBuffer (queue, request->Data); @@ -184,11 +265,23 @@ static VOID IoThreadProc (PVOID threadAr } else { - KeSetEvent (&request->Item->IoRequestCompletedEvent, 0, FALSE); + InterlockedDecrement (&request->Item->OutstandingRequestCount); + KeSetEvent (&queue->RequestCompletedEvent, IO_DISK_INCREMENT, FALSE); } TCfree (request); } + else + { + if (NT_SUCCESS (request->Item->Status)) + memcpy (request->OrigDataBufferFragment, request->Data, request->Length); + + ReleaseFragmentBuffer (queue, request->Data); + request->Data = request->OrigDataBufferFragment; + + ExInterlockedInsertTailList (&queue->CompletionThreadQueue, &request->CompletionListEntry, &queue->CompletionThreadQueueLock); + KeSetEvent (&queue->CompletionThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE); + } } } @@ -201,7 +294,7 @@ static NTSTATUS OnPassedIrpCompleted (PD if (irp->PendingReturned) IoMarkIrpPending (irp); - OnItemCompleted (item); + OnItemCompleted (item, TRUE); return STATUS_CONTINUE_COMPLETION; } @@ -211,7 +304,6 @@ static VOID MainThreadProc (PVOID thread EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg; PLIST_ENTRY listEntry; EncryptedIoQueueItem *item; - NTSTATUS status; LARGE_INTEGER fragmentOffset; ULONG dataRemaining; @@ -221,6 +313,13 @@ static VOID MainThreadProc (PVOID thread uint64 intersectStart; uint32 intersectLength; + int mdlWaitTime; + LARGE_INTEGER mdlWaitInterval; + mdlWaitInterval.QuadPart = TC_ENC_IO_QUEUE_MEM_ALLOC_RETRY_DELAY * -10000; + + if (IsEncryptionThreadPoolRunning()) + KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY); + while (!queue->ThreadExitRequested) { if (!NT_SUCCESS (KeWaitForSingleObject (&queue->MainThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL))) @@ -228,42 +327,133 @@ static VOID MainThreadProc (PVOID thread while ((listEntry = ExInterlockedRemoveHeadList (&queue->MainThreadQueue, &queue->MainThreadQueueLock))) { - item = CONTAINING_RECORD (listEntry, EncryptedIoQueueItem, ListEntry); - + PIRP irp = CONTAINING_RECORD (listEntry, IRP, Tail.Overlay.ListEntry); + PIO_STACK_LOCATION irpSp = IoGetCurrentIrpStackLocation (irp); + if (queue->Suspended) - { KeWaitForSingleObject (&queue->QueueResumedEvent, Executive, KernelMode, FALSE, NULL); + + item = AllocateMemoryWithTimeout (sizeof (EncryptedIoQueueItem), TC_ENC_IO_QUEUE_MEM_ALLOC_RETRY_DELAY, TC_ENC_IO_QUEUE_MEM_ALLOC_TIMEOUT); + if (!item) + { + EncryptedIoQueueItem stackItem; + stackItem.Queue = queue; + stackItem.OriginalIrp = irp; + stackItem.Status = STATUS_INSUFFICIENT_RESOURCES; + + TCCompleteDiskIrp (irp, STATUS_INSUFFICIENT_RESOURCES, 0); + OnItemCompleted (&stackItem, FALSE); + continue; } - - IoSetCancelRoutine (item->OriginalIrp, NULL); - if (item->OriginalIrp->Cancel) + + item->Queue = queue; + item->OriginalIrp = irp; + item->OutstandingRequestCount = 0; + item->Status = STATUS_SUCCESS; + + IoSetCancelRoutine (irp, NULL); + if (irp->Cancel) { CompleteOriginalIrp (item, STATUS_CANCELLED, 0); continue; } + switch (irpSp->MajorFunction) + { + case IRP_MJ_READ: + item->Write = FALSE; + item->OriginalOffset = irpSp->Parameters.Read.ByteOffset; + item->OriginalLength = irpSp->Parameters.Read.Length; + break; + + case IRP_MJ_WRITE: + item->Write = TRUE; + item->OriginalOffset = irpSp->Parameters.Write.ByteOffset; + item->OriginalLength = irpSp->Parameters.Write.Length; + break; + + default: + CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); + continue; + } + // Pass the IRP if the drive is not encrypted if (queue->IsFilterDevice && (queue->EncryptedAreaStart == -1 || queue->EncryptedAreaEnd == -1)) { - - IoCopyCurrentIrpStackLocationToNext (item->OriginalIrp); - IoSetCompletionRoutine (item->OriginalIrp, OnPassedIrpCompleted, item, TRUE, TRUE, TRUE); - IoCallDriver (queue->LowerDeviceObject, item->OriginalIrp); + IoCopyCurrentIrpStackLocationToNext (irp); + IoSetCompletionRoutine (irp, OnPassedIrpCompleted, item, TRUE, TRUE, TRUE); + IoCallDriver (queue->LowerDeviceObject, irp); continue; } - //Dump ("--- Queue %c %I64d (%I64d) %d out=%d\n", item->Write ? 'W' : 'R', item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart / 1024 / 1024, item->OriginalLength, queue->OutstandingIoCount); + // Handle misaligned reads to support Windows System Assessment Tool which reads from disk devices at offsets not aligned on sector boundaries + if (queue->IsFilterDevice + && !item->Write + && item->OriginalLength > 0 + && (item->OriginalLength & (ENCRYPTION_DATA_UNIT_SIZE - 1)) == 0 + && (item->OriginalOffset.QuadPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0) + { + byte *buffer; + ULONG alignedLength = item->OriginalLength + ENCRYPTION_DATA_UNIT_SIZE; + LARGE_INTEGER alignedOffset; + alignedOffset.QuadPart = item->OriginalOffset.QuadPart & ~((LONGLONG) ENCRYPTION_DATA_UNIT_SIZE - 1); + + buffer = TCalloc (alignedLength); + if (!buffer) + { + CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); + continue; + } + + item->Status = TCReadDevice (queue->LowerDeviceObject, buffer, alignedOffset, alignedLength); + + if (NT_SUCCESS (item->Status)) + { + UINT64_STRUCT dataUnit; + + dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, HighPagePriority); + if (!dataBuffer) + { + TCfree (buffer); + CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); + continue; + } + + GetIntersection (alignedOffset.QuadPart, alignedLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength); + if (intersectLength > 0) + { + dataUnit.Value = intersectStart / ENCRYPTION_DATA_UNIT_SIZE; + DecryptDataUnits (buffer + (intersectStart - alignedOffset.QuadPart), &dataUnit, intersectLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo); + } + + memcpy (dataBuffer, buffer + (item->OriginalOffset.LowPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)), item->OriginalLength); + } + + TCfree (buffer); + CompleteOriginalIrp (item, item->Status, NT_SUCCESS (item->Status) ? item->OriginalLength : 0); + continue; + } // Validate offset and length - if (item->OriginalLength == 0 || (item->OriginalLength & (ENCRYPTION_DATA_UNIT_SIZE - 1)) + if (item->OriginalLength == 0 + || (item->OriginalLength & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0 + || (item->OriginalOffset.QuadPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0 || (!queue->IsFilterDevice && item->OriginalOffset.QuadPart + item->OriginalLength > queue->VirtualDeviceLength)) { CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); continue; } + //Dump ("--- Queue %c %I64d (%I64d) %d out=%d\n", item->Write ? 'W' : 'R', item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart / 1024 / 1024, item->OriginalLength, queue->OutstandingIoCount); + if (!queue->IsFilterDevice) { + // Adjust the offset for host file or device + if (queue->CryptoInfo->hiddenVolume) + item->OriginalOffset.QuadPart += queue->CryptoInfo->hiddenVolumeOffset; + else + item->OriginalOffset.QuadPart += queue->CryptoInfo->volDataAreaOffset; + // Hidden volume protection if (item->Write && queue->CryptoInfo->bProtectHiddenVolume) { @@ -279,11 +469,12 @@ static VOID MainThreadProc (PVOID thread } // Verify that no byte is going to be written to the hidden volume area - if (RegionsOverlap ((unsigned __int64) item->OriginalOffset.QuadPart + HEADER_SIZE, - (unsigned __int64) item->OriginalOffset.QuadPart + HEADER_SIZE + item->OriginalLength - 1, + if (RegionsOverlap ((unsigned __int64) item->OriginalOffset.QuadPart, + (unsigned __int64) item->OriginalOffset.QuadPart + item->OriginalLength - 1, queue->CryptoInfo->hiddenVolumeOffset, - (unsigned __int64) queue->VirtualDeviceLength + HEADER_SIZE - (HIDDEN_VOL_HEADER_OFFSET - HEADER_SIZE) - 1)) + (unsigned __int64) queue->CryptoInfo->hiddenVolumeOffset + queue->CryptoInfo->hiddenVolumeProtectedSize - 1)) { + Dump ("Hidden volume protection triggered: write %I64d-%I64d (protected %I64d-%I64d)\n", item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, queue->CryptoInfo->hiddenVolumeOffset, queue->CryptoInfo->hiddenVolumeOffset + queue->CryptoInfo->hiddenVolumeProtectedSize - 1); queue->CryptoInfo->bHiddenVolProtectionAction = TRUE; // Deny this write operation to prevent the hidden volume from being overwritten @@ -291,16 +482,29 @@ static VOID MainThreadProc (PVOID thread continue; } } - - // Adjust the offset for host file or device - if (queue->CryptoInfo->hiddenVolume) - item->OriginalOffset.QuadPart += queue->CryptoInfo->hiddenVolumeOffset; - else - item->OriginalOffset.QuadPart += HEADER_SIZE; + } + else if (item->Write && IsHiddenSystemRunning() + && (RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, SECTOR_SIZE, TC_BOOT_LOADER_AREA_SECTOR_COUNT * SECTOR_SIZE - 1) + || RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, GetBootDriveLength(), _I64_MAX))) + { + Dump ("Preventing write to boot loader or host protected area\n"); + CompleteOriginalIrp (item, STATUS_MEDIA_WRITE_PROTECTED, 0); + continue; } // Original IRP data buffer - dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (item->OriginalIrp->MdlAddress, HighPagePriority); + mdlWaitTime = 0; + while (TRUE) + { + dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, HighPagePriority); + + if (dataBuffer || mdlWaitTime >= TC_ENC_IO_QUEUE_MEM_ALLOC_TIMEOUT) + break; + + KeDelayExecutionThread (KernelMode, FALSE, &mdlWaitInterval); + mdlWaitTime += TC_ENC_IO_QUEUE_MEM_ALLOC_RETRY_DELAY; + } + if (dataBuffer == NULL) { CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); @@ -314,16 +518,18 @@ static VOID MainThreadProc (PVOID thread while (dataRemaining > 0) { - BOOL isFirstFragment = fragmentOffset.QuadPart == item->OriginalOffset.QuadPart; BOOL isLastFragment = dataRemaining <= TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; ULONG dataFragmentLength = isLastFragment ? dataRemaining : TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; activeFragmentBuffer = (activeFragmentBuffer == queue->FragmentBufferA ? queue->FragmentBufferB : queue->FragmentBufferA); // Create IO request - request = (EncryptedIoRequest *) TCalloc (sizeof (EncryptedIoRequest)); + request = (EncryptedIoRequest *) AllocateMemoryWithTimeout (sizeof (EncryptedIoRequest), TC_ENC_IO_QUEUE_MEM_ALLOC_RETRY_DELAY, TC_ENC_IO_QUEUE_MEM_ALLOC_TIMEOUT); if (!request) { + while (InterlockedExchangeAdd (&item->OutstandingRequestCount, 0) > 0) + KeWaitForSingleObject (&queue->RequestCompletedEvent, Executive, KernelMode, FALSE, NULL); + CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); break; } @@ -362,30 +568,21 @@ static VOID MainThreadProc (PVOID thread ASSERT (request->EncryptedOffset + request->EncryptedLength <= request->Offset.QuadPart + request->Length); dataUnit.Value = (request->Offset.QuadPart + request->EncryptedOffset) / ENCRYPTION_DATA_UNIT_SIZE; - EncryptDataUnits (activeFragmentBuffer + request->EncryptedOffset, &dataUnit, request->EncryptedLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo); - } - } - - if (!isFirstFragment) - { - // Wait for completion of previous fragment IO - - status = KeWaitForSingleObject (&item->IoRequestCompletedEvent, Executive, KernelMode, FALSE, NULL); - if (!NT_SUCCESS (status)) - TC_BUG_CHECK (status); - if (!NT_SUCCESS (item->Status)) - { - // If the previous fragment IO failed, stop processing remaining fragments and complete the IRP - ReleaseFragmentBuffer (queue, activeFragmentBuffer); - CompleteOriginalIrp (item, item->Status, 0); - break; + if (queue->CryptoInfo->bPartitionInInactiveSysEncScope) + dataUnit.Value += queue->CryptoInfo->FirstDataUnitNo.Value; + else if (queue->RemapEncryptedArea) + dataUnit.Value += queue->RemappedAreaDataUnitOffset; + + EncryptDataUnits (activeFragmentBuffer + request->EncryptedOffset, &dataUnit, request->EncryptedLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo); } } // Queue IO request + InterlockedIncrement (&item->OutstandingRequestCount); + ExInterlockedInsertTailList (&queue->IoThreadQueue, &request->ListEntry, &queue->IoThreadQueueLock); - KeSetEvent (&queue->IoThreadQueueNotEmptyEvent, 0, FALSE); + KeSetEvent (&queue->IoThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE); if (isLastFragment) break; @@ -403,8 +600,6 @@ static VOID MainThreadProc (PVOID thread NTSTATUS EncryptedIoQueueAddIrp (EncryptedIoQueue *queue, PIRP irp) { - EncryptedIoQueueItem *item; - PIO_STACK_LOCATION origIrpSp = IoGetCurrentIrpStackLocation (irp); NTSTATUS status; InterlockedIncrement (&queue->OutstandingIoCount); @@ -422,45 +617,11 @@ NTSTATUS EncryptedIoQueueAddIrp (Encrypt goto err; } - item = TCalloc (sizeof (EncryptedIoQueueItem)); - if (!item) - { - status = STATUS_INSUFFICIENT_RESOURCES; - goto err; - } - - memset (item, 0, sizeof (EncryptedIoQueueItem)); - - switch (origIrpSp->MajorFunction) - { - case IRP_MJ_READ: - item->Write = FALSE; - item->OriginalOffset = origIrpSp->Parameters.Read.ByteOffset; - item->OriginalLength = origIrpSp->Parameters.Read.Length; - break; - - case IRP_MJ_WRITE: - item->Write = TRUE; - item->OriginalOffset = origIrpSp->Parameters.Write.ByteOffset; - item->OriginalLength = origIrpSp->Parameters.Write.Length; - break; - - default: - TCfree (item); - status = STATUS_INVALID_PARAMETER; - goto err; - } - - item->Queue = queue; - item->OriginalIrp = irp; - KeInitializeEvent (&item->IoRequestCompletedEvent, SynchronizationEvent, FALSE); - + //Dump ("Queue irp %p out=%d\n", irp, queue->OutstandingIoCount); IoMarkIrpPending (irp); - //Dump ("Queue add %I64d %I64d out=%d\n", item->OriginalOffset, item->OriginalLength, queue->OutstandingIoCount); - - ExInterlockedInsertTailList (&queue->MainThreadQueue, &item->ListEntry, &queue->MainThreadQueueLock); - KeSetEvent (&queue->MainThreadQueueNotEmptyEvent, 0, FALSE); + ExInterlockedInsertTailList (&queue->MainThreadQueue, &irp->Tail.Overlay.ListEntry, &queue->MainThreadQueueLock); + KeSetEvent (&queue->MainThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE); return STATUS_PENDING; @@ -491,6 +652,10 @@ NTSTATUS EncryptedIoQueueHoldWhenIdle (E if (!NT_SUCCESS (status)) return status; + + TCSleep (1); + if (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0) + return STATUS_UNSUCCESSFUL; } KeClearEvent (&queue->QueueResumedEvent); @@ -500,7 +665,7 @@ NTSTATUS EncryptedIoQueueHoldWhenIdle (E break; queue->Suspended = FALSE; - KeSetEvent (&queue->QueueResumedEvent, 0, FALSE); + KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE); } @@ -528,7 +693,7 @@ NTSTATUS EncryptedIoQueueResumeFromHold ASSERT (queue->Suspended); queue->Suspended = FALSE; - KeSetEvent (&queue->QueueResumedEvent, 0, FALSE); + KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE); //Dump ("Queue resumed out=%d\n", queue->OutstandingIoCount); @@ -542,6 +707,7 @@ NTSTATUS EncryptedIoQueueStart (Encrypte queue->ThreadExitRequested = FALSE; KeInitializeEvent (&queue->NoOutstandingIoEvent, SynchronizationEvent, FALSE); + KeInitializeEvent (&queue->RequestCompletedEvent, SynchronizationEvent, FALSE); KeInitializeEvent (&queue->QueueResumedEvent, SynchronizationEvent, FALSE); queue->FragmentBufferA = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE);