--- truecrypt/driver/encryptedioqueue.c 2018/04/24 16:59:51 1.1.1.7 +++ truecrypt/driver/encryptedioqueue.c 2018/04/24 17:09:18 1.1.1.12 @@ -1,9 +1,9 @@ /* - Copyright (c) 2008 TrueCrypt Foundation. All rights reserved. + Copyright (c) 2008-2009 TrueCrypt Developers Association. All rights reserved. - Governed by the TrueCrypt License 2.6 the full text of which is contained - in the file License.txt included in TrueCrypt binary and source code - distribution packages. + Governed by the TrueCrypt License 3.0 the full text of which is contained in + the file License.txt included in TrueCrypt binary and source code distribution + packages. */ #include "TCdefs.h" @@ -15,6 +15,137 @@ #include "Volumes.h" +static void AcquireBufferPoolMutex (EncryptedIoQueue *queue) +{ + NTSTATUS status; + + status = KeWaitForMutexObject (&queue->BufferPoolMutex, Executive, KernelMode, FALSE, NULL); + if (!NT_SUCCESS (status)) + TC_BUG_CHECK (status); +} + + +static void ReleaseBufferPoolMutex (EncryptedIoQueue *queue) +{ + KeReleaseMutex (&queue->BufferPoolMutex, FALSE); +} + + +static void *GetPoolBuffer (EncryptedIoQueue *queue, ULONG requestedSize) +{ + EncryptedIoQueueBuffer *buffer; + void *bufferAddress = NULL; + BOOL requestedSizePresentInPool = FALSE; + + while (TRUE) + { + AcquireBufferPoolMutex (queue); + + for (buffer = queue->FirstPoolBuffer; ; buffer = buffer->NextBuffer) + { + if (buffer && buffer->Size == requestedSize) + { + requestedSizePresentInPool = TRUE; + + if (!buffer->InUse) + { + // Reuse a free buffer + buffer->InUse = TRUE; + bufferAddress = buffer->Address; + break; + } + } + + if (!buffer || !buffer->NextBuffer) + { + EncryptedIoQueueBuffer *newBuffer; + + if (requestedSizePresentInPool && !queue->StartPending) + break; + + // Allocate a new buffer + newBuffer = TCalloc (sizeof (EncryptedIoQueueBuffer)); + if (!newBuffer) + { + bufferAddress = NULL; + break; + } + + bufferAddress = TCalloc (requestedSize); + if (bufferAddress) + { + newBuffer->NextBuffer = NULL; + newBuffer->Address = bufferAddress; + newBuffer->Size = requestedSize; + newBuffer->InUse = TRUE; + + if (!buffer) + queue->FirstPoolBuffer = newBuffer; + else + buffer->NextBuffer = newBuffer; + } + else + TCfree (newBuffer); + + break; + } + } + + ReleaseBufferPoolMutex (queue); + + if (bufferAddress || !requestedSizePresentInPool || queue->StartPending) + break; + + KeWaitForSingleObject (&queue->PoolBufferFreeEvent, Executive, KernelMode, FALSE, NULL); + } + + return bufferAddress; +} + + +static void ReleasePoolBuffer (EncryptedIoQueue *queue, void *address) +{ + EncryptedIoQueueBuffer *buffer; + AcquireBufferPoolMutex (queue); + + for (buffer = queue->FirstPoolBuffer; buffer != NULL; buffer = buffer->NextBuffer) + { + if (buffer->Address == address) + { + ASSERT (buffer->InUse); + + buffer->InUse = FALSE; + break; + } + } + + ReleaseBufferPoolMutex (queue); + KeSetEvent (&queue->PoolBufferFreeEvent, IO_DISK_INCREMENT, FALSE); +} + + +static void FreePoolBuffers (EncryptedIoQueue *queue) +{ + EncryptedIoQueueBuffer *buffer; + AcquireBufferPoolMutex (queue); + + for (buffer = queue->FirstPoolBuffer; buffer != NULL; ) + { + EncryptedIoQueueBuffer *nextBuffer = buffer->NextBuffer; + + ASSERT (!buffer->InUse); + + TCfree (buffer->Address); + TCfree (buffer); + + buffer = nextBuffer; + } + + queue->FirstPoolBuffer = NULL; + ReleaseBufferPoolMutex (queue); +} + + static void DecrementOutstandingIoCount (EncryptedIoQueue *queue) { if (InterlockedDecrement (&queue->OutstandingIoCount) == 0 && (queue->SuspendPending || queue->StopPending)) @@ -25,9 +156,7 @@ static void DecrementOutstandingIoCount static void OnItemCompleted (EncryptedIoQueueItem *item, BOOL freeItem) { DecrementOutstandingIoCount (item->Queue); - - if (item->Queue->IsFilterDevice) - IoReleaseRemoveLock (&item->Queue->RemoveLock, item->OriginalIrp); + IoReleaseRemoveLock (&item->Queue->RemoveLock, item->OriginalIrp); if (NT_SUCCESS (item->Status)) { @@ -38,14 +167,16 @@ static void OnItemCompleted (EncryptedIo } if (freeItem) - TCfree (item); + ReleasePoolBuffer (item->Queue, item); } static NTSTATUS CompleteOriginalIrp (EncryptedIoQueueItem *item, NTSTATUS status, ULONG_PTR information) { - //Dump ("Queue comp offset=%I64d status=%x info=%p out=%d\n", item->OriginalOffset, status, information, item->Queue->OutstandingIoCount - 1); - +#ifdef TC_TRACE_IO_QUEUE + Dump ("< %I64d [%I64d] %c status=%x info=%I64d\n", item->OriginalIrpOffset, GetElapsedTime (&item->Queue->LastPerformanceCounter), item->Write ? 'W' : 'R', status, (int64) information); +#endif + TCCompleteDiskIrp (item->OriginalIrp, status, information); item->Status = status; @@ -130,13 +261,8 @@ static VOID CompletionThreadProc (PVOID CompleteOriginalIrp (request->Item, request->Item->Status, NT_SUCCESS (request->Item->Status) ? request->Item->OriginalLength : 0); } - else - { - InterlockedDecrement (&request->Item->OutstandingRequestCount); - KeSetEvent (&queue->RequestCompletedEvent, IO_DISK_INCREMENT, FALSE); - } - TCfree (request); + ReleasePoolBuffer (queue, request); } } @@ -144,6 +270,31 @@ static VOID CompletionThreadProc (PVOID } +static NTSTATUS TCCachedRead (EncryptedIoQueue *queue, IO_STATUS_BLOCK *ioStatus, PVOID buffer, LARGE_INTEGER offset, ULONG length) +{ + queue->LastReadOffset = offset; + queue->LastReadLength = length; + + if (queue->ReadAheadBufferValid && queue->ReadAheadOffset.QuadPart == offset.QuadPart && queue->ReadAheadLength >= length) + { + memcpy (buffer, queue->ReadAheadBuffer, length); + + if (!queue->IsFilterDevice) + { + ioStatus->Information = length; + ioStatus->Status = STATUS_SUCCESS; + } + + return STATUS_SUCCESS; + } + + if (queue->IsFilterDevice) + return TCReadDevice (queue->LowerDeviceObject, buffer, offset, length); + + return ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, ioStatus, buffer, length, &offset, NULL); +} + + static VOID IoThreadProc (PVOID threadArg) { EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg; @@ -171,8 +322,13 @@ static VOID IoThreadProc (PVOID threadAr while ((listEntry = ExInterlockedRemoveHeadList (&queue->IoThreadQueue, &queue->IoThreadQueueLock))) { + InterlockedDecrement (&queue->IoThreadPendingRequestCount); request = CONTAINING_RECORD (listEntry, EncryptedIoRequest, ListEntry); +#ifdef TC_TRACE_IO_QUEUE + Dump ("%c %I64d [%I64d] roff=%I64d rlen=%d\n", request->Item->Write ? 'W' : 'R', request->Item->OriginalIrpOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), request->Offset.QuadPart, request->Length); +#endif + // Perform IO request if no preceding request of the item failed if (NT_SUCCESS (request->Item->Status)) { @@ -214,7 +370,7 @@ static VOID IoThreadProc (PVOID threadAr if (request->Item->Write) request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, subFragmentData, subFragmentOffset, subFragmentLength); else - request->Item->Status = TCReadDevice (queue->LowerDeviceObject, subFragmentData, subFragmentOffset, subFragmentLength); + request->Item->Status = TCCachedRead (queue, NULL, subFragmentData, subFragmentOffset, subFragmentLength); subFragmentData += subFragmentLength; } @@ -229,7 +385,7 @@ static VOID IoThreadProc (PVOID threadAr if (request->Item->Write) request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, remappedOffset, request->Length); else - request->Item->Status = TCReadDevice (queue->LowerDeviceObject, request->Data, remappedOffset, request->Length); + request->Item->Status = TCCachedRead (queue, NULL, request->Data, remappedOffset, request->Length); } } else @@ -237,7 +393,7 @@ static VOID IoThreadProc (PVOID threadAr if (request->Item->Write) request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, request->Offset, request->Length); else - request->Item->Status = TCReadDevice (queue->LowerDeviceObject, request->Data, request->Offset, request->Length); + request->Item->Status = TCCachedRead (queue, NULL, request->Data, request->Offset, request->Length); } } else @@ -247,7 +403,7 @@ static VOID IoThreadProc (PVOID threadAr if (request->Item->Write) request->Item->Status = ZwWriteFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, request->Data, request->Length, &request->Offset, NULL); else - request->Item->Status = ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, request->Data, request->Length, &request->Offset, NULL); + request->Item->Status = TCCachedRead (queue, &ioStatus, request->Data, request->Offset, request->Length); if (NT_SUCCESS (request->Item->Status) && ioStatus.Information != request->Length) request->Item->Status = STATUS_END_OF_FILE; @@ -256,6 +412,8 @@ static VOID IoThreadProc (PVOID threadAr if (request->Item->Write) { + queue->ReadAheadBufferValid = FALSE; + ReleaseFragmentBuffer (queue, request->Data); if (request->CompleteOriginalIrp) @@ -263,24 +421,56 @@ static VOID IoThreadProc (PVOID threadAr CompleteOriginalIrp (request->Item, request->Item->Status, NT_SUCCESS (request->Item->Status) ? request->Item->OriginalLength : 0); } - else - { - InterlockedDecrement (&request->Item->OutstandingRequestCount); - KeSetEvent (&queue->RequestCompletedEvent, IO_DISK_INCREMENT, FALSE); - } - TCfree (request); + ReleasePoolBuffer (queue, request); } else { + BOOL readAhead = FALSE; + if (NT_SUCCESS (request->Item->Status)) memcpy (request->OrigDataBufferFragment, request->Data, request->Length); ReleaseFragmentBuffer (queue, request->Data); request->Data = request->OrigDataBufferFragment; + if (request->CompleteOriginalIrp + && queue->LastReadLength > 0 + && NT_SUCCESS (request->Item->Status) + && InterlockedExchangeAdd (&queue->IoThreadPendingRequestCount, 0) == 0) + { + readAhead = TRUE; + InterlockedIncrement (&queue->OutstandingIoCount); + } + ExInterlockedInsertTailList (&queue->CompletionThreadQueue, &request->CompletionListEntry, &queue->CompletionThreadQueueLock); KeSetEvent (&queue->CompletionThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE); + + if (readAhead) + { + queue->ReadAheadBufferValid = FALSE; + queue->ReadAheadOffset.QuadPart = queue->LastReadOffset.QuadPart + queue->LastReadLength; + queue->ReadAheadLength = queue->LastReadLength; + + if (queue->ReadAheadOffset.QuadPart + queue->ReadAheadLength <= queue->MaxReadAheadOffset.QuadPart) + { +#ifdef TC_TRACE_IO_QUEUE + Dump ("A %I64d [%I64d] roff=%I64d rlen=%d\n", request->Item->OriginalIrpOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), queue->ReadAheadOffset, queue->ReadAheadLength); +#endif + if (queue->IsFilterDevice) + { + queue->ReadAheadBufferValid = NT_SUCCESS (TCReadDevice (queue->LowerDeviceObject, queue->ReadAheadBuffer, queue->ReadAheadOffset, queue->ReadAheadLength)); + } + else + { + IO_STATUS_BLOCK ioStatus; + queue->ReadAheadBufferValid = NT_SUCCESS (ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, queue->ReadAheadBuffer, queue->ReadAheadLength, &queue->ReadAheadOffset, NULL)); + queue->ReadAheadLength = (ULONG) ioStatus.Information; + } + } + + DecrementOutstandingIoCount (queue); + } } } } @@ -289,16 +479,6 @@ static VOID IoThreadProc (PVOID threadAr } -static NTSTATUS OnPassedIrpCompleted (PDEVICE_OBJECT filterDeviceObject, PIRP irp, EncryptedIoQueueItem *item) -{ - if (irp->PendingReturned) - IoMarkIrpPending (irp); - - OnItemCompleted (item, TRUE); - return STATUS_CONTINUE_COMPLETION; -} - - static VOID MainThreadProc (PVOID threadArg) { EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg; @@ -313,10 +493,6 @@ static VOID MainThreadProc (PVOID thread uint64 intersectStart; uint32 intersectLength; - int mdlWaitTime; - LARGE_INTEGER mdlWaitInterval; - mdlWaitInterval.QuadPart = TC_ENC_IO_QUEUE_MEM_ALLOC_RETRY_DELAY * -10000; - if (IsEncryptionThreadPoolRunning()) KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY); @@ -333,22 +509,9 @@ static VOID MainThreadProc (PVOID thread if (queue->Suspended) KeWaitForSingleObject (&queue->QueueResumedEvent, Executive, KernelMode, FALSE, NULL); - item = AllocateMemoryWithTimeout (sizeof (EncryptedIoQueueItem), TC_ENC_IO_QUEUE_MEM_ALLOC_RETRY_DELAY, TC_ENC_IO_QUEUE_MEM_ALLOC_TIMEOUT); - if (!item) - { - EncryptedIoQueueItem stackItem; - stackItem.Queue = queue; - stackItem.OriginalIrp = irp; - stackItem.Status = STATUS_INSUFFICIENT_RESOURCES; - - TCCompleteDiskIrp (irp, STATUS_INSUFFICIENT_RESOURCES, 0); - OnItemCompleted (&stackItem, FALSE); - continue; - } - + item = GetPoolBuffer (queue, sizeof (EncryptedIoQueueItem)); item->Queue = queue; item->OriginalIrp = irp; - item->OutstandingRequestCount = 0; item->Status = STATUS_SUCCESS; IoSetCancelRoutine (irp, NULL); @@ -377,16 +540,11 @@ static VOID MainThreadProc (PVOID thread continue; } - // Pass the IRP if the drive is not encrypted - if (queue->IsFilterDevice && (queue->EncryptedAreaStart == -1 || queue->EncryptedAreaEnd == -1)) - { - IoCopyCurrentIrpStackLocationToNext (irp); - IoSetCompletionRoutine (irp, OnPassedIrpCompleted, item, TRUE, TRUE, TRUE); - IoCallDriver (queue->LowerDeviceObject, irp); - continue; - } +#ifdef TC_TRACE_IO_QUEUE + item->OriginalIrpOffset = item->OriginalOffset; +#endif - // Handle misaligned reads to support Windows System Assessment Tool which reads from disk devices at offsets not aligned on sector boundaries + // Handle misaligned read operations to work around a bug in Windows System Assessment Tool which does not follow FILE_FLAG_NO_BUFFERING requirements when benchmarking disk devices if (queue->IsFilterDevice && !item->Write && item->OriginalLength > 0 @@ -419,11 +577,14 @@ static VOID MainThreadProc (PVOID thread continue; } - GetIntersection (alignedOffset.QuadPart, alignedLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength); - if (intersectLength > 0) + if (queue->EncryptedAreaStart != -1 && queue->EncryptedAreaEnd != -1) { - dataUnit.Value = intersectStart / ENCRYPTION_DATA_UNIT_SIZE; - DecryptDataUnits (buffer + (intersectStart - alignedOffset.QuadPart), &dataUnit, intersectLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo); + GetIntersection (alignedOffset.QuadPart, alignedLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength); + if (intersectLength > 0) + { + dataUnit.Value = intersectStart / ENCRYPTION_DATA_UNIT_SIZE; + DecryptDataUnits (buffer + (intersectStart - alignedOffset.QuadPart), &dataUnit, intersectLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo); + } } memcpy (dataBuffer, buffer + (item->OriginalOffset.LowPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)), item->OriginalLength); @@ -444,7 +605,9 @@ static VOID MainThreadProc (PVOID thread continue; } - //Dump ("--- Queue %c %I64d (%I64d) %d out=%d\n", item->Write ? 'W' : 'R', item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart / 1024 / 1024, item->OriginalLength, queue->OutstandingIoCount); +#ifdef TC_TRACE_IO_QUEUE + Dump ("Q %I64d [%I64d] %c len=%d\n", item->OriginalOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), item->Write ? 'W' : 'R', item->OriginalLength); +#endif if (!queue->IsFilterDevice) { @@ -483,8 +646,16 @@ static VOID MainThreadProc (PVOID thread } } } + else if (item->Write + && RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, TC_BOOT_VOLUME_HEADER_SECTOR_OFFSET, TC_BOOT_VOLUME_HEADER_SECTOR_OFFSET + TC_BOOT_ENCRYPTION_VOLUME_HEADER_SIZE - 1)) + { + // Prevent inappropriately designed software from damaging important data that may be out of sync with the backup on the Rescue Disk (such as the end of the encrypted area). + Dump ("Preventing write to the system encryption key data area\n"); + CompleteOriginalIrp (item, STATUS_MEDIA_WRITE_PROTECTED, 0); + continue; + } else if (item->Write && IsHiddenSystemRunning() - && (RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, SECTOR_SIZE, TC_BOOT_LOADER_AREA_SECTOR_COUNT * SECTOR_SIZE - 1) + && (RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, TC_SECTOR_SIZE_BIOS, TC_BOOT_LOADER_AREA_SECTOR_COUNT * TC_SECTOR_SIZE_BIOS - 1) || RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, GetBootDriveLength(), _I64_MAX))) { Dump ("Preventing write to boot loader or host protected area\n"); @@ -492,18 +663,7 @@ static VOID MainThreadProc (PVOID thread continue; } - // Original IRP data buffer - mdlWaitTime = 0; - while (TRUE) - { - dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, HighPagePriority); - - if (dataBuffer || mdlWaitTime >= TC_ENC_IO_QUEUE_MEM_ALLOC_TIMEOUT) - break; - - KeDelayExecutionThread (KernelMode, FALSE, &mdlWaitInterval); - mdlWaitTime += TC_ENC_IO_QUEUE_MEM_ALLOC_RETRY_DELAY; - } + dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, HighPagePriority); if (dataBuffer == NULL) { @@ -523,17 +683,10 @@ static VOID MainThreadProc (PVOID thread ULONG dataFragmentLength = isLastFragment ? dataRemaining : TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; activeFragmentBuffer = (activeFragmentBuffer == queue->FragmentBufferA ? queue->FragmentBufferB : queue->FragmentBufferA); - // Create IO request - request = (EncryptedIoRequest *) AllocateMemoryWithTimeout (sizeof (EncryptedIoRequest), TC_ENC_IO_QUEUE_MEM_ALLOC_RETRY_DELAY, TC_ENC_IO_QUEUE_MEM_ALLOC_TIMEOUT); - if (!request) - { - while (InterlockedExchangeAdd (&item->OutstandingRequestCount, 0) > 0) - KeWaitForSingleObject (&queue->RequestCompletedEvent, Executive, KernelMode, FALSE, NULL); - - CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); - break; - } + InterlockedIncrement (&queue->IoThreadPendingRequestCount); + // Create IO request + request = GetPoolBuffer (queue, sizeof (EncryptedIoRequest)); request->Item = item; request->CompleteOriginalIrp = isLastFragment; request->Offset = fragmentOffset; @@ -543,11 +696,18 @@ static VOID MainThreadProc (PVOID thread if (queue->IsFilterDevice) { - // Get intersection of data fragment with encrypted area - GetIntersection (fragmentOffset.QuadPart, dataFragmentLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength); + if (queue->EncryptedAreaStart == -1 || queue->EncryptedAreaEnd == -1) + { + request->EncryptedLength = 0; + } + else + { + // Get intersection of data fragment with encrypted area + GetIntersection (fragmentOffset.QuadPart, dataFragmentLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength); - request->EncryptedOffset = intersectStart - fragmentOffset.QuadPart; - request->EncryptedLength = intersectLength; + request->EncryptedOffset = intersectStart - fragmentOffset.QuadPart; + request->EncryptedLength = intersectLength; + } } else { @@ -579,8 +739,6 @@ static VOID MainThreadProc (PVOID thread } // Queue IO request - InterlockedIncrement (&item->OutstandingRequestCount); - ExInterlockedInsertTailList (&queue->IoThreadQueue, &request->ListEntry, &queue->IoThreadQueueLock); KeSetEvent (&queue->IoThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE); @@ -610,14 +768,17 @@ NTSTATUS EncryptedIoQueueAddIrp (Encrypt goto err; } - if (queue->IsFilterDevice) + status = IoAcquireRemoveLock (&queue->RemoveLock, irp); + if (!NT_SUCCESS (status)) + goto err; + +#ifdef TC_TRACE_IO_QUEUE { - status = IoAcquireRemoveLock (&queue->RemoveLock, irp); - if (!NT_SUCCESS (status)) - goto err; + PIO_STACK_LOCATION irpSp = IoGetCurrentIrpStackLocation (irp); + Dump ("* %I64d [%I64d] %c len=%d out=%d\n", irpSp->MajorFunction == IRP_MJ_WRITE ? irpSp->Parameters.Write.ByteOffset : irpSp->Parameters.Read.ByteOffset, GetElapsedTime (&queue->LastPerformanceCounter), irpSp->MajorFunction == IRP_MJ_WRITE ? 'W' : 'R', irpSp->MajorFunction == IRP_MJ_WRITE ? irpSp->Parameters.Write.Length : irpSp->Parameters.Read.Length, queue->OutstandingIoCount); } +#endif - //Dump ("Queue irp %p out=%d\n", irp, queue->OutstandingIoCount); IoMarkIrpPending (irp); ExInterlockedInsertTailList (&queue->MainThreadQueue, &irp->Tail.Overlay.ListEntry, &queue->MainThreadQueueLock); @@ -651,11 +812,17 @@ NTSTATUS EncryptedIoQueueHoldWhenIdle (E status = STATUS_UNSUCCESSFUL; if (!NT_SUCCESS (status)) + { + queue->SuspendPending = FALSE; return status; + } TCSleep (1); if (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0) + { + queue->SuspendPending = FALSE; return STATUS_UNSUCCESSFUL; + } } KeClearEvent (&queue->QueueResumedEvent); @@ -666,12 +833,11 @@ NTSTATUS EncryptedIoQueueHoldWhenIdle (E queue->Suspended = FALSE; KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE); - } - queue->SuspendPending = FALSE; - //Dump ("Queue suspended out=%d\n", queue->OutstandingIoCount); + queue->ReadAheadBufferValid = FALSE; + queue->SuspendPending = FALSE; return STATUS_SUCCESS; } @@ -695,8 +861,6 @@ NTSTATUS EncryptedIoQueueResumeFromHold queue->Suspended = FALSE; KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE); - //Dump ("Queue resumed out=%d\n", queue->OutstandingIoCount); - return STATUS_SUCCESS; } @@ -704,10 +868,20 @@ NTSTATUS EncryptedIoQueueResumeFromHold NTSTATUS EncryptedIoQueueStart (EncryptedIoQueue *queue) { NTSTATUS status; + EncryptedIoQueueBuffer *buffer; + int i; + + queue->StartPending = TRUE; queue->ThreadExitRequested = FALSE; + queue->OutstandingIoCount = 0; + queue->IoThreadPendingRequestCount = 0; + + queue->FirstPoolBuffer = NULL; + KeInitializeMutex (&queue->BufferPoolMutex, 0); + KeInitializeEvent (&queue->NoOutstandingIoEvent, SynchronizationEvent, FALSE); - KeInitializeEvent (&queue->RequestCompletedEvent, SynchronizationEvent, FALSE); + KeInitializeEvent (&queue->PoolBufferFreeEvent, SynchronizationEvent, FALSE); KeInitializeEvent (&queue->QueueResumedEvent, SynchronizationEvent, FALSE); queue->FragmentBufferA = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE); @@ -721,6 +895,26 @@ NTSTATUS EncryptedIoQueueStart (Encrypte KeInitializeEvent (&queue->FragmentBufferAFreeEvent, SynchronizationEvent, TRUE); KeInitializeEvent (&queue->FragmentBufferBFreeEvent, SynchronizationEvent, TRUE); + queue->ReadAheadBufferValid = FALSE; + queue->ReadAheadBuffer = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE); + if (!queue->ReadAheadBuffer) + goto noMemory; + + // Preallocate buffers + for (i = 0; i < TC_ENC_IO_QUEUE_PREALLOCATED_IO_REQUEST_COUNT; ++i) + { + if (i < TC_ENC_IO_QUEUE_PREALLOCATED_ITEM_COUNT && !GetPoolBuffer (queue, sizeof (EncryptedIoQueueItem))) + goto noMemory; + + if (!GetPoolBuffer (queue, sizeof (EncryptedIoRequest))) + goto noMemory; + } + + for (buffer = queue->FirstPoolBuffer; buffer != NULL; buffer = buffer->NextBuffer) + { + buffer->InUse = FALSE; + } + // Main thread InitializeListHead (&queue->MainThreadQueue); KeInitializeSpinLock (&queue->MainThreadQueueLock); @@ -757,7 +951,13 @@ NTSTATUS EncryptedIoQueueStart (Encrypte goto err; } +#ifdef TC_TRACE_IO_QUEUE + GetElapsedTimeInit (&queue->LastPerformanceCounter); +#endif + queue->StopPending = FALSE; + queue->StartPending = FALSE; + Dump ("Queue started\n"); return STATUS_SUCCESS; @@ -770,6 +970,9 @@ err: if (queue->FragmentBufferB) TCfree (queue->FragmentBufferB); + FreePoolBuffers (queue); + + queue->StartPending = FALSE; return status; } @@ -794,6 +997,9 @@ NTSTATUS EncryptedIoQueueStop (Encrypted TCfree (queue->FragmentBufferA); TCfree (queue->FragmentBufferB); + TCfree (queue->ReadAheadBuffer); + + FreePoolBuffers (queue); Dump ("Queue stopped out=%d\n", queue->OutstandingIoCount); return STATUS_SUCCESS;