![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to encryptedioqueue.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [truecrypt] / truecrypt / driver|
Return to encryptedioqueue.c CVS log | Up to [truecrypt] / truecrypt / driver |
1.1 root 1: /*
2: Copyright (c) 2008 TrueCrypt Foundation. All rights reserved.
3:
1.1.1.6 root 4: Governed by the TrueCrypt License 2.6 the full text of which is contained
1.1 root 5: in the file License.txt included in TrueCrypt binary and source code
6: distribution packages.
7: */
8:
9: #include "TCdefs.h"
10: #include "Apidrvr.h"
11: #include "Ntdriver.h"
1.1.1.6 root 12: #include "DriveFilter.h"
1.1 root 13: #include "EncryptedIoQueue.h"
1.1.1.5 root 14: #include "EncryptionThreadPool.h"
15: #include "Volumes.h"
1.1 root 16:
17:
18: static void DecrementOutstandingIoCount (EncryptedIoQueue *queue)
19: {
20: if (InterlockedDecrement (&queue->OutstandingIoCount) == 0 && (queue->SuspendPending || queue->StopPending))
1.1.1.3 root 21: KeSetEvent (&queue->NoOutstandingIoEvent, IO_DISK_INCREMENT, FALSE);
1.1 root 22: }
23:
24:
1.1.1.5 root 25: static void OnItemCompleted (EncryptedIoQueueItem *item, BOOL freeItem)
1.1 root 26: {
27: DecrementOutstandingIoCount (item->Queue);
28:
29: if (item->Queue->IsFilterDevice)
30: IoReleaseRemoveLock (&item->Queue->RemoveLock, item->OriginalIrp);
31:
32: if (NT_SUCCESS (item->Status))
33: {
34: if (item->Write)
35: item->Queue->TotalBytesWritten += item->OriginalLength;
36: else
37: item->Queue->TotalBytesRead += item->OriginalLength;
38: }
39:
1.1.1.5 root 40: if (freeItem)
41: TCfree (item);
1.1 root 42: }
43:
44:
45: static NTSTATUS CompleteOriginalIrp (EncryptedIoQueueItem *item, NTSTATUS status, ULONG_PTR information)
46: {
47: //Dump ("Queue comp offset=%I64d status=%x info=%p out=%d\n", item->OriginalOffset, status, information, item->Queue->OutstandingIoCount - 1);
1.1.1.5 root 48:
1.1 root 49: TCCompleteDiskIrp (item->OriginalIrp, status, information);
1.1.1.5 root 50:
51: item->Status = status;
52: OnItemCompleted (item, TRUE);
53:
1.1 root 54: return status;
55: }
56:
57:
58: static void AcquireFragmentBuffer (EncryptedIoQueue *queue, byte *buffer)
59: {
60: NTSTATUS status = STATUS_INVALID_PARAMETER;
61:
62: if (buffer == queue->FragmentBufferA)
63: {
64: status = KeWaitForSingleObject (&queue->FragmentBufferAFreeEvent, Executive, KernelMode, FALSE, NULL);
65: }
66: else if (buffer == queue->FragmentBufferB)
67: {
68: status = KeWaitForSingleObject (&queue->FragmentBufferBFreeEvent, Executive, KernelMode, FALSE, NULL);
69: }
70:
71: if (!NT_SUCCESS (status))
72: TC_BUG_CHECK (status);
73: }
74:
75:
76: static void ReleaseFragmentBuffer (EncryptedIoQueue *queue, byte *buffer)
77: {
78: if (buffer == queue->FragmentBufferA)
79: {
1.1.1.3 root 80: KeSetEvent (&queue->FragmentBufferAFreeEvent, IO_DISK_INCREMENT, FALSE);
1.1 root 81: }
82: else if (buffer == queue->FragmentBufferB)
83: {
1.1.1.3 root 84: KeSetEvent (&queue->FragmentBufferBFreeEvent, IO_DISK_INCREMENT, FALSE);
1.1 root 85: }
86: else
87: {
88: TC_BUG_CHECK (STATUS_INVALID_PARAMETER);
89: }
90: }
91:
92:
93: static VOID CompletionThreadProc (PVOID threadArg)
94: {
95: EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg;
96: PLIST_ENTRY listEntry;
97: EncryptedIoRequest *request;
98: UINT64_STRUCT dataUnit;
99:
1.1.1.5 root 100: if (IsEncryptionThreadPoolRunning())
101: KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY);
102:
1.1 root 103: while (!queue->ThreadExitRequested)
104: {
105: if (!NT_SUCCESS (KeWaitForSingleObject (&queue->CompletionThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL)))
106: continue;
107:
108: if (queue->ThreadExitRequested)
109: break;
110:
111: while ((listEntry = ExInterlockedRemoveHeadList (&queue->CompletionThreadQueue, &queue->CompletionThreadQueueLock)))
112: {
113: request = CONTAINING_RECORD (listEntry, EncryptedIoRequest, CompletionListEntry);
114:
1.1.1.3 root 115: if (request->EncryptedLength > 0 && NT_SUCCESS (request->Item->Status))
1.1 root 116: {
117: ASSERT (request->EncryptedOffset + request->EncryptedLength <= request->Offset.QuadPart + request->Length);
118: dataUnit.Value = (request->Offset.QuadPart + request->EncryptedOffset) / ENCRYPTION_DATA_UNIT_SIZE;
1.1.1.3 root 119:
120: if (queue->CryptoInfo->bPartitionInInactiveSysEncScope)
121: dataUnit.Value += queue->CryptoInfo->FirstDataUnitNo.Value;
1.1.1.5 root 122: else if (queue->RemapEncryptedArea)
123: dataUnit.Value += queue->RemappedAreaDataUnitOffset;
1.1.1.3 root 124:
1.1 root 125: DecryptDataUnits (request->Data + request->EncryptedOffset, &dataUnit, request->EncryptedLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo);
126: }
127:
128: if (request->CompleteOriginalIrp)
129: {
130: CompleteOriginalIrp (request->Item, request->Item->Status,
131: NT_SUCCESS (request->Item->Status) ? request->Item->OriginalLength : 0);
132: }
1.1.1.3 root 133: else
134: {
135: InterlockedDecrement (&request->Item->OutstandingRequestCount);
136: KeSetEvent (&queue->RequestCompletedEvent, IO_DISK_INCREMENT, FALSE);
137: }
1.1 root 138:
139: TCfree (request);
140: }
141: }
142:
143: PsTerminateSystemThread (STATUS_SUCCESS);
144: }
145:
146:
147: static VOID IoThreadProc (PVOID threadArg)
148: {
149: EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg;
150: PLIST_ENTRY listEntry;
151: EncryptedIoRequest *request;
152:
1.1.1.4 root 153: KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY);
154:
1.1.1.6 root 155: if (!queue->IsFilterDevice && queue->SecurityClientContext)
156: {
157: #ifdef DEBUG
158: NTSTATUS status =
159: #endif
160: SeImpersonateClientEx (queue->SecurityClientContext, NULL);
161: ASSERT (NT_SUCCESS (status));
162: }
163:
1.1 root 164: while (!queue->ThreadExitRequested)
165: {
166: if (!NT_SUCCESS (KeWaitForSingleObject (&queue->IoThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL)))
167: continue;
168:
169: if (queue->ThreadExitRequested)
170: break;
171:
172: while ((listEntry = ExInterlockedRemoveHeadList (&queue->IoThreadQueue, &queue->IoThreadQueueLock)))
173: {
174: request = CONTAINING_RECORD (listEntry, EncryptedIoRequest, ListEntry);
1.1.1.3 root 175:
176: // Perform IO request if no preceding request of the item failed
177: if (NT_SUCCESS (request->Item->Status))
1.1 root 178: {
1.1.1.3 root 179: if (queue->IsFilterDevice)
180: {
1.1.1.5 root 181: if (queue->RemapEncryptedArea && request->EncryptedLength > 0)
182: {
183: if (request->EncryptedLength != request->Length)
184: {
185: // Up to three subfragments may be required to handle a partially remapped fragment
186: int subFragment;
187: byte *subFragmentData = request->Data;
188:
189: for (subFragment = 0 ; subFragment < 3; ++subFragment)
190: {
191: LARGE_INTEGER subFragmentOffset;
192: ULONG subFragmentLength;
193: subFragmentOffset.QuadPart = request->Offset.QuadPart;
194:
195: switch (subFragment)
196: {
197: case 0:
198: subFragmentLength = (ULONG) request->EncryptedOffset;
199: break;
200:
201: case 1:
202: subFragmentOffset.QuadPart += request->EncryptedOffset + queue->RemappedAreaOffset;
203: subFragmentLength = request->EncryptedLength;
204: break;
205:
206: case 2:
207: subFragmentOffset.QuadPart += request->EncryptedOffset + request->EncryptedLength;
208: subFragmentLength = (ULONG) (request->Length - (request->EncryptedOffset + request->EncryptedLength));
209: break;
210: }
211:
212: if (subFragmentLength > 0)
213: {
214: if (request->Item->Write)
215: request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, subFragmentData, subFragmentOffset, subFragmentLength);
216: else
217: request->Item->Status = TCReadDevice (queue->LowerDeviceObject, subFragmentData, subFragmentOffset, subFragmentLength);
218:
219: subFragmentData += subFragmentLength;
220: }
221: }
222: }
223: else
224: {
225: // Remap the fragment
226: LARGE_INTEGER remappedOffset;
227: remappedOffset.QuadPart = request->Offset.QuadPart + queue->RemappedAreaOffset;
228:
229: if (request->Item->Write)
230: request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, remappedOffset, request->Length);
231: else
232: request->Item->Status = TCReadDevice (queue->LowerDeviceObject, request->Data, remappedOffset, request->Length);
233: }
234: }
1.1.1.3 root 235: else
1.1.1.5 root 236: {
237: if (request->Item->Write)
238: request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, request->Offset, request->Length);
239: else
240: request->Item->Status = TCReadDevice (queue->LowerDeviceObject, request->Data, request->Offset, request->Length);
241: }
1.1.1.3 root 242: }
1.1 root 243: else
1.1.1.3 root 244: {
245: IO_STATUS_BLOCK ioStatus;
1.1 root 246:
1.1.1.3 root 247: if (request->Item->Write)
248: request->Item->Status = ZwWriteFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, request->Data, request->Length, &request->Offset, NULL);
249: else
250: request->Item->Status = ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, request->Data, request->Length, &request->Offset, NULL);
1.1.1.5 root 251:
252: if (NT_SUCCESS (request->Item->Status) && ioStatus.Information != request->Length)
253: request->Item->Status = STATUS_END_OF_FILE;
1.1.1.3 root 254: }
1.1 root 255: }
256:
1.1.1.3 root 257: if (request->Item->Write)
1.1 root 258: {
259: ReleaseFragmentBuffer (queue, request->Data);
260:
261: if (request->CompleteOriginalIrp)
262: {
263: CompleteOriginalIrp (request->Item, request->Item->Status,
264: NT_SUCCESS (request->Item->Status) ? request->Item->OriginalLength : 0);
265: }
266: else
267: {
1.1.1.3 root 268: InterlockedDecrement (&request->Item->OutstandingRequestCount);
269: KeSetEvent (&queue->RequestCompletedEvent, IO_DISK_INCREMENT, FALSE);
1.1 root 270: }
271:
272: TCfree (request);
273: }
1.1.1.3 root 274: else
275: {
276: if (NT_SUCCESS (request->Item->Status))
277: memcpy (request->OrigDataBufferFragment, request->Data, request->Length);
278:
279: ReleaseFragmentBuffer (queue, request->Data);
280: request->Data = request->OrigDataBufferFragment;
281:
282: ExInterlockedInsertTailList (&queue->CompletionThreadQueue, &request->CompletionListEntry, &queue->CompletionThreadQueueLock);
283: KeSetEvent (&queue->CompletionThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE);
284: }
1.1 root 285: }
286: }
287:
288: PsTerminateSystemThread (STATUS_SUCCESS);
289: }
290:
291:
292: static NTSTATUS OnPassedIrpCompleted (PDEVICE_OBJECT filterDeviceObject, PIRP irp, EncryptedIoQueueItem *item)
293: {
294: if (irp->PendingReturned)
295: IoMarkIrpPending (irp);
296:
1.1.1.5 root 297: OnItemCompleted (item, TRUE);
1.1 root 298: return STATUS_CONTINUE_COMPLETION;
299: }
300:
301:
302: static VOID MainThreadProc (PVOID threadArg)
303: {
304: EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg;
305: PLIST_ENTRY listEntry;
306: EncryptedIoQueueItem *item;
307:
308: LARGE_INTEGER fragmentOffset;
309: ULONG dataRemaining;
310: PUCHAR activeFragmentBuffer = queue->FragmentBufferA;
311: PUCHAR dataBuffer;
312: EncryptedIoRequest *request;
313: uint64 intersectStart;
314: uint32 intersectLength;
315:
1.1.1.5 root 316: int mdlWaitTime;
317: LARGE_INTEGER mdlWaitInterval;
318: mdlWaitInterval.QuadPart = TC_ENC_IO_QUEUE_MEM_ALLOC_RETRY_DELAY * -10000;
319:
320: if (IsEncryptionThreadPoolRunning())
321: KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY);
322:
1.1 root 323: while (!queue->ThreadExitRequested)
324: {
325: if (!NT_SUCCESS (KeWaitForSingleObject (&queue->MainThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL)))
326: continue;
327:
328: while ((listEntry = ExInterlockedRemoveHeadList (&queue->MainThreadQueue, &queue->MainThreadQueueLock)))
329: {
1.1.1.5 root 330: PIRP irp = CONTAINING_RECORD (listEntry, IRP, Tail.Overlay.ListEntry);
331: PIO_STACK_LOCATION irpSp = IoGetCurrentIrpStackLocation (irp);
332:
1.1 root 333: if (queue->Suspended)
334: KeWaitForSingleObject (&queue->QueueResumedEvent, Executive, KernelMode, FALSE, NULL);
1.1.1.5 root 335:
336: item = AllocateMemoryWithTimeout (sizeof (EncryptedIoQueueItem), TC_ENC_IO_QUEUE_MEM_ALLOC_RETRY_DELAY, TC_ENC_IO_QUEUE_MEM_ALLOC_TIMEOUT);
337: if (!item)
338: {
339: EncryptedIoQueueItem stackItem;
340: stackItem.Queue = queue;
341: stackItem.OriginalIrp = irp;
342: stackItem.Status = STATUS_INSUFFICIENT_RESOURCES;
343:
344: TCCompleteDiskIrp (irp, STATUS_INSUFFICIENT_RESOURCES, 0);
345: OnItemCompleted (&stackItem, FALSE);
346: continue;
1.1 root 347: }
1.1.1.5 root 348:
349: item->Queue = queue;
350: item->OriginalIrp = irp;
351: item->OutstandingRequestCount = 0;
352: item->Status = STATUS_SUCCESS;
353:
354: IoSetCancelRoutine (irp, NULL);
355: if (irp->Cancel)
1.1 root 356: {
357: CompleteOriginalIrp (item, STATUS_CANCELLED, 0);
358: continue;
359: }
360:
1.1.1.5 root 361: switch (irpSp->MajorFunction)
362: {
363: case IRP_MJ_READ:
364: item->Write = FALSE;
365: item->OriginalOffset = irpSp->Parameters.Read.ByteOffset;
366: item->OriginalLength = irpSp->Parameters.Read.Length;
367: break;
368:
369: case IRP_MJ_WRITE:
370: item->Write = TRUE;
371: item->OriginalOffset = irpSp->Parameters.Write.ByteOffset;
372: item->OriginalLength = irpSp->Parameters.Write.Length;
373: break;
374:
375: default:
376: CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0);
377: continue;
378: }
379:
1.1 root 380: // Pass the IRP if the drive is not encrypted
381: if (queue->IsFilterDevice && (queue->EncryptedAreaStart == -1 || queue->EncryptedAreaEnd == -1))
382: {
1.1.1.5 root 383: IoCopyCurrentIrpStackLocationToNext (irp);
384: IoSetCompletionRoutine (irp, OnPassedIrpCompleted, item, TRUE, TRUE, TRUE);
385: IoCallDriver (queue->LowerDeviceObject, irp);
1.1 root 386: continue;
387: }
388:
1.1.1.7 ! root 389: // Handle misaligned reads to support Windows System Assessment Tool which reads from disk devices at offsets not aligned on sector boundaries
! 390: if (queue->IsFilterDevice
! 391: && !item->Write
! 392: && item->OriginalLength > 0
! 393: && (item->OriginalLength & (ENCRYPTION_DATA_UNIT_SIZE - 1)) == 0
! 394: && (item->OriginalOffset.QuadPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0)
! 395: {
! 396: byte *buffer;
! 397: ULONG alignedLength = item->OriginalLength + ENCRYPTION_DATA_UNIT_SIZE;
! 398: LARGE_INTEGER alignedOffset;
! 399: alignedOffset.QuadPart = item->OriginalOffset.QuadPart & ~((LONGLONG) ENCRYPTION_DATA_UNIT_SIZE - 1);
! 400:
! 401: buffer = TCalloc (alignedLength);
! 402: if (!buffer)
! 403: {
! 404: CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0);
! 405: continue;
! 406: }
! 407:
! 408: item->Status = TCReadDevice (queue->LowerDeviceObject, buffer, alignedOffset, alignedLength);
! 409:
! 410: if (NT_SUCCESS (item->Status))
! 411: {
! 412: UINT64_STRUCT dataUnit;
! 413:
! 414: dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, HighPagePriority);
! 415: if (!dataBuffer)
! 416: {
! 417: TCfree (buffer);
! 418: CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0);
! 419: continue;
! 420: }
! 421:
! 422: GetIntersection (alignedOffset.QuadPart, alignedLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength);
! 423: if (intersectLength > 0)
! 424: {
! 425: dataUnit.Value = intersectStart / ENCRYPTION_DATA_UNIT_SIZE;
! 426: DecryptDataUnits (buffer + (intersectStart - alignedOffset.QuadPart), &dataUnit, intersectLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo);
! 427: }
! 428:
! 429: memcpy (dataBuffer, buffer + (item->OriginalOffset.LowPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)), item->OriginalLength);
! 430: }
! 431:
! 432: TCfree (buffer);
! 433: CompleteOriginalIrp (item, item->Status, NT_SUCCESS (item->Status) ? item->OriginalLength : 0);
! 434: continue;
! 435: }
! 436:
1.1 root 437: // Validate offset and length
1.1.1.3 root 438: if (item->OriginalLength == 0
439: || (item->OriginalLength & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0
440: || (item->OriginalOffset.QuadPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0
1.1 root 441: || (!queue->IsFilterDevice && item->OriginalOffset.QuadPart + item->OriginalLength > queue->VirtualDeviceLength))
442: {
443: CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0);
444: continue;
445: }
446:
1.1.1.5 root 447: //Dump ("--- Queue %c %I64d (%I64d) %d out=%d\n", item->Write ? 'W' : 'R', item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart / 1024 / 1024, item->OriginalLength, queue->OutstandingIoCount);
448:
1.1 root 449: if (!queue->IsFilterDevice)
450: {
1.1.1.3 root 451: // Adjust the offset for host file or device
452: if (queue->CryptoInfo->hiddenVolume)
453: item->OriginalOffset.QuadPart += queue->CryptoInfo->hiddenVolumeOffset;
454: else
455: item->OriginalOffset.QuadPart += queue->CryptoInfo->volDataAreaOffset;
456:
1.1 root 457: // Hidden volume protection
458: if (item->Write && queue->CryptoInfo->bProtectHiddenVolume)
459: {
460: // If there has already been a write operation denied in order to protect the
461: // hidden volume (since the volume mount time)
462: if (queue->CryptoInfo->bHiddenVolProtectionAction)
463: {
464: // Do not allow writing to this volume anymore. This is to fake a complete volume
465: // or system failure (otherwise certain kinds of inconsistency within the file
466: // system could indicate that this volume has used hidden volume protection).
467: CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0);
468: continue;
469: }
470:
471: // Verify that no byte is going to be written to the hidden volume area
1.1.1.3 root 472: if (RegionsOverlap ((unsigned __int64) item->OriginalOffset.QuadPart,
473: (unsigned __int64) item->OriginalOffset.QuadPart + item->OriginalLength - 1,
1.1 root 474: queue->CryptoInfo->hiddenVolumeOffset,
1.1.1.5 root 475: (unsigned __int64) queue->CryptoInfo->hiddenVolumeOffset + queue->CryptoInfo->hiddenVolumeProtectedSize - 1))
1.1 root 476: {
1.1.1.5 root 477: Dump ("Hidden volume protection triggered: write %I64d-%I64d (protected %I64d-%I64d)\n", item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, queue->CryptoInfo->hiddenVolumeOffset, queue->CryptoInfo->hiddenVolumeOffset + queue->CryptoInfo->hiddenVolumeProtectedSize - 1);
1.1 root 478: queue->CryptoInfo->bHiddenVolProtectionAction = TRUE;
479:
480: // Deny this write operation to prevent the hidden volume from being overwritten
481: CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0);
482: continue;
483: }
484: }
485: }
1.1.1.6 root 486: else if (item->Write && IsHiddenSystemRunning()
487: && (RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, SECTOR_SIZE, TC_BOOT_LOADER_AREA_SECTOR_COUNT * SECTOR_SIZE - 1)
488: || RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, GetBootDriveLength(), _I64_MAX)))
489: {
490: Dump ("Preventing write to boot loader or host protected area\n");
491: CompleteOriginalIrp (item, STATUS_MEDIA_WRITE_PROTECTED, 0);
492: continue;
493: }
1.1 root 494:
495: // Original IRP data buffer
1.1.1.5 root 496: mdlWaitTime = 0;
497: while (TRUE)
498: {
499: dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, HighPagePriority);
500:
501: if (dataBuffer || mdlWaitTime >= TC_ENC_IO_QUEUE_MEM_ALLOC_TIMEOUT)
502: break;
503:
504: KeDelayExecutionThread (KernelMode, FALSE, &mdlWaitInterval);
505: mdlWaitTime += TC_ENC_IO_QUEUE_MEM_ALLOC_RETRY_DELAY;
506: }
507:
1.1 root 508: if (dataBuffer == NULL)
509: {
510: CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0);
511: continue;
512: }
513:
514: // Divide data block to fragments to enable efficient overlapping of encryption and IO operations
515:
516: dataRemaining = item->OriginalLength;
517: fragmentOffset = item->OriginalOffset;
518:
519: while (dataRemaining > 0)
520: {
521: BOOL isLastFragment = dataRemaining <= TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE;
522:
523: ULONG dataFragmentLength = isLastFragment ? dataRemaining : TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE;
524: activeFragmentBuffer = (activeFragmentBuffer == queue->FragmentBufferA ? queue->FragmentBufferB : queue->FragmentBufferA);
525:
526: // Create IO request
1.1.1.5 root 527: request = (EncryptedIoRequest *) AllocateMemoryWithTimeout (sizeof (EncryptedIoRequest), TC_ENC_IO_QUEUE_MEM_ALLOC_RETRY_DELAY, TC_ENC_IO_QUEUE_MEM_ALLOC_TIMEOUT);
1.1 root 528: if (!request)
529: {
1.1.1.3 root 530: while (InterlockedExchangeAdd (&item->OutstandingRequestCount, 0) > 0)
531: KeWaitForSingleObject (&queue->RequestCompletedEvent, Executive, KernelMode, FALSE, NULL);
532:
1.1 root 533: CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0);
534: break;
535: }
536:
537: request->Item = item;
538: request->CompleteOriginalIrp = isLastFragment;
539: request->Offset = fragmentOffset;
540: request->Data = activeFragmentBuffer;
541: request->OrigDataBufferFragment = dataBuffer;
542: request->Length = dataFragmentLength;
543:
544: if (queue->IsFilterDevice)
545: {
546: // Get intersection of data fragment with encrypted area
547: GetIntersection (fragmentOffset.QuadPart, dataFragmentLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength);
548:
549: request->EncryptedOffset = intersectStart - fragmentOffset.QuadPart;
550: request->EncryptedLength = intersectLength;
551: }
552: else
553: {
554: request->EncryptedOffset = 0;
555: request->EncryptedLength = dataFragmentLength;
556: }
557:
558: AcquireFragmentBuffer (queue, activeFragmentBuffer);
559:
560: if (item->Write)
561: {
562: // Encrypt data
563: memcpy (activeFragmentBuffer, dataBuffer, dataFragmentLength);
564:
565: if (request->EncryptedLength > 0)
566: {
567: UINT64_STRUCT dataUnit;
568: ASSERT (request->EncryptedOffset + request->EncryptedLength <= request->Offset.QuadPart + request->Length);
569:
570: dataUnit.Value = (request->Offset.QuadPart + request->EncryptedOffset) / ENCRYPTION_DATA_UNIT_SIZE;
571:
1.1.1.3 root 572: if (queue->CryptoInfo->bPartitionInInactiveSysEncScope)
573: dataUnit.Value += queue->CryptoInfo->FirstDataUnitNo.Value;
1.1.1.5 root 574: else if (queue->RemapEncryptedArea)
575: dataUnit.Value += queue->RemappedAreaDataUnitOffset;
576:
1.1.1.3 root 577: EncryptDataUnits (activeFragmentBuffer + request->EncryptedOffset, &dataUnit, request->EncryptedLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo);
1.1 root 578: }
579: }
580:
581: // Queue IO request
1.1.1.3 root 582: InterlockedIncrement (&item->OutstandingRequestCount);
583:
1.1 root 584: ExInterlockedInsertTailList (&queue->IoThreadQueue, &request->ListEntry, &queue->IoThreadQueueLock);
1.1.1.3 root 585: KeSetEvent (&queue->IoThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE);
1.1 root 586:
587: if (isLastFragment)
588: break;
589:
590: dataRemaining -= TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE;
591: dataBuffer += TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE;
592: fragmentOffset.QuadPart += TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE;
593: }
594: }
595: }
596:
597: PsTerminateSystemThread (STATUS_SUCCESS);
598: }
599:
600:
601: NTSTATUS EncryptedIoQueueAddIrp (EncryptedIoQueue *queue, PIRP irp)
602: {
603: NTSTATUS status;
604:
605: InterlockedIncrement (&queue->OutstandingIoCount);
606: if (queue->StopPending)
607: {
608: Dump ("STATUS_DEVICE_NOT_READY out=%d\n", queue->OutstandingIoCount);
609: status = STATUS_DEVICE_NOT_READY;
610: goto err;
611: }
612:
613: if (queue->IsFilterDevice)
614: {
615: status = IoAcquireRemoveLock (&queue->RemoveLock, irp);
616: if (!NT_SUCCESS (status))
617: goto err;
618: }
619:
1.1.1.5 root 620: //Dump ("Queue irp %p out=%d\n", irp, queue->OutstandingIoCount);
1.1 root 621: IoMarkIrpPending (irp);
622:
1.1.1.5 root 623: ExInterlockedInsertTailList (&queue->MainThreadQueue, &irp->Tail.Overlay.ListEntry, &queue->MainThreadQueueLock);
1.1.1.3 root 624: KeSetEvent (&queue->MainThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE);
1.1 root 625:
626: return STATUS_PENDING;
627:
628: err:
629: DecrementOutstandingIoCount (queue);
630: return status;
631: }
632:
633:
634: NTSTATUS EncryptedIoQueueHoldWhenIdle (EncryptedIoQueue *queue, int64 timeout)
635: {
636: NTSTATUS status;
637: ASSERT (!queue->Suspended);
638:
639: queue->SuspendPending = TRUE;
640:
641: while (TRUE)
642: {
643: while (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0)
644: {
645: LARGE_INTEGER waitTimeout;
646:
647: waitTimeout.QuadPart = timeout * -10000;
648: status = KeWaitForSingleObject (&queue->NoOutstandingIoEvent, Executive, KernelMode, FALSE, timeout != 0 ? &waitTimeout : NULL);
649:
650: if (status == STATUS_TIMEOUT)
651: status = STATUS_UNSUCCESSFUL;
652:
653: if (!NT_SUCCESS (status))
654: return status;
1.1.1.6 root 655:
1.1.1.7 ! root 656: TCSleep (1);
1.1.1.6 root 657: if (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0)
658: return STATUS_UNSUCCESSFUL;
1.1 root 659: }
660:
661: KeClearEvent (&queue->QueueResumedEvent);
662: queue->Suspended = TRUE;
663:
664: if (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) == 0)
665: break;
666:
667: queue->Suspended = FALSE;
1.1.1.3 root 668: KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE);
1.1 root 669:
670: }
671:
672: queue->SuspendPending = FALSE;
673: //Dump ("Queue suspended out=%d\n", queue->OutstandingIoCount);
674:
675: return STATUS_SUCCESS;
676: }
677:
678:
679: BOOL EncryptedIoQueueIsSuspended (EncryptedIoQueue *queue)
680: {
681: return queue->Suspended;
682: }
683:
684:
685: BOOL EncryptedIoQueueIsRunning (EncryptedIoQueue *queue)
686: {
687: return !queue->StopPending;
688: }
689:
690:
691: NTSTATUS EncryptedIoQueueResumeFromHold (EncryptedIoQueue *queue)
692: {
693: ASSERT (queue->Suspended);
694:
695: queue->Suspended = FALSE;
1.1.1.3 root 696: KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE);
1.1 root 697:
698: //Dump ("Queue resumed out=%d\n", queue->OutstandingIoCount);
699:
700: return STATUS_SUCCESS;
701: }
702:
703:
1.1.1.6 root 704: NTSTATUS EncryptedIoQueueStart (EncryptedIoQueue *queue)
1.1 root 705: {
706: NTSTATUS status;
707: queue->ThreadExitRequested = FALSE;
708:
709: KeInitializeEvent (&queue->NoOutstandingIoEvent, SynchronizationEvent, FALSE);
1.1.1.3 root 710: KeInitializeEvent (&queue->RequestCompletedEvent, SynchronizationEvent, FALSE);
1.1 root 711: KeInitializeEvent (&queue->QueueResumedEvent, SynchronizationEvent, FALSE);
712:
713: queue->FragmentBufferA = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE);
714: if (!queue->FragmentBufferA)
715: goto noMemory;
716:
717: queue->FragmentBufferB = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE);
718: if (!queue->FragmentBufferB)
719: goto noMemory;
720:
721: KeInitializeEvent (&queue->FragmentBufferAFreeEvent, SynchronizationEvent, TRUE);
722: KeInitializeEvent (&queue->FragmentBufferBFreeEvent, SynchronizationEvent, TRUE);
723:
724: // Main thread
725: InitializeListHead (&queue->MainThreadQueue);
726: KeInitializeSpinLock (&queue->MainThreadQueueLock);
727: KeInitializeEvent (&queue->MainThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE);
728:
729: status = TCStartThread (MainThreadProc, queue, &queue->MainThread);
730: if (!NT_SUCCESS (status))
731: goto err;
732:
733: // IO thread
734: InitializeListHead (&queue->IoThreadQueue);
735: KeInitializeSpinLock (&queue->IoThreadQueueLock);
736: KeInitializeEvent (&queue->IoThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE);
737:
1.1.1.6 root 738: status = TCStartThread (IoThreadProc, queue, &queue->IoThread);
1.1 root 739: if (!NT_SUCCESS (status))
740: {
741: queue->ThreadExitRequested = TRUE;
742: TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent);
743: goto err;
744: }
745:
746: // Completion thread
747: InitializeListHead (&queue->CompletionThreadQueue);
748: KeInitializeSpinLock (&queue->CompletionThreadQueueLock);
749: KeInitializeEvent (&queue->CompletionThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE);
750:
751: status = TCStartThread (CompletionThreadProc, queue, &queue->CompletionThread);
752: if (!NT_SUCCESS (status))
753: {
754: queue->ThreadExitRequested = TRUE;
755: TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent);
756: TCStopThread (queue->IoThread, &queue->IoThreadQueueNotEmptyEvent);
757: goto err;
758: }
759:
760: queue->StopPending = FALSE;
761: Dump ("Queue started\n");
762: return STATUS_SUCCESS;
763:
764: noMemory:
765: status = STATUS_INSUFFICIENT_RESOURCES;
766:
767: err:
768: if (queue->FragmentBufferA)
769: TCfree (queue->FragmentBufferA);
770: if (queue->FragmentBufferB)
771: TCfree (queue->FragmentBufferB);
772:
773: return status;
774: }
775:
776:
777: NTSTATUS EncryptedIoQueueStop (EncryptedIoQueue *queue)
778: {
779: ASSERT (!queue->StopPending);
780: queue->StopPending = TRUE;
781:
782: while (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0)
783: {
784: KeWaitForSingleObject (&queue->NoOutstandingIoEvent, Executive, KernelMode, FALSE, NULL);
785: }
786:
787: Dump ("Queue stopping out=%d\n", queue->OutstandingIoCount);
788:
789: queue->ThreadExitRequested = TRUE;
790:
791: TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent);
792: TCStopThread (queue->IoThread, &queue->IoThreadQueueNotEmptyEvent);
793: TCStopThread (queue->CompletionThread, &queue->CompletionThreadQueueNotEmptyEvent);
794:
795: TCfree (queue->FragmentBufferA);
796: TCfree (queue->FragmentBufferB);
797:
798: Dump ("Queue stopped out=%d\n", queue->OutstandingIoCount);
799: return STATUS_SUCCESS;
800: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.