![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to encryptedioqueue.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [truecrypt] / truecrypt / driver|
Return to encryptedioqueue.c CVS log | Up to [truecrypt] / truecrypt / driver |
1.1 root 1: /*
1.1.1.11 root 2: Copyright (c) 2008-2009 TrueCrypt Developers Association. All rights reserved.
1.1 root 3:
1.1.1.12 root 4: Governed by the TrueCrypt License 3.0 the full text of which is contained in
1.1.1.11 root 5: the file License.txt included in TrueCrypt binary and source code distribution
6: packages.
1.1 root 7: */
8:
9: #include "TCdefs.h"
10: #include "Apidrvr.h"
11: #include "Ntdriver.h"
1.1.1.6 root 12: #include "DriveFilter.h"
1.1 root 13: #include "EncryptedIoQueue.h"
1.1.1.5 root 14: #include "EncryptionThreadPool.h"
15: #include "Volumes.h"
1.1 root 16:
17:
1.1.1.8 root 18: static void AcquireBufferPoolMutex (EncryptedIoQueue *queue)
19: {
20: NTSTATUS status;
21:
22: status = KeWaitForMutexObject (&queue->BufferPoolMutex, Executive, KernelMode, FALSE, NULL);
23: if (!NT_SUCCESS (status))
24: TC_BUG_CHECK (status);
25: }
26:
27:
28: static void ReleaseBufferPoolMutex (EncryptedIoQueue *queue)
29: {
30: KeReleaseMutex (&queue->BufferPoolMutex, FALSE);
31: }
32:
33:
34: static void *GetPoolBuffer (EncryptedIoQueue *queue, ULONG requestedSize)
35: {
36: EncryptedIoQueueBuffer *buffer;
37: void *bufferAddress = NULL;
38: BOOL requestedSizePresentInPool = FALSE;
39:
40: while (TRUE)
41: {
42: AcquireBufferPoolMutex (queue);
43:
44: for (buffer = queue->FirstPoolBuffer; ; buffer = buffer->NextBuffer)
45: {
46: if (buffer && buffer->Size == requestedSize)
47: {
48: requestedSizePresentInPool = TRUE;
49:
50: if (!buffer->InUse)
51: {
52: // Reuse a free buffer
53: buffer->InUse = TRUE;
54: bufferAddress = buffer->Address;
55: break;
56: }
57: }
58:
59: if (!buffer || !buffer->NextBuffer)
60: {
1.1.1.9 root 61: EncryptedIoQueueBuffer *newBuffer;
62:
63: if (requestedSizePresentInPool && !queue->StartPending)
64: break;
65:
1.1.1.8 root 66: // Allocate a new buffer
1.1.1.9 root 67: newBuffer = TCalloc (sizeof (EncryptedIoQueueBuffer));
1.1.1.8 root 68: if (!newBuffer)
69: {
70: bufferAddress = NULL;
71: break;
72: }
73:
74: bufferAddress = TCalloc (requestedSize);
75: if (bufferAddress)
76: {
77: newBuffer->NextBuffer = NULL;
78: newBuffer->Address = bufferAddress;
79: newBuffer->Size = requestedSize;
80: newBuffer->InUse = TRUE;
81:
82: if (!buffer)
83: queue->FirstPoolBuffer = newBuffer;
84: else
85: buffer->NextBuffer = newBuffer;
86: }
87: else
88: TCfree (newBuffer);
89:
90: break;
91: }
92: }
93:
94: ReleaseBufferPoolMutex (queue);
95:
96: if (bufferAddress || !requestedSizePresentInPool || queue->StartPending)
97: break;
98:
1.1.1.9 root 99: KeWaitForSingleObject (&queue->PoolBufferFreeEvent, Executive, KernelMode, FALSE, NULL);
1.1.1.8 root 100: }
101:
102: return bufferAddress;
103: }
104:
105:
106: static void ReleasePoolBuffer (EncryptedIoQueue *queue, void *address)
107: {
108: EncryptedIoQueueBuffer *buffer;
109: AcquireBufferPoolMutex (queue);
110:
111: for (buffer = queue->FirstPoolBuffer; buffer != NULL; buffer = buffer->NextBuffer)
112: {
113: if (buffer->Address == address)
114: {
115: ASSERT (buffer->InUse);
116:
117: buffer->InUse = FALSE;
118: break;
119: }
120: }
121:
122: ReleaseBufferPoolMutex (queue);
1.1.1.9 root 123: KeSetEvent (&queue->PoolBufferFreeEvent, IO_DISK_INCREMENT, FALSE);
1.1.1.8 root 124: }
125:
126:
127: static void FreePoolBuffers (EncryptedIoQueue *queue)
128: {
129: EncryptedIoQueueBuffer *buffer;
130: AcquireBufferPoolMutex (queue);
131:
132: for (buffer = queue->FirstPoolBuffer; buffer != NULL; )
133: {
134: EncryptedIoQueueBuffer *nextBuffer = buffer->NextBuffer;
135:
1.1.1.13! root 136: ASSERT (!buffer->InUse || queue->StartPending);
1.1.1.8 root 137:
138: TCfree (buffer->Address);
139: TCfree (buffer);
140:
141: buffer = nextBuffer;
142: }
143:
144: queue->FirstPoolBuffer = NULL;
145: ReleaseBufferPoolMutex (queue);
146: }
147:
148:
1.1 root 149: static void DecrementOutstandingIoCount (EncryptedIoQueue *queue)
150: {
151: if (InterlockedDecrement (&queue->OutstandingIoCount) == 0 && (queue->SuspendPending || queue->StopPending))
1.1.1.3 root 152: KeSetEvent (&queue->NoOutstandingIoEvent, IO_DISK_INCREMENT, FALSE);
1.1 root 153: }
154:
155:
1.1.1.5 root 156: static void OnItemCompleted (EncryptedIoQueueItem *item, BOOL freeItem)
1.1 root 157: {
158: DecrementOutstandingIoCount (item->Queue);
1.1.1.10 root 159: IoReleaseRemoveLock (&item->Queue->RemoveLock, item->OriginalIrp);
1.1 root 160:
161: if (NT_SUCCESS (item->Status))
162: {
163: if (item->Write)
164: item->Queue->TotalBytesWritten += item->OriginalLength;
165: else
166: item->Queue->TotalBytesRead += item->OriginalLength;
167: }
168:
1.1.1.5 root 169: if (freeItem)
1.1.1.8 root 170: ReleasePoolBuffer (item->Queue, item);
1.1 root 171: }
172:
173:
174: static NTSTATUS CompleteOriginalIrp (EncryptedIoQueueItem *item, NTSTATUS status, ULONG_PTR information)
175: {
1.1.1.8 root 176: #ifdef TC_TRACE_IO_QUEUE
177: Dump ("< %I64d [%I64d] %c status=%x info=%I64d\n", item->OriginalIrpOffset, GetElapsedTime (&item->Queue->LastPerformanceCounter), item->Write ? 'W' : 'R', status, (int64) information);
178: #endif
179:
1.1 root 180: TCCompleteDiskIrp (item->OriginalIrp, status, information);
1.1.1.5 root 181:
182: item->Status = status;
183: OnItemCompleted (item, TRUE);
184:
1.1 root 185: return status;
186: }
187:
188:
189: static void AcquireFragmentBuffer (EncryptedIoQueue *queue, byte *buffer)
190: {
191: NTSTATUS status = STATUS_INVALID_PARAMETER;
192:
193: if (buffer == queue->FragmentBufferA)
194: {
195: status = KeWaitForSingleObject (&queue->FragmentBufferAFreeEvent, Executive, KernelMode, FALSE, NULL);
196: }
197: else if (buffer == queue->FragmentBufferB)
198: {
199: status = KeWaitForSingleObject (&queue->FragmentBufferBFreeEvent, Executive, KernelMode, FALSE, NULL);
200: }
201:
202: if (!NT_SUCCESS (status))
203: TC_BUG_CHECK (status);
204: }
205:
206:
207: static void ReleaseFragmentBuffer (EncryptedIoQueue *queue, byte *buffer)
208: {
209: if (buffer == queue->FragmentBufferA)
210: {
1.1.1.3 root 211: KeSetEvent (&queue->FragmentBufferAFreeEvent, IO_DISK_INCREMENT, FALSE);
1.1 root 212: }
213: else if (buffer == queue->FragmentBufferB)
214: {
1.1.1.3 root 215: KeSetEvent (&queue->FragmentBufferBFreeEvent, IO_DISK_INCREMENT, FALSE);
1.1 root 216: }
217: else
218: {
219: TC_BUG_CHECK (STATUS_INVALID_PARAMETER);
220: }
221: }
222:
223:
224: static VOID CompletionThreadProc (PVOID threadArg)
225: {
226: EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg;
227: PLIST_ENTRY listEntry;
228: EncryptedIoRequest *request;
229: UINT64_STRUCT dataUnit;
230:
1.1.1.5 root 231: if (IsEncryptionThreadPoolRunning())
232: KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY);
233:
1.1 root 234: while (!queue->ThreadExitRequested)
235: {
236: if (!NT_SUCCESS (KeWaitForSingleObject (&queue->CompletionThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL)))
237: continue;
238:
239: if (queue->ThreadExitRequested)
240: break;
241:
242: while ((listEntry = ExInterlockedRemoveHeadList (&queue->CompletionThreadQueue, &queue->CompletionThreadQueueLock)))
243: {
244: request = CONTAINING_RECORD (listEntry, EncryptedIoRequest, CompletionListEntry);
245:
1.1.1.3 root 246: if (request->EncryptedLength > 0 && NT_SUCCESS (request->Item->Status))
1.1 root 247: {
248: ASSERT (request->EncryptedOffset + request->EncryptedLength <= request->Offset.QuadPart + request->Length);
249: dataUnit.Value = (request->Offset.QuadPart + request->EncryptedOffset) / ENCRYPTION_DATA_UNIT_SIZE;
1.1.1.3 root 250:
251: if (queue->CryptoInfo->bPartitionInInactiveSysEncScope)
252: dataUnit.Value += queue->CryptoInfo->FirstDataUnitNo.Value;
1.1.1.5 root 253: else if (queue->RemapEncryptedArea)
254: dataUnit.Value += queue->RemappedAreaDataUnitOffset;
1.1.1.3 root 255:
1.1 root 256: DecryptDataUnits (request->Data + request->EncryptedOffset, &dataUnit, request->EncryptedLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo);
257: }
258:
259: if (request->CompleteOriginalIrp)
260: {
261: CompleteOriginalIrp (request->Item, request->Item->Status,
262: NT_SUCCESS (request->Item->Status) ? request->Item->OriginalLength : 0);
263: }
264:
1.1.1.8 root 265: ReleasePoolBuffer (queue, request);
1.1 root 266: }
267: }
268:
269: PsTerminateSystemThread (STATUS_SUCCESS);
270: }
271:
272:
1.1.1.8 root 273: static NTSTATUS TCCachedRead (EncryptedIoQueue *queue, IO_STATUS_BLOCK *ioStatus, PVOID buffer, LARGE_INTEGER offset, ULONG length)
274: {
275: queue->LastReadOffset = offset;
276: queue->LastReadLength = length;
277:
278: if (queue->ReadAheadBufferValid && queue->ReadAheadOffset.QuadPart == offset.QuadPart && queue->ReadAheadLength >= length)
279: {
280: memcpy (buffer, queue->ReadAheadBuffer, length);
281:
282: if (!queue->IsFilterDevice)
283: {
284: ioStatus->Information = length;
285: ioStatus->Status = STATUS_SUCCESS;
286: }
287:
288: return STATUS_SUCCESS;
289: }
290:
291: if (queue->IsFilterDevice)
292: return TCReadDevice (queue->LowerDeviceObject, buffer, offset, length);
293:
294: return ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, ioStatus, buffer, length, &offset, NULL);
295: }
296:
297:
1.1 root 298: static VOID IoThreadProc (PVOID threadArg)
299: {
300: EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg;
301: PLIST_ENTRY listEntry;
302: EncryptedIoRequest *request;
303:
1.1.1.4 root 304: KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY);
305:
1.1.1.6 root 306: if (!queue->IsFilterDevice && queue->SecurityClientContext)
307: {
308: #ifdef DEBUG
309: NTSTATUS status =
310: #endif
311: SeImpersonateClientEx (queue->SecurityClientContext, NULL);
312: ASSERT (NT_SUCCESS (status));
313: }
314:
1.1 root 315: while (!queue->ThreadExitRequested)
316: {
317: if (!NT_SUCCESS (KeWaitForSingleObject (&queue->IoThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL)))
318: continue;
319:
320: if (queue->ThreadExitRequested)
321: break;
322:
323: while ((listEntry = ExInterlockedRemoveHeadList (&queue->IoThreadQueue, &queue->IoThreadQueueLock)))
324: {
1.1.1.8 root 325: InterlockedDecrement (&queue->IoThreadPendingRequestCount);
1.1 root 326: request = CONTAINING_RECORD (listEntry, EncryptedIoRequest, ListEntry);
1.1.1.3 root 327:
1.1.1.8 root 328: #ifdef TC_TRACE_IO_QUEUE
329: Dump ("%c %I64d [%I64d] roff=%I64d rlen=%d\n", request->Item->Write ? 'W' : 'R', request->Item->OriginalIrpOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), request->Offset.QuadPart, request->Length);
330: #endif
331:
1.1.1.3 root 332: // Perform IO request if no preceding request of the item failed
333: if (NT_SUCCESS (request->Item->Status))
1.1 root 334: {
1.1.1.3 root 335: if (queue->IsFilterDevice)
336: {
1.1.1.5 root 337: if (queue->RemapEncryptedArea && request->EncryptedLength > 0)
338: {
339: if (request->EncryptedLength != request->Length)
340: {
341: // Up to three subfragments may be required to handle a partially remapped fragment
342: int subFragment;
343: byte *subFragmentData = request->Data;
344:
345: for (subFragment = 0 ; subFragment < 3; ++subFragment)
346: {
347: LARGE_INTEGER subFragmentOffset;
348: ULONG subFragmentLength;
349: subFragmentOffset.QuadPart = request->Offset.QuadPart;
350:
351: switch (subFragment)
352: {
353: case 0:
354: subFragmentLength = (ULONG) request->EncryptedOffset;
355: break;
356:
357: case 1:
358: subFragmentOffset.QuadPart += request->EncryptedOffset + queue->RemappedAreaOffset;
359: subFragmentLength = request->EncryptedLength;
360: break;
361:
362: case 2:
363: subFragmentOffset.QuadPart += request->EncryptedOffset + request->EncryptedLength;
364: subFragmentLength = (ULONG) (request->Length - (request->EncryptedOffset + request->EncryptedLength));
365: break;
366: }
367:
368: if (subFragmentLength > 0)
369: {
370: if (request->Item->Write)
371: request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, subFragmentData, subFragmentOffset, subFragmentLength);
372: else
1.1.1.8 root 373: request->Item->Status = TCCachedRead (queue, NULL, subFragmentData, subFragmentOffset, subFragmentLength);
1.1.1.5 root 374:
375: subFragmentData += subFragmentLength;
376: }
377: }
378: }
379: else
380: {
381: // Remap the fragment
382: LARGE_INTEGER remappedOffset;
383: remappedOffset.QuadPart = request->Offset.QuadPart + queue->RemappedAreaOffset;
384:
385: if (request->Item->Write)
386: request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, remappedOffset, request->Length);
387: else
1.1.1.8 root 388: request->Item->Status = TCCachedRead (queue, NULL, request->Data, remappedOffset, request->Length);
1.1.1.5 root 389: }
390: }
1.1.1.3 root 391: else
1.1.1.5 root 392: {
393: if (request->Item->Write)
394: request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, request->Offset, request->Length);
395: else
1.1.1.8 root 396: request->Item->Status = TCCachedRead (queue, NULL, request->Data, request->Offset, request->Length);
1.1.1.5 root 397: }
1.1.1.3 root 398: }
1.1 root 399: else
1.1.1.3 root 400: {
401: IO_STATUS_BLOCK ioStatus;
1.1 root 402:
1.1.1.3 root 403: if (request->Item->Write)
404: request->Item->Status = ZwWriteFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, request->Data, request->Length, &request->Offset, NULL);
405: else
1.1.1.8 root 406: request->Item->Status = TCCachedRead (queue, &ioStatus, request->Data, request->Offset, request->Length);
1.1.1.5 root 407:
408: if (NT_SUCCESS (request->Item->Status) && ioStatus.Information != request->Length)
409: request->Item->Status = STATUS_END_OF_FILE;
1.1.1.3 root 410: }
1.1 root 411: }
412:
1.1.1.3 root 413: if (request->Item->Write)
1.1 root 414: {
1.1.1.8 root 415: queue->ReadAheadBufferValid = FALSE;
416:
1.1 root 417: ReleaseFragmentBuffer (queue, request->Data);
418:
419: if (request->CompleteOriginalIrp)
420: {
421: CompleteOriginalIrp (request->Item, request->Item->Status,
422: NT_SUCCESS (request->Item->Status) ? request->Item->OriginalLength : 0);
423: }
424:
1.1.1.8 root 425: ReleasePoolBuffer (queue, request);
1.1 root 426: }
1.1.1.3 root 427: else
428: {
1.1.1.8 root 429: BOOL readAhead = FALSE;
430:
1.1.1.3 root 431: if (NT_SUCCESS (request->Item->Status))
432: memcpy (request->OrigDataBufferFragment, request->Data, request->Length);
433:
434: ReleaseFragmentBuffer (queue, request->Data);
435: request->Data = request->OrigDataBufferFragment;
436:
1.1.1.8 root 437: if (request->CompleteOriginalIrp
438: && queue->LastReadLength > 0
439: && NT_SUCCESS (request->Item->Status)
440: && InterlockedExchangeAdd (&queue->IoThreadPendingRequestCount, 0) == 0)
441: {
442: readAhead = TRUE;
443: InterlockedIncrement (&queue->OutstandingIoCount);
444: }
445:
1.1.1.3 root 446: ExInterlockedInsertTailList (&queue->CompletionThreadQueue, &request->CompletionListEntry, &queue->CompletionThreadQueueLock);
447: KeSetEvent (&queue->CompletionThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE);
1.1.1.8 root 448:
449: if (readAhead)
450: {
451: queue->ReadAheadBufferValid = FALSE;
452: queue->ReadAheadOffset.QuadPart = queue->LastReadOffset.QuadPart + queue->LastReadLength;
453: queue->ReadAheadLength = queue->LastReadLength;
454:
455: if (queue->ReadAheadOffset.QuadPart + queue->ReadAheadLength <= queue->MaxReadAheadOffset.QuadPart)
456: {
1.1.1.11 root 457: #ifdef TC_TRACE_IO_QUEUE
458: Dump ("A %I64d [%I64d] roff=%I64d rlen=%d\n", request->Item->OriginalIrpOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), queue->ReadAheadOffset, queue->ReadAheadLength);
459: #endif
1.1.1.8 root 460: if (queue->IsFilterDevice)
461: {
462: queue->ReadAheadBufferValid = NT_SUCCESS (TCReadDevice (queue->LowerDeviceObject, queue->ReadAheadBuffer, queue->ReadAheadOffset, queue->ReadAheadLength));
463: }
464: else
465: {
466: IO_STATUS_BLOCK ioStatus;
467: queue->ReadAheadBufferValid = NT_SUCCESS (ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, queue->ReadAheadBuffer, queue->ReadAheadLength, &queue->ReadAheadOffset, NULL));
468: queue->ReadAheadLength = (ULONG) ioStatus.Information;
469: }
470: }
471:
472: DecrementOutstandingIoCount (queue);
473: }
1.1.1.3 root 474: }
1.1 root 475: }
476: }
477:
478: PsTerminateSystemThread (STATUS_SUCCESS);
479: }
480:
481:
482: static VOID MainThreadProc (PVOID threadArg)
483: {
484: EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg;
485: PLIST_ENTRY listEntry;
486: EncryptedIoQueueItem *item;
487:
488: LARGE_INTEGER fragmentOffset;
489: ULONG dataRemaining;
490: PUCHAR activeFragmentBuffer = queue->FragmentBufferA;
491: PUCHAR dataBuffer;
492: EncryptedIoRequest *request;
493: uint64 intersectStart;
494: uint32 intersectLength;
495:
1.1.1.5 root 496: if (IsEncryptionThreadPoolRunning())
497: KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY);
498:
1.1 root 499: while (!queue->ThreadExitRequested)
500: {
501: if (!NT_SUCCESS (KeWaitForSingleObject (&queue->MainThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL)))
502: continue;
503:
504: while ((listEntry = ExInterlockedRemoveHeadList (&queue->MainThreadQueue, &queue->MainThreadQueueLock)))
505: {
1.1.1.5 root 506: PIRP irp = CONTAINING_RECORD (listEntry, IRP, Tail.Overlay.ListEntry);
507: PIO_STACK_LOCATION irpSp = IoGetCurrentIrpStackLocation (irp);
508:
1.1 root 509: if (queue->Suspended)
510: KeWaitForSingleObject (&queue->QueueResumedEvent, Executive, KernelMode, FALSE, NULL);
1.1.1.5 root 511:
1.1.1.8 root 512: item = GetPoolBuffer (queue, sizeof (EncryptedIoQueueItem));
1.1.1.5 root 513: item->Queue = queue;
514: item->OriginalIrp = irp;
515: item->Status = STATUS_SUCCESS;
516:
517: IoSetCancelRoutine (irp, NULL);
518: if (irp->Cancel)
1.1 root 519: {
520: CompleteOriginalIrp (item, STATUS_CANCELLED, 0);
521: continue;
522: }
523:
1.1.1.5 root 524: switch (irpSp->MajorFunction)
525: {
526: case IRP_MJ_READ:
527: item->Write = FALSE;
528: item->OriginalOffset = irpSp->Parameters.Read.ByteOffset;
529: item->OriginalLength = irpSp->Parameters.Read.Length;
530: break;
531:
532: case IRP_MJ_WRITE:
533: item->Write = TRUE;
534: item->OriginalOffset = irpSp->Parameters.Write.ByteOffset;
535: item->OriginalLength = irpSp->Parameters.Write.Length;
536: break;
537:
538: default:
539: CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0);
540: continue;
541: }
542:
1.1.1.8 root 543: #ifdef TC_TRACE_IO_QUEUE
544: item->OriginalIrpOffset = item->OriginalOffset;
545: #endif
1.1 root 546:
1.1.1.8 root 547: // Handle misaligned read operations to work around a bug in Windows System Assessment Tool which does not follow FILE_FLAG_NO_BUFFERING requirements when benchmarking disk devices
1.1.1.7 root 548: if (queue->IsFilterDevice
549: && !item->Write
550: && item->OriginalLength > 0
551: && (item->OriginalLength & (ENCRYPTION_DATA_UNIT_SIZE - 1)) == 0
552: && (item->OriginalOffset.QuadPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0)
553: {
554: byte *buffer;
555: ULONG alignedLength = item->OriginalLength + ENCRYPTION_DATA_UNIT_SIZE;
556: LARGE_INTEGER alignedOffset;
557: alignedOffset.QuadPart = item->OriginalOffset.QuadPart & ~((LONGLONG) ENCRYPTION_DATA_UNIT_SIZE - 1);
558:
559: buffer = TCalloc (alignedLength);
560: if (!buffer)
561: {
562: CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0);
563: continue;
564: }
565:
566: item->Status = TCReadDevice (queue->LowerDeviceObject, buffer, alignedOffset, alignedLength);
567:
568: if (NT_SUCCESS (item->Status))
569: {
570: UINT64_STRUCT dataUnit;
571:
572: dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, HighPagePriority);
573: if (!dataBuffer)
574: {
575: TCfree (buffer);
576: CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0);
577: continue;
578: }
579:
1.1.1.8 root 580: if (queue->EncryptedAreaStart != -1 && queue->EncryptedAreaEnd != -1)
1.1.1.7 root 581: {
1.1.1.8 root 582: GetIntersection (alignedOffset.QuadPart, alignedLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength);
583: if (intersectLength > 0)
584: {
585: dataUnit.Value = intersectStart / ENCRYPTION_DATA_UNIT_SIZE;
586: DecryptDataUnits (buffer + (intersectStart - alignedOffset.QuadPart), &dataUnit, intersectLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo);
587: }
1.1.1.7 root 588: }
589:
590: memcpy (dataBuffer, buffer + (item->OriginalOffset.LowPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)), item->OriginalLength);
591: }
592:
593: TCfree (buffer);
594: CompleteOriginalIrp (item, item->Status, NT_SUCCESS (item->Status) ? item->OriginalLength : 0);
595: continue;
596: }
597:
1.1 root 598: // Validate offset and length
1.1.1.3 root 599: if (item->OriginalLength == 0
600: || (item->OriginalLength & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0
601: || (item->OriginalOffset.QuadPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0
1.1 root 602: || (!queue->IsFilterDevice && item->OriginalOffset.QuadPart + item->OriginalLength > queue->VirtualDeviceLength))
603: {
604: CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0);
605: continue;
606: }
607:
1.1.1.8 root 608: #ifdef TC_TRACE_IO_QUEUE
609: Dump ("Q %I64d [%I64d] %c len=%d\n", item->OriginalOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), item->Write ? 'W' : 'R', item->OriginalLength);
610: #endif
1.1.1.5 root 611:
1.1 root 612: if (!queue->IsFilterDevice)
613: {
1.1.1.3 root 614: // Adjust the offset for host file or device
615: if (queue->CryptoInfo->hiddenVolume)
616: item->OriginalOffset.QuadPart += queue->CryptoInfo->hiddenVolumeOffset;
617: else
618: item->OriginalOffset.QuadPart += queue->CryptoInfo->volDataAreaOffset;
619:
1.1 root 620: // Hidden volume protection
621: if (item->Write && queue->CryptoInfo->bProtectHiddenVolume)
622: {
623: // If there has already been a write operation denied in order to protect the
624: // hidden volume (since the volume mount time)
625: if (queue->CryptoInfo->bHiddenVolProtectionAction)
626: {
627: // Do not allow writing to this volume anymore. This is to fake a complete volume
628: // or system failure (otherwise certain kinds of inconsistency within the file
629: // system could indicate that this volume has used hidden volume protection).
630: CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0);
631: continue;
632: }
633:
634: // Verify that no byte is going to be written to the hidden volume area
1.1.1.3 root 635: if (RegionsOverlap ((unsigned __int64) item->OriginalOffset.QuadPart,
636: (unsigned __int64) item->OriginalOffset.QuadPart + item->OriginalLength - 1,
1.1 root 637: queue->CryptoInfo->hiddenVolumeOffset,
1.1.1.5 root 638: (unsigned __int64) queue->CryptoInfo->hiddenVolumeOffset + queue->CryptoInfo->hiddenVolumeProtectedSize - 1))
1.1 root 639: {
1.1.1.5 root 640: Dump ("Hidden volume protection triggered: write %I64d-%I64d (protected %I64d-%I64d)\n", item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, queue->CryptoInfo->hiddenVolumeOffset, queue->CryptoInfo->hiddenVolumeOffset + queue->CryptoInfo->hiddenVolumeProtectedSize - 1);
1.1 root 641: queue->CryptoInfo->bHiddenVolProtectionAction = TRUE;
642:
643: // Deny this write operation to prevent the hidden volume from being overwritten
644: CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0);
645: continue;
646: }
647: }
648: }
1.1.1.10 root 649: else if (item->Write
650: && RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, TC_BOOT_VOLUME_HEADER_SECTOR_OFFSET, TC_BOOT_VOLUME_HEADER_SECTOR_OFFSET + TC_BOOT_ENCRYPTION_VOLUME_HEADER_SIZE - 1))
651: {
652: // Prevent inappropriately designed software from damaging important data that may be out of sync with the backup on the Rescue Disk (such as the end of the encrypted area).
653: Dump ("Preventing write to the system encryption key data area\n");
654: CompleteOriginalIrp (item, STATUS_MEDIA_WRITE_PROTECTED, 0);
655: continue;
656: }
1.1.1.6 root 657: else if (item->Write && IsHiddenSystemRunning()
1.1.1.12 root 658: && (RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, TC_SECTOR_SIZE_BIOS, TC_BOOT_LOADER_AREA_SECTOR_COUNT * TC_SECTOR_SIZE_BIOS - 1)
1.1.1.6 root 659: || RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, GetBootDriveLength(), _I64_MAX)))
660: {
661: Dump ("Preventing write to boot loader or host protected area\n");
662: CompleteOriginalIrp (item, STATUS_MEDIA_WRITE_PROTECTED, 0);
663: continue;
664: }
1.1 root 665:
1.1.1.10 root 666: dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, HighPagePriority);
1.1.1.5 root 667:
1.1 root 668: if (dataBuffer == NULL)
669: {
670: CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0);
671: continue;
672: }
673:
674: // Divide data block to fragments to enable efficient overlapping of encryption and IO operations
675:
676: dataRemaining = item->OriginalLength;
677: fragmentOffset = item->OriginalOffset;
678:
679: while (dataRemaining > 0)
680: {
681: BOOL isLastFragment = dataRemaining <= TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE;
682:
683: ULONG dataFragmentLength = isLastFragment ? dataRemaining : TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE;
684: activeFragmentBuffer = (activeFragmentBuffer == queue->FragmentBufferA ? queue->FragmentBufferB : queue->FragmentBufferA);
685:
1.1.1.8 root 686: InterlockedIncrement (&queue->IoThreadPendingRequestCount);
687:
1.1.1.10 root 688: // Create IO request
689: request = GetPoolBuffer (queue, sizeof (EncryptedIoRequest));
1.1 root 690: request->Item = item;
691: request->CompleteOriginalIrp = isLastFragment;
692: request->Offset = fragmentOffset;
693: request->Data = activeFragmentBuffer;
694: request->OrigDataBufferFragment = dataBuffer;
695: request->Length = dataFragmentLength;
696:
697: if (queue->IsFilterDevice)
698: {
1.1.1.8 root 699: if (queue->EncryptedAreaStart == -1 || queue->EncryptedAreaEnd == -1)
700: {
701: request->EncryptedLength = 0;
702: }
703: else
704: {
705: // Get intersection of data fragment with encrypted area
706: GetIntersection (fragmentOffset.QuadPart, dataFragmentLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength);
1.1 root 707:
1.1.1.8 root 708: request->EncryptedOffset = intersectStart - fragmentOffset.QuadPart;
709: request->EncryptedLength = intersectLength;
710: }
1.1 root 711: }
712: else
713: {
714: request->EncryptedOffset = 0;
715: request->EncryptedLength = dataFragmentLength;
716: }
717:
718: AcquireFragmentBuffer (queue, activeFragmentBuffer);
719:
720: if (item->Write)
721: {
722: // Encrypt data
723: memcpy (activeFragmentBuffer, dataBuffer, dataFragmentLength);
724:
725: if (request->EncryptedLength > 0)
726: {
727: UINT64_STRUCT dataUnit;
728: ASSERT (request->EncryptedOffset + request->EncryptedLength <= request->Offset.QuadPart + request->Length);
729:
730: dataUnit.Value = (request->Offset.QuadPart + request->EncryptedOffset) / ENCRYPTION_DATA_UNIT_SIZE;
731:
1.1.1.3 root 732: if (queue->CryptoInfo->bPartitionInInactiveSysEncScope)
733: dataUnit.Value += queue->CryptoInfo->FirstDataUnitNo.Value;
1.1.1.5 root 734: else if (queue->RemapEncryptedArea)
735: dataUnit.Value += queue->RemappedAreaDataUnitOffset;
736:
1.1.1.3 root 737: EncryptDataUnits (activeFragmentBuffer + request->EncryptedOffset, &dataUnit, request->EncryptedLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo);
1.1 root 738: }
739: }
740:
741: // Queue IO request
742: ExInterlockedInsertTailList (&queue->IoThreadQueue, &request->ListEntry, &queue->IoThreadQueueLock);
1.1.1.3 root 743: KeSetEvent (&queue->IoThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE);
1.1 root 744:
745: if (isLastFragment)
746: break;
747:
748: dataRemaining -= TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE;
749: dataBuffer += TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE;
750: fragmentOffset.QuadPart += TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE;
751: }
752: }
753: }
754:
755: PsTerminateSystemThread (STATUS_SUCCESS);
756: }
757:
758:
759: NTSTATUS EncryptedIoQueueAddIrp (EncryptedIoQueue *queue, PIRP irp)
760: {
761: NTSTATUS status;
762:
763: InterlockedIncrement (&queue->OutstandingIoCount);
764: if (queue->StopPending)
765: {
766: Dump ("STATUS_DEVICE_NOT_READY out=%d\n", queue->OutstandingIoCount);
767: status = STATUS_DEVICE_NOT_READY;
768: goto err;
769: }
770:
1.1.1.10 root 771: status = IoAcquireRemoveLock (&queue->RemoveLock, irp);
772: if (!NT_SUCCESS (status))
773: goto err;
1.1 root 774:
1.1.1.8 root 775: #ifdef TC_TRACE_IO_QUEUE
776: {
777: PIO_STACK_LOCATION irpSp = IoGetCurrentIrpStackLocation (irp);
778: Dump ("* %I64d [%I64d] %c len=%d out=%d\n", irpSp->MajorFunction == IRP_MJ_WRITE ? irpSp->Parameters.Write.ByteOffset : irpSp->Parameters.Read.ByteOffset, GetElapsedTime (&queue->LastPerformanceCounter), irpSp->MajorFunction == IRP_MJ_WRITE ? 'W' : 'R', irpSp->MajorFunction == IRP_MJ_WRITE ? irpSp->Parameters.Write.Length : irpSp->Parameters.Read.Length, queue->OutstandingIoCount);
779: }
780: #endif
781:
1.1 root 782: IoMarkIrpPending (irp);
783:
1.1.1.5 root 784: ExInterlockedInsertTailList (&queue->MainThreadQueue, &irp->Tail.Overlay.ListEntry, &queue->MainThreadQueueLock);
1.1.1.3 root 785: KeSetEvent (&queue->MainThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE);
1.1 root 786:
787: return STATUS_PENDING;
788:
789: err:
790: DecrementOutstandingIoCount (queue);
791: return status;
792: }
793:
794:
795: NTSTATUS EncryptedIoQueueHoldWhenIdle (EncryptedIoQueue *queue, int64 timeout)
796: {
797: NTSTATUS status;
798: ASSERT (!queue->Suspended);
799:
800: queue->SuspendPending = TRUE;
801:
802: while (TRUE)
803: {
804: while (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0)
805: {
806: LARGE_INTEGER waitTimeout;
807:
808: waitTimeout.QuadPart = timeout * -10000;
809: status = KeWaitForSingleObject (&queue->NoOutstandingIoEvent, Executive, KernelMode, FALSE, timeout != 0 ? &waitTimeout : NULL);
810:
811: if (status == STATUS_TIMEOUT)
812: status = STATUS_UNSUCCESSFUL;
813:
814: if (!NT_SUCCESS (status))
1.1.1.8 root 815: {
816: queue->SuspendPending = FALSE;
1.1 root 817: return status;
1.1.1.8 root 818: }
1.1.1.6 root 819:
1.1.1.7 root 820: TCSleep (1);
1.1.1.6 root 821: if (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0)
1.1.1.8 root 822: {
823: queue->SuspendPending = FALSE;
1.1.1.6 root 824: return STATUS_UNSUCCESSFUL;
1.1.1.8 root 825: }
1.1 root 826: }
827:
828: KeClearEvent (&queue->QueueResumedEvent);
829: queue->Suspended = TRUE;
830:
831: if (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) == 0)
832: break;
833:
834: queue->Suspended = FALSE;
1.1.1.3 root 835: KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE);
1.1 root 836: }
837:
1.1.1.8 root 838: queue->ReadAheadBufferValid = FALSE;
1.1 root 839:
1.1.1.8 root 840: queue->SuspendPending = FALSE;
1.1 root 841: return STATUS_SUCCESS;
842: }
843:
844:
845: BOOL EncryptedIoQueueIsSuspended (EncryptedIoQueue *queue)
846: {
847: return queue->Suspended;
848: }
849:
850:
851: BOOL EncryptedIoQueueIsRunning (EncryptedIoQueue *queue)
852: {
853: return !queue->StopPending;
854: }
855:
856:
857: NTSTATUS EncryptedIoQueueResumeFromHold (EncryptedIoQueue *queue)
858: {
859: ASSERT (queue->Suspended);
860:
861: queue->Suspended = FALSE;
1.1.1.3 root 862: KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE);
1.1 root 863:
864: return STATUS_SUCCESS;
865: }
866:
867:
1.1.1.6 root 868: NTSTATUS EncryptedIoQueueStart (EncryptedIoQueue *queue)
1.1 root 869: {
870: NTSTATUS status;
1.1.1.8 root 871: EncryptedIoQueueBuffer *buffer;
872: int i;
873:
874: queue->StartPending = TRUE;
1.1 root 875: queue->ThreadExitRequested = FALSE;
876:
1.1.1.8 root 877: queue->OutstandingIoCount = 0;
878: queue->IoThreadPendingRequestCount = 0;
879:
880: queue->FirstPoolBuffer = NULL;
881: KeInitializeMutex (&queue->BufferPoolMutex, 0);
882:
1.1 root 883: KeInitializeEvent (&queue->NoOutstandingIoEvent, SynchronizationEvent, FALSE);
1.1.1.9 root 884: KeInitializeEvent (&queue->PoolBufferFreeEvent, SynchronizationEvent, FALSE);
1.1 root 885: KeInitializeEvent (&queue->QueueResumedEvent, SynchronizationEvent, FALSE);
886:
887: queue->FragmentBufferA = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE);
888: if (!queue->FragmentBufferA)
889: goto noMemory;
890:
891: queue->FragmentBufferB = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE);
892: if (!queue->FragmentBufferB)
893: goto noMemory;
894:
895: KeInitializeEvent (&queue->FragmentBufferAFreeEvent, SynchronizationEvent, TRUE);
896: KeInitializeEvent (&queue->FragmentBufferBFreeEvent, SynchronizationEvent, TRUE);
897:
1.1.1.8 root 898: queue->ReadAheadBufferValid = FALSE;
899: queue->ReadAheadBuffer = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE);
900: if (!queue->ReadAheadBuffer)
901: goto noMemory;
902:
903: // Preallocate buffers
904: for (i = 0; i < TC_ENC_IO_QUEUE_PREALLOCATED_IO_REQUEST_COUNT; ++i)
905: {
906: if (i < TC_ENC_IO_QUEUE_PREALLOCATED_ITEM_COUNT && !GetPoolBuffer (queue, sizeof (EncryptedIoQueueItem)))
907: goto noMemory;
908:
909: if (!GetPoolBuffer (queue, sizeof (EncryptedIoRequest)))
910: goto noMemory;
911: }
912:
913: for (buffer = queue->FirstPoolBuffer; buffer != NULL; buffer = buffer->NextBuffer)
914: {
915: buffer->InUse = FALSE;
916: }
917:
1.1 root 918: // Main thread
919: InitializeListHead (&queue->MainThreadQueue);
920: KeInitializeSpinLock (&queue->MainThreadQueueLock);
921: KeInitializeEvent (&queue->MainThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE);
922:
923: status = TCStartThread (MainThreadProc, queue, &queue->MainThread);
924: if (!NT_SUCCESS (status))
925: goto err;
926:
927: // IO thread
928: InitializeListHead (&queue->IoThreadQueue);
929: KeInitializeSpinLock (&queue->IoThreadQueueLock);
930: KeInitializeEvent (&queue->IoThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE);
931:
1.1.1.6 root 932: status = TCStartThread (IoThreadProc, queue, &queue->IoThread);
1.1 root 933: if (!NT_SUCCESS (status))
934: {
935: queue->ThreadExitRequested = TRUE;
936: TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent);
937: goto err;
938: }
939:
940: // Completion thread
941: InitializeListHead (&queue->CompletionThreadQueue);
942: KeInitializeSpinLock (&queue->CompletionThreadQueueLock);
943: KeInitializeEvent (&queue->CompletionThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE);
944:
945: status = TCStartThread (CompletionThreadProc, queue, &queue->CompletionThread);
946: if (!NT_SUCCESS (status))
947: {
948: queue->ThreadExitRequested = TRUE;
949: TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent);
950: TCStopThread (queue->IoThread, &queue->IoThreadQueueNotEmptyEvent);
951: goto err;
952: }
953:
1.1.1.8 root 954: #ifdef TC_TRACE_IO_QUEUE
955: GetElapsedTimeInit (&queue->LastPerformanceCounter);
956: #endif
957:
1.1 root 958: queue->StopPending = FALSE;
1.1.1.8 root 959: queue->StartPending = FALSE;
960:
1.1 root 961: Dump ("Queue started\n");
962: return STATUS_SUCCESS;
963:
964: noMemory:
965: status = STATUS_INSUFFICIENT_RESOURCES;
966:
967: err:
968: if (queue->FragmentBufferA)
969: TCfree (queue->FragmentBufferA);
970: if (queue->FragmentBufferB)
971: TCfree (queue->FragmentBufferB);
1.1.1.13! root 972: if (queue->ReadAheadBuffer)
! 973: TCfree (queue->ReadAheadBuffer);
1.1 root 974:
1.1.1.8 root 975: FreePoolBuffers (queue);
976:
977: queue->StartPending = FALSE;
1.1 root 978: return status;
979: }
980:
981:
982: NTSTATUS EncryptedIoQueueStop (EncryptedIoQueue *queue)
983: {
984: ASSERT (!queue->StopPending);
985: queue->StopPending = TRUE;
986:
987: while (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0)
988: {
989: KeWaitForSingleObject (&queue->NoOutstandingIoEvent, Executive, KernelMode, FALSE, NULL);
990: }
991:
992: Dump ("Queue stopping out=%d\n", queue->OutstandingIoCount);
993:
994: queue->ThreadExitRequested = TRUE;
995:
996: TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent);
997: TCStopThread (queue->IoThread, &queue->IoThreadQueueNotEmptyEvent);
998: TCStopThread (queue->CompletionThread, &queue->CompletionThreadQueueNotEmptyEvent);
999:
1000: TCfree (queue->FragmentBufferA);
1001: TCfree (queue->FragmentBufferB);
1.1.1.8 root 1002: TCfree (queue->ReadAheadBuffer);
1003:
1004: FreePoolBuffers (queue);
1.1 root 1005:
1006: Dump ("Queue stopped out=%d\n", queue->OutstandingIoCount);
1007: return STATUS_SUCCESS;
1008: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.