From fc37cc4a02ed13d1a73b941a9f80975600fd1b99 Mon Sep 17 00:00:00 2001 From: David Foerster Date: Tue, 10 May 2016 20:20:14 +0200 Subject: Normalize all line terminators --- src/Driver/EncryptedIoQueue.c | 2098 ++++++++++++++++++++--------------------- 1 file changed, 1049 insertions(+), 1049 deletions(-) (limited to 'src/Driver/EncryptedIoQueue.c') diff --git a/src/Driver/EncryptedIoQueue.c b/src/Driver/EncryptedIoQueue.c index 1f57ad5c..637f8504 100644 --- a/src/Driver/EncryptedIoQueue.c +++ b/src/Driver/EncryptedIoQueue.c @@ -1,1049 +1,1049 @@ -/* - Derived from source code of TrueCrypt 7.1a, which is - Copyright (c) 2008-2012 TrueCrypt Developers Association and which is governed - by the TrueCrypt License 3.0. - - Modifications and additions to the original source code (contained in this file) - and all other portions of this file are Copyright (c) 2013-2016 IDRIX - and are governed by the Apache License 2.0 the full text of which is - contained in the file License.txt included in VeraCrypt binary and source - code distribution packages. -*/ - -#include "TCdefs.h" -#include "Apidrvr.h" -#include "Ntdriver.h" -#include "DriveFilter.h" -#include "EncryptedIoQueue.h" -#include "EncryptionThreadPool.h" -#include "Volumes.h" -#include - - -static void AcquireBufferPoolMutex (EncryptedIoQueue *queue) -{ - NTSTATUS status; - - status = KeWaitForMutexObject (&queue->BufferPoolMutex, Executive, KernelMode, FALSE, NULL); - if (!NT_SUCCESS (status)) - TC_BUG_CHECK (status); -} - - -static void ReleaseBufferPoolMutex (EncryptedIoQueue *queue) -{ - KeReleaseMutex (&queue->BufferPoolMutex, FALSE); -} - - -static void *GetPoolBuffer (EncryptedIoQueue *queue, ULONG requestedSize) -{ - EncryptedIoQueueBuffer *buffer; - void *bufferAddress = NULL; - BOOL requestedSizePresentInPool = FALSE; - - while (TRUE) - { - AcquireBufferPoolMutex (queue); - - for (buffer = queue->FirstPoolBuffer; ; buffer = buffer->NextBuffer) - { - if (buffer && buffer->Size == requestedSize) - { - requestedSizePresentInPool = TRUE; - - if (!buffer->InUse) - { - // Reuse a free buffer - buffer->InUse = TRUE; - bufferAddress = buffer->Address; - break; - } - } - - if (!buffer || !buffer->NextBuffer) - { - EncryptedIoQueueBuffer *newBuffer; - - if (requestedSizePresentInPool && !queue->StartPending) - break; - - // Allocate a new buffer - newBuffer = TCalloc (sizeof (EncryptedIoQueueBuffer)); - if (!newBuffer) - { - bufferAddress = NULL; - break; - } - - bufferAddress = TCalloc (requestedSize); - if (bufferAddress) - { - newBuffer->NextBuffer = NULL; - newBuffer->Address = bufferAddress; - newBuffer->Size = requestedSize; - newBuffer->InUse = TRUE; - - if (!buffer) - queue->FirstPoolBuffer = newBuffer; - else - buffer->NextBuffer = newBuffer; - } - else - TCfree (newBuffer); - - break; - } - } - - ReleaseBufferPoolMutex (queue); - - if (bufferAddress || !requestedSizePresentInPool || queue->StartPending) - break; - - KeWaitForSingleObject (&queue->PoolBufferFreeEvent, Executive, KernelMode, FALSE, NULL); - } - - return bufferAddress; -} - - -static void ReleasePoolBuffer (EncryptedIoQueue *queue, void *address) -{ - EncryptedIoQueueBuffer *buffer; - AcquireBufferPoolMutex (queue); - - for (buffer = queue->FirstPoolBuffer; buffer != NULL; buffer = buffer->NextBuffer) - { - if (buffer->Address == address) - { - ASSERT (buffer->InUse); - - buffer->InUse = FALSE; - break; - } - } - - ReleaseBufferPoolMutex (queue); - KeSetEvent (&queue->PoolBufferFreeEvent, IO_DISK_INCREMENT, FALSE); -} - - -static void FreePoolBuffers (EncryptedIoQueue *queue) -{ - EncryptedIoQueueBuffer *buffer; - AcquireBufferPoolMutex (queue); - - for (buffer = queue->FirstPoolBuffer; buffer != NULL; ) - { - EncryptedIoQueueBuffer *nextBuffer = buffer->NextBuffer; - - ASSERT (!buffer->InUse || queue->StartPending); - - TCfree (buffer->Address); - TCfree (buffer); - - buffer = nextBuffer; - } - - queue->FirstPoolBuffer = NULL; - ReleaseBufferPoolMutex (queue); -} - - -static void DecrementOutstandingIoCount (EncryptedIoQueue *queue) -{ - if (InterlockedDecrement (&queue->OutstandingIoCount) == 0 && (queue->SuspendPending || queue->StopPending)) - KeSetEvent (&queue->NoOutstandingIoEvent, IO_DISK_INCREMENT, FALSE); -} - - -static void OnItemCompleted (EncryptedIoQueueItem *item, BOOL freeItem) -{ - DecrementOutstandingIoCount (item->Queue); - IoReleaseRemoveLock (&item->Queue->RemoveLock, item->OriginalIrp); - - if (NT_SUCCESS (item->Status)) - { - if (item->Write) - item->Queue->TotalBytesWritten += item->OriginalLength; - else - item->Queue->TotalBytesRead += item->OriginalLength; - } - - if (freeItem) - ReleasePoolBuffer (item->Queue, item); -} - - -static NTSTATUS CompleteOriginalIrp (EncryptedIoQueueItem *item, NTSTATUS status, ULONG_PTR information) -{ -#ifdef TC_TRACE_IO_QUEUE - Dump ("< %I64d [%I64d] %c status=%x info=%I64d\n", item->OriginalIrpOffset, GetElapsedTime (&item->Queue->LastPerformanceCounter), item->Write ? 'W' : 'R', status, (int64) information); -#endif - - TCCompleteDiskIrp (item->OriginalIrp, status, information); - - item->Status = status; - OnItemCompleted (item, TRUE); - - return status; -} - - -static void AcquireFragmentBuffer (EncryptedIoQueue *queue, byte *buffer) -{ - NTSTATUS status = STATUS_INVALID_PARAMETER; - - if (buffer == queue->FragmentBufferA) - { - status = KeWaitForSingleObject (&queue->FragmentBufferAFreeEvent, Executive, KernelMode, FALSE, NULL); - } - else if (buffer == queue->FragmentBufferB) - { - status = KeWaitForSingleObject (&queue->FragmentBufferBFreeEvent, Executive, KernelMode, FALSE, NULL); - } - - if (!NT_SUCCESS (status)) - TC_BUG_CHECK (status); -} - - -static void ReleaseFragmentBuffer (EncryptedIoQueue *queue, byte *buffer) -{ - if (buffer == queue->FragmentBufferA) - { - KeSetEvent (&queue->FragmentBufferAFreeEvent, IO_DISK_INCREMENT, FALSE); - } - else if (buffer == queue->FragmentBufferB) - { - KeSetEvent (&queue->FragmentBufferBFreeEvent, IO_DISK_INCREMENT, FALSE); - } - else - { - TC_BUG_CHECK (STATUS_INVALID_PARAMETER); - } -} - - -static VOID CompletionThreadProc (PVOID threadArg) -{ - EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg; - PLIST_ENTRY listEntry; - EncryptedIoRequest *request; - UINT64_STRUCT dataUnit; - - if (IsEncryptionThreadPoolRunning()) - KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY); - - while (!queue->ThreadExitRequested) - { - if (!NT_SUCCESS (KeWaitForSingleObject (&queue->CompletionThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL))) - continue; - - if (queue->ThreadExitRequested) - break; - - while ((listEntry = ExInterlockedRemoveHeadList (&queue->CompletionThreadQueue, &queue->CompletionThreadQueueLock))) - { - request = CONTAINING_RECORD (listEntry, EncryptedIoRequest, CompletionListEntry); - - if (request->EncryptedLength > 0 && NT_SUCCESS (request->Item->Status)) - { - ASSERT (request->EncryptedOffset + request->EncryptedLength <= request->Offset.QuadPart + request->Length); - dataUnit.Value = (request->Offset.QuadPart + request->EncryptedOffset) / ENCRYPTION_DATA_UNIT_SIZE; - - if (queue->CryptoInfo->bPartitionInInactiveSysEncScope) - dataUnit.Value += queue->CryptoInfo->FirstDataUnitNo.Value; - else if (queue->RemapEncryptedArea) - dataUnit.Value += queue->RemappedAreaDataUnitOffset; - - DecryptDataUnits (request->Data + request->EncryptedOffset, &dataUnit, request->EncryptedLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo); - } - - if (request->CompleteOriginalIrp) - { - CompleteOriginalIrp (request->Item, request->Item->Status, - NT_SUCCESS (request->Item->Status) ? request->Item->OriginalLength : 0); - } - - ReleasePoolBuffer (queue, request); - } - } - - PsTerminateSystemThread (STATUS_SUCCESS); -} - - -static NTSTATUS TCCachedRead (EncryptedIoQueue *queue, IO_STATUS_BLOCK *ioStatus, PVOID buffer, LARGE_INTEGER offset, ULONG length) -{ - queue->LastReadOffset = offset; - queue->LastReadLength = length; - - if (queue->ReadAheadBufferValid && queue->ReadAheadOffset.QuadPart == offset.QuadPart && queue->ReadAheadLength >= length) - { - memcpy (buffer, queue->ReadAheadBuffer, length); - - if (!queue->IsFilterDevice) - { - ioStatus->Information = length; - ioStatus->Status = STATUS_SUCCESS; - } - - return STATUS_SUCCESS; - } - - if (queue->IsFilterDevice) - return TCReadDevice (queue->LowerDeviceObject, buffer, offset, length); - - return ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, ioStatus, buffer, length, &offset, NULL); -} - - -static VOID IoThreadProc (PVOID threadArg) -{ - EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg; - PLIST_ENTRY listEntry; - EncryptedIoRequest *request; - - KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY); - - if (!queue->IsFilterDevice && queue->SecurityClientContext) - { -#ifdef DEBUG - NTSTATUS status = -#endif - SeImpersonateClientEx (queue->SecurityClientContext, NULL); - ASSERT (NT_SUCCESS (status)); - } - - while (!queue->ThreadExitRequested) - { - if (!NT_SUCCESS (KeWaitForSingleObject (&queue->IoThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL))) - continue; - - if (queue->ThreadExitRequested) - break; - - while ((listEntry = ExInterlockedRemoveHeadList (&queue->IoThreadQueue, &queue->IoThreadQueueLock))) - { - InterlockedDecrement (&queue->IoThreadPendingRequestCount); - request = CONTAINING_RECORD (listEntry, EncryptedIoRequest, ListEntry); - -#ifdef TC_TRACE_IO_QUEUE - Dump ("%c %I64d [%I64d] roff=%I64d rlen=%d\n", request->Item->Write ? 'W' : 'R', request->Item->OriginalIrpOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), request->Offset.QuadPart, request->Length); -#endif - - // Perform IO request if no preceding request of the item failed - if (NT_SUCCESS (request->Item->Status)) - { - if (queue->IsFilterDevice) - { - if (queue->RemapEncryptedArea && request->EncryptedLength > 0) - { - if (request->EncryptedLength != request->Length) - { - // Up to three subfragments may be required to handle a partially remapped fragment - int subFragment; - byte *subFragmentData = request->Data; - - for (subFragment = 0 ; subFragment < 3; ++subFragment) - { - LARGE_INTEGER subFragmentOffset; - ULONG subFragmentLength; - subFragmentOffset.QuadPart = request->Offset.QuadPart; - - switch (subFragment) - { - case 0: - subFragmentLength = (ULONG) request->EncryptedOffset; - break; - - case 1: - subFragmentOffset.QuadPart += request->EncryptedOffset + queue->RemappedAreaOffset; - subFragmentLength = request->EncryptedLength; - break; - - case 2: - subFragmentOffset.QuadPart += request->EncryptedOffset + request->EncryptedLength; - subFragmentLength = (ULONG) (request->Length - (request->EncryptedOffset + request->EncryptedLength)); - break; - } - - if (subFragmentLength > 0) - { - if (request->Item->Write) - request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, subFragmentData, subFragmentOffset, subFragmentLength); - else - request->Item->Status = TCCachedRead (queue, NULL, subFragmentData, subFragmentOffset, subFragmentLength); - - subFragmentData += subFragmentLength; - } - } - } - else - { - // Remap the fragment - LARGE_INTEGER remappedOffset; - remappedOffset.QuadPart = request->Offset.QuadPart + queue->RemappedAreaOffset; - - if (request->Item->Write) - request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, remappedOffset, request->Length); - else - request->Item->Status = TCCachedRead (queue, NULL, request->Data, remappedOffset, request->Length); - } - } - else - { - if (request->Item->Write) - request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, request->Offset, request->Length); - else - request->Item->Status = TCCachedRead (queue, NULL, request->Data, request->Offset, request->Length); - } - } - else - { - IO_STATUS_BLOCK ioStatus; - - if (request->Item->Write) - request->Item->Status = ZwWriteFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, request->Data, request->Length, &request->Offset, NULL); - else - request->Item->Status = TCCachedRead (queue, &ioStatus, request->Data, request->Offset, request->Length); - - if (NT_SUCCESS (request->Item->Status) && ioStatus.Information != request->Length) - request->Item->Status = STATUS_END_OF_FILE; - } - } - - if (request->Item->Write) - { - queue->ReadAheadBufferValid = FALSE; - - ReleaseFragmentBuffer (queue, request->Data); - - if (request->CompleteOriginalIrp) - { - CompleteOriginalIrp (request->Item, request->Item->Status, - NT_SUCCESS (request->Item->Status) ? request->Item->OriginalLength : 0); - } - - ReleasePoolBuffer (queue, request); - } - else - { - BOOL readAhead = FALSE; - - if (NT_SUCCESS (request->Item->Status)) - memcpy (request->OrigDataBufferFragment, request->Data, request->Length); - - ReleaseFragmentBuffer (queue, request->Data); - request->Data = request->OrigDataBufferFragment; - - if (request->CompleteOriginalIrp - && queue->LastReadLength > 0 - && NT_SUCCESS (request->Item->Status) - && InterlockedExchangeAdd (&queue->IoThreadPendingRequestCount, 0) == 0) - { - readAhead = TRUE; - InterlockedIncrement (&queue->OutstandingIoCount); - } - - ExInterlockedInsertTailList (&queue->CompletionThreadQueue, &request->CompletionListEntry, &queue->CompletionThreadQueueLock); - KeSetEvent (&queue->CompletionThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE); - - if (readAhead) - { - queue->ReadAheadBufferValid = FALSE; - queue->ReadAheadOffset.QuadPart = queue->LastReadOffset.QuadPart + queue->LastReadLength; - queue->ReadAheadLength = queue->LastReadLength; - - if (queue->ReadAheadOffset.QuadPart + queue->ReadAheadLength <= queue->MaxReadAheadOffset.QuadPart) - { -#ifdef TC_TRACE_IO_QUEUE - Dump ("A %I64d [%I64d] roff=%I64d rlen=%d\n", request->Item->OriginalIrpOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), queue->ReadAheadOffset, queue->ReadAheadLength); -#endif - if (queue->IsFilterDevice) - { - queue->ReadAheadBufferValid = NT_SUCCESS (TCReadDevice (queue->LowerDeviceObject, queue->ReadAheadBuffer, queue->ReadAheadOffset, queue->ReadAheadLength)); - } - else - { - IO_STATUS_BLOCK ioStatus; - queue->ReadAheadBufferValid = NT_SUCCESS (ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, queue->ReadAheadBuffer, queue->ReadAheadLength, &queue->ReadAheadOffset, NULL)); - queue->ReadAheadLength = (ULONG) ioStatus.Information; - } - } - - DecrementOutstandingIoCount (queue); - } - } - } - } - - PsTerminateSystemThread (STATUS_SUCCESS); -} - - -static VOID MainThreadProc (PVOID threadArg) -{ - EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg; - PLIST_ENTRY listEntry; - EncryptedIoQueueItem *item; - - LARGE_INTEGER fragmentOffset; - ULONG dataRemaining; - PUCHAR activeFragmentBuffer = queue->FragmentBufferA; - PUCHAR dataBuffer; - EncryptedIoRequest *request; - uint64 intersectStart; - uint32 intersectLength; - ULONGLONG addResult; - HRESULT hResult; - - if (IsEncryptionThreadPoolRunning()) - KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY); - - while (!queue->ThreadExitRequested) - { - if (!NT_SUCCESS (KeWaitForSingleObject (&queue->MainThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL))) - continue; - - while ((listEntry = ExInterlockedRemoveHeadList (&queue->MainThreadQueue, &queue->MainThreadQueueLock))) - { - PIRP irp = CONTAINING_RECORD (listEntry, IRP, Tail.Overlay.ListEntry); - PIO_STACK_LOCATION irpSp = IoGetCurrentIrpStackLocation (irp); - - if (queue->Suspended) - KeWaitForSingleObject (&queue->QueueResumedEvent, Executive, KernelMode, FALSE, NULL); - - item = GetPoolBuffer (queue, sizeof (EncryptedIoQueueItem)); - if (!item) - { - TCCompleteDiskIrp (irp, STATUS_INSUFFICIENT_RESOURCES, 0); - DecrementOutstandingIoCount (queue); - IoReleaseRemoveLock (&queue->RemoveLock, irp); - - continue; - } - - item->Queue = queue; - item->OriginalIrp = irp; - item->Status = STATUS_SUCCESS; - - IoSetCancelRoutine (irp, NULL); - if (irp->Cancel) - { - CompleteOriginalIrp (item, STATUS_CANCELLED, 0); - continue; - } - - switch (irpSp->MajorFunction) - { - case IRP_MJ_READ: - item->Write = FALSE; - item->OriginalOffset = irpSp->Parameters.Read.ByteOffset; - item->OriginalLength = irpSp->Parameters.Read.Length; - break; - - case IRP_MJ_WRITE: - item->Write = TRUE; - item->OriginalOffset = irpSp->Parameters.Write.ByteOffset; - item->OriginalLength = irpSp->Parameters.Write.Length; - break; - - default: - CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); - continue; - } - -#ifdef TC_TRACE_IO_QUEUE - item->OriginalIrpOffset = item->OriginalOffset; -#endif - - // Handle misaligned read operations to work around a bug in Windows System Assessment Tool which does not follow FILE_FLAG_NO_BUFFERING requirements when benchmarking disk devices - if (queue->IsFilterDevice - && !item->Write - && item->OriginalLength > 0 - && (item->OriginalLength & (ENCRYPTION_DATA_UNIT_SIZE - 1)) == 0 - && (item->OriginalOffset.QuadPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0) - { - byte *buffer; - ULONG alignedLength; - LARGE_INTEGER alignedOffset; - hResult = ULongAdd(item->OriginalLength, ENCRYPTION_DATA_UNIT_SIZE, &alignedLength); - if (hResult != S_OK) - { - CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); - continue; - } - - alignedOffset.QuadPart = item->OriginalOffset.QuadPart & ~((LONGLONG) ENCRYPTION_DATA_UNIT_SIZE - 1); - - buffer = TCalloc (alignedLength); - if (!buffer) - { - CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); - continue; - } - - item->Status = TCReadDevice (queue->LowerDeviceObject, buffer, alignedOffset, alignedLength); - - if (NT_SUCCESS (item->Status)) - { - UINT64_STRUCT dataUnit; - - dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, HighPagePriority); - if (!dataBuffer) - { - TCfree (buffer); - CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); - continue; - } - - if (queue->EncryptedAreaStart != -1 && queue->EncryptedAreaEnd != -1) - { - GetIntersection (alignedOffset.QuadPart, alignedLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength); - if (intersectLength > 0) - { - dataUnit.Value = intersectStart / ENCRYPTION_DATA_UNIT_SIZE; - DecryptDataUnits (buffer + (intersectStart - alignedOffset.QuadPart), &dataUnit, intersectLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo); - } - } - - memcpy (dataBuffer, buffer + (item->OriginalOffset.LowPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)), item->OriginalLength); - } - - TCfree (buffer); - CompleteOriginalIrp (item, item->Status, NT_SUCCESS (item->Status) ? item->OriginalLength : 0); - continue; - } - - // Validate offset and length - if (item->OriginalLength == 0 - || (item->OriginalLength & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0 - || (item->OriginalOffset.QuadPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0 - || ( !queue->IsFilterDevice && - ( (S_OK != ULongLongAdd(item->OriginalOffset.QuadPart, item->OriginalLength, &addResult)) - || (addResult > (ULONGLONG) queue->VirtualDeviceLength) - ) - ) - ) - { - CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); - continue; - } - -#ifdef TC_TRACE_IO_QUEUE - Dump ("Q %I64d [%I64d] %c len=%d\n", item->OriginalOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), item->Write ? 'W' : 'R', item->OriginalLength); -#endif - - if (!queue->IsFilterDevice) - { - // Adjust the offset for host file or device - if (queue->CryptoInfo->hiddenVolume) - hResult = ULongLongAdd(item->OriginalOffset.QuadPart, queue->CryptoInfo->hiddenVolumeOffset, &addResult); - else - hResult = ULongLongAdd(item->OriginalOffset.QuadPart, queue->CryptoInfo->volDataAreaOffset, &addResult); - - if (hResult != S_OK) - { - CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); - continue; - } - else - item->OriginalOffset.QuadPart = addResult; - - // Hidden volume protection - if (item->Write && queue->CryptoInfo->bProtectHiddenVolume) - { - // If there has already been a write operation denied in order to protect the - // hidden volume (since the volume mount time) - if (queue->CryptoInfo->bHiddenVolProtectionAction) - { - // Do not allow writing to this volume anymore. This is to fake a complete volume - // or system failure (otherwise certain kinds of inconsistency within the file - // system could indicate that this volume has used hidden volume protection). - CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); - continue; - } - - // Verify that no byte is going to be written to the hidden volume area - if (RegionsOverlap ((unsigned __int64) item->OriginalOffset.QuadPart, - (unsigned __int64) item->OriginalOffset.QuadPart + item->OriginalLength - 1, - queue->CryptoInfo->hiddenVolumeOffset, - (unsigned __int64) queue->CryptoInfo->hiddenVolumeOffset + queue->CryptoInfo->hiddenVolumeProtectedSize - 1)) - { - Dump ("Hidden volume protection triggered: write %I64d-%I64d (protected %I64d-%I64d)\n", item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, queue->CryptoInfo->hiddenVolumeOffset, queue->CryptoInfo->hiddenVolumeOffset + queue->CryptoInfo->hiddenVolumeProtectedSize - 1); - queue->CryptoInfo->bHiddenVolProtectionAction = TRUE; - - // Deny this write operation to prevent the hidden volume from being overwritten - CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); - continue; - } - } - } - else if (item->Write - && RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, TC_BOOT_VOLUME_HEADER_SECTOR_OFFSET, TC_BOOT_VOLUME_HEADER_SECTOR_OFFSET + TC_BOOT_ENCRYPTION_VOLUME_HEADER_SIZE - 1)) - { - // Prevent inappropriately designed software from damaging important data that may be out of sync with the backup on the Rescue Disk (such as the end of the encrypted area). - Dump ("Preventing write to the system encryption key data area\n"); - CompleteOriginalIrp (item, STATUS_MEDIA_WRITE_PROTECTED, 0); - continue; - } - else if (item->Write && IsHiddenSystemRunning() - && (RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, TC_SECTOR_SIZE_BIOS, TC_BOOT_LOADER_AREA_SECTOR_COUNT * TC_SECTOR_SIZE_BIOS - 1) - || RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, GetBootDriveLength(), _I64_MAX))) - { - Dump ("Preventing write to boot loader or host protected area\n"); - CompleteOriginalIrp (item, STATUS_MEDIA_WRITE_PROTECTED, 0); - continue; - } - - dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, HighPagePriority); - - if (dataBuffer == NULL) - { - CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); - continue; - } - - // Divide data block to fragments to enable efficient overlapping of encryption and IO operations - - dataRemaining = item->OriginalLength; - fragmentOffset = item->OriginalOffset; - - while (dataRemaining > 0) - { - BOOL isLastFragment = dataRemaining <= TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; - - ULONG dataFragmentLength = isLastFragment ? dataRemaining : TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; - activeFragmentBuffer = (activeFragmentBuffer == queue->FragmentBufferA ? queue->FragmentBufferB : queue->FragmentBufferA); - - InterlockedIncrement (&queue->IoThreadPendingRequestCount); - - // Create IO request - request = GetPoolBuffer (queue, sizeof (EncryptedIoRequest)); - if (!request) - { - CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); - break; - } - request->Item = item; - request->CompleteOriginalIrp = isLastFragment; - request->Offset = fragmentOffset; - request->Data = activeFragmentBuffer; - request->OrigDataBufferFragment = dataBuffer; - request->Length = dataFragmentLength; - - if (queue->IsFilterDevice) - { - if (queue->EncryptedAreaStart == -1 || queue->EncryptedAreaEnd == -1) - { - request->EncryptedLength = 0; - } - else - { - // Get intersection of data fragment with encrypted area - GetIntersection (fragmentOffset.QuadPart, dataFragmentLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength); - - request->EncryptedOffset = intersectStart - fragmentOffset.QuadPart; - request->EncryptedLength = intersectLength; - } - } - else - { - request->EncryptedOffset = 0; - request->EncryptedLength = dataFragmentLength; - } - - AcquireFragmentBuffer (queue, activeFragmentBuffer); - - if (item->Write) - { - // Encrypt data - memcpy (activeFragmentBuffer, dataBuffer, dataFragmentLength); - - if (request->EncryptedLength > 0) - { - UINT64_STRUCT dataUnit; - ASSERT (request->EncryptedOffset + request->EncryptedLength <= request->Offset.QuadPart + request->Length); - - dataUnit.Value = (request->Offset.QuadPart + request->EncryptedOffset) / ENCRYPTION_DATA_UNIT_SIZE; - - if (queue->CryptoInfo->bPartitionInInactiveSysEncScope) - dataUnit.Value += queue->CryptoInfo->FirstDataUnitNo.Value; - else if (queue->RemapEncryptedArea) - dataUnit.Value += queue->RemappedAreaDataUnitOffset; - - EncryptDataUnits (activeFragmentBuffer + request->EncryptedOffset, &dataUnit, request->EncryptedLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo); - } - } - - // Queue IO request - ExInterlockedInsertTailList (&queue->IoThreadQueue, &request->ListEntry, &queue->IoThreadQueueLock); - KeSetEvent (&queue->IoThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE); - - if (isLastFragment) - break; - - dataRemaining -= TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; - dataBuffer += TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; - fragmentOffset.QuadPart += TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; - } - } - } - - PsTerminateSystemThread (STATUS_SUCCESS); -} - - -NTSTATUS EncryptedIoQueueAddIrp (EncryptedIoQueue *queue, PIRP irp) -{ - NTSTATUS status; - - InterlockedIncrement (&queue->OutstandingIoCount); - if (queue->StopPending) - { - Dump ("STATUS_DEVICE_NOT_READY out=%d\n", queue->OutstandingIoCount); - status = STATUS_DEVICE_NOT_READY; - goto err; - } - - status = IoAcquireRemoveLock (&queue->RemoveLock, irp); - if (!NT_SUCCESS (status)) - goto err; - -#ifdef TC_TRACE_IO_QUEUE - { - PIO_STACK_LOCATION irpSp = IoGetCurrentIrpStackLocation (irp); - Dump ("* %I64d [%I64d] %c len=%d out=%d\n", irpSp->MajorFunction == IRP_MJ_WRITE ? irpSp->Parameters.Write.ByteOffset : irpSp->Parameters.Read.ByteOffset, GetElapsedTime (&queue->LastPerformanceCounter), irpSp->MajorFunction == IRP_MJ_WRITE ? 'W' : 'R', irpSp->MajorFunction == IRP_MJ_WRITE ? irpSp->Parameters.Write.Length : irpSp->Parameters.Read.Length, queue->OutstandingIoCount); - } -#endif - - IoMarkIrpPending (irp); - - ExInterlockedInsertTailList (&queue->MainThreadQueue, &irp->Tail.Overlay.ListEntry, &queue->MainThreadQueueLock); - KeSetEvent (&queue->MainThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE); - - return STATUS_PENDING; - -err: - DecrementOutstandingIoCount (queue); - return status; -} - - -NTSTATUS EncryptedIoQueueHoldWhenIdle (EncryptedIoQueue *queue, int64 timeout) -{ - NTSTATUS status; - ASSERT (!queue->Suspended); - - queue->SuspendPending = TRUE; - - while (TRUE) - { - while (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0) - { - LARGE_INTEGER waitTimeout; - - waitTimeout.QuadPart = timeout * -10000; - status = KeWaitForSingleObject (&queue->NoOutstandingIoEvent, Executive, KernelMode, FALSE, timeout != 0 ? &waitTimeout : NULL); - - if (status == STATUS_TIMEOUT) - status = STATUS_UNSUCCESSFUL; - - if (!NT_SUCCESS (status)) - { - queue->SuspendPending = FALSE; - return status; - } - - TCSleep (1); - if (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0) - { - queue->SuspendPending = FALSE; - return STATUS_UNSUCCESSFUL; - } - } - - KeClearEvent (&queue->QueueResumedEvent); - queue->Suspended = TRUE; - - if (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) == 0) - break; - - queue->Suspended = FALSE; - KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE); - } - - queue->ReadAheadBufferValid = FALSE; - - queue->SuspendPending = FALSE; - return STATUS_SUCCESS; -} - - -BOOL EncryptedIoQueueIsSuspended (EncryptedIoQueue *queue) -{ - return queue->Suspended; -} - - -BOOL EncryptedIoQueueIsRunning (EncryptedIoQueue *queue) -{ - return !queue->StopPending; -} - - -NTSTATUS EncryptedIoQueueResumeFromHold (EncryptedIoQueue *queue) -{ - ASSERT (queue->Suspended); - - queue->Suspended = FALSE; - KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE); - - return STATUS_SUCCESS; -} - - -NTSTATUS EncryptedIoQueueStart (EncryptedIoQueue *queue) -{ - NTSTATUS status; - EncryptedIoQueueBuffer *buffer; - int i; - - queue->StartPending = TRUE; - queue->ThreadExitRequested = FALSE; - - queue->OutstandingIoCount = 0; - queue->IoThreadPendingRequestCount = 0; - - queue->FirstPoolBuffer = NULL; - KeInitializeMutex (&queue->BufferPoolMutex, 0); - - KeInitializeEvent (&queue->NoOutstandingIoEvent, SynchronizationEvent, FALSE); - KeInitializeEvent (&queue->PoolBufferFreeEvent, SynchronizationEvent, FALSE); - KeInitializeEvent (&queue->QueueResumedEvent, SynchronizationEvent, FALSE); - - queue->FragmentBufferA = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE); - if (!queue->FragmentBufferA) - goto noMemory; - - queue->FragmentBufferB = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE); - if (!queue->FragmentBufferB) - goto noMemory; - - KeInitializeEvent (&queue->FragmentBufferAFreeEvent, SynchronizationEvent, TRUE); - KeInitializeEvent (&queue->FragmentBufferBFreeEvent, SynchronizationEvent, TRUE); - - queue->ReadAheadBufferValid = FALSE; - queue->ReadAheadBuffer = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE); - if (!queue->ReadAheadBuffer) - goto noMemory; - - // Preallocate buffers - for (i = 0; i < TC_ENC_IO_QUEUE_PREALLOCATED_IO_REQUEST_COUNT; ++i) - { - if (i < TC_ENC_IO_QUEUE_PREALLOCATED_ITEM_COUNT && !GetPoolBuffer (queue, sizeof (EncryptedIoQueueItem))) - goto noMemory; - - if (!GetPoolBuffer (queue, sizeof (EncryptedIoRequest))) - goto noMemory; - } - - for (buffer = queue->FirstPoolBuffer; buffer != NULL; buffer = buffer->NextBuffer) - { - buffer->InUse = FALSE; - } - - // Main thread - InitializeListHead (&queue->MainThreadQueue); - KeInitializeSpinLock (&queue->MainThreadQueueLock); - KeInitializeEvent (&queue->MainThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE); - - status = TCStartThread (MainThreadProc, queue, &queue->MainThread); - if (!NT_SUCCESS (status)) - goto err; - - // IO thread - InitializeListHead (&queue->IoThreadQueue); - KeInitializeSpinLock (&queue->IoThreadQueueLock); - KeInitializeEvent (&queue->IoThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE); - - status = TCStartThread (IoThreadProc, queue, &queue->IoThread); - if (!NT_SUCCESS (status)) - { - queue->ThreadExitRequested = TRUE; - TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent); - goto err; - } - - // Completion thread - InitializeListHead (&queue->CompletionThreadQueue); - KeInitializeSpinLock (&queue->CompletionThreadQueueLock); - KeInitializeEvent (&queue->CompletionThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE); - - status = TCStartThread (CompletionThreadProc, queue, &queue->CompletionThread); - if (!NT_SUCCESS (status)) - { - queue->ThreadExitRequested = TRUE; - TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent); - TCStopThread (queue->IoThread, &queue->IoThreadQueueNotEmptyEvent); - goto err; - } - -#ifdef TC_TRACE_IO_QUEUE - GetElapsedTimeInit (&queue->LastPerformanceCounter); -#endif - - queue->StopPending = FALSE; - queue->StartPending = FALSE; - - Dump ("Queue started\n"); - return STATUS_SUCCESS; - -noMemory: - status = STATUS_INSUFFICIENT_RESOURCES; - -err: - if (queue->FragmentBufferA) - TCfree (queue->FragmentBufferA); - if (queue->FragmentBufferB) - TCfree (queue->FragmentBufferB); - if (queue->ReadAheadBuffer) - TCfree (queue->ReadAheadBuffer); - - FreePoolBuffers (queue); - - queue->StartPending = FALSE; - return status; -} - - -NTSTATUS EncryptedIoQueueStop (EncryptedIoQueue *queue) -{ - ASSERT (!queue->StopPending); - queue->StopPending = TRUE; - - while (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0) - { - KeWaitForSingleObject (&queue->NoOutstandingIoEvent, Executive, KernelMode, FALSE, NULL); - } - - Dump ("Queue stopping out=%d\n", queue->OutstandingIoCount); - - queue->ThreadExitRequested = TRUE; - - TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent); - TCStopThread (queue->IoThread, &queue->IoThreadQueueNotEmptyEvent); - TCStopThread (queue->CompletionThread, &queue->CompletionThreadQueueNotEmptyEvent); - - TCfree (queue->FragmentBufferA); - TCfree (queue->FragmentBufferB); - TCfree (queue->ReadAheadBuffer); - - FreePoolBuffers (queue); - - Dump ("Queue stopped out=%d\n", queue->OutstandingIoCount); - return STATUS_SUCCESS; -} +/* + Derived from source code of TrueCrypt 7.1a, which is + Copyright (c) 2008-2012 TrueCrypt Developers Association and which is governed + by the TrueCrypt License 3.0. + + Modifications and additions to the original source code (contained in this file) + and all other portions of this file are Copyright (c) 2013-2016 IDRIX + and are governed by the Apache License 2.0 the full text of which is + contained in the file License.txt included in VeraCrypt binary and source + code distribution packages. +*/ + +#include "TCdefs.h" +#include "Apidrvr.h" +#include "Ntdriver.h" +#include "DriveFilter.h" +#include "EncryptedIoQueue.h" +#include "EncryptionThreadPool.h" +#include "Volumes.h" +#include + + +static void AcquireBufferPoolMutex (EncryptedIoQueue *queue) +{ + NTSTATUS status; + + status = KeWaitForMutexObject (&queue->BufferPoolMutex, Executive, KernelMode, FALSE, NULL); + if (!NT_SUCCESS (status)) + TC_BUG_CHECK (status); +} + + +static void ReleaseBufferPoolMutex (EncryptedIoQueue *queue) +{ + KeReleaseMutex (&queue->BufferPoolMutex, FALSE); +} + + +static void *GetPoolBuffer (EncryptedIoQueue *queue, ULONG requestedSize) +{ + EncryptedIoQueueBuffer *buffer; + void *bufferAddress = NULL; + BOOL requestedSizePresentInPool = FALSE; + + while (TRUE) + { + AcquireBufferPoolMutex (queue); + + for (buffer = queue->FirstPoolBuffer; ; buffer = buffer->NextBuffer) + { + if (buffer && buffer->Size == requestedSize) + { + requestedSizePresentInPool = TRUE; + + if (!buffer->InUse) + { + // Reuse a free buffer + buffer->InUse = TRUE; + bufferAddress = buffer->Address; + break; + } + } + + if (!buffer || !buffer->NextBuffer) + { + EncryptedIoQueueBuffer *newBuffer; + + if (requestedSizePresentInPool && !queue->StartPending) + break; + + // Allocate a new buffer + newBuffer = TCalloc (sizeof (EncryptedIoQueueBuffer)); + if (!newBuffer) + { + bufferAddress = NULL; + break; + } + + bufferAddress = TCalloc (requestedSize); + if (bufferAddress) + { + newBuffer->NextBuffer = NULL; + newBuffer->Address = bufferAddress; + newBuffer->Size = requestedSize; + newBuffer->InUse = TRUE; + + if (!buffer) + queue->FirstPoolBuffer = newBuffer; + else + buffer->NextBuffer = newBuffer; + } + else + TCfree (newBuffer); + + break; + } + } + + ReleaseBufferPoolMutex (queue); + + if (bufferAddress || !requestedSizePresentInPool || queue->StartPending) + break; + + KeWaitForSingleObject (&queue->PoolBufferFreeEvent, Executive, KernelMode, FALSE, NULL); + } + + return bufferAddress; +} + + +static void ReleasePoolBuffer (EncryptedIoQueue *queue, void *address) +{ + EncryptedIoQueueBuffer *buffer; + AcquireBufferPoolMutex (queue); + + for (buffer = queue->FirstPoolBuffer; buffer != NULL; buffer = buffer->NextBuffer) + { + if (buffer->Address == address) + { + ASSERT (buffer->InUse); + + buffer->InUse = FALSE; + break; + } + } + + ReleaseBufferPoolMutex (queue); + KeSetEvent (&queue->PoolBufferFreeEvent, IO_DISK_INCREMENT, FALSE); +} + + +static void FreePoolBuffers (EncryptedIoQueue *queue) +{ + EncryptedIoQueueBuffer *buffer; + AcquireBufferPoolMutex (queue); + + for (buffer = queue->FirstPoolBuffer; buffer != NULL; ) + { + EncryptedIoQueueBuffer *nextBuffer = buffer->NextBuffer; + + ASSERT (!buffer->InUse || queue->StartPending); + + TCfree (buffer->Address); + TCfree (buffer); + + buffer = nextBuffer; + } + + queue->FirstPoolBuffer = NULL; + ReleaseBufferPoolMutex (queue); +} + + +static void DecrementOutstandingIoCount (EncryptedIoQueue *queue) +{ + if (InterlockedDecrement (&queue->OutstandingIoCount) == 0 && (queue->SuspendPending || queue->StopPending)) + KeSetEvent (&queue->NoOutstandingIoEvent, IO_DISK_INCREMENT, FALSE); +} + + +static void OnItemCompleted (EncryptedIoQueueItem *item, BOOL freeItem) +{ + DecrementOutstandingIoCount (item->Queue); + IoReleaseRemoveLock (&item->Queue->RemoveLock, item->OriginalIrp); + + if (NT_SUCCESS (item->Status)) + { + if (item->Write) + item->Queue->TotalBytesWritten += item->OriginalLength; + else + item->Queue->TotalBytesRead += item->OriginalLength; + } + + if (freeItem) + ReleasePoolBuffer (item->Queue, item); +} + + +static NTSTATUS CompleteOriginalIrp (EncryptedIoQueueItem *item, NTSTATUS status, ULONG_PTR information) +{ +#ifdef TC_TRACE_IO_QUEUE + Dump ("< %I64d [%I64d] %c status=%x info=%I64d\n", item->OriginalIrpOffset, GetElapsedTime (&item->Queue->LastPerformanceCounter), item->Write ? 'W' : 'R', status, (int64) information); +#endif + + TCCompleteDiskIrp (item->OriginalIrp, status, information); + + item->Status = status; + OnItemCompleted (item, TRUE); + + return status; +} + + +static void AcquireFragmentBuffer (EncryptedIoQueue *queue, byte *buffer) +{ + NTSTATUS status = STATUS_INVALID_PARAMETER; + + if (buffer == queue->FragmentBufferA) + { + status = KeWaitForSingleObject (&queue->FragmentBufferAFreeEvent, Executive, KernelMode, FALSE, NULL); + } + else if (buffer == queue->FragmentBufferB) + { + status = KeWaitForSingleObject (&queue->FragmentBufferBFreeEvent, Executive, KernelMode, FALSE, NULL); + } + + if (!NT_SUCCESS (status)) + TC_BUG_CHECK (status); +} + + +static void ReleaseFragmentBuffer (EncryptedIoQueue *queue, byte *buffer) +{ + if (buffer == queue->FragmentBufferA) + { + KeSetEvent (&queue->FragmentBufferAFreeEvent, IO_DISK_INCREMENT, FALSE); + } + else if (buffer == queue->FragmentBufferB) + { + KeSetEvent (&queue->FragmentBufferBFreeEvent, IO_DISK_INCREMENT, FALSE); + } + else + { + TC_BUG_CHECK (STATUS_INVALID_PARAMETER); + } +} + + +static VOID CompletionThreadProc (PVOID threadArg) +{ + EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg; + PLIST_ENTRY listEntry; + EncryptedIoRequest *request; + UINT64_STRUCT dataUnit; + + if (IsEncryptionThreadPoolRunning()) + KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY); + + while (!queue->ThreadExitRequested) + { + if (!NT_SUCCESS (KeWaitForSingleObject (&queue->CompletionThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL))) + continue; + + if (queue->ThreadExitRequested) + break; + + while ((listEntry = ExInterlockedRemoveHeadList (&queue->CompletionThreadQueue, &queue->CompletionThreadQueueLock))) + { + request = CONTAINING_RECORD (listEntry, EncryptedIoRequest, CompletionListEntry); + + if (request->EncryptedLength > 0 && NT_SUCCESS (request->Item->Status)) + { + ASSERT (request->EncryptedOffset + request->EncryptedLength <= request->Offset.QuadPart + request->Length); + dataUnit.Value = (request->Offset.QuadPart + request->EncryptedOffset) / ENCRYPTION_DATA_UNIT_SIZE; + + if (queue->CryptoInfo->bPartitionInInactiveSysEncScope) + dataUnit.Value += queue->CryptoInfo->FirstDataUnitNo.Value; + else if (queue->RemapEncryptedArea) + dataUnit.Value += queue->RemappedAreaDataUnitOffset; + + DecryptDataUnits (request->Data + request->EncryptedOffset, &dataUnit, request->EncryptedLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo); + } + + if (request->CompleteOriginalIrp) + { + CompleteOriginalIrp (request->Item, request->Item->Status, + NT_SUCCESS (request->Item->Status) ? request->Item->OriginalLength : 0); + } + + ReleasePoolBuffer (queue, request); + } + } + + PsTerminateSystemThread (STATUS_SUCCESS); +} + + +static NTSTATUS TCCachedRead (EncryptedIoQueue *queue, IO_STATUS_BLOCK *ioStatus, PVOID buffer, LARGE_INTEGER offset, ULONG length) +{ + queue->LastReadOffset = offset; + queue->LastReadLength = length; + + if (queue->ReadAheadBufferValid && queue->ReadAheadOffset.QuadPart == offset.QuadPart && queue->ReadAheadLength >= length) + { + memcpy (buffer, queue->ReadAheadBuffer, length); + + if (!queue->IsFilterDevice) + { + ioStatus->Information = length; + ioStatus->Status = STATUS_SUCCESS; + } + + return STATUS_SUCCESS; + } + + if (queue->IsFilterDevice) + return TCReadDevice (queue->LowerDeviceObject, buffer, offset, length); + + return ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, ioStatus, buffer, length, &offset, NULL); +} + + +static VOID IoThreadProc (PVOID threadArg) +{ + EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg; + PLIST_ENTRY listEntry; + EncryptedIoRequest *request; + + KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY); + + if (!queue->IsFilterDevice && queue->SecurityClientContext) + { +#ifdef DEBUG + NTSTATUS status = +#endif + SeImpersonateClientEx (queue->SecurityClientContext, NULL); + ASSERT (NT_SUCCESS (status)); + } + + while (!queue->ThreadExitRequested) + { + if (!NT_SUCCESS (KeWaitForSingleObject (&queue->IoThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL))) + continue; + + if (queue->ThreadExitRequested) + break; + + while ((listEntry = ExInterlockedRemoveHeadList (&queue->IoThreadQueue, &queue->IoThreadQueueLock))) + { + InterlockedDecrement (&queue->IoThreadPendingRequestCount); + request = CONTAINING_RECORD (listEntry, EncryptedIoRequest, ListEntry); + +#ifdef TC_TRACE_IO_QUEUE + Dump ("%c %I64d [%I64d] roff=%I64d rlen=%d\n", request->Item->Write ? 'W' : 'R', request->Item->OriginalIrpOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), request->Offset.QuadPart, request->Length); +#endif + + // Perform IO request if no preceding request of the item failed + if (NT_SUCCESS (request->Item->Status)) + { + if (queue->IsFilterDevice) + { + if (queue->RemapEncryptedArea && request->EncryptedLength > 0) + { + if (request->EncryptedLength != request->Length) + { + // Up to three subfragments may be required to handle a partially remapped fragment + int subFragment; + byte *subFragmentData = request->Data; + + for (subFragment = 0 ; subFragment < 3; ++subFragment) + { + LARGE_INTEGER subFragmentOffset; + ULONG subFragmentLength; + subFragmentOffset.QuadPart = request->Offset.QuadPart; + + switch (subFragment) + { + case 0: + subFragmentLength = (ULONG) request->EncryptedOffset; + break; + + case 1: + subFragmentOffset.QuadPart += request->EncryptedOffset + queue->RemappedAreaOffset; + subFragmentLength = request->EncryptedLength; + break; + + case 2: + subFragmentOffset.QuadPart += request->EncryptedOffset + request->EncryptedLength; + subFragmentLength = (ULONG) (request->Length - (request->EncryptedOffset + request->EncryptedLength)); + break; + } + + if (subFragmentLength > 0) + { + if (request->Item->Write) + request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, subFragmentData, subFragmentOffset, subFragmentLength); + else + request->Item->Status = TCCachedRead (queue, NULL, subFragmentData, subFragmentOffset, subFragmentLength); + + subFragmentData += subFragmentLength; + } + } + } + else + { + // Remap the fragment + LARGE_INTEGER remappedOffset; + remappedOffset.QuadPart = request->Offset.QuadPart + queue->RemappedAreaOffset; + + if (request->Item->Write) + request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, remappedOffset, request->Length); + else + request->Item->Status = TCCachedRead (queue, NULL, request->Data, remappedOffset, request->Length); + } + } + else + { + if (request->Item->Write) + request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, request->Offset, request->Length); + else + request->Item->Status = TCCachedRead (queue, NULL, request->Data, request->Offset, request->Length); + } + } + else + { + IO_STATUS_BLOCK ioStatus; + + if (request->Item->Write) + request->Item->Status = ZwWriteFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, request->Data, request->Length, &request->Offset, NULL); + else + request->Item->Status = TCCachedRead (queue, &ioStatus, request->Data, request->Offset, request->Length); + + if (NT_SUCCESS (request->Item->Status) && ioStatus.Information != request->Length) + request->Item->Status = STATUS_END_OF_FILE; + } + } + + if (request->Item->Write) + { + queue->ReadAheadBufferValid = FALSE; + + ReleaseFragmentBuffer (queue, request->Data); + + if (request->CompleteOriginalIrp) + { + CompleteOriginalIrp (request->Item, request->Item->Status, + NT_SUCCESS (request->Item->Status) ? request->Item->OriginalLength : 0); + } + + ReleasePoolBuffer (queue, request); + } + else + { + BOOL readAhead = FALSE; + + if (NT_SUCCESS (request->Item->Status)) + memcpy (request->OrigDataBufferFragment, request->Data, request->Length); + + ReleaseFragmentBuffer (queue, request->Data); + request->Data = request->OrigDataBufferFragment; + + if (request->CompleteOriginalIrp + && queue->LastReadLength > 0 + && NT_SUCCESS (request->Item->Status) + && InterlockedExchangeAdd (&queue->IoThreadPendingRequestCount, 0) == 0) + { + readAhead = TRUE; + InterlockedIncrement (&queue->OutstandingIoCount); + } + + ExInterlockedInsertTailList (&queue->CompletionThreadQueue, &request->CompletionListEntry, &queue->CompletionThreadQueueLock); + KeSetEvent (&queue->CompletionThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE); + + if (readAhead) + { + queue->ReadAheadBufferValid = FALSE; + queue->ReadAheadOffset.QuadPart = queue->LastReadOffset.QuadPart + queue->LastReadLength; + queue->ReadAheadLength = queue->LastReadLength; + + if (queue->ReadAheadOffset.QuadPart + queue->ReadAheadLength <= queue->MaxReadAheadOffset.QuadPart) + { +#ifdef TC_TRACE_IO_QUEUE + Dump ("A %I64d [%I64d] roff=%I64d rlen=%d\n", request->Item->OriginalIrpOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), queue->ReadAheadOffset, queue->ReadAheadLength); +#endif + if (queue->IsFilterDevice) + { + queue->ReadAheadBufferValid = NT_SUCCESS (TCReadDevice (queue->LowerDeviceObject, queue->ReadAheadBuffer, queue->ReadAheadOffset, queue->ReadAheadLength)); + } + else + { + IO_STATUS_BLOCK ioStatus; + queue->ReadAheadBufferValid = NT_SUCCESS (ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, queue->ReadAheadBuffer, queue->ReadAheadLength, &queue->ReadAheadOffset, NULL)); + queue->ReadAheadLength = (ULONG) ioStatus.Information; + } + } + + DecrementOutstandingIoCount (queue); + } + } + } + } + + PsTerminateSystemThread (STATUS_SUCCESS); +} + + +static VOID MainThreadProc (PVOID threadArg) +{ + EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg; + PLIST_ENTRY listEntry; + EncryptedIoQueueItem *item; + + LARGE_INTEGER fragmentOffset; + ULONG dataRemaining; + PUCHAR activeFragmentBuffer = queue->FragmentBufferA; + PUCHAR dataBuffer; + EncryptedIoRequest *request; + uint64 intersectStart; + uint32 intersectLength; + ULONGLONG addResult; + HRESULT hResult; + + if (IsEncryptionThreadPoolRunning()) + KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY); + + while (!queue->ThreadExitRequested) + { + if (!NT_SUCCESS (KeWaitForSingleObject (&queue->MainThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL))) + continue; + + while ((listEntry = ExInterlockedRemoveHeadList (&queue->MainThreadQueue, &queue->MainThreadQueueLock))) + { + PIRP irp = CONTAINING_RECORD (listEntry, IRP, Tail.Overlay.ListEntry); + PIO_STACK_LOCATION irpSp = IoGetCurrentIrpStackLocation (irp); + + if (queue->Suspended) + KeWaitForSingleObject (&queue->QueueResumedEvent, Executive, KernelMode, FALSE, NULL); + + item = GetPoolBuffer (queue, sizeof (EncryptedIoQueueItem)); + if (!item) + { + TCCompleteDiskIrp (irp, STATUS_INSUFFICIENT_RESOURCES, 0); + DecrementOutstandingIoCount (queue); + IoReleaseRemoveLock (&queue->RemoveLock, irp); + + continue; + } + + item->Queue = queue; + item->OriginalIrp = irp; + item->Status = STATUS_SUCCESS; + + IoSetCancelRoutine (irp, NULL); + if (irp->Cancel) + { + CompleteOriginalIrp (item, STATUS_CANCELLED, 0); + continue; + } + + switch (irpSp->MajorFunction) + { + case IRP_MJ_READ: + item->Write = FALSE; + item->OriginalOffset = irpSp->Parameters.Read.ByteOffset; + item->OriginalLength = irpSp->Parameters.Read.Length; + break; + + case IRP_MJ_WRITE: + item->Write = TRUE; + item->OriginalOffset = irpSp->Parameters.Write.ByteOffset; + item->OriginalLength = irpSp->Parameters.Write.Length; + break; + + default: + CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); + continue; + } + +#ifdef TC_TRACE_IO_QUEUE + item->OriginalIrpOffset = item->OriginalOffset; +#endif + + // Handle misaligned read operations to work around a bug in Windows System Assessment Tool which does not follow FILE_FLAG_NO_BUFFERING requirements when benchmarking disk devices + if (queue->IsFilterDevice + && !item->Write + && item->OriginalLength > 0 + && (item->OriginalLength & (ENCRYPTION_DATA_UNIT_SIZE - 1)) == 0 + && (item->OriginalOffset.QuadPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0) + { + byte *buffer; + ULONG alignedLength; + LARGE_INTEGER alignedOffset; + hResult = ULongAdd(item->OriginalLength, ENCRYPTION_DATA_UNIT_SIZE, &alignedLength); + if (hResult != S_OK) + { + CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); + continue; + } + + alignedOffset.QuadPart = item->OriginalOffset.QuadPart & ~((LONGLONG) ENCRYPTION_DATA_UNIT_SIZE - 1); + + buffer = TCalloc (alignedLength); + if (!buffer) + { + CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); + continue; + } + + item->Status = TCReadDevice (queue->LowerDeviceObject, buffer, alignedOffset, alignedLength); + + if (NT_SUCCESS (item->Status)) + { + UINT64_STRUCT dataUnit; + + dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, HighPagePriority); + if (!dataBuffer) + { + TCfree (buffer); + CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); + continue; + } + + if (queue->EncryptedAreaStart != -1 && queue->EncryptedAreaEnd != -1) + { + GetIntersection (alignedOffset.QuadPart, alignedLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength); + if (intersectLength > 0) + { + dataUnit.Value = intersectStart / ENCRYPTION_DATA_UNIT_SIZE; + DecryptDataUnits (buffer + (intersectStart - alignedOffset.QuadPart), &dataUnit, intersectLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo); + } + } + + memcpy (dataBuffer, buffer + (item->OriginalOffset.LowPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)), item->OriginalLength); + } + + TCfree (buffer); + CompleteOriginalIrp (item, item->Status, NT_SUCCESS (item->Status) ? item->OriginalLength : 0); + continue; + } + + // Validate offset and length + if (item->OriginalLength == 0 + || (item->OriginalLength & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0 + || (item->OriginalOffset.QuadPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0 + || ( !queue->IsFilterDevice && + ( (S_OK != ULongLongAdd(item->OriginalOffset.QuadPart, item->OriginalLength, &addResult)) + || (addResult > (ULONGLONG) queue->VirtualDeviceLength) + ) + ) + ) + { + CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); + continue; + } + +#ifdef TC_TRACE_IO_QUEUE + Dump ("Q %I64d [%I64d] %c len=%d\n", item->OriginalOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), item->Write ? 'W' : 'R', item->OriginalLength); +#endif + + if (!queue->IsFilterDevice) + { + // Adjust the offset for host file or device + if (queue->CryptoInfo->hiddenVolume) + hResult = ULongLongAdd(item->OriginalOffset.QuadPart, queue->CryptoInfo->hiddenVolumeOffset, &addResult); + else + hResult = ULongLongAdd(item->OriginalOffset.QuadPart, queue->CryptoInfo->volDataAreaOffset, &addResult); + + if (hResult != S_OK) + { + CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); + continue; + } + else + item->OriginalOffset.QuadPart = addResult; + + // Hidden volume protection + if (item->Write && queue->CryptoInfo->bProtectHiddenVolume) + { + // If there has already been a write operation denied in order to protect the + // hidden volume (since the volume mount time) + if (queue->CryptoInfo->bHiddenVolProtectionAction) + { + // Do not allow writing to this volume anymore. This is to fake a complete volume + // or system failure (otherwise certain kinds of inconsistency within the file + // system could indicate that this volume has used hidden volume protection). + CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); + continue; + } + + // Verify that no byte is going to be written to the hidden volume area + if (RegionsOverlap ((unsigned __int64) item->OriginalOffset.QuadPart, + (unsigned __int64) item->OriginalOffset.QuadPart + item->OriginalLength - 1, + queue->CryptoInfo->hiddenVolumeOffset, + (unsigned __int64) queue->CryptoInfo->hiddenVolumeOffset + queue->CryptoInfo->hiddenVolumeProtectedSize - 1)) + { + Dump ("Hidden volume protection triggered: write %I64d-%I64d (protected %I64d-%I64d)\n", item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, queue->CryptoInfo->hiddenVolumeOffset, queue->CryptoInfo->hiddenVolumeOffset + queue->CryptoInfo->hiddenVolumeProtectedSize - 1); + queue->CryptoInfo->bHiddenVolProtectionAction = TRUE; + + // Deny this write operation to prevent the hidden volume from being overwritten + CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0); + continue; + } + } + } + else if (item->Write + && RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, TC_BOOT_VOLUME_HEADER_SECTOR_OFFSET, TC_BOOT_VOLUME_HEADER_SECTOR_OFFSET + TC_BOOT_ENCRYPTION_VOLUME_HEADER_SIZE - 1)) + { + // Prevent inappropriately designed software from damaging important data that may be out of sync with the backup on the Rescue Disk (such as the end of the encrypted area). + Dump ("Preventing write to the system encryption key data area\n"); + CompleteOriginalIrp (item, STATUS_MEDIA_WRITE_PROTECTED, 0); + continue; + } + else if (item->Write && IsHiddenSystemRunning() + && (RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, TC_SECTOR_SIZE_BIOS, TC_BOOT_LOADER_AREA_SECTOR_COUNT * TC_SECTOR_SIZE_BIOS - 1) + || RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, GetBootDriveLength(), _I64_MAX))) + { + Dump ("Preventing write to boot loader or host protected area\n"); + CompleteOriginalIrp (item, STATUS_MEDIA_WRITE_PROTECTED, 0); + continue; + } + + dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, HighPagePriority); + + if (dataBuffer == NULL) + { + CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); + continue; + } + + // Divide data block to fragments to enable efficient overlapping of encryption and IO operations + + dataRemaining = item->OriginalLength; + fragmentOffset = item->OriginalOffset; + + while (dataRemaining > 0) + { + BOOL isLastFragment = dataRemaining <= TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; + + ULONG dataFragmentLength = isLastFragment ? dataRemaining : TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; + activeFragmentBuffer = (activeFragmentBuffer == queue->FragmentBufferA ? queue->FragmentBufferB : queue->FragmentBufferA); + + InterlockedIncrement (&queue->IoThreadPendingRequestCount); + + // Create IO request + request = GetPoolBuffer (queue, sizeof (EncryptedIoRequest)); + if (!request) + { + CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); + break; + } + request->Item = item; + request->CompleteOriginalIrp = isLastFragment; + request->Offset = fragmentOffset; + request->Data = activeFragmentBuffer; + request->OrigDataBufferFragment = dataBuffer; + request->Length = dataFragmentLength; + + if (queue->IsFilterDevice) + { + if (queue->EncryptedAreaStart == -1 || queue->EncryptedAreaEnd == -1) + { + request->EncryptedLength = 0; + } + else + { + // Get intersection of data fragment with encrypted area + GetIntersection (fragmentOffset.QuadPart, dataFragmentLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength); + + request->EncryptedOffset = intersectStart - fragmentOffset.QuadPart; + request->EncryptedLength = intersectLength; + } + } + else + { + request->EncryptedOffset = 0; + request->EncryptedLength = dataFragmentLength; + } + + AcquireFragmentBuffer (queue, activeFragmentBuffer); + + if (item->Write) + { + // Encrypt data + memcpy (activeFragmentBuffer, dataBuffer, dataFragmentLength); + + if (request->EncryptedLength > 0) + { + UINT64_STRUCT dataUnit; + ASSERT (request->EncryptedOffset + request->EncryptedLength <= request->Offset.QuadPart + request->Length); + + dataUnit.Value = (request->Offset.QuadPart + request->EncryptedOffset) / ENCRYPTION_DATA_UNIT_SIZE; + + if (queue->CryptoInfo->bPartitionInInactiveSysEncScope) + dataUnit.Value += queue->CryptoInfo->FirstDataUnitNo.Value; + else if (queue->RemapEncryptedArea) + dataUnit.Value += queue->RemappedAreaDataUnitOffset; + + EncryptDataUnits (activeFragmentBuffer + request->EncryptedOffset, &dataUnit, request->EncryptedLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo); + } + } + + // Queue IO request + ExInterlockedInsertTailList (&queue->IoThreadQueue, &request->ListEntry, &queue->IoThreadQueueLock); + KeSetEvent (&queue->IoThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE); + + if (isLastFragment) + break; + + dataRemaining -= TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; + dataBuffer += TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; + fragmentOffset.QuadPart += TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE; + } + } + } + + PsTerminateSystemThread (STATUS_SUCCESS); +} + + +NTSTATUS EncryptedIoQueueAddIrp (EncryptedIoQueue *queue, PIRP irp) +{ + NTSTATUS status; + + InterlockedIncrement (&queue->OutstandingIoCount); + if (queue->StopPending) + { + Dump ("STATUS_DEVICE_NOT_READY out=%d\n", queue->OutstandingIoCount); + status = STATUS_DEVICE_NOT_READY; + goto err; + } + + status = IoAcquireRemoveLock (&queue->RemoveLock, irp); + if (!NT_SUCCESS (status)) + goto err; + +#ifdef TC_TRACE_IO_QUEUE + { + PIO_STACK_LOCATION irpSp = IoGetCurrentIrpStackLocation (irp); + Dump ("* %I64d [%I64d] %c len=%d out=%d\n", irpSp->MajorFunction == IRP_MJ_WRITE ? irpSp->Parameters.Write.ByteOffset : irpSp->Parameters.Read.ByteOffset, GetElapsedTime (&queue->LastPerformanceCounter), irpSp->MajorFunction == IRP_MJ_WRITE ? 'W' : 'R', irpSp->MajorFunction == IRP_MJ_WRITE ? irpSp->Parameters.Write.Length : irpSp->Parameters.Read.Length, queue->OutstandingIoCount); + } +#endif + + IoMarkIrpPending (irp); + + ExInterlockedInsertTailList (&queue->MainThreadQueue, &irp->Tail.Overlay.ListEntry, &queue->MainThreadQueueLock); + KeSetEvent (&queue->MainThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE); + + return STATUS_PENDING; + +err: + DecrementOutstandingIoCount (queue); + return status; +} + + +NTSTATUS EncryptedIoQueueHoldWhenIdle (EncryptedIoQueue *queue, int64 timeout) +{ + NTSTATUS status; + ASSERT (!queue->Suspended); + + queue->SuspendPending = TRUE; + + while (TRUE) + { + while (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0) + { + LARGE_INTEGER waitTimeout; + + waitTimeout.QuadPart = timeout * -10000; + status = KeWaitForSingleObject (&queue->NoOutstandingIoEvent, Executive, KernelMode, FALSE, timeout != 0 ? &waitTimeout : NULL); + + if (status == STATUS_TIMEOUT) + status = STATUS_UNSUCCESSFUL; + + if (!NT_SUCCESS (status)) + { + queue->SuspendPending = FALSE; + return status; + } + + TCSleep (1); + if (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0) + { + queue->SuspendPending = FALSE; + return STATUS_UNSUCCESSFUL; + } + } + + KeClearEvent (&queue->QueueResumedEvent); + queue->Suspended = TRUE; + + if (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) == 0) + break; + + queue->Suspended = FALSE; + KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE); + } + + queue->ReadAheadBufferValid = FALSE; + + queue->SuspendPending = FALSE; + return STATUS_SUCCESS; +} + + +BOOL EncryptedIoQueueIsSuspended (EncryptedIoQueue *queue) +{ + return queue->Suspended; +} + + +BOOL EncryptedIoQueueIsRunning (EncryptedIoQueue *queue) +{ + return !queue->StopPending; +} + + +NTSTATUS EncryptedIoQueueResumeFromHold (EncryptedIoQueue *queue) +{ + ASSERT (queue->Suspended); + + queue->Suspended = FALSE; + KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE); + + return STATUS_SUCCESS; +} + + +NTSTATUS EncryptedIoQueueStart (EncryptedIoQueue *queue) +{ + NTSTATUS status; + EncryptedIoQueueBuffer *buffer; + int i; + + queue->StartPending = TRUE; + queue->ThreadExitRequested = FALSE; + + queue->OutstandingIoCount = 0; + queue->IoThreadPendingRequestCount = 0; + + queue->FirstPoolBuffer = NULL; + KeInitializeMutex (&queue->BufferPoolMutex, 0); + + KeInitializeEvent (&queue->NoOutstandingIoEvent, SynchronizationEvent, FALSE); + KeInitializeEvent (&queue->PoolBufferFreeEvent, SynchronizationEvent, FALSE); + KeInitializeEvent (&queue->QueueResumedEvent, SynchronizationEvent, FALSE); + + queue->FragmentBufferA = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE); + if (!queue->FragmentBufferA) + goto noMemory; + + queue->FragmentBufferB = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE); + if (!queue->FragmentBufferB) + goto noMemory; + + KeInitializeEvent (&queue->FragmentBufferAFreeEvent, SynchronizationEvent, TRUE); + KeInitializeEvent (&queue->FragmentBufferBFreeEvent, SynchronizationEvent, TRUE); + + queue->ReadAheadBufferValid = FALSE; + queue->ReadAheadBuffer = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE); + if (!queue->ReadAheadBuffer) + goto noMemory; + + // Preallocate buffers + for (i = 0; i < TC_ENC_IO_QUEUE_PREALLOCATED_IO_REQUEST_COUNT; ++i) + { + if (i < TC_ENC_IO_QUEUE_PREALLOCATED_ITEM_COUNT && !GetPoolBuffer (queue, sizeof (EncryptedIoQueueItem))) + goto noMemory; + + if (!GetPoolBuffer (queue, sizeof (EncryptedIoRequest))) + goto noMemory; + } + + for (buffer = queue->FirstPoolBuffer; buffer != NULL; buffer = buffer->NextBuffer) + { + buffer->InUse = FALSE; + } + + // Main thread + InitializeListHead (&queue->MainThreadQueue); + KeInitializeSpinLock (&queue->MainThreadQueueLock); + KeInitializeEvent (&queue->MainThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE); + + status = TCStartThread (MainThreadProc, queue, &queue->MainThread); + if (!NT_SUCCESS (status)) + goto err; + + // IO thread + InitializeListHead (&queue->IoThreadQueue); + KeInitializeSpinLock (&queue->IoThreadQueueLock); + KeInitializeEvent (&queue->IoThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE); + + status = TCStartThread (IoThreadProc, queue, &queue->IoThread); + if (!NT_SUCCESS (status)) + { + queue->ThreadExitRequested = TRUE; + TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent); + goto err; + } + + // Completion thread + InitializeListHead (&queue->CompletionThreadQueue); + KeInitializeSpinLock (&queue->CompletionThreadQueueLock); + KeInitializeEvent (&queue->CompletionThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE); + + status = TCStartThread (CompletionThreadProc, queue, &queue->CompletionThread); + if (!NT_SUCCESS (status)) + { + queue->ThreadExitRequested = TRUE; + TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent); + TCStopThread (queue->IoThread, &queue->IoThreadQueueNotEmptyEvent); + goto err; + } + +#ifdef TC_TRACE_IO_QUEUE + GetElapsedTimeInit (&queue->LastPerformanceCounter); +#endif + + queue->StopPending = FALSE; + queue->StartPending = FALSE; + + Dump ("Queue started\n"); + return STATUS_SUCCESS; + +noMemory: + status = STATUS_INSUFFICIENT_RESOURCES; + +err: + if (queue->FragmentBufferA) + TCfree (queue->FragmentBufferA); + if (queue->FragmentBufferB) + TCfree (queue->FragmentBufferB); + if (queue->ReadAheadBuffer) + TCfree (queue->ReadAheadBuffer); + + FreePoolBuffers (queue); + + queue->StartPending = FALSE; + return status; +} + + +NTSTATUS EncryptedIoQueueStop (EncryptedIoQueue *queue) +{ + ASSERT (!queue->StopPending); + queue->StopPending = TRUE; + + while (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0) + { + KeWaitForSingleObject (&queue->NoOutstandingIoEvent, Executive, KernelMode, FALSE, NULL); + } + + Dump ("Queue stopping out=%d\n", queue->OutstandingIoCount); + + queue->ThreadExitRequested = TRUE; + + TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent); + TCStopThread (queue->IoThread, &queue->IoThreadQueueNotEmptyEvent); + TCStopThread (queue->CompletionThread, &queue->CompletionThreadQueueNotEmptyEvent); + + TCfree (queue->FragmentBufferA); + TCfree (queue->FragmentBufferB); + TCfree (queue->ReadAheadBuffer); + + FreePoolBuffers (queue); + + Dump ("Queue stopped out=%d\n", queue->OutstandingIoCount); + return STATUS_SUCCESS; +} -- cgit v1.2.3