--- truecrypt/common/crypto.c 2018/04/24 16:51:07 1.1.1.14 +++ truecrypt/common/crypto.c 2018/04/24 17:08:22 1.1.1.21 @@ -3,11 +3,11 @@ derived from the source code of Encryption for the Masses 2.02a, which is Copyright (c) 1998-2000 Paul Le Roux and which is governed by the 'License Agreement for Encryption for the Masses'. Modifications and additions to - the original source code (contained in this file) and all other portions of - this file are Copyright (c) 2003-2008 TrueCrypt Foundation and are governed - by the TrueCrypt License 2.4 the full text of which is contained in the - file License.txt included in TrueCrypt binary and source code distribution - packages. */ + the original source code (contained in this file) and all other portions + of this file are Copyright (c) 2003-2010 TrueCrypt Developers Association + and are governed by the TrueCrypt License 3.0 the full text of which is + contained in the file License.txt included in TrueCrypt binary and source + code distribution packages. */ #include "Tcdefs.h" #include "Crypto.h" @@ -15,6 +15,10 @@ #include "Crc.h" #include "Common/Endian.h" #include +#ifndef TC_WINDOWS_BOOT +#include "EncryptionThreadPool.h" +#endif +#include "Volumes.h" /* Update the following when adding a new cipher or EA: @@ -42,10 +46,9 @@ static Cipher Ciphers[] = { SERPENT, "Serpent", 16, 32, 140*4 }, { TWOFISH, "Twofish", 16, 32, TWOFISH_KS }, #ifndef TC_WINDOWS_BOOT - { BLOWFISH, "Blowfish", 8, 56, 4168 }, // Deprecated/legacy - { CAST, "CAST5", 8, 16, 128 }, // Deprecated/legacy - { DES56, "DES", 8, 7, 128 }, // Deprecated/legacy - { TRIPLEDES,"Triple DES", 8, 8*3, 128*3 }, // Deprecated/legacy + { BLOWFISH, "Blowfish", 8, 56, sizeof (BF_KEY) }, // Deprecated/legacy + { CAST, "CAST5", 8, 16, sizeof (CAST_KEY) }, // Deprecated/legacy + { TRIPLEDES,"Triple DES", 8, 8*3, sizeof (TDES_KEY) }, // Deprecated/legacy #endif { 0, 0, 0, 0, 0 } }; @@ -139,52 +142,17 @@ int CipherInit (int cipher, unsigned cha case BLOWFISH: /* Deprecated/legacy */ - BF_set_key ((BF_KEY *)ks, CipherGetKeySize(BLOWFISH), key); - break; - - case DES56: - /* Deprecated/legacy */ - switch (des_key_sched ((des_cblock *) key, (struct des_ks_struct *) ks)) - { - case -1: - return ERR_CIPHER_INIT_FAILURE; - case -2: - retVal = ERR_CIPHER_INIT_WEAK_KEY; // Non-fatal error - break; - } + BlowfishSetKey ((BF_KEY *)ks, CipherGetKeySize(BLOWFISH), key); break; case CAST: /* Deprecated/legacy */ - CAST_set_key((CAST_KEY *) ks, CipherGetKeySize(CAST), key); + Cast5SetKey ((CAST_KEY *) ks, CipherGetKeySize(CAST), key); break; case TRIPLEDES: /* Deprecated/legacy */ - switch (des_key_sched ((des_cblock *) key, (struct des_ks_struct *) ks)) - { - case -1: - return ERR_CIPHER_INIT_FAILURE; - case -2: - retVal = ERR_CIPHER_INIT_WEAK_KEY; // Non-fatal error - break; - } - switch (des_key_sched ((des_cblock *) ((char*)(key)+8), (struct des_ks_struct *) (ks + CipherGetKeyScheduleSize (DES56)))) - { - case -1: - return ERR_CIPHER_INIT_FAILURE; - case -2: - retVal = ERR_CIPHER_INIT_WEAK_KEY; // Non-fatal error - break; - } - switch (des_key_sched ((des_cblock *) ((char*)(key)+16), (struct des_ks_struct *) (ks + CipherGetKeyScheduleSize (DES56) * 2))) - { - case -1: - return ERR_CIPHER_INIT_FAILURE; - case -2: - retVal = ERR_CIPHER_INIT_WEAK_KEY; // Non-fatal error - break; - } + TripleDesSetKey (key, CipherGetKeySize (TRIPLEDES), (TDES_KEY *) ks); // Verify whether all three DES keys are mutually different if (((*((__int64 *) key) ^ *((__int64 *) key+1)) & 0xFEFEFEFEFEFEFEFEULL) == 0 @@ -208,20 +176,69 @@ void EncipherBlock(int cipher, void *dat { switch (cipher) { - case AES: aes_encrypt (data, data, ks); break; + case AES: + // In 32-bit kernel mode, due to KeSaveFloatingPointState() overhead, AES instructions can be used only when processing the whole data unit. +#if (defined (_WIN64) || !defined (TC_WINDOWS_DRIVER)) && !defined (TC_WINDOWS_BOOT) + if (IsAesHwCpuSupported()) + aes_hw_cpu_encrypt (ks, data); + else +#endif + aes_encrypt (data, data, ks); + break; + case TWOFISH: twofish_encrypt (ks, data, data); break; case SERPENT: serpent_encrypt (data, data, ks); break; #ifndef TC_WINDOWS_BOOT - case BLOWFISH: BF_ecb_le_encrypt (data, data, ks, 1); break; // Deprecated/legacy - case DES56: des_encrypt (data, ks, 1); break; // Deprecated/legacy - case CAST: CAST_ecb_encrypt (data, data, ks, 1); break; // Deprecated/legacy - case TRIPLEDES: des_ecb3_encrypt (data, data, ks, // Deprecated/legacy - (void*)((char*) ks + CipherGetKeyScheduleSize (DES56)), (void*)((char*) ks + CipherGetKeyScheduleSize (DES56) * 2), 1); break; + case BLOWFISH: BlowfishEncryptLE (data, data, ks, 1); break; // Deprecated/legacy + case CAST: Cast5Encrypt (data, data, ks); break; // Deprecated/legacy + case TRIPLEDES: TripleDesEncrypt (data, data, ks, 1); break; // Deprecated/legacy #endif default: TC_THROW_FATAL_EXCEPTION; // Unknown/wrong ID } } +#ifndef TC_WINDOWS_BOOT + +void EncipherBlocks (int cipher, void *dataPtr, void *ks, size_t blockCount) +{ + byte *data = dataPtr; +#if defined (TC_WINDOWS_DRIVER) && !defined (_WIN64) + KFLOATING_SAVE floatingPointState; +#endif + + if (cipher == AES + && (blockCount & (32 - 1)) == 0 + && IsAesHwCpuSupported() +#if defined (TC_WINDOWS_DRIVER) && !defined (_WIN64) + && NT_SUCCESS (KeSaveFloatingPointState (&floatingPointState)) +#endif + ) + { + while (blockCount > 0) + { + aes_hw_cpu_encrypt_32_blocks (ks, data); + + data += 32 * 16; + blockCount -= 32; + } + +#if defined (TC_WINDOWS_DRIVER) && !defined (_WIN64) + KeRestoreFloatingPointState (&floatingPointState); +#endif + } + else + { + size_t blockSize = CipherGetBlockSize (cipher); + while (blockCount-- > 0) + { + EncipherBlock (cipher, data, ks); + data += blockSize; + } + } +} + +#endif // !TC_WINDOWS_BOOT + void DecipherBlock(int cipher, void *data, void *ks) { switch (cipher) @@ -229,13 +246,19 @@ void DecipherBlock(int cipher, void *dat case SERPENT: serpent_decrypt (data, data, ks); break; case TWOFISH: twofish_decrypt (ks, data, data); break; #ifndef TC_WINDOWS_BOOT - case AES: aes_decrypt (data, data, (void *) ((char *) ks + sizeof(aes_encrypt_ctx))); break; - case BLOWFISH: BF_ecb_le_encrypt (data, data, ks, 0); break; // Deprecated/legacy - case DES56: des_encrypt (data, ks, 0); break; // Deprecated/legacy - case CAST: CAST_ecb_encrypt (data, data, ks,0); break; // Deprecated/legacy - case TRIPLEDES: des_ecb3_encrypt (data, data, ks, // Deprecated/legacy - (void*)((char*) ks + CipherGetKeyScheduleSize (DES56)), - (void*)((char*) ks + CipherGetKeyScheduleSize (DES56) * 2), 0); break; + + case AES: +#if defined (_WIN64) || !defined (TC_WINDOWS_DRIVER) + if (IsAesHwCpuSupported()) + aes_hw_cpu_decrypt ((byte *) ks + sizeof (aes_encrypt_ctx), data); + else +#endif + aes_decrypt (data, data, (void *) ((char *) ks + sizeof(aes_encrypt_ctx))); + break; + + case BLOWFISH: BlowfishEncryptLE (data, data, ks, 0); break; // Deprecated/legacy + case CAST: Cast5Decrypt (data, data, ks); break; // Deprecated/legacy + case TRIPLEDES: TripleDesEncrypt (data, data, ks, 0); break; // Deprecated/legacy #else case AES: aes_decrypt (data, data, ks); break; #endif @@ -243,6 +266,49 @@ void DecipherBlock(int cipher, void *dat } } +#ifndef TC_WINDOWS_BOOT + +void DecipherBlocks (int cipher, void *dataPtr, void *ks, size_t blockCount) +{ + byte *data = dataPtr; +#if defined (TC_WINDOWS_DRIVER) && !defined (_WIN64) + KFLOATING_SAVE floatingPointState; +#endif + + if (cipher == AES + && (blockCount & (32 - 1)) == 0 + && IsAesHwCpuSupported() +#if defined (TC_WINDOWS_DRIVER) && !defined (_WIN64) + && NT_SUCCESS (KeSaveFloatingPointState (&floatingPointState)) +#endif + ) + { + while (blockCount > 0) + { + aes_hw_cpu_decrypt_32_blocks ((byte *) ks + sizeof (aes_encrypt_ctx), data); + + data += 32 * 16; + blockCount -= 32; + } + +#if defined (TC_WINDOWS_DRIVER) && !defined (_WIN64) + KeRestoreFloatingPointState (&floatingPointState); +#endif + } + else + { + size_t blockSize = CipherGetBlockSize (cipher); + while (blockCount-- > 0) + { + DecipherBlock (cipher, data, ks); + data += blockSize; + } + } +} + +#endif // !TC_WINDOWS_BOOT + + // Ciphers support Cipher *CipherGet (int id) @@ -275,6 +341,15 @@ int CipherGetKeyScheduleSize (int cipher return CipherGet (cipherId) -> KeyScheduleSize; } +#ifndef TC_WINDOWS_BOOT + +BOOL CipherSupportsIntraDataUnitParallelization (int cipher) +{ + return cipher == AES && IsAesHwCpuSupported(); +} + +#endif + // Encryption algorithms support @@ -332,7 +407,7 @@ int EAInit (int ea, unsigned char *key, #ifndef TC_WINDOWS_BOOT -int EAInitMode (PCRYPTO_INFO ci) +BOOL EAInitMode (PCRYPTO_INFO ci) { switch (ci->mode) { @@ -671,10 +746,8 @@ PCRYPTO_INFO crypto_open () memset (cryptoInfo, 0, sizeof (CRYPTO_INFO)); #ifndef DEVICE_DRIVER -#ifdef _WIN32 VirtualLock (cryptoInfo, sizeof (CRYPTO_INFO)); #endif -#endif if (cryptoInfo == NULL) return NULL; @@ -709,10 +782,8 @@ void crypto_close (PCRYPTO_INFO cryptoIn { burn (cryptoInfo, sizeof (CRYPTO_INFO)); #ifndef DEVICE_DRIVER -#ifdef _WIN32 VirtualUnlock (cryptoInfo, sizeof (CRYPTO_INFO)); #endif -#endif TCfree (cryptoInfo); } @@ -742,7 +813,7 @@ void Xor64 (unsigned __int64 *a, unsigne } -void EncryptBufferLRW128 (unsigned __int8 *buffer, unsigned __int64 length, unsigned __int64 blockIndex, PCRYPTO_INFO cryptoInfo) +void EncryptBufferLRW128 (byte *buffer, uint64 length, uint64 blockIndex, PCRYPTO_INFO cryptoInfo) { /* Deprecated/legacy */ @@ -797,7 +868,7 @@ void EncryptBufferLRW128 (unsigned __int } -void EncryptBufferLRW64 (unsigned __int8 *buffer, unsigned __int64 length, unsigned __int64 blockIndex, PCRYPTO_INFO cryptoInfo) +void EncryptBufferLRW64 (byte *buffer, uint64 length, uint64 blockIndex, PCRYPTO_INFO cryptoInfo) { /* Deprecated/legacy */ @@ -834,7 +905,7 @@ void EncryptBufferLRW64 (unsigned __int8 } -void DecryptBufferLRW128 (unsigned __int8 *buffer, unsigned __int64 length, unsigned __int64 blockIndex, PCRYPTO_INFO cryptoInfo) +void DecryptBufferLRW128 (byte *buffer, uint64 length, uint64 blockIndex, PCRYPTO_INFO cryptoInfo) { /* Deprecated/legacy */ @@ -891,7 +962,7 @@ void DecryptBufferLRW128 (unsigned __int -void DecryptBufferLRW64 (unsigned __int8 *buffer, unsigned __int64 length, unsigned __int64 blockIndex, PCRYPTO_INFO cryptoInfo) +void DecryptBufferLRW64 (byte *buffer, uint64 length, uint64 blockIndex, PCRYPTO_INFO cryptoInfo) { /* Deprecated/legacy */ @@ -1166,12 +1237,10 @@ DecryptBufferCBC (unsigned __int32 *data // EncryptBuffer // -// buf: data to be encrypted -// len: number of bytes to encrypt; must be divisible by the block size (for cascaded -// ciphers divisible by the largest block size used within the cascade) -void EncryptBuffer (unsigned __int8 *buf, - TC_LARGEST_COMPILER_UINT len, - PCRYPTO_INFO cryptoInfo) +// buf: data to be encrypted; the start of the buffer is assumed to be aligned with the start of a data unit. +// len: number of bytes to encrypt; must be divisible by the block size (for cascaded ciphers, divisible +// by the largest block size used within the cascade) +void EncryptBuffer (unsigned __int8 *buf, TC_LARGEST_COMPILER_UINT len, PCRYPTO_INFO cryptoInfo) { switch (cryptoInfo->mode) { @@ -1184,7 +1253,7 @@ void EncryptBuffer (unsigned __int8 *buf // When encrypting/decrypting a buffer (typically a volume header) the sequential number // of the first XTS data unit in the buffer is always 0 and the start of the buffer is - // always considered aligned with the start of a data unit. + // always assumed to be aligned with the start of a data unit. dataUnitNo.LowPart = 0; dataUnitNo.HighPart = 0; @@ -1269,14 +1338,14 @@ void EncryptBuffer (unsigned __int8 *buf #ifndef TC_NO_COMPILER_INT64 // Converts a data unit number to the index of the first LRW block in the data unit. // Note that the maximum supported volume size is 8589934592 GB (i.e., 2^63 bytes). -unsigned __int64 DataUnit2LRWIndex (unsigned __int64 dataUnit, int blockSize, PCRYPTO_INFO ci) +uint64 DataUnit2LRWIndex (uint64 dataUnit, int blockSize, PCRYPTO_INFO ci) { /* Deprecated/legacy */ if (ci->hiddenVolume) dataUnit -= ci->hiddenVolumeOffset / ENCRYPTION_DATA_UNIT_SIZE; else - dataUnit -= HEADER_SIZE / ENCRYPTION_DATA_UNIT_SIZE; // Compensate for the volume header size + dataUnit -= TC_VOLUME_HEADER_SIZE_LEGACY / ENCRYPTION_DATA_UNIT_SIZE; // Compensate for the volume header size switch (blockSize) { @@ -1298,7 +1367,14 @@ unsigned __int64 DataUnit2LRWIndex (unsi // buf: data to be encrypted // unitNo: sequential number of the data unit with which the buffer starts // nbrUnits: number of data units in the buffer -void EncryptDataUnits (unsigned __int8 *buf, const UINT64_STRUCT *structUnitNo, TC_LARGEST_COMPILER_UINT nbrUnits, PCRYPTO_INFO ci) +void EncryptDataUnits (unsigned __int8 *buf, const UINT64_STRUCT *structUnitNo, uint32 nbrUnits, PCRYPTO_INFO ci) +#ifndef TC_WINDOWS_BOOT +{ + EncryptionThreadPoolDoWork (EncryptDataUnitsWork, buf, structUnitNo, nbrUnits, ci); +} + +void EncryptDataUnitsCurrentThread (unsigned __int8 *buf, const UINT64_STRUCT *structUnitNo, TC_LARGEST_COMPILER_UINT nbrUnits, PCRYPTO_INFO ci) +#endif // !TC_WINDOWS_BOOT { int ea = ci->ea; unsigned __int8 *ks = ci->ks; @@ -1414,12 +1490,10 @@ void EncryptDataUnits (unsigned __int8 * // DecryptBuffer // -// buf: data to be decrypted -// len: number of bytes to decrypt; must be divisible by the block size (for cascaded -// ciphers divisible by the largest block size used within the cascade) -void DecryptBuffer (unsigned __int8 *buf, - TC_LARGEST_COMPILER_UINT len, - PCRYPTO_INFO cryptoInfo) +// buf: data to be decrypted; the start of the buffer is assumed to be aligned with the start of a data unit. +// len: number of bytes to decrypt; must be divisible by the block size (for cascaded ciphers, divisible +// by the largest block size used within the cascade) +void DecryptBuffer (unsigned __int8 *buf, TC_LARGEST_COMPILER_UINT len, PCRYPTO_INFO cryptoInfo) { switch (cryptoInfo->mode) { @@ -1432,7 +1506,7 @@ void DecryptBuffer (unsigned __int8 *buf // When encrypting/decrypting a buffer (typically a volume header) the sequential number // of the first XTS data unit in the buffer is always 0 and the start of the buffer is - // always considered aligned with the start of the data unit 0. + // always assumed to be aligned with the start of the data unit 0. dataUnitNo.LowPart = 0; dataUnitNo.HighPart = 0; @@ -1516,7 +1590,14 @@ void DecryptBuffer (unsigned __int8 *buf // buf: data to be decrypted // unitNo: sequential number of the data unit with which the buffer starts // nbrUnits: number of data units in the buffer -void DecryptDataUnits (unsigned __int8 *buf, const UINT64_STRUCT *structUnitNo, TC_LARGEST_COMPILER_UINT nbrUnits, PCRYPTO_INFO ci) +void DecryptDataUnits (unsigned __int8 *buf, const UINT64_STRUCT *structUnitNo, uint32 nbrUnits, PCRYPTO_INFO ci) +#ifndef TC_WINDOWS_BOOT +{ + EncryptionThreadPoolDoWork (DecryptDataUnitsWork, buf, structUnitNo, nbrUnits, ci); +} + +void DecryptDataUnitsCurrentThread (unsigned __int8 *buf, const UINT64_STRUCT *structUnitNo, TC_LARGEST_COMPILER_UINT nbrUnits, PCRYPTO_INFO ci) +#endif // !TC_WINDOWS_BOOT { int ea = ci->ea; unsigned __int8 *ks = ci->ks; @@ -1640,10 +1721,6 @@ int GetMaxPkcs5OutSize (void) { int size = 32; -#ifndef max -#define max(a,b) (((a) > (b)) ? (a) : (b)) -#endif - size = max (size, EAGetLargestKeyForMode (XTS) * 2); // Sizes of primary + secondary keys #ifndef TC_WINDOWS_BOOT @@ -1667,7 +1744,10 @@ int GetMaxPkcs5OutSize (void) void EncipherBlock(int cipher, void *data, void *ks) { #ifdef TC_WINDOWS_BOOT_AES - aes_encrypt (data, data, ks); + if (IsAesHwCpuSupported()) + aes_hw_cpu_encrypt ((byte *) ks, data); + else + aes_encrypt (data, data, ks); #elif defined (TC_WINDOWS_BOOT_SERPENT) serpent_encrypt (data, data, ks); #elif defined (TC_WINDOWS_BOOT_TWOFISH) @@ -1678,7 +1758,10 @@ void EncipherBlock(int cipher, void *dat void DecipherBlock(int cipher, void *data, void *ks) { #ifdef TC_WINDOWS_BOOT_AES - aes_decrypt (data, data, (aes_decrypt_ctx *) ((byte *) ks + sizeof(aes_encrypt_ctx))); + if (IsAesHwCpuSupported()) + aes_hw_cpu_decrypt ((byte *) ks + sizeof (aes_encrypt_ctx) + 14 * 16, data); + else + aes_decrypt (data, data, (aes_decrypt_ctx *) ((byte *) ks + sizeof(aes_encrypt_ctx))); #elif defined (TC_WINDOWS_BOOT_SERPENT) serpent_decrypt (data, data, ks); #elif defined (TC_WINDOWS_BOOT_TWOFISH) @@ -1750,3 +1833,39 @@ void DecryptDataUnits (unsigned __int8 * } #endif // TC_WINDOWS_BOOT_SINGLE_CIPHER_MODE + + +#if !defined (TC_WINDOWS_BOOT) || defined (TC_WINDOWS_BOOT_AES) + +static BOOL HwEncryptionDisabled = FALSE; + +BOOL IsAesHwCpuSupported () +{ + static BOOL state = FALSE; + static BOOL stateValid = FALSE; + + if (!stateValid) + { + state = is_aes_hw_cpu_supported() ? TRUE : FALSE; + stateValid = TRUE; + } + + return state && !HwEncryptionDisabled; +} + +void EnableHwEncryption (BOOL enable) +{ +#if defined (TC_WINDOWS_BOOT) + if (enable) + aes_hw_cpu_enable_sse(); +#endif + + HwEncryptionDisabled = !enable; +} + +BOOL IsHwEncryptionEnabled () +{ + return !HwEncryptionDisabled; +} + +#endif // !TC_WINDOWS_BOOT