--- truecrypt/linux/kernel/dm-target.c 2018/04/24 16:44:05 1.1.1.2 +++ truecrypt/linux/kernel/dm-target.c 2018/04/24 16:46:48 1.1.1.5 @@ -1,19 +1,22 @@ -/* -Copyright (c) 2004-2005 TrueCrypt Foundation. All rights reserved. +/* + Copyright (c) TrueCrypt Foundation. All rights reserved. -Covered by TrueCrypt License 2.0 the full text of which is contained in the file -License.txt included in TrueCrypt binary and source code distribution archives. + Covered by the TrueCrypt License 2.2 the full text of which is contained + in the file License.txt included in TrueCrypt binary and source code + distribution packages. */ #include +#include #include +#include #include +#include #include #include #include #include - -#include "dm.h" +#include #include "Tcdefs.h" #include "Crypto.h" @@ -33,6 +36,10 @@ int trace_level = 0; #define MIN_POOL_SIZE 16 +#ifndef __GFP_NOMEMALLOC +#define __GFP_NOMEMALLOC 0 +#endif + struct target_ctx { struct dm_dev *dev; @@ -40,10 +47,11 @@ struct target_ctx char *volume_path; mempool_t *bio_ctx_pool; mempool_t *pg_pool; - sector_t read_only_start; - sector_t read_only_end; - u64 mtime; - u64 atime; + unsigned long long read_only_start; + unsigned long long read_only_end; + unsigned long long uid; + unsigned long long mtime; + unsigned long long atime; int flags; PCRYPTO_INFO ci; }; @@ -59,10 +67,15 @@ struct bio_ctx }; static struct workqueue_struct *work_queue = NULL; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,20) +static struct kmem_cache *bio_ctx_cache = NULL; +#else static kmem_cache_t *bio_ctx_cache = NULL; +#endif -#define READ_ONLY(tc) (tc->flags & FLAG_READ_ONLY) -#define HID_VOL_PROT(tc) (tc->flags & FLAG_HIDDEN_VOLUME_PROTECTION) +#define READ_ONLY(tc) (tc->flags & TC_READ_ONLY) +#define HID_VOL_PROT(tc) (tc->flags & TC_HIDDEN_VOLUME_PROTECTION) static int hex2bin (char *hex_string, u8 *byte_buf, int max_length) @@ -71,8 +84,6 @@ static int hex2bin (char *hex_string, u8 char s[3]; s[2] = 0; - trace (3, "hex2bin (%p, %p, %d)\n", hex_string, byte_buf, max_length); - while (i < max_length && (s[0] = *hex_string++) && (s[1] = *hex_string++)) @@ -85,23 +96,46 @@ static int hex2bin (char *hex_string, u8 return i; } -#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,12) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19) +#define congestion_wait blk_congestion_wait +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,17) + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,14) +static void *mempool_alloc_pg (gfp_t gfp_mask, void *pool_data) +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,12) static void *mempool_alloc_pg (unsigned int gfp_mask, void *pool_data) #else static void *mempool_alloc_pg (int gfp_mask, void *pool_data) #endif { - trace (3, "mempool_alloc_pg (%d, %p)\n", gfp_mask, pool_data); return alloc_page (gfp_mask); } - static void mempool_free_pg (void *element, void *pool_data) { - trace (3, "mempool_free_pg (%p, %p)\n", element, pool_data); __free_page (element); } +#endif // LINUX_VERSION_CODE < KERNEL_VERSION(2,6,17) + + +static void *malloc_wait (mempool_t *pool, int direction) +{ + void *p; + while (1) + { + p = mempool_alloc (pool, GFP_NOIO | __GFP_NOMEMALLOC); + + if (p) + return p; + + congestion_wait (direction, HZ / 50); + } +} + static void wipe_args (unsigned int argc, char **argv) { @@ -119,10 +153,9 @@ static int truecrypt_ctr (struct dm_targ struct target_ctx *tc; int key_size; int error = -EINVAL; + unsigned long long sector; - trace (3, "truecrypt_ctr (%p, %d, %p)\n", ti, argc, argv); - - if (argc != LAST_ARG + 1) + if (argc != TC_LAST_ARG + 1) { ti->error = "truecrypt: Usage: truecrypt "; return -EINVAL; @@ -145,7 +178,12 @@ static int truecrypt_ctr (struct dm_targ goto err; } +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,17) + tc->bio_ctx_pool = mempool_create_slab_pool (MIN_POOL_SIZE, bio_ctx_cache); +#else tc->bio_ctx_pool = mempool_create (MIN_POOL_SIZE, mempool_alloc_slab, mempool_free_slab, bio_ctx_cache); +#endif + if (!tc->bio_ctx_pool) { ti->error = "truecrypt: Cannot create bio context memory pool"; @@ -153,7 +191,11 @@ static int truecrypt_ctr (struct dm_targ goto err; } +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,17) + tc->pg_pool = mempool_create_page_pool (MIN_POOL_SIZE, 0); +#else tc->pg_pool = mempool_create (MIN_POOL_SIZE, mempool_alloc_pg, mempool_free_pg, NULL); +#endif if (!tc->pg_pool) { ti->error = "truecrypt: Cannot create page memory pool"; @@ -161,13 +203,14 @@ static int truecrypt_ctr (struct dm_targ goto err; } - if (sscanf (argv[ARG_SEC], SECTOR_FORMAT, &tc->start) != 1) + if (sscanf (argv[TC_ARG_SEC], "%llu", §or) != 1) { ti->error = "truecrypt: Invalid device sector"; goto err; } + tc->start = sector; - if (dm_get_device (ti, argv[ARG_DEV], tc->start, ti->len, dm_table_get_mode (ti->table), &tc->dev)) + if (dm_get_device (ti, argv[TC_ARG_DEV], tc->start, ti->len, dm_table_get_mode (ti->table), &tc->dev)) { ti->error = "truecrypt: Device lookup failed"; goto err; @@ -175,7 +218,7 @@ static int truecrypt_ctr (struct dm_targ // Encryption algorithm tc->ci->ea = 0; - if (sscanf (argv[ARG_EA], "%d", &tc->ci->ea) != 1 + if (sscanf (argv[TC_ARG_EA], "%d", &tc->ci->ea) != 1 || tc->ci->ea < EAGetFirst () || tc->ci->ea > EAGetCount ()) { @@ -185,7 +228,7 @@ static int truecrypt_ctr (struct dm_targ // Mode of operation tc->ci->mode = 0; - if (sscanf (argv[ARG_MODE], "%d", &tc->ci->mode) != 1 + if (sscanf (argv[TC_ARG_MODE], "%d", &tc->ci->mode) != 1 || tc->ci->mode < 1 || tc->ci->mode >= INVALID_MODE) { @@ -195,14 +238,13 @@ static int truecrypt_ctr (struct dm_targ // Key key_size = EAGetKeySize (tc->ci->ea); - if (hex2bin (argv[ARG_KEY], tc->ci->master_key, key_size) != key_size) + if (hex2bin (argv[TC_ARG_KEY], tc->ci->master_key, key_size) != key_size) { ti->error = "truecrypt: Invalid key"; goto err; } // EA init - trace (2, "EAInit (%d, %p, %p)\n", tc->ci->ea, tc->ci->master_key, tc->ci->ks); if (EAInit (tc->ci->ea, tc->ci->master_key, tc->ci->ks) == ERR_CIPHER_INIT_FAILURE) { ti->error = "truecrypt: Encryption algorithm initialization failed"; @@ -210,7 +252,7 @@ static int truecrypt_ctr (struct dm_targ } // Key2 / IV - if (hex2bin (argv[ARG_IV], tc->ci->iv, sizeof (tc->ci->iv)) != sizeof (tc->ci->iv)) + if (hex2bin (argv[TC_ARG_IV], tc->ci->iv, sizeof (tc->ci->iv)) != sizeof (tc->ci->iv)) { ti->error = "truecrypt: Invalid IV"; goto err; @@ -224,49 +266,56 @@ static int truecrypt_ctr (struct dm_targ } // Read-only start sector - if (sscanf (argv[ARG_RO_START], SECTOR_FORMAT, &tc->read_only_start) != 1) + if (sscanf (argv[TC_ARG_RO_START], "%llu", &tc->read_only_start) != 1) { ti->error = "truecrypt: Invalid read-only start sector"; goto err; } // Read-only end sector - if (sscanf (argv[ARG_RO_END], SECTOR_FORMAT, &tc->read_only_end) != 1) + if (sscanf (argv[TC_ARG_RO_END], "%llu", &tc->read_only_end) != 1) { ti->error = "truecrypt: Invalid read-only end sector"; goto err; } + // User ID + if (sscanf (argv[TC_ARG_UID], "%llu", &tc->uid) != 1) + { + ti->error = "truecrypt: Invalid user ID"; + goto err; + } + // Modification time - if (sscanf (argv[ARG_MTIME], "%Ld", &tc->mtime) != 1) + if (sscanf (argv[TC_ARG_MTIME], "%llu", &tc->mtime) != 1) { ti->error = "truecrypt: Invalid modification time"; goto err; } // Access time - if (sscanf (argv[ARG_ATIME], "%Ld", &tc->atime) != 1) + if (sscanf (argv[TC_ARG_ATIME], "%llu", &tc->atime) != 1) { ti->error = "truecrypt: Invalid access time"; goto err; } // Flags - if (sscanf (argv[ARG_FLAGS], "%d", &tc->flags) != 1) + if (sscanf (argv[TC_ARG_FLAGS], "%d", &tc->flags) != 1) { ti->error = "truecrypt: Invalid flags"; goto err; } // Volume path - tc->volume_path = kmalloc (strlen (argv[ARG_VOL]) + 1, GFP_KERNEL); + tc->volume_path = kmalloc (strlen (argv[TC_ARG_VOL]) + 1, GFP_KERNEL); if (tc->volume_path == NULL) { ti->error = "truecrypt: Cannot allocate volume path buffer"; error = -ENOMEM; goto err; } - strcpy (tc->volume_path, argv[ARG_VOL]); + strcpy (tc->volume_path, argv[TC_ARG_VOL]); // Hidden volume if (tc->start > 1) @@ -281,8 +330,6 @@ static int truecrypt_ctr (struct dm_targ return 0; err: - trace (3, "truecrypt_ctr: error\n"); - if (tc) { if (tc->ci) @@ -305,8 +352,6 @@ static void truecrypt_dtr (struct dm_tar { struct target_ctx *tc = (struct target_ctx *) ti->private; - trace (3, "truecrypt_dtr (%p)\n", ti); - mempool_destroy (tc->bio_ctx_pool); mempool_destroy (tc->pg_pool); crypto_close (tc->ci); @@ -317,7 +362,7 @@ static void truecrypt_dtr (struct dm_tar // Checks if two regions overlap (borders are parts of regions) -static int RegionsOverlap (sector_t start1, sector_t end1, sector_t start2, sector_t end2) +static int RegionsOverlap (u64 start1, u64 end1, u64 start2, u64 end2) { return (start1 < start2) ? (end1 >= start2) : (start1 <= end2); } @@ -326,41 +371,46 @@ static int RegionsOverlap (sector_t star static void dereference_bio_ctx (struct bio_ctx *bc) { struct target_ctx *tc = (struct target_ctx *) bc->target->private; - trace (3, "dereference_bio_ctx (%p)\n", bc); if (!atomic_dec_and_test (&bc->ref_count)) return; bio_endio (bc->orig_bio, bc->orig_bio->bi_size, bc->error); - trace (3, "dereference_bio_ctx: mempool_free (%p)\n", bc); mempool_free (bc, tc->bio_ctx_pool); } -static void work_process (void *data) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,20) +static void work_process (struct work_struct *qdata) +{ + struct bio_ctx *bc = container_of(qdata, struct bio_ctx, work); +#else +static void work_process (void *qdata) { - struct bio_ctx *bc = (struct bio_ctx *) data; + struct bio_ctx *bc = (struct bio_ctx *) qdata; +#endif + struct target_ctx *tc = (struct target_ctx *) bc->target->private; struct bio_vec *bv; - sector_t sec_no = bc->crypto_sector; + u64 sec_no = bc->crypto_sector; int seg_no; unsigned long flags; - trace (3, "work_process (%p)\n", data); - // Decrypt queued data bio_for_each_segment (bv, bc->orig_bio, seg_no) { unsigned int secs = bv->bv_len / SECTOR_SIZE; char *data = bvec_kmap_irq (bv, &flags); - trace (2, "DecryptSectors (%Ld, %d)\n", sec_no, secs); DecryptSectors ((unsigned __int32 *)data, sec_no, secs, tc->ci); sec_no += secs; flush_dcache_page (bv->bv_page); bvec_kunmap_irq (data, &flags); + + if (seg_no + 1 < bc->orig_bio->bi_vcnt) + cond_resched (); } dereference_bio_ctx (bc); @@ -374,9 +424,8 @@ static int truecrypt_endio (struct bio * struct bio_vec *bv; int seg_no; - trace (3, "truecrypt_endio (%p, %d, %d)\n", bio, bytes_done, error); - trace (1, "end: sc=" SECTOR_FORMAT " fl=%ld rw=%ld sz=%d ix=%hd vc=%hd dn=%d er=%d\n", - bio->bi_sector, bio->bi_flags, bio->bi_rw, bio->bi_size, bio->bi_idx, bio->bi_vcnt, bytes_done, error); + trace (1, "end: sc=%llu fl=%ld rw=%ld sz=%d ix=%hd vc=%hd dn=%d er=%d\n", + (unsigned long long) bio->bi_sector, bio->bi_flags, bio->bi_rw, bio->bi_size, bio->bi_idx, bio->bi_vcnt, bytes_done, error); if (error != 0) bc->error = error; @@ -392,8 +441,11 @@ static int truecrypt_endio (struct bio * bio_put (bio); // Queue decryption to leave completion interrupt ASAP +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,20) + INIT_WORK (&bc->work, work_process); +#else INIT_WORK (&bc->work, work_process, bc); - trace (3, "queue_work (%p)\n", work_queue); +#endif queue_work (work_queue, &bc->work); return error; } @@ -401,7 +453,6 @@ static int truecrypt_endio (struct bio * // Free pages allocated for encryption bio_for_each_segment (bv, bio, seg_no) { - trace (3, "mempool_free (%p, %p)\n", bv->bv_page, tc->pg_pool); mempool_free (bv->bv_page, tc->pg_pool); } @@ -419,9 +470,8 @@ static int truecrypt_map (struct dm_targ struct bio_vec *bv; int seg_no; - trace (3, "truecrypt_map (%p, %p, %p)\n", ti, bio, map_context); - trace (1, "map: sc=" SECTOR_FORMAT " fl=%ld rw=%ld sz=%d ix=%hd vc=%hd\n", - bio->bi_sector, bio->bi_flags, bio->bi_rw, bio->bi_size, bio->bi_idx, bio->bi_vcnt); + trace (1, "map: sc=%llu fl=%ld rw=%ld sz=%d ix=%hd vc=%hd\n", + (unsigned long long) bio->bi_sector, bio->bi_flags, bio->bi_rw, bio->bi_size, bio->bi_idx, bio->bi_vcnt); // Write protection if (bio_data_dir (bio) == WRITE && READ_ONLY (tc)) @@ -432,20 +482,19 @@ static int truecrypt_map (struct dm_targ { if (bv->bv_len & (SECTOR_SIZE - 1)) { - error ("unsupported segment size %d (%ld %d %hd %hd)\n", + error ("unsupported bio segment size %d (%ld %d %hd %hd)\n", bv->bv_len, bio->bi_rw, bio->bi_size, bio->bi_idx, bio->bi_vcnt); return -EINVAL; } } // Bio context - bc = mempool_alloc (tc->bio_ctx_pool, GFP_NOIO); + bc = malloc_wait (tc->bio_ctx_pool, bio_data_dir (bio)); if (!bc) { error ("bio context allocation failed\n"); return -ENOMEM; } - trace (3, "truecrypt_map: mempool_alloc bc: %p\n", bc); atomic_set (&bc->ref_count, 1); bc->orig_bio = bio; @@ -454,14 +503,9 @@ static int truecrypt_map (struct dm_targ bc->crypto_sector = tc->start + (bio->bi_sector - ti->begin); // New bio for encrypted device - trace (3, "bio_alloc (%hd)\n", bio_segments (bio)); - bion = bio_alloc (GFP_NOIO, bio_segments (bio)); - if (!bion) - { - error ("bio allocation failed\n"); - bc->error = -ENOMEM; - dereference_bio_ctx (bc); - return 0; + while (!(bion = bio_alloc (GFP_NOIO | __GFP_NOMEMALLOC, bio_segments (bio)))) + { + congestion_wait (bio_data_dir (bio), HZ / 50); } bion->bi_bdev = tc->dev->bdev; @@ -482,10 +526,12 @@ static int truecrypt_map (struct dm_targ } else { + u64 sec_no = bc->crypto_sector; + int seg_no; + // Encrypt data to be written unsigned long flags, copyFlags; char *data, *copy; - long long sec_no = bc->crypto_sector; memset (bion->bi_io_vec, 0, sizeof (struct bio_vec) * bion->bi_vcnt); @@ -498,19 +544,12 @@ static int truecrypt_map (struct dm_targ if (!READ_ONLY (tc) && HID_VOL_PROT (tc) && RegionsOverlap (sec_no, sec_no + secs - 1, tc->read_only_start, tc->read_only_end)) { - tc->flags |= FLAG_READ_ONLY | FLAG_PROTECTION_ACTIVATED; + tc->flags |= TC_READ_ONLY | TC_PROTECTION_ACTIVATED; } - if (!READ_ONLY (tc)) + if (READ_ONLY (tc)) { - cbv->bv_page = mempool_alloc (tc->pg_pool, GFP_NOIO); - if (cbv->bv_page == NULL) - error ("page allocation failed during write\n"); - } - - if (READ_ONLY (tc) || cbv->bv_page == NULL) - { - // Write not permitted or no memory + // Write not permitted bio_for_each_segment (cbv, bion, seg_no) { if (cbv->bv_page != NULL) @@ -522,32 +561,31 @@ static int truecrypt_map (struct dm_targ dereference_bio_ctx (bc); return 0; } - trace (3, "truecrypt_map: mempool_alloc pg: %p\n", cbv->bv_page); + + cbv->bv_page = malloc_wait (tc->pg_pool, bio_data_dir (bion)); cbv->bv_offset = 0; cbv->bv_len = bv->bv_len; - data = bvec_kmap_irq (bv, &flags); copy = bvec_kmap_irq (cbv, ©Flags); + data = bvec_kmap_irq (bv, &flags); memcpy (copy, data, bv->bv_len); - flush_dcache_page (bv->bv_page); - bvec_kunmap_irq (data, &flags); - - trace (2, "EncryptSectors (%Ld, %d)\n", sec_no, secs); - - EncryptSectors ((unsigned __int32 *)copy, sec_no, secs, tc->ci); + EncryptSectors ((unsigned __int32 *) copy, sec_no, secs, tc->ci); sec_no += secs; - flush_dcache_page (cbv->bv_page); + bvec_kunmap_irq (data, &flags); bvec_kunmap_irq (copy, ©Flags); + flush_dcache_page (bv->bv_page); + flush_dcache_page (cbv->bv_page); + + if (seg_no + 1 < bio->bi_vcnt) + cond_resched(); } } atomic_inc (&bc->ref_count); - - trace (3, "generic_make_request (rw=%ld sc=" SECTOR_FORMAT ")\n", bion->bi_rw, bion->bi_sector); generic_make_request (bion); dereference_bio_ctx (bc); @@ -569,13 +607,14 @@ static int truecrypt_status (struct dm_t { char name[32]; format_dev_t (name, tc->dev->bdev->bd_dev); - snprintf (result, maxlen, "%d %d 0 0 %s " SECTOR_FORMAT " " SECTOR_FORMAT " " SECTOR_FORMAT " %Ld %Ld %d %s", + snprintf (result, maxlen, "%d %d 0 0 %s %llu %llu %llu %llu %llu %llu %d %s", tc->ci->ea, tc->ci->mode, name, - tc->start, + (unsigned long long) tc->start, tc->read_only_start, tc->read_only_end, + tc->uid, tc->mtime, tc->atime, tc->flags, @@ -590,7 +629,7 @@ static int truecrypt_status (struct dm_t static struct target_type truecrypt_target = { .name = "truecrypt", - .version= {VERSION_NUM1, VERSION_NUM2, VERSION_NUM3}, + .version= {TC_VERSION_NUM1, TC_VERSION_NUM2, TC_VERSION_NUM3}, .module = THIS_MODULE, .ctr = truecrypt_ctr, .dtr = truecrypt_dtr, @@ -602,11 +641,10 @@ static struct target_type truecrypt_targ int __init dm_truecrypt_init(void) { int r; - trace (3, "dm_truecrypt_init (trace_level=%d)\n", trace_level); if (!AutoTestAlgorithms ()) { - DMERR ("truecrypt: self-test of algorithms failed"); + error ("self-test of algorithms failed"); return -ERANGE; } @@ -614,21 +652,21 @@ int __init dm_truecrypt_init(void) if (!work_queue) { - DMERR ("truecrypt: create_workqueue creation failed"); + error ("create_workqueue failed"); goto err; } bio_ctx_cache = kmem_cache_create ("truecrypt-bioctx", sizeof (struct bio_ctx), 0, 0, NULL, NULL); if (!bio_ctx_cache) { - DMERR ("truecrypt: kmem_cache_create failed"); + error ("kmem_cache_create failed"); goto err; } r = dm_register_target (&truecrypt_target); if (r < 0) { - DMERR ("truecrypt: register failed %d", r); + error ("register failed %d", r); goto err; } @@ -647,12 +685,11 @@ err: void __exit dm_truecrypt_exit(void) { int r; - trace (3, "dm_truecrypt_exit ()\n"); r = dm_unregister_target (&truecrypt_target); if (r < 0) - DMERR ("truecrypt: unregister failed %d", r); + error ("unregister failed %d", r); destroy_workqueue (work_queue); kmem_cache_destroy (bio_ctx_cache); @@ -664,6 +701,6 @@ module_exit(dm_truecrypt_exit); module_param_named(trace, trace_level, int, 0); MODULE_AUTHOR("TrueCrypt Foundation"); -MODULE_DESCRIPTION(DM_NAME " target for encryption and decryption of TrueCrypt volumes"); +MODULE_DESCRIPTION("device-mapper target for encryption and decryption of TrueCrypt volumes"); MODULE_PARM_DESC(trace, "Trace level"); -MODULE_LICENSE("GPL and additional rights"); // Kernel thinks only GPL/BSD/MPL != closed-source code +MODULE_LICENSE("GPL and additional rights");