1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * This contains encryption functions for per-file encryption.
4  *
5  * Copyright (C) 2015, Google, Inc.
6  * Copyright (C) 2015, Motorola Mobility
7  *
8  * Written by Michael Halcrow, 2014.
9  *
10  * Filename encryption additions
11  *	Uday Savagaonkar, 2014
12  * Encryption policy handling additions
13  *	Ildar Muslukhov, 2014
14  * Add fscrypt_pullback_bio_page()
15  *	Jaegeuk Kim, 2015.
16  *
17  * This has not yet undergone a rigorous security audit.
18  *
19  * The usage of AES-XTS should conform to recommendations in NIST
20  * Special Publication 800-38E and IEEE P1619/D16.
21  */
22 
23 #include <linux/pagemap.h>
24 #include <linux/mempool.h>
25 #include <linux/module.h>
26 #include <linux/scatterlist.h>
27 #include <linux/ratelimit.h>
28 #include <crypto/skcipher.h>
29 #include "fscrypt_private.h"
30 
31 static unsigned int num_prealloc_crypto_pages = 32;
32 
33 module_param(num_prealloc_crypto_pages, uint, 0444);
34 MODULE_PARM_DESC(num_prealloc_crypto_pages,
35 		"Number of crypto pages to preallocate");
36 
37 static mempool_t *fscrypt_bounce_page_pool = NULL;
38 
39 static struct workqueue_struct *fscrypt_read_workqueue;
40 static DEFINE_MUTEX(fscrypt_init_mutex);
41 
42 struct kmem_cache *fscrypt_info_cachep;
43 
fscrypt_enqueue_decrypt_work(struct work_struct * work)44 void fscrypt_enqueue_decrypt_work(struct work_struct *work)
45 {
46 	queue_work(fscrypt_read_workqueue, work);
47 }
48 EXPORT_SYMBOL(fscrypt_enqueue_decrypt_work);
49 
fscrypt_alloc_bounce_page(gfp_t gfp_flags)50 struct page *fscrypt_alloc_bounce_page(gfp_t gfp_flags)
51 {
52 	return mempool_alloc(fscrypt_bounce_page_pool, gfp_flags);
53 }
54 
55 /**
56  * fscrypt_free_bounce_page() - free a ciphertext bounce page
57  * @bounce_page: the bounce page to free, or NULL
58  *
59  * Free a bounce page that was allocated by fscrypt_encrypt_pagecache_blocks(),
60  * or by fscrypt_alloc_bounce_page() directly.
61  */
fscrypt_free_bounce_page(struct page * bounce_page)62 void fscrypt_free_bounce_page(struct page *bounce_page)
63 {
64 	if (!bounce_page)
65 		return;
66 	set_page_private(bounce_page, (unsigned long)NULL);
67 	ClearPagePrivate(bounce_page);
68 	mempool_free(bounce_page, fscrypt_bounce_page_pool);
69 }
70 EXPORT_SYMBOL(fscrypt_free_bounce_page);
71 
fscrypt_generate_iv(union fscrypt_iv * iv,u64 lblk_num,const struct fscrypt_info * ci)72 void fscrypt_generate_iv(union fscrypt_iv *iv, u64 lblk_num,
73 			 const struct fscrypt_info *ci)
74 {
75 	u8 flags = fscrypt_policy_flags(&ci->ci_policy);
76 
77 	memset(iv, 0, ci->ci_mode->ivsize);
78 
79 	if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64) {
80 		WARN_ON_ONCE(lblk_num > U32_MAX);
81 		WARN_ON_ONCE(ci->ci_inode->i_ino > U32_MAX);
82 		lblk_num |= (u64)ci->ci_inode->i_ino << 32;
83 	} else if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32) {
84 		WARN_ON_ONCE(lblk_num > U32_MAX);
85 		lblk_num = (u32)(ci->ci_hashed_ino + lblk_num);
86 	} else if (flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY) {
87 		memcpy(iv->nonce, ci->ci_nonce, FSCRYPT_FILE_NONCE_SIZE);
88 	}
89 	iv->lblk_num = cpu_to_le64(lblk_num);
90 }
91 
92 /* Encrypt or decrypt a single filesystem block of file contents */
fscrypt_crypt_block(const struct inode * inode,fscrypt_direction_t rw,u64 lblk_num,struct page * src_page,struct page * dest_page,unsigned int len,unsigned int offs,gfp_t gfp_flags)93 int fscrypt_crypt_block(const struct inode *inode, fscrypt_direction_t rw,
94 			u64 lblk_num, struct page *src_page,
95 			struct page *dest_page, unsigned int len,
96 			unsigned int offs, gfp_t gfp_flags)
97 {
98 	union fscrypt_iv iv;
99 	struct skcipher_request *req = NULL;
100 	DECLARE_CRYPTO_WAIT(wait);
101 	struct scatterlist dst, src;
102 	struct fscrypt_info *ci = inode->i_crypt_info;
103 	struct crypto_skcipher *tfm = ci->ci_enc_key.tfm;
104 	int res = 0;
105 
106 	if (WARN_ON_ONCE(len <= 0))
107 		return -EINVAL;
108 	if (WARN_ON_ONCE(len % FS_CRYPTO_BLOCK_SIZE != 0))
109 		return -EINVAL;
110 
111 	fscrypt_generate_iv(&iv, lblk_num, ci);
112 
113 	req = skcipher_request_alloc(tfm, gfp_flags);
114 	if (!req)
115 		return -ENOMEM;
116 
117 	skcipher_request_set_callback(
118 		req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
119 		crypto_req_done, &wait);
120 
121 	sg_init_table(&dst, 1);
122 	sg_set_page(&dst, dest_page, len, offs);
123 	sg_init_table(&src, 1);
124 	sg_set_page(&src, src_page, len, offs);
125 	skcipher_request_set_crypt(req, &src, &dst, len, &iv);
126 	if (rw == FS_DECRYPT)
127 		res = crypto_wait_req(crypto_skcipher_decrypt(req), &wait);
128 	else
129 		res = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
130 	skcipher_request_free(req);
131 	if (res) {
132 		fscrypt_err(inode, "%scryption failed for block %llu: %d",
133 			    (rw == FS_DECRYPT ? "De" : "En"), lblk_num, res);
134 		return res;
135 	}
136 	return 0;
137 }
138 
139 /**
140  * fscrypt_encrypt_pagecache_blocks() - Encrypt filesystem blocks from a
141  *					pagecache page
142  * @page:      The locked pagecache page containing the block(s) to encrypt
143  * @len:       Total size of the block(s) to encrypt.  Must be a nonzero
144  *		multiple of the filesystem's block size.
145  * @offs:      Byte offset within @page of the first block to encrypt.  Must be
146  *		a multiple of the filesystem's block size.
147  * @gfp_flags: Memory allocation flags.  See details below.
148  *
149  * A new bounce page is allocated, and the specified block(s) are encrypted into
150  * it.  In the bounce page, the ciphertext block(s) will be located at the same
151  * offsets at which the plaintext block(s) were located in the source page; any
152  * other parts of the bounce page will be left uninitialized.  However, normally
153  * blocksize == PAGE_SIZE and the whole page is encrypted at once.
154  *
155  * This is for use by the filesystem's ->writepages() method.
156  *
157  * The bounce page allocation is mempool-backed, so it will always succeed when
158  * @gfp_flags includes __GFP_DIRECT_RECLAIM, e.g. when it's GFP_NOFS.  However,
159  * only the first page of each bio can be allocated this way.  To prevent
160  * deadlocks, for any additional pages a mask like GFP_NOWAIT must be used.
161  *
162  * Return: the new encrypted bounce page on success; an ERR_PTR() on failure
163  */
fscrypt_encrypt_pagecache_blocks(struct page * page,unsigned int len,unsigned int offs,gfp_t gfp_flags)164 struct page *fscrypt_encrypt_pagecache_blocks(struct page *page,
165 					      unsigned int len,
166 					      unsigned int offs,
167 					      gfp_t gfp_flags)
168 
169 {
170 	const struct inode *inode = page->mapping->host;
171 	const unsigned int blockbits = inode->i_blkbits;
172 	const unsigned int blocksize = 1 << blockbits;
173 	struct page *ciphertext_page;
174 	u64 lblk_num = ((u64)page->index << (PAGE_SHIFT - blockbits)) +
175 		       (offs >> blockbits);
176 	unsigned int i;
177 	int err;
178 
179 	if (WARN_ON_ONCE(!PageLocked(page)))
180 		return ERR_PTR(-EINVAL);
181 
182 	if (WARN_ON_ONCE(len <= 0 || !IS_ALIGNED(len | offs, blocksize)))
183 		return ERR_PTR(-EINVAL);
184 
185 	ciphertext_page = fscrypt_alloc_bounce_page(gfp_flags);
186 	if (!ciphertext_page)
187 		return ERR_PTR(-ENOMEM);
188 
189 	for (i = offs; i < offs + len; i += blocksize, lblk_num++) {
190 		err = fscrypt_crypt_block(inode, FS_ENCRYPT, lblk_num,
191 					  page, ciphertext_page,
192 					  blocksize, i, gfp_flags);
193 		if (err) {
194 			fscrypt_free_bounce_page(ciphertext_page);
195 			return ERR_PTR(err);
196 		}
197 	}
198 	SetPagePrivate(ciphertext_page);
199 	set_page_private(ciphertext_page, (unsigned long)page);
200 	return ciphertext_page;
201 }
202 EXPORT_SYMBOL(fscrypt_encrypt_pagecache_blocks);
203 
204 /**
205  * fscrypt_encrypt_block_inplace() - Encrypt a filesystem block in-place
206  * @inode:     The inode to which this block belongs
207  * @page:      The page containing the block to encrypt
208  * @len:       Size of block to encrypt.  Doesn't need to be a multiple of the
209  *		fs block size, but must be a multiple of FS_CRYPTO_BLOCK_SIZE.
210  * @offs:      Byte offset within @page at which the block to encrypt begins
211  * @lblk_num:  Filesystem logical block number of the block, i.e. the 0-based
212  *		number of the block within the file
213  * @gfp_flags: Memory allocation flags
214  *
215  * Encrypt a possibly-compressed filesystem block that is located in an
216  * arbitrary page, not necessarily in the original pagecache page.  The @inode
217  * and @lblk_num must be specified, as they can't be determined from @page.
218  *
219  * Return: 0 on success; -errno on failure
220  */
fscrypt_encrypt_block_inplace(const struct inode * inode,struct page * page,unsigned int len,unsigned int offs,u64 lblk_num,gfp_t gfp_flags)221 int fscrypt_encrypt_block_inplace(const struct inode *inode, struct page *page,
222 				  unsigned int len, unsigned int offs,
223 				  u64 lblk_num, gfp_t gfp_flags)
224 {
225 	return fscrypt_crypt_block(inode, FS_ENCRYPT, lblk_num, page, page,
226 				   len, offs, gfp_flags);
227 }
228 EXPORT_SYMBOL(fscrypt_encrypt_block_inplace);
229 
230 /**
231  * fscrypt_decrypt_pagecache_blocks() - Decrypt filesystem blocks in a
232  *					pagecache page
233  * @page:      The locked pagecache page containing the block(s) to decrypt
234  * @len:       Total size of the block(s) to decrypt.  Must be a nonzero
235  *		multiple of the filesystem's block size.
236  * @offs:      Byte offset within @page of the first block to decrypt.  Must be
237  *		a multiple of the filesystem's block size.
238  *
239  * The specified block(s) are decrypted in-place within the pagecache page,
240  * which must still be locked and not uptodate.  Normally, blocksize ==
241  * PAGE_SIZE and the whole page is decrypted at once.
242  *
243  * This is for use by the filesystem's ->readpages() method.
244  *
245  * Return: 0 on success; -errno on failure
246  */
fscrypt_decrypt_pagecache_blocks(struct page * page,unsigned int len,unsigned int offs)247 int fscrypt_decrypt_pagecache_blocks(struct page *page, unsigned int len,
248 				     unsigned int offs)
249 {
250 	const struct inode *inode = page->mapping->host;
251 	const unsigned int blockbits = inode->i_blkbits;
252 	const unsigned int blocksize = 1 << blockbits;
253 	u64 lblk_num = ((u64)page->index << (PAGE_SHIFT - blockbits)) +
254 		       (offs >> blockbits);
255 	unsigned int i;
256 	int err;
257 
258 	if (WARN_ON_ONCE(!PageLocked(page)))
259 		return -EINVAL;
260 
261 	if (WARN_ON_ONCE(len <= 0 || !IS_ALIGNED(len | offs, blocksize)))
262 		return -EINVAL;
263 
264 	for (i = offs; i < offs + len; i += blocksize, lblk_num++) {
265 		err = fscrypt_crypt_block(inode, FS_DECRYPT, lblk_num, page,
266 					  page, blocksize, i, GFP_NOFS);
267 		if (err)
268 			return err;
269 	}
270 	return 0;
271 }
272 EXPORT_SYMBOL(fscrypt_decrypt_pagecache_blocks);
273 
274 /**
275  * fscrypt_decrypt_block_inplace() - Decrypt a filesystem block in-place
276  * @inode:     The inode to which this block belongs
277  * @page:      The page containing the block to decrypt
278  * @len:       Size of block to decrypt.  Doesn't need to be a multiple of the
279  *		fs block size, but must be a multiple of FS_CRYPTO_BLOCK_SIZE.
280  * @offs:      Byte offset within @page at which the block to decrypt begins
281  * @lblk_num:  Filesystem logical block number of the block, i.e. the 0-based
282  *		number of the block within the file
283  *
284  * Decrypt a possibly-compressed filesystem block that is located in an
285  * arbitrary page, not necessarily in the original pagecache page.  The @inode
286  * and @lblk_num must be specified, as they can't be determined from @page.
287  *
288  * Return: 0 on success; -errno on failure
289  */
fscrypt_decrypt_block_inplace(const struct inode * inode,struct page * page,unsigned int len,unsigned int offs,u64 lblk_num)290 int fscrypt_decrypt_block_inplace(const struct inode *inode, struct page *page,
291 				  unsigned int len, unsigned int offs,
292 				  u64 lblk_num)
293 {
294 	return fscrypt_crypt_block(inode, FS_DECRYPT, lblk_num, page, page,
295 				   len, offs, GFP_NOFS);
296 }
297 EXPORT_SYMBOL(fscrypt_decrypt_block_inplace);
298 
299 /**
300  * fscrypt_initialize() - allocate major buffers for fs encryption.
301  * @cop_flags:  fscrypt operations flags
302  *
303  * We only call this when we start accessing encrypted files, since it
304  * results in memory getting allocated that wouldn't otherwise be used.
305  *
306  * Return: 0 on success; -errno on failure
307  */
fscrypt_initialize(unsigned int cop_flags)308 int fscrypt_initialize(unsigned int cop_flags)
309 {
310 	int err = 0;
311 
312 	/* No need to allocate a bounce page pool if this FS won't use it. */
313 	if (cop_flags & FS_CFLG_OWN_PAGES)
314 		return 0;
315 
316 	mutex_lock(&fscrypt_init_mutex);
317 	if (fscrypt_bounce_page_pool)
318 		goto out_unlock;
319 
320 	err = -ENOMEM;
321 	fscrypt_bounce_page_pool =
322 		mempool_create_page_pool(num_prealloc_crypto_pages, 0);
323 	if (!fscrypt_bounce_page_pool)
324 		goto out_unlock;
325 
326 	err = 0;
327 out_unlock:
328 	mutex_unlock(&fscrypt_init_mutex);
329 	return err;
330 }
331 
fscrypt_msg(const struct inode * inode,const char * level,const char * fmt,...)332 void fscrypt_msg(const struct inode *inode, const char *level,
333 		 const char *fmt, ...)
334 {
335 	static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
336 				      DEFAULT_RATELIMIT_BURST);
337 	struct va_format vaf;
338 	va_list args;
339 
340 	if (!__ratelimit(&rs))
341 		return;
342 
343 	va_start(args, fmt);
344 	vaf.fmt = fmt;
345 	vaf.va = &args;
346 	if (inode && inode->i_ino)
347 		printk("%sfscrypt (%s, inode %lu): %pV\n",
348 		       level, inode->i_sb->s_id, inode->i_ino, &vaf);
349 	else if (inode)
350 		printk("%sfscrypt (%s): %pV\n", level, inode->i_sb->s_id, &vaf);
351 	else
352 		printk("%sfscrypt: %pV\n", level, &vaf);
353 	va_end(args);
354 }
355 
356 /**
357  * fscrypt_init() - Set up for fs encryption.
358  *
359  * Return: 0 on success; -errno on failure
360  */
fscrypt_init(void)361 static int __init fscrypt_init(void)
362 {
363 	int err = -ENOMEM;
364 
365 	/*
366 	 * Use an unbound workqueue to allow bios to be decrypted in parallel
367 	 * even when they happen to complete on the same CPU.  This sacrifices
368 	 * locality, but it's worthwhile since decryption is CPU-intensive.
369 	 *
370 	 * Also use a high-priority workqueue to prioritize decryption work,
371 	 * which blocks reads from completing, over regular application tasks.
372 	 */
373 	fscrypt_read_workqueue = alloc_workqueue("fscrypt_read_queue",
374 						 WQ_UNBOUND | WQ_HIGHPRI,
375 						 num_online_cpus());
376 	if (!fscrypt_read_workqueue)
377 		goto fail;
378 
379 	fscrypt_info_cachep = KMEM_CACHE(fscrypt_info, SLAB_RECLAIM_ACCOUNT);
380 	if (!fscrypt_info_cachep)
381 		goto fail_free_queue;
382 
383 	err = fscrypt_init_keyring();
384 	if (err)
385 		goto fail_free_info;
386 
387 	return 0;
388 
389 fail_free_info:
390 	kmem_cache_destroy(fscrypt_info_cachep);
391 fail_free_queue:
392 	destroy_workqueue(fscrypt_read_workqueue);
393 fail:
394 	return err;
395 }
396 late_initcall(fscrypt_init)
397