From 82a22311b7a68a78709699dc8c098953b70e4fd2 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 7 Nov 2019 22:49:11 -0500 Subject: XArray: Fix xas_pause at ULONG_MAX If we were unlucky enough to call xas_pause() when the index was at ULONG_MAX (or a multi-slot entry which ends at ULONG_MAX), we would wrap the index back around to 0 and restart the iteration from the beginning. Use the XAS_BOUNDS state to indicate that we should just stop the iteration. Signed-off-by: Matthew Wilcox (Oracle) --- lib/test_xarray.c | 22 ++++++++++++++++++++++ lib/xarray.c | 8 +++++--- 2 files changed, 27 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/test_xarray.c b/lib/test_xarray.c index 7df4f7f395bf..c477f22a95cd 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -1132,6 +1132,27 @@ static noinline void check_move_tiny(struct xarray *xa) XA_BUG_ON(xa, !xa_empty(xa)); } +static noinline void check_move_max(struct xarray *xa) +{ + XA_STATE(xas, xa, 0); + + xa_store_index(xa, ULONG_MAX, GFP_KERNEL); + rcu_read_lock(); + XA_BUG_ON(xa, xas_find(&xas, ULONG_MAX) != xa_mk_index(ULONG_MAX)); + XA_BUG_ON(xa, xas_find(&xas, ULONG_MAX) != NULL); + rcu_read_unlock(); + + xas_set(&xas, 0); + rcu_read_lock(); + XA_BUG_ON(xa, xas_find(&xas, ULONG_MAX) != xa_mk_index(ULONG_MAX)); + xas_pause(&xas); + XA_BUG_ON(xa, xas_find(&xas, ULONG_MAX) != NULL); + rcu_read_unlock(); + + xa_erase_index(xa, ULONG_MAX); + XA_BUG_ON(xa, !xa_empty(xa)); +} + static noinline void check_move_small(struct xarray *xa, unsigned long idx) { XA_STATE(xas, xa, 0); @@ -1240,6 +1261,7 @@ static noinline void check_move(struct xarray *xa) xa_destroy(xa); check_move_tiny(xa); + check_move_max(xa); for (i = 0; i < 16; i++) check_move_small(xa, 1UL << i); diff --git a/lib/xarray.c b/lib/xarray.c index 1237c213f52b..bfaaa2c1f3fe 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -967,6 +967,7 @@ void xas_pause(struct xa_state *xas) if (xas_invalid(xas)) return; + xas->xa_node = XAS_RESTART; if (node) { unsigned int offset = xas->xa_offset; while (++offset < XA_CHUNK_SIZE) { @@ -974,10 +975,11 @@ void xas_pause(struct xa_state *xas) break; } xas->xa_index += (offset - xas->xa_offset) << node->shift; + if (xas->xa_index == 0) + xas->xa_node = XAS_BOUNDS; } else { xas->xa_index++; } - xas->xa_node = XAS_RESTART; } EXPORT_SYMBOL_GPL(xas_pause); @@ -1079,13 +1081,13 @@ void *xas_find(struct xa_state *xas, unsigned long max) { void *entry; - if (xas_error(xas)) + if (xas_error(xas) || xas->xa_node == XAS_BOUNDS) return NULL; if (!xas->xa_node) { xas->xa_index = 1; return set_bounds(xas); - } else if (xas_top(xas->xa_node)) { + } else if (xas->xa_node == XAS_RESTART) { entry = xas_load(xas); if (entry || xas_not_node(xas->xa_node)) return entry; -- cgit v1.2.3 From 9f47286924f2f868760a528c14596370e32d86b4 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Tue, 15 Oct 2019 21:18:19 +0200 Subject: sched/core: Use CONFIG_PREEMPTION CONFIG_PREEMPTION is selected by CONFIG_PREEMPT and by CONFIG_PREEMPT_RT. Both PREEMPT and PREEMPT_RT require the same functionality which today depends on CONFIG_PREEMPT. Let DEBUG_PREEMPT depend on CONFIG_PREEMPTION. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Cc: Andrew Morton Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20191015191821.11479-33-bigeasy@linutronix.de Signed-off-by: Ingo Molnar --- lib/Kconfig.debug | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 2f6fb96405af..d03e2b1570ca 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1051,7 +1051,7 @@ config DEBUG_TIMEKEEPING config DEBUG_PREEMPT bool "Debug preemptible kernel" - depends on DEBUG_KERNEL && PREEMPT && TRACE_IRQFLAGS_SUPPORT + depends on DEBUG_KERNEL && PREEMPTION && TRACE_IRQFLAGS_SUPPORT default y help If you say Y here then the kernel will use a debug variant of the -- cgit v1.2.3 From aa127963f1cab2b93c74c9b128a84610203fb674 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Mon, 16 Dec 2019 19:53:26 +0100 Subject: crypto: lib/curve25519 - re-add selftests Somehow these were dropped when Zinc was being integrated, which is problematic, because testing the library interface for Curve25519 is important.. This commit simply adds them back and wires them in in the same way that the blake2s selftests are wired in. Signed-off-by: Jason A. Donenfeld Signed-off-by: Herbert Xu --- lib/crypto/Makefile | 1 + lib/crypto/curve25519-selftest.c | 1321 ++++++++++++++++++++++++++++++++++++++ lib/crypto/curve25519.c | 17 + 3 files changed, 1339 insertions(+) create mode 100644 lib/crypto/curve25519-selftest.c (limited to 'lib') diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 34a701ab8b92..f97f9b941110 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -36,4 +36,5 @@ libsha256-y := sha256.o ifneq ($(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS),y) libblake2s-y += blake2s-selftest.o libchacha20poly1305-y += chacha20poly1305-selftest.o +libcurve25519-y += curve25519-selftest.o endif diff --git a/lib/crypto/curve25519-selftest.c b/lib/crypto/curve25519-selftest.c new file mode 100644 index 000000000000..c85e85381e78 --- /dev/null +++ b/lib/crypto/curve25519-selftest.c @@ -0,0 +1,1321 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include + +struct curve25519_test_vector { + u8 private[CURVE25519_KEY_SIZE]; + u8 public[CURVE25519_KEY_SIZE]; + u8 result[CURVE25519_KEY_SIZE]; + bool valid; +}; +static const struct curve25519_test_vector curve25519_test_vectors[] __initconst = { + { + .private = { 0x77, 0x07, 0x6d, 0x0a, 0x73, 0x18, 0xa5, 0x7d, + 0x3c, 0x16, 0xc1, 0x72, 0x51, 0xb2, 0x66, 0x45, + 0xdf, 0x4c, 0x2f, 0x87, 0xeb, 0xc0, 0x99, 0x2a, + 0xb1, 0x77, 0xfb, 0xa5, 0x1d, 0xb9, 0x2c, 0x2a }, + .public = { 0xde, 0x9e, 0xdb, 0x7d, 0x7b, 0x7d, 0xc1, 0xb4, + 0xd3, 0x5b, 0x61, 0xc2, 0xec, 0xe4, 0x35, 0x37, + 0x3f, 0x83, 0x43, 0xc8, 0x5b, 0x78, 0x67, 0x4d, + 0xad, 0xfc, 0x7e, 0x14, 0x6f, 0x88, 0x2b, 0x4f }, + .result = { 0x4a, 0x5d, 0x9d, 0x5b, 0xa4, 0xce, 0x2d, 0xe1, + 0x72, 0x8e, 0x3b, 0xf4, 0x80, 0x35, 0x0f, 0x25, + 0xe0, 0x7e, 0x21, 0xc9, 0x47, 0xd1, 0x9e, 0x33, + 0x76, 0xf0, 0x9b, 0x3c, 0x1e, 0x16, 0x17, 0x42 }, + .valid = true + }, + { + .private = { 0x5d, 0xab, 0x08, 0x7e, 0x62, 0x4a, 0x8a, 0x4b, + 0x79, 0xe1, 0x7f, 0x8b, 0x83, 0x80, 0x0e, 0xe6, + 0x6f, 0x3b, 0xb1, 0x29, 0x26, 0x18, 0xb6, 0xfd, + 0x1c, 0x2f, 0x8b, 0x27, 0xff, 0x88, 0xe0, 0xeb }, + .public = { 0x85, 0x20, 0xf0, 0x09, 0x89, 0x30, 0xa7, 0x54, + 0x74, 0x8b, 0x7d, 0xdc, 0xb4, 0x3e, 0xf7, 0x5a, + 0x0d, 0xbf, 0x3a, 0x0d, 0x26, 0x38, 0x1a, 0xf4, + 0xeb, 0xa4, 0xa9, 0x8e, 0xaa, 0x9b, 0x4e, 0x6a }, + .result = { 0x4a, 0x5d, 0x9d, 0x5b, 0xa4, 0xce, 0x2d, 0xe1, + 0x72, 0x8e, 0x3b, 0xf4, 0x80, 0x35, 0x0f, 0x25, + 0xe0, 0x7e, 0x21, 0xc9, 0x47, 0xd1, 0x9e, 0x33, + 0x76, 0xf0, 0x9b, 0x3c, 0x1e, 0x16, 0x17, 0x42 }, + .valid = true + }, + { + .private = { 1 }, + .public = { 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x3c, 0x77, 0x77, 0xca, 0xf9, 0x97, 0xb2, 0x64, + 0x41, 0x60, 0x77, 0x66, 0x5b, 0x4e, 0x22, 0x9d, + 0x0b, 0x95, 0x48, 0xdc, 0x0c, 0xd8, 0x19, 0x98, + 0xdd, 0xcd, 0xc5, 0xc8, 0x53, 0x3c, 0x79, 0x7f }, + .valid = true + }, + { + .private = { 1 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0xb3, 0x2d, 0x13, 0x62, 0xc2, 0x48, 0xd6, 0x2f, + 0xe6, 0x26, 0x19, 0xcf, 0xf0, 0x4d, 0xd4, 0x3d, + 0xb7, 0x3f, 0xfc, 0x1b, 0x63, 0x08, 0xed, 0xe3, + 0x0b, 0x78, 0xd8, 0x73, 0x80, 0xf1, 0xe8, 0x34 }, + .valid = true + }, + { + .private = { 0xa5, 0x46, 0xe3, 0x6b, 0xf0, 0x52, 0x7c, 0x9d, + 0x3b, 0x16, 0x15, 0x4b, 0x82, 0x46, 0x5e, 0xdd, + 0x62, 0x14, 0x4c, 0x0a, 0xc1, 0xfc, 0x5a, 0x18, + 0x50, 0x6a, 0x22, 0x44, 0xba, 0x44, 0x9a, 0xc4 }, + .public = { 0xe6, 0xdb, 0x68, 0x67, 0x58, 0x30, 0x30, 0xdb, + 0x35, 0x94, 0xc1, 0xa4, 0x24, 0xb1, 0x5f, 0x7c, + 0x72, 0x66, 0x24, 0xec, 0x26, 0xb3, 0x35, 0x3b, + 0x10, 0xa9, 0x03, 0xa6, 0xd0, 0xab, 0x1c, 0x4c }, + .result = { 0xc3, 0xda, 0x55, 0x37, 0x9d, 0xe9, 0xc6, 0x90, + 0x8e, 0x94, 0xea, 0x4d, 0xf2, 0x8d, 0x08, 0x4f, + 0x32, 0xec, 0xcf, 0x03, 0x49, 0x1c, 0x71, 0xf7, + 0x54, 0xb4, 0x07, 0x55, 0x77, 0xa2, 0x85, 0x52 }, + .valid = true + }, + { + .private = { 1, 2, 3, 4 }, + .public = { 0 }, + .result = { 0 }, + .valid = false + }, + { + .private = { 2, 4, 6, 8 }, + .public = { 0xe0, 0xeb, 0x7a, 0x7c, 0x3b, 0x41, 0xb8, 0xae, + 0x16, 0x56, 0xe3, 0xfa, 0xf1, 0x9f, 0xc4, 0x6a, + 0xda, 0x09, 0x8d, 0xeb, 0x9c, 0x32, 0xb1, 0xfd, + 0x86, 0x62, 0x05, 0x16, 0x5f, 0x49, 0xb8 }, + .result = { 0 }, + .valid = false + }, + { + .private = { 0xff, 0xff, 0xff, 0xff, 0x0a, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x0a, 0x00, 0xfb, 0x9f }, + .result = { 0x77, 0x52, 0xb6, 0x18, 0xc1, 0x2d, 0x48, 0xd2, + 0xc6, 0x93, 0x46, 0x83, 0x81, 0x7c, 0xc6, 0x57, + 0xf3, 0x31, 0x03, 0x19, 0x49, 0x48, 0x20, 0x05, + 0x42, 0x2b, 0x4e, 0xae, 0x8d, 0x1d, 0x43, 0x23 }, + .valid = true + }, + { + .private = { 0x8e, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .public = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8e, 0x06 }, + .result = { 0x5a, 0xdf, 0xaa, 0x25, 0x86, 0x8e, 0x32, 0x3d, + 0xae, 0x49, 0x62, 0xc1, 0x01, 0x5c, 0xb3, 0x12, + 0xe1, 0xc5, 0xc7, 0x9e, 0x95, 0x3f, 0x03, 0x99, + 0xb0, 0xba, 0x16, 0x22, 0xf3, 0xb6, 0xf7, 0x0c }, + .valid = true + }, + /* wycheproof - normal case */ + { + .private = { 0x48, 0x52, 0x83, 0x4d, 0x9d, 0x6b, 0x77, 0xda, + 0xde, 0xab, 0xaa, 0xf2, 0xe1, 0x1d, 0xca, 0x66, + 0xd1, 0x9f, 0xe7, 0x49, 0x93, 0xa7, 0xbe, 0xc3, + 0x6c, 0x6e, 0x16, 0xa0, 0x98, 0x3f, 0xea, 0xba }, + .public = { 0x9c, 0x64, 0x7d, 0x9a, 0xe5, 0x89, 0xb9, 0xf5, + 0x8f, 0xdc, 0x3c, 0xa4, 0x94, 0x7e, 0xfb, 0xc9, + 0x15, 0xc4, 0xb2, 0xe0, 0x8e, 0x74, 0x4a, 0x0e, + 0xdf, 0x46, 0x9d, 0xac, 0x59, 0xc8, 0xf8, 0x5a }, + .result = { 0x87, 0xb7, 0xf2, 0x12, 0xb6, 0x27, 0xf7, 0xa5, + 0x4c, 0xa5, 0xe0, 0xbc, 0xda, 0xdd, 0xd5, 0x38, + 0x9d, 0x9d, 0xe6, 0x15, 0x6c, 0xdb, 0xcf, 0x8e, + 0xbe, 0x14, 0xff, 0xbc, 0xfb, 0x43, 0x65, 0x51 }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0x58, 0x8c, 0x06, 0x1a, 0x50, 0x80, 0x4a, 0xc4, + 0x88, 0xad, 0x77, 0x4a, 0xc7, 0x16, 0xc3, 0xf5, + 0xba, 0x71, 0x4b, 0x27, 0x12, 0xe0, 0x48, 0x49, + 0x13, 0x79, 0xa5, 0x00, 0x21, 0x19, 0x98, 0xa8 }, + .public = { 0x63, 0xaa, 0x40, 0xc6, 0xe3, 0x83, 0x46, 0xc5, + 0xca, 0xf2, 0x3a, 0x6d, 0xf0, 0xa5, 0xe6, 0xc8, + 0x08, 0x89, 0xa0, 0x86, 0x47, 0xe5, 0x51, 0xb3, + 0x56, 0x34, 0x49, 0xbe, 0xfc, 0xfc, 0x97, 0x33 }, + .result = { 0xb1, 0xa7, 0x07, 0x51, 0x94, 0x95, 0xff, 0xff, + 0xb2, 0x98, 0xff, 0x94, 0x17, 0x16, 0xb0, 0x6d, + 0xfa, 0xb8, 0x7c, 0xf8, 0xd9, 0x11, 0x23, 0xfe, + 0x2b, 0xe9, 0xa2, 0x33, 0xdd, 0xa2, 0x22, 0x12 }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0xb0, 0x5b, 0xfd, 0x32, 0xe5, 0x53, 0x25, 0xd9, + 0xfd, 0x64, 0x8c, 0xb3, 0x02, 0x84, 0x80, 0x39, + 0x00, 0x0b, 0x39, 0x0e, 0x44, 0xd5, 0x21, 0xe5, + 0x8a, 0xab, 0x3b, 0x29, 0xa6, 0x96, 0x0b, 0xa8 }, + .public = { 0x0f, 0x83, 0xc3, 0x6f, 0xde, 0xd9, 0xd3, 0x2f, + 0xad, 0xf4, 0xef, 0xa3, 0xae, 0x93, 0xa9, 0x0b, + 0xb5, 0xcf, 0xa6, 0x68, 0x93, 0xbc, 0x41, 0x2c, + 0x43, 0xfa, 0x72, 0x87, 0xdb, 0xb9, 0x97, 0x79 }, + .result = { 0x67, 0xdd, 0x4a, 0x6e, 0x16, 0x55, 0x33, 0x53, + 0x4c, 0x0e, 0x3f, 0x17, 0x2e, 0x4a, 0xb8, 0x57, + 0x6b, 0xca, 0x92, 0x3a, 0x5f, 0x07, 0xb2, 0xc0, + 0x69, 0xb4, 0xc3, 0x10, 0xff, 0x2e, 0x93, 0x5b }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0x70, 0xe3, 0x4b, 0xcb, 0xe1, 0xf4, 0x7f, 0xbc, + 0x0f, 0xdd, 0xfd, 0x7c, 0x1e, 0x1a, 0xa5, 0x3d, + 0x57, 0xbf, 0xe0, 0xf6, 0x6d, 0x24, 0x30, 0x67, + 0xb4, 0x24, 0xbb, 0x62, 0x10, 0xbe, 0xd1, 0x9c }, + .public = { 0x0b, 0x82, 0x11, 0xa2, 0xb6, 0x04, 0x90, 0x97, + 0xf6, 0x87, 0x1c, 0x6c, 0x05, 0x2d, 0x3c, 0x5f, + 0xc1, 0xba, 0x17, 0xda, 0x9e, 0x32, 0xae, 0x45, + 0x84, 0x03, 0xb0, 0x5b, 0xb2, 0x83, 0x09, 0x2a }, + .result = { 0x4a, 0x06, 0x38, 0xcf, 0xaa, 0x9e, 0xf1, 0x93, + 0x3b, 0x47, 0xf8, 0x93, 0x92, 0x96, 0xa6, 0xb2, + 0x5b, 0xe5, 0x41, 0xef, 0x7f, 0x70, 0xe8, 0x44, + 0xc0, 0xbc, 0xc0, 0x0b, 0x13, 0x4d, 0xe6, 0x4a }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0x68, 0xc1, 0xf3, 0xa6, 0x53, 0xa4, 0xcd, 0xb1, + 0xd3, 0x7b, 0xba, 0x94, 0x73, 0x8f, 0x8b, 0x95, + 0x7a, 0x57, 0xbe, 0xb2, 0x4d, 0x64, 0x6e, 0x99, + 0x4d, 0xc2, 0x9a, 0x27, 0x6a, 0xad, 0x45, 0x8d }, + .public = { 0x34, 0x3a, 0xc2, 0x0a, 0x3b, 0x9c, 0x6a, 0x27, + 0xb1, 0x00, 0x81, 0x76, 0x50, 0x9a, 0xd3, 0x07, + 0x35, 0x85, 0x6e, 0xc1, 0xc8, 0xd8, 0xfc, 0xae, + 0x13, 0x91, 0x2d, 0x08, 0xd1, 0x52, 0xf4, 0x6c }, + .result = { 0x39, 0x94, 0x91, 0xfc, 0xe8, 0xdf, 0xab, 0x73, + 0xb4, 0xf9, 0xf6, 0x11, 0xde, 0x8e, 0xa0, 0xb2, + 0x7b, 0x28, 0xf8, 0x59, 0x94, 0x25, 0x0b, 0x0f, + 0x47, 0x5d, 0x58, 0x5d, 0x04, 0x2a, 0xc2, 0x07 }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0xd8, 0x77, 0xb2, 0x6d, 0x06, 0xdf, 0xf9, 0xd9, + 0xf7, 0xfd, 0x4c, 0x5b, 0x37, 0x69, 0xf8, 0xcd, + 0xd5, 0xb3, 0x05, 0x16, 0xa5, 0xab, 0x80, 0x6b, + 0xe3, 0x24, 0xff, 0x3e, 0xb6, 0x9e, 0xa0, 0xb2 }, + .public = { 0xfa, 0x69, 0x5f, 0xc7, 0xbe, 0x8d, 0x1b, 0xe5, + 0xbf, 0x70, 0x48, 0x98, 0xf3, 0x88, 0xc4, 0x52, + 0xba, 0xfd, 0xd3, 0xb8, 0xea, 0xe8, 0x05, 0xf8, + 0x68, 0x1a, 0x8d, 0x15, 0xc2, 0xd4, 0xe1, 0x42 }, + .result = { 0x2c, 0x4f, 0xe1, 0x1d, 0x49, 0x0a, 0x53, 0x86, + 0x17, 0x76, 0xb1, 0x3b, 0x43, 0x54, 0xab, 0xd4, + 0xcf, 0x5a, 0x97, 0x69, 0x9d, 0xb6, 0xe6, 0xc6, + 0x8c, 0x16, 0x26, 0xd0, 0x76, 0x62, 0xf7, 0x58 }, + .valid = true + }, + /* wycheproof - public key = 0 */ + { + .private = { 0x20, 0x74, 0x94, 0x03, 0x8f, 0x2b, 0xb8, 0x11, + 0xd4, 0x78, 0x05, 0xbc, 0xdf, 0x04, 0xa2, 0xac, + 0x58, 0x5a, 0xda, 0x7f, 0x2f, 0x23, 0x38, 0x9b, + 0xfd, 0x46, 0x58, 0xf9, 0xdd, 0xd4, 0xde, 0xbc }, + .public = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key = 1 */ + { + .private = { 0x20, 0x2e, 0x89, 0x72, 0xb6, 0x1c, 0x7e, 0x61, + 0x93, 0x0e, 0xb9, 0x45, 0x0b, 0x50, 0x70, 0xea, + 0xe1, 0xc6, 0x70, 0x47, 0x56, 0x85, 0x54, 0x1f, + 0x04, 0x76, 0x21, 0x7e, 0x48, 0x18, 0xcf, 0xab }, + .public = { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x38, 0xdd, 0xe9, 0xf3, 0xe7, 0xb7, 0x99, 0x04, + 0x5f, 0x9a, 0xc3, 0x79, 0x3d, 0x4a, 0x92, 0x77, + 0xda, 0xde, 0xad, 0xc4, 0x1b, 0xec, 0x02, 0x90, + 0xf8, 0x1f, 0x74, 0x4f, 0x73, 0x77, 0x5f, 0x84 }, + .public = { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x9a, 0x2c, 0xfe, 0x84, 0xff, 0x9c, 0x4a, 0x97, + 0x39, 0x62, 0x5c, 0xae, 0x4a, 0x3b, 0x82, 0xa9, + 0x06, 0x87, 0x7a, 0x44, 0x19, 0x46, 0xf8, 0xd7, + 0xb3, 0xd7, 0x95, 0xfe, 0x8f, 0x5d, 0x16, 0x39 }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x98, 0x57, 0xa9, 0x14, 0xe3, 0xc2, 0x90, 0x36, + 0xfd, 0x9a, 0x44, 0x2b, 0xa5, 0x26, 0xb5, 0xcd, + 0xcd, 0xf2, 0x82, 0x16, 0x15, 0x3e, 0x63, 0x6c, + 0x10, 0x67, 0x7a, 0xca, 0xb6, 0xbd, 0x6a, 0xa5 }, + .public = { 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x4d, 0xa4, 0xe0, 0xaa, 0x07, 0x2c, 0x23, 0x2e, + 0xe2, 0xf0, 0xfa, 0x4e, 0x51, 0x9a, 0xe5, 0x0b, + 0x52, 0xc1, 0xed, 0xd0, 0x8a, 0x53, 0x4d, 0x4e, + 0xf3, 0x46, 0xc2, 0xe1, 0x06, 0xd2, 0x1d, 0x60 }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x48, 0xe2, 0x13, 0x0d, 0x72, 0x33, 0x05, 0xed, + 0x05, 0xe6, 0xe5, 0x89, 0x4d, 0x39, 0x8a, 0x5e, + 0x33, 0x36, 0x7a, 0x8c, 0x6a, 0xac, 0x8f, 0xcd, + 0xf0, 0xa8, 0x8e, 0x4b, 0x42, 0x82, 0x0d, 0xb7 }, + .public = { 0xff, 0xff, 0xff, 0x03, 0x00, 0x00, 0xf8, 0xff, + 0xff, 0x1f, 0x00, 0x00, 0xc0, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0xfe, 0xff, 0xff, 0x07, 0x00, + 0x00, 0xf0, 0xff, 0xff, 0x3f, 0x00, 0x00, 0x00 }, + .result = { 0x9e, 0xd1, 0x0c, 0x53, 0x74, 0x7f, 0x64, 0x7f, + 0x82, 0xf4, 0x51, 0x25, 0xd3, 0xde, 0x15, 0xa1, + 0xe6, 0xb8, 0x24, 0x49, 0x6a, 0xb4, 0x04, 0x10, + 0xff, 0xcc, 0x3c, 0xfe, 0x95, 0x76, 0x0f, 0x3b }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x28, 0xf4, 0x10, 0x11, 0x69, 0x18, 0x51, 0xb3, + 0xa6, 0x2b, 0x64, 0x15, 0x53, 0xb3, 0x0d, 0x0d, + 0xfd, 0xdc, 0xb8, 0xff, 0xfc, 0xf5, 0x37, 0x00, + 0xa7, 0xbe, 0x2f, 0x6a, 0x87, 0x2e, 0x9f, 0xb0 }, + .public = { 0x00, 0x00, 0x00, 0xfc, 0xff, 0xff, 0x07, 0x00, + 0x00, 0xe0, 0xff, 0xff, 0x3f, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0x01, 0x00, 0x00, 0xf8, 0xff, + 0xff, 0x0f, 0x00, 0x00, 0xc0, 0xff, 0xff, 0x7f }, + .result = { 0xcf, 0x72, 0xb4, 0xaa, 0x6a, 0xa1, 0xc9, 0xf8, + 0x94, 0xf4, 0x16, 0x5b, 0x86, 0x10, 0x9a, 0xa4, + 0x68, 0x51, 0x76, 0x48, 0xe1, 0xf0, 0xcc, 0x70, + 0xe1, 0xab, 0x08, 0x46, 0x01, 0x76, 0x50, 0x6b }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x18, 0xa9, 0x3b, 0x64, 0x99, 0xb9, 0xf6, 0xb3, + 0x22, 0x5c, 0xa0, 0x2f, 0xef, 0x41, 0x0e, 0x0a, + 0xde, 0xc2, 0x35, 0x32, 0x32, 0x1d, 0x2d, 0x8e, + 0xf1, 0xa6, 0xd6, 0x02, 0xa8, 0xc6, 0x5b, 0x83 }, + .public = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x5d, 0x50, 0xb6, 0x28, 0x36, 0xbb, 0x69, 0x57, + 0x94, 0x10, 0x38, 0x6c, 0xf7, 0xbb, 0x81, 0x1c, + 0x14, 0xbf, 0x85, 0xb1, 0xc7, 0xb1, 0x7e, 0x59, + 0x24, 0xc7, 0xff, 0xea, 0x91, 0xef, 0x9e, 0x12 }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0xc0, 0x1d, 0x13, 0x05, 0xa1, 0x33, 0x8a, 0x1f, + 0xca, 0xc2, 0xba, 0x7e, 0x2e, 0x03, 0x2b, 0x42, + 0x7e, 0x0b, 0x04, 0x90, 0x31, 0x65, 0xac, 0xa9, + 0x57, 0xd8, 0xd0, 0x55, 0x3d, 0x87, 0x17, 0xb0 }, + .public = { 0xea, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x19, 0x23, 0x0e, 0xb1, 0x48, 0xd5, 0xd6, 0x7c, + 0x3c, 0x22, 0xab, 0x1d, 0xae, 0xff, 0x80, 0xa5, + 0x7e, 0xae, 0x42, 0x65, 0xce, 0x28, 0x72, 0x65, + 0x7b, 0x2c, 0x80, 0x99, 0xfc, 0x69, 0x8e, 0x50 }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0x38, 0x6f, 0x7f, 0x16, 0xc5, 0x07, 0x31, 0xd6, + 0x4f, 0x82, 0xe6, 0xa1, 0x70, 0xb1, 0x42, 0xa4, + 0xe3, 0x4f, 0x31, 0xfd, 0x77, 0x68, 0xfc, 0xb8, + 0x90, 0x29, 0x25, 0xe7, 0xd1, 0xe2, 0x1a, 0xbe }, + .public = { 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x0f, 0xca, 0xb5, 0xd8, 0x42, 0xa0, 0x78, 0xd7, + 0xa7, 0x1f, 0xc5, 0x9b, 0x57, 0xbf, 0xb4, 0xca, + 0x0b, 0xe6, 0x87, 0x3b, 0x49, 0xdc, 0xdb, 0x9f, + 0x44, 0xe1, 0x4a, 0xe8, 0xfb, 0xdf, 0xa5, 0x42 }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0xe0, 0x23, 0xa2, 0x89, 0xbd, 0x5e, 0x90, 0xfa, + 0x28, 0x04, 0xdd, 0xc0, 0x19, 0xa0, 0x5e, 0xf3, + 0xe7, 0x9d, 0x43, 0x4b, 0xb6, 0xea, 0x2f, 0x52, + 0x2e, 0xcb, 0x64, 0x3a, 0x75, 0x29, 0x6e, 0x95 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x54, 0xce, 0x8f, 0x22, 0x75, 0xc0, 0x77, 0xe3, + 0xb1, 0x30, 0x6a, 0x39, 0x39, 0xc5, 0xe0, 0x3e, + 0xef, 0x6b, 0xbb, 0x88, 0x06, 0x05, 0x44, 0x75, + 0x8d, 0x9f, 0xef, 0x59, 0xb0, 0xbc, 0x3e, 0x4f }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0x68, 0xf0, 0x10, 0xd6, 0x2e, 0xe8, 0xd9, 0x26, + 0x05, 0x3a, 0x36, 0x1c, 0x3a, 0x75, 0xc6, 0xea, + 0x4e, 0xbd, 0xc8, 0x60, 0x6a, 0xb2, 0x85, 0x00, + 0x3a, 0x6f, 0x8f, 0x40, 0x76, 0xb0, 0x1e, 0x83 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03 }, + .result = { 0xf1, 0x36, 0x77, 0x5c, 0x5b, 0xeb, 0x0a, 0xf8, + 0x11, 0x0a, 0xf1, 0x0b, 0x20, 0x37, 0x23, 0x32, + 0x04, 0x3c, 0xab, 0x75, 0x24, 0x19, 0x67, 0x87, + 0x75, 0xa2, 0x23, 0xdf, 0x57, 0xc9, 0xd3, 0x0d }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0x58, 0xeb, 0xcb, 0x35, 0xb0, 0xf8, 0x84, 0x5c, + 0xaf, 0x1e, 0xc6, 0x30, 0xf9, 0x65, 0x76, 0xb6, + 0x2c, 0x4b, 0x7b, 0x6c, 0x36, 0xb2, 0x9d, 0xeb, + 0x2c, 0xb0, 0x08, 0x46, 0x51, 0x75, 0x5c, 0x96 }, + .public = { 0xff, 0xff, 0xff, 0xfb, 0xff, 0xff, 0xfb, 0xff, + 0xff, 0xdf, 0xff, 0xff, 0xdf, 0xff, 0xff, 0xff, + 0xfe, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xf7, 0xff, + 0xff, 0xf7, 0xff, 0xff, 0xbf, 0xff, 0xff, 0x3f }, + .result = { 0xbf, 0x9a, 0xff, 0xd0, 0x6b, 0x84, 0x40, 0x85, + 0x58, 0x64, 0x60, 0x96, 0x2e, 0xf2, 0x14, 0x6f, + 0xf3, 0xd4, 0x53, 0x3d, 0x94, 0x44, 0xaa, 0xb0, + 0x06, 0xeb, 0x88, 0xcc, 0x30, 0x54, 0x40, 0x7d }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0x18, 0x8c, 0x4b, 0xc5, 0xb9, 0xc4, 0x4b, 0x38, + 0xbb, 0x65, 0x8b, 0x9b, 0x2a, 0xe8, 0x2d, 0x5b, + 0x01, 0x01, 0x5e, 0x09, 0x31, 0x84, 0xb1, 0x7c, + 0xb7, 0x86, 0x35, 0x03, 0xa7, 0x83, 0xe1, 0xbb }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .result = { 0xd4, 0x80, 0xde, 0x04, 0xf6, 0x99, 0xcb, 0x3b, + 0xe0, 0x68, 0x4a, 0x9c, 0xc2, 0xe3, 0x12, 0x81, + 0xea, 0x0b, 0xc5, 0xa9, 0xdc, 0xc1, 0x57, 0xd3, + 0xd2, 0x01, 0x58, 0xd4, 0x6c, 0xa5, 0x24, 0x6d }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0xe0, 0x6c, 0x11, 0xbb, 0x2e, 0x13, 0xce, 0x3d, + 0xc7, 0x67, 0x3f, 0x67, 0xf5, 0x48, 0x22, 0x42, + 0x90, 0x94, 0x23, 0xa9, 0xae, 0x95, 0xee, 0x98, + 0x6a, 0x98, 0x8d, 0x98, 0xfa, 0xee, 0x23, 0xa2 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f }, + .result = { 0x4c, 0x44, 0x01, 0xcc, 0xe6, 0xb5, 0x1e, 0x4c, + 0xb1, 0x8f, 0x27, 0x90, 0x24, 0x6c, 0x9b, 0xf9, + 0x14, 0xdb, 0x66, 0x77, 0x50, 0xa1, 0xcb, 0x89, + 0x06, 0x90, 0x92, 0xaf, 0x07, 0x29, 0x22, 0x76 }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0xc0, 0x65, 0x8c, 0x46, 0xdd, 0xe1, 0x81, 0x29, + 0x29, 0x38, 0x77, 0x53, 0x5b, 0x11, 0x62, 0xb6, + 0xf9, 0xf5, 0x41, 0x4a, 0x23, 0xcf, 0x4d, 0x2c, + 0xbc, 0x14, 0x0a, 0x4d, 0x99, 0xda, 0x2b, 0x8f }, + .public = { 0xeb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x57, 0x8b, 0xa8, 0xcc, 0x2d, 0xbd, 0xc5, 0x75, + 0xaf, 0xcf, 0x9d, 0xf2, 0xb3, 0xee, 0x61, 0x89, + 0xf5, 0x33, 0x7d, 0x68, 0x54, 0xc7, 0x9b, 0x4c, + 0xe1, 0x65, 0xea, 0x12, 0x29, 0x3b, 0x3a, 0x0f }, + .valid = true + }, + /* wycheproof - public key with low order */ + { + .private = { 0x10, 0x25, 0x5c, 0x92, 0x30, 0xa9, 0x7a, 0x30, + 0xa4, 0x58, 0xca, 0x28, 0x4a, 0x62, 0x96, 0x69, + 0x29, 0x3a, 0x31, 0x89, 0x0c, 0xda, 0x9d, 0x14, + 0x7f, 0xeb, 0xc7, 0xd1, 0xe2, 0x2d, 0x6b, 0xb1 }, + .public = { 0xe0, 0xeb, 0x7a, 0x7c, 0x3b, 0x41, 0xb8, 0xae, + 0x16, 0x56, 0xe3, 0xfa, 0xf1, 0x9f, 0xc4, 0x6a, + 0xda, 0x09, 0x8d, 0xeb, 0x9c, 0x32, 0xb1, 0xfd, + 0x86, 0x62, 0x05, 0x16, 0x5f, 0x49, 0xb8, 0x00 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x78, 0xf1, 0xe8, 0xed, 0xf1, 0x44, 0x81, 0xb3, + 0x89, 0x44, 0x8d, 0xac, 0x8f, 0x59, 0xc7, 0x0b, + 0x03, 0x8e, 0x7c, 0xf9, 0x2e, 0xf2, 0xc7, 0xef, + 0xf5, 0x7a, 0x72, 0x46, 0x6e, 0x11, 0x52, 0x96 }, + .public = { 0x5f, 0x9c, 0x95, 0xbc, 0xa3, 0x50, 0x8c, 0x24, + 0xb1, 0xd0, 0xb1, 0x55, 0x9c, 0x83, 0xef, 0x5b, + 0x04, 0x44, 0x5c, 0xc4, 0x58, 0x1c, 0x8e, 0x86, + 0xd8, 0x22, 0x4e, 0xdd, 0xd0, 0x9f, 0x11, 0x57 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0xa0, 0xa0, 0x5a, 0x3e, 0x8f, 0x9f, 0x44, 0x20, + 0x4d, 0x5f, 0x80, 0x59, 0xa9, 0x4a, 0xc7, 0xdf, + 0xc3, 0x9a, 0x49, 0xac, 0x01, 0x6d, 0xd7, 0x43, + 0xdb, 0xfa, 0x43, 0xc5, 0xd6, 0x71, 0xfd, 0x88 }, + .public = { 0xec, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0xd0, 0xdb, 0xb3, 0xed, 0x19, 0x06, 0x66, 0x3f, + 0x15, 0x42, 0x0a, 0xf3, 0x1f, 0x4e, 0xaf, 0x65, + 0x09, 0xd9, 0xa9, 0x94, 0x97, 0x23, 0x50, 0x06, + 0x05, 0xad, 0x7c, 0x1c, 0x6e, 0x74, 0x50, 0xa9 }, + .public = { 0xed, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0xc0, 0xb1, 0xd0, 0xeb, 0x22, 0xb2, 0x44, 0xfe, + 0x32, 0x91, 0x14, 0x00, 0x72, 0xcd, 0xd9, 0xd9, + 0x89, 0xb5, 0xf0, 0xec, 0xd9, 0x6c, 0x10, 0x0f, + 0xeb, 0x5b, 0xca, 0x24, 0x1c, 0x1d, 0x9f, 0x8f }, + .public = { 0xee, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x48, 0x0b, 0xf4, 0x5f, 0x59, 0x49, 0x42, 0xa8, + 0xbc, 0x0f, 0x33, 0x53, 0xc6, 0xe8, 0xb8, 0x85, + 0x3d, 0x77, 0xf3, 0x51, 0xf1, 0xc2, 0xca, 0x6c, + 0x2d, 0x1a, 0xbf, 0x8a, 0x00, 0xb4, 0x22, 0x9c }, + .public = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x30, 0xf9, 0x93, 0xfc, 0xf8, 0x51, 0x4f, 0xc8, + 0x9b, 0xd8, 0xdb, 0x14, 0xcd, 0x43, 0xba, 0x0d, + 0x4b, 0x25, 0x30, 0xe7, 0x3c, 0x42, 0x76, 0xa0, + 0x5e, 0x1b, 0x14, 0x5d, 0x42, 0x0c, 0xed, 0xb4 }, + .public = { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0xc0, 0x49, 0x74, 0xb7, 0x58, 0x38, 0x0e, 0x2a, + 0x5b, 0x5d, 0xf6, 0xeb, 0x09, 0xbb, 0x2f, 0x6b, + 0x34, 0x34, 0xf9, 0x82, 0x72, 0x2a, 0x8e, 0x67, + 0x6d, 0x3d, 0xa2, 0x51, 0xd1, 0xb3, 0xde, 0x83 }, + .public = { 0xe0, 0xeb, 0x7a, 0x7c, 0x3b, 0x41, 0xb8, 0xae, + 0x16, 0x56, 0xe3, 0xfa, 0xf1, 0x9f, 0xc4, 0x6a, + 0xda, 0x09, 0x8d, 0xeb, 0x9c, 0x32, 0xb1, 0xfd, + 0x86, 0x62, 0x05, 0x16, 0x5f, 0x49, 0xb8, 0x80 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x50, 0x2a, 0x31, 0x37, 0x3d, 0xb3, 0x24, 0x46, + 0x84, 0x2f, 0xe5, 0xad, 0xd3, 0xe0, 0x24, 0x02, + 0x2e, 0xa5, 0x4f, 0x27, 0x41, 0x82, 0xaf, 0xc3, + 0xd9, 0xf1, 0xbb, 0x3d, 0x39, 0x53, 0x4e, 0xb5 }, + .public = { 0x5f, 0x9c, 0x95, 0xbc, 0xa3, 0x50, 0x8c, 0x24, + 0xb1, 0xd0, 0xb1, 0x55, 0x9c, 0x83, 0xef, 0x5b, + 0x04, 0x44, 0x5c, 0xc4, 0x58, 0x1c, 0x8e, 0x86, + 0xd8, 0x22, 0x4e, 0xdd, 0xd0, 0x9f, 0x11, 0xd7 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x90, 0xfa, 0x64, 0x17, 0xb0, 0xe3, 0x70, 0x30, + 0xfd, 0x6e, 0x43, 0xef, 0xf2, 0xab, 0xae, 0xf1, + 0x4c, 0x67, 0x93, 0x11, 0x7a, 0x03, 0x9c, 0xf6, + 0x21, 0x31, 0x8b, 0xa9, 0x0f, 0x4e, 0x98, 0xbe }, + .public = { 0xec, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x78, 0xad, 0x3f, 0x26, 0x02, 0x7f, 0x1c, 0x9f, + 0xdd, 0x97, 0x5a, 0x16, 0x13, 0xb9, 0x47, 0x77, + 0x9b, 0xad, 0x2c, 0xf2, 0xb7, 0x41, 0xad, 0xe0, + 0x18, 0x40, 0x88, 0x5a, 0x30, 0xbb, 0x97, 0x9c }, + .public = { 0xed, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x98, 0xe2, 0x3d, 0xe7, 0xb1, 0xe0, 0x92, 0x6e, + 0xd9, 0xc8, 0x7e, 0x7b, 0x14, 0xba, 0xf5, 0x5f, + 0x49, 0x7a, 0x1d, 0x70, 0x96, 0xf9, 0x39, 0x77, + 0x68, 0x0e, 0x44, 0xdc, 0x1c, 0x7b, 0x7b, 0x8b }, + .public = { 0xee, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key >= p */ + { + .private = { 0xf0, 0x1e, 0x48, 0xda, 0xfa, 0xc9, 0xd7, 0xbc, + 0xf5, 0x89, 0xcb, 0xc3, 0x82, 0xc8, 0x78, 0xd1, + 0x8b, 0xda, 0x35, 0x50, 0x58, 0x9f, 0xfb, 0x5d, + 0x50, 0xb5, 0x23, 0xbe, 0xbe, 0x32, 0x9d, 0xae }, + .public = { 0xef, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0xbd, 0x36, 0xa0, 0x79, 0x0e, 0xb8, 0x83, 0x09, + 0x8c, 0x98, 0x8b, 0x21, 0x78, 0x67, 0x73, 0xde, + 0x0b, 0x3a, 0x4d, 0xf1, 0x62, 0x28, 0x2c, 0xf1, + 0x10, 0xde, 0x18, 0xdd, 0x48, 0x4c, 0xe7, 0x4b }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x28, 0x87, 0x96, 0xbc, 0x5a, 0xff, 0x4b, 0x81, + 0xa3, 0x75, 0x01, 0x75, 0x7b, 0xc0, 0x75, 0x3a, + 0x3c, 0x21, 0x96, 0x47, 0x90, 0xd3, 0x86, 0x99, + 0x30, 0x8d, 0xeb, 0xc1, 0x7a, 0x6e, 0xaf, 0x8d }, + .public = { 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0xb4, 0xe0, 0xdd, 0x76, 0xda, 0x7b, 0x07, 0x17, + 0x28, 0xb6, 0x1f, 0x85, 0x67, 0x71, 0xaa, 0x35, + 0x6e, 0x57, 0xed, 0xa7, 0x8a, 0x5b, 0x16, 0x55, + 0xcc, 0x38, 0x20, 0xfb, 0x5f, 0x85, 0x4c, 0x5c }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x98, 0xdf, 0x84, 0x5f, 0x66, 0x51, 0xbf, 0x11, + 0x38, 0x22, 0x1f, 0x11, 0x90, 0x41, 0xf7, 0x2b, + 0x6d, 0xbc, 0x3c, 0x4a, 0xce, 0x71, 0x43, 0xd9, + 0x9f, 0xd5, 0x5a, 0xd8, 0x67, 0x48, 0x0d, 0xa8 }, + .public = { 0xf1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x6f, 0xdf, 0x6c, 0x37, 0x61, 0x1d, 0xbd, 0x53, + 0x04, 0xdc, 0x0f, 0x2e, 0xb7, 0xc9, 0x51, 0x7e, + 0xb3, 0xc5, 0x0e, 0x12, 0xfd, 0x05, 0x0a, 0xc6, + 0xde, 0xc2, 0x70, 0x71, 0xd4, 0xbf, 0xc0, 0x34 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xf0, 0x94, 0x98, 0xe4, 0x6f, 0x02, 0xf8, 0x78, + 0x82, 0x9e, 0x78, 0xb8, 0x03, 0xd3, 0x16, 0xa2, + 0xed, 0x69, 0x5d, 0x04, 0x98, 0xa0, 0x8a, 0xbd, + 0xf8, 0x27, 0x69, 0x30, 0xe2, 0x4e, 0xdc, 0xb0 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x4c, 0x8f, 0xc4, 0xb1, 0xc6, 0xab, 0x88, 0xfb, + 0x21, 0xf1, 0x8f, 0x6d, 0x4c, 0x81, 0x02, 0x40, + 0xd4, 0xe9, 0x46, 0x51, 0xba, 0x44, 0xf7, 0xa2, + 0xc8, 0x63, 0xce, 0xc7, 0xdc, 0x56, 0x60, 0x2d }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x18, 0x13, 0xc1, 0x0a, 0x5c, 0x7f, 0x21, 0xf9, + 0x6e, 0x17, 0xf2, 0x88, 0xc0, 0xcc, 0x37, 0x60, + 0x7c, 0x04, 0xc5, 0xf5, 0xae, 0xa2, 0xdb, 0x13, + 0x4f, 0x9e, 0x2f, 0xfc, 0x66, 0xbd, 0x9d, 0xb8 }, + .public = { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x1c, 0xd0, 0xb2, 0x82, 0x67, 0xdc, 0x54, 0x1c, + 0x64, 0x2d, 0x6d, 0x7d, 0xca, 0x44, 0xa8, 0xb3, + 0x8a, 0x63, 0x73, 0x6e, 0xef, 0x5c, 0x4e, 0x65, + 0x01, 0xff, 0xbb, 0xb1, 0x78, 0x0c, 0x03, 0x3c }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x78, 0x57, 0xfb, 0x80, 0x86, 0x53, 0x64, 0x5a, + 0x0b, 0xeb, 0x13, 0x8a, 0x64, 0xf5, 0xf4, 0xd7, + 0x33, 0xa4, 0x5e, 0xa8, 0x4c, 0x3c, 0xda, 0x11, + 0xa9, 0xc0, 0x6f, 0x7e, 0x71, 0x39, 0x14, 0x9e }, + .public = { 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x87, 0x55, 0xbe, 0x01, 0xc6, 0x0a, 0x7e, 0x82, + 0x5c, 0xff, 0x3e, 0x0e, 0x78, 0xcb, 0x3a, 0xa4, + 0x33, 0x38, 0x61, 0x51, 0x6a, 0xa5, 0x9b, 0x1c, + 0x51, 0xa8, 0xb2, 0xa5, 0x43, 0xdf, 0xa8, 0x22 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xe0, 0x3a, 0xa8, 0x42, 0xe2, 0xab, 0xc5, 0x6e, + 0x81, 0xe8, 0x7b, 0x8b, 0x9f, 0x41, 0x7b, 0x2a, + 0x1e, 0x59, 0x13, 0xc7, 0x23, 0xee, 0xd2, 0x8d, + 0x75, 0x2f, 0x8d, 0x47, 0xa5, 0x9f, 0x49, 0x8f }, + .public = { 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x54, 0xc9, 0xa1, 0xed, 0x95, 0xe5, 0x46, 0xd2, + 0x78, 0x22, 0xa3, 0x60, 0x93, 0x1d, 0xda, 0x60, + 0xa1, 0xdf, 0x04, 0x9d, 0xa6, 0xf9, 0x04, 0x25, + 0x3c, 0x06, 0x12, 0xbb, 0xdc, 0x08, 0x74, 0x76 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xf8, 0xf7, 0x07, 0xb7, 0x99, 0x9b, 0x18, 0xcb, + 0x0d, 0x6b, 0x96, 0x12, 0x4f, 0x20, 0x45, 0x97, + 0x2c, 0xa2, 0x74, 0xbf, 0xc1, 0x54, 0xad, 0x0c, + 0x87, 0x03, 0x8c, 0x24, 0xc6, 0xd0, 0xd4, 0xb2 }, + .public = { 0xda, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0xcc, 0x1f, 0x40, 0xd7, 0x43, 0xcd, 0xc2, 0x23, + 0x0e, 0x10, 0x43, 0xda, 0xba, 0x8b, 0x75, 0xe8, + 0x10, 0xf1, 0xfb, 0xab, 0x7f, 0x25, 0x52, 0x69, + 0xbd, 0x9e, 0xbb, 0x29, 0xe6, 0xbf, 0x49, 0x4f }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xa0, 0x34, 0xf6, 0x84, 0xfa, 0x63, 0x1e, 0x1a, + 0x34, 0x81, 0x18, 0xc1, 0xce, 0x4c, 0x98, 0x23, + 0x1f, 0x2d, 0x9e, 0xec, 0x9b, 0xa5, 0x36, 0x5b, + 0x4a, 0x05, 0xd6, 0x9a, 0x78, 0x5b, 0x07, 0x96 }, + .public = { 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x54, 0x99, 0x8e, 0xe4, 0x3a, 0x5b, 0x00, 0x7b, + 0xf4, 0x99, 0xf0, 0x78, 0xe7, 0x36, 0x52, 0x44, + 0x00, 0xa8, 0xb5, 0xc7, 0xe9, 0xb9, 0xb4, 0x37, + 0x71, 0x74, 0x8c, 0x7c, 0xdf, 0x88, 0x04, 0x12 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x30, 0xb6, 0xc6, 0xa0, 0xf2, 0xff, 0xa6, 0x80, + 0x76, 0x8f, 0x99, 0x2b, 0xa8, 0x9e, 0x15, 0x2d, + 0x5b, 0xc9, 0x89, 0x3d, 0x38, 0xc9, 0x11, 0x9b, + 0xe4, 0xf7, 0x67, 0xbf, 0xab, 0x6e, 0x0c, 0xa5 }, + .public = { 0xdc, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0xea, 0xd9, 0xb3, 0x8e, 0xfd, 0xd7, 0x23, 0x63, + 0x79, 0x34, 0xe5, 0x5a, 0xb7, 0x17, 0xa7, 0xae, + 0x09, 0xeb, 0x86, 0xa2, 0x1d, 0xc3, 0x6a, 0x3f, + 0xee, 0xb8, 0x8b, 0x75, 0x9e, 0x39, 0x1e, 0x09 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x90, 0x1b, 0x9d, 0xcf, 0x88, 0x1e, 0x01, 0xe0, + 0x27, 0x57, 0x50, 0x35, 0xd4, 0x0b, 0x43, 0xbd, + 0xc1, 0xc5, 0x24, 0x2e, 0x03, 0x08, 0x47, 0x49, + 0x5b, 0x0c, 0x72, 0x86, 0x46, 0x9b, 0x65, 0x91 }, + .public = { 0xea, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x60, 0x2f, 0xf4, 0x07, 0x89, 0xb5, 0x4b, 0x41, + 0x80, 0x59, 0x15, 0xfe, 0x2a, 0x62, 0x21, 0xf0, + 0x7a, 0x50, 0xff, 0xc2, 0xc3, 0xfc, 0x94, 0xcf, + 0x61, 0xf1, 0x3d, 0x79, 0x04, 0xe8, 0x8e, 0x0e }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x80, 0x46, 0x67, 0x7c, 0x28, 0xfd, 0x82, 0xc9, + 0xa1, 0xbd, 0xb7, 0x1a, 0x1a, 0x1a, 0x34, 0xfa, + 0xba, 0x12, 0x25, 0xe2, 0x50, 0x7f, 0xe3, 0xf5, + 0x4d, 0x10, 0xbd, 0x5b, 0x0d, 0x86, 0x5f, 0x8e }, + .public = { 0xeb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0xe0, 0x0a, 0xe8, 0xb1, 0x43, 0x47, 0x12, 0x47, + 0xba, 0x24, 0xf1, 0x2c, 0x88, 0x55, 0x36, 0xc3, + 0xcb, 0x98, 0x1b, 0x58, 0xe1, 0xe5, 0x6b, 0x2b, + 0xaf, 0x35, 0xc1, 0x2a, 0xe1, 0xf7, 0x9c, 0x26 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x60, 0x2f, 0x7e, 0x2f, 0x68, 0xa8, 0x46, 0xb8, + 0x2c, 0xc2, 0x69, 0xb1, 0xd4, 0x8e, 0x93, 0x98, + 0x86, 0xae, 0x54, 0xfd, 0x63, 0x6c, 0x1f, 0xe0, + 0x74, 0xd7, 0x10, 0x12, 0x7d, 0x47, 0x24, 0x91 }, + .public = { 0xef, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x98, 0xcb, 0x9b, 0x50, 0xdd, 0x3f, 0xc2, 0xb0, + 0xd4, 0xf2, 0xd2, 0xbf, 0x7c, 0x5c, 0xfd, 0xd1, + 0x0c, 0x8f, 0xcd, 0x31, 0xfc, 0x40, 0xaf, 0x1a, + 0xd4, 0x4f, 0x47, 0xc1, 0x31, 0x37, 0x63, 0x62 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x60, 0x88, 0x7b, 0x3d, 0xc7, 0x24, 0x43, 0x02, + 0x6e, 0xbe, 0xdb, 0xbb, 0xb7, 0x06, 0x65, 0xf4, + 0x2b, 0x87, 0xad, 0xd1, 0x44, 0x0e, 0x77, 0x68, + 0xfb, 0xd7, 0xe8, 0xe2, 0xce, 0x5f, 0x63, 0x9d }, + .public = { 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x38, 0xd6, 0x30, 0x4c, 0x4a, 0x7e, 0x6d, 0x9f, + 0x79, 0x59, 0x33, 0x4f, 0xb5, 0x24, 0x5b, 0xd2, + 0xc7, 0x54, 0x52, 0x5d, 0x4c, 0x91, 0xdb, 0x95, + 0x02, 0x06, 0x92, 0x62, 0x34, 0xc1, 0xf6, 0x33 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x78, 0xd3, 0x1d, 0xfa, 0x85, 0x44, 0x97, 0xd7, + 0x2d, 0x8d, 0xef, 0x8a, 0x1b, 0x7f, 0xb0, 0x06, + 0xce, 0xc2, 0xd8, 0xc4, 0x92, 0x46, 0x47, 0xc9, + 0x38, 0x14, 0xae, 0x56, 0xfa, 0xed, 0xa4, 0x95 }, + .public = { 0xf1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x78, 0x6c, 0xd5, 0x49, 0x96, 0xf0, 0x14, 0xa5, + 0xa0, 0x31, 0xec, 0x14, 0xdb, 0x81, 0x2e, 0xd0, + 0x83, 0x55, 0x06, 0x1f, 0xdb, 0x5d, 0xe6, 0x80, + 0xa8, 0x00, 0xac, 0x52, 0x1f, 0x31, 0x8e, 0x23 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xc0, 0x4c, 0x5b, 0xae, 0xfa, 0x83, 0x02, 0xdd, + 0xde, 0xd6, 0xa4, 0xbb, 0x95, 0x77, 0x61, 0xb4, + 0xeb, 0x97, 0xae, 0xfa, 0x4f, 0xc3, 0xb8, 0x04, + 0x30, 0x85, 0xf9, 0x6a, 0x56, 0x59, 0xb3, 0xa5 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x29, 0xae, 0x8b, 0xc7, 0x3e, 0x9b, 0x10, 0xa0, + 0x8b, 0x4f, 0x68, 0x1c, 0x43, 0xc3, 0xe0, 0xac, + 0x1a, 0x17, 0x1d, 0x31, 0xb3, 0x8f, 0x1a, 0x48, + 0xef, 0xba, 0x29, 0xae, 0x63, 0x9e, 0xa1, 0x34 }, + .valid = true + }, + /* wycheproof - RFC 7748 */ + { + .private = { 0xa0, 0x46, 0xe3, 0x6b, 0xf0, 0x52, 0x7c, 0x9d, + 0x3b, 0x16, 0x15, 0x4b, 0x82, 0x46, 0x5e, 0xdd, + 0x62, 0x14, 0x4c, 0x0a, 0xc1, 0xfc, 0x5a, 0x18, + 0x50, 0x6a, 0x22, 0x44, 0xba, 0x44, 0x9a, 0x44 }, + .public = { 0xe6, 0xdb, 0x68, 0x67, 0x58, 0x30, 0x30, 0xdb, + 0x35, 0x94, 0xc1, 0xa4, 0x24, 0xb1, 0x5f, 0x7c, + 0x72, 0x66, 0x24, 0xec, 0x26, 0xb3, 0x35, 0x3b, + 0x10, 0xa9, 0x03, 0xa6, 0xd0, 0xab, 0x1c, 0x4c }, + .result = { 0xc3, 0xda, 0x55, 0x37, 0x9d, 0xe9, 0xc6, 0x90, + 0x8e, 0x94, 0xea, 0x4d, 0xf2, 0x8d, 0x08, 0x4f, + 0x32, 0xec, 0xcf, 0x03, 0x49, 0x1c, 0x71, 0xf7, + 0x54, 0xb4, 0x07, 0x55, 0x77, 0xa2, 0x85, 0x52 }, + .valid = true + }, + /* wycheproof - RFC 7748 */ + { + .private = { 0x48, 0x66, 0xe9, 0xd4, 0xd1, 0xb4, 0x67, 0x3c, + 0x5a, 0xd2, 0x26, 0x91, 0x95, 0x7d, 0x6a, 0xf5, + 0xc1, 0x1b, 0x64, 0x21, 0xe0, 0xea, 0x01, 0xd4, + 0x2c, 0xa4, 0x16, 0x9e, 0x79, 0x18, 0xba, 0x4d }, + .public = { 0xe5, 0x21, 0x0f, 0x12, 0x78, 0x68, 0x11, 0xd3, + 0xf4, 0xb7, 0x95, 0x9d, 0x05, 0x38, 0xae, 0x2c, + 0x31, 0xdb, 0xe7, 0x10, 0x6f, 0xc0, 0x3c, 0x3e, + 0xfc, 0x4c, 0xd5, 0x49, 0xc7, 0x15, 0xa4, 0x13 }, + .result = { 0x95, 0xcb, 0xde, 0x94, 0x76, 0xe8, 0x90, 0x7d, + 0x7a, 0xad, 0xe4, 0x5c, 0xb4, 0xb8, 0x73, 0xf8, + 0x8b, 0x59, 0x5a, 0x68, 0x79, 0x9f, 0xa1, 0x52, + 0xe6, 0xf8, 0xf7, 0x64, 0x7a, 0xac, 0x79, 0x57 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x0a, 0xb4, 0xe7, 0x63, 0x80, 0xd8, 0x4d, 0xde, + 0x4f, 0x68, 0x33, 0xc5, 0x8f, 0x2a, 0x9f, 0xb8, + 0xf8, 0x3b, 0xb0, 0x16, 0x9b, 0x17, 0x2b, 0xe4, + 0xb6, 0xe0, 0x59, 0x28, 0x87, 0x74, 0x1a, 0x36 }, + .result = { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x89, 0xe1, 0x0d, 0x57, 0x01, 0xb4, 0x33, 0x7d, + 0x2d, 0x03, 0x21, 0x81, 0x53, 0x8b, 0x10, 0x64, + 0xbd, 0x40, 0x84, 0x40, 0x1c, 0xec, 0xa1, 0xfd, + 0x12, 0x66, 0x3a, 0x19, 0x59, 0x38, 0x80, 0x00 }, + .result = { 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x2b, 0x55, 0xd3, 0xaa, 0x4a, 0x8f, 0x80, 0xc8, + 0xc0, 0xb2, 0xae, 0x5f, 0x93, 0x3e, 0x85, 0xaf, + 0x49, 0xbe, 0xac, 0x36, 0xc2, 0xfa, 0x73, 0x94, + 0xba, 0xb7, 0x6c, 0x89, 0x33, 0xf8, 0xf8, 0x1d }, + .result = { 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x63, 0xe5, 0xb1, 0xfe, 0x96, 0x01, 0xfe, 0x84, + 0x38, 0x5d, 0x88, 0x66, 0xb0, 0x42, 0x12, 0x62, + 0xf7, 0x8f, 0xbf, 0xa5, 0xaf, 0xf9, 0x58, 0x5e, + 0x62, 0x66, 0x79, 0xb1, 0x85, 0x47, 0xd9, 0x59 }, + .result = { 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0xe4, 0x28, 0xf3, 0xda, 0xc1, 0x78, 0x09, 0xf8, + 0x27, 0xa5, 0x22, 0xce, 0x32, 0x35, 0x50, 0x58, + 0xd0, 0x73, 0x69, 0x36, 0x4a, 0xa7, 0x89, 0x02, + 0xee, 0x10, 0x13, 0x9b, 0x9f, 0x9d, 0xd6, 0x53 }, + .result = { 0xfc, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0xb3, 0xb5, 0x0e, 0x3e, 0xd3, 0xa4, 0x07, 0xb9, + 0x5d, 0xe9, 0x42, 0xef, 0x74, 0x57, 0x5b, 0x5a, + 0xb8, 0xa1, 0x0c, 0x09, 0xee, 0x10, 0x35, 0x44, + 0xd6, 0x0b, 0xdf, 0xed, 0x81, 0x38, 0xab, 0x2b }, + .result = { 0xf9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x21, 0x3f, 0xff, 0xe9, 0x3d, 0x5e, 0xa8, 0xcd, + 0x24, 0x2e, 0x46, 0x28, 0x44, 0x02, 0x99, 0x22, + 0xc4, 0x3c, 0x77, 0xc9, 0xe3, 0xe4, 0x2f, 0x56, + 0x2f, 0x48, 0x5d, 0x24, 0xc5, 0x01, 0xa2, 0x0b }, + .result = { 0xf3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x91, 0xb2, 0x32, 0xa1, 0x78, 0xb3, 0xcd, 0x53, + 0x09, 0x32, 0x44, 0x1e, 0x61, 0x39, 0x41, 0x8f, + 0x72, 0x17, 0x22, 0x92, 0xf1, 0xda, 0x4c, 0x18, + 0x34, 0xfc, 0x5e, 0xbf, 0xef, 0xb5, 0x1e, 0x3f }, + .result = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x04, 0x5c, 0x6e, 0x11, 0xc5, 0xd3, 0x32, 0x55, + 0x6c, 0x78, 0x22, 0xfe, 0x94, 0xeb, 0xf8, 0x9b, + 0x56, 0xa3, 0x87, 0x8d, 0xc2, 0x7c, 0xa0, 0x79, + 0x10, 0x30, 0x58, 0x84, 0x9f, 0xab, 0xcb, 0x4f }, + .result = { 0xe5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x1c, 0xa2, 0x19, 0x0b, 0x71, 0x16, 0x35, 0x39, + 0x06, 0x3c, 0x35, 0x77, 0x3b, 0xda, 0x0c, 0x9c, + 0x92, 0x8e, 0x91, 0x36, 0xf0, 0x62, 0x0a, 0xeb, + 0x09, 0x3f, 0x09, 0x91, 0x97, 0xb7, 0xf7, 0x4e }, + .result = { 0xe3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0xf7, 0x6e, 0x90, 0x10, 0xac, 0x33, 0xc5, 0x04, + 0x3b, 0x2d, 0x3b, 0x76, 0xa8, 0x42, 0x17, 0x10, + 0x00, 0xc4, 0x91, 0x62, 0x22, 0xe9, 0xe8, 0x58, + 0x97, 0xa0, 0xae, 0xc7, 0xf6, 0x35, 0x0b, 0x3c }, + .result = { 0xdd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0xbb, 0x72, 0x68, 0x8d, 0x8f, 0x8a, 0xa7, 0xa3, + 0x9c, 0xd6, 0x06, 0x0c, 0xd5, 0xc8, 0x09, 0x3c, + 0xde, 0xc6, 0xfe, 0x34, 0x19, 0x37, 0xc3, 0x88, + 0x6a, 0x99, 0x34, 0x6c, 0xd0, 0x7f, 0xaa, 0x55 }, + .result = { 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x88, 0xfd, 0xde, 0xa1, 0x93, 0x39, 0x1c, 0x6a, + 0x59, 0x33, 0xef, 0x9b, 0x71, 0x90, 0x15, 0x49, + 0x44, 0x72, 0x05, 0xaa, 0xe9, 0xda, 0x92, 0x8a, + 0x6b, 0x91, 0xa3, 0x52, 0xba, 0x10, 0xf4, 0x1f }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x30, 0x3b, 0x39, 0x2f, 0x15, 0x31, 0x16, 0xca, + 0xd9, 0xcc, 0x68, 0x2a, 0x00, 0xcc, 0xc4, 0x4c, + 0x95, 0xff, 0x0d, 0x3b, 0xbe, 0x56, 0x8b, 0xeb, + 0x6c, 0x4e, 0x73, 0x9b, 0xaf, 0xdc, 0x2c, 0x68 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00 }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0xfd, 0x30, 0x0a, 0xeb, 0x40, 0xe1, 0xfa, 0x58, + 0x25, 0x18, 0x41, 0x2b, 0x49, 0xb2, 0x08, 0xa7, + 0x84, 0x2b, 0x1e, 0x1f, 0x05, 0x6a, 0x04, 0x01, + 0x78, 0xea, 0x41, 0x41, 0x53, 0x4f, 0x65, 0x2d }, + .result = { 0xb7, 0x34, 0x10, 0x5d, 0xc2, 0x57, 0x58, 0x5d, + 0x73, 0xb5, 0x66, 0xcc, 0xb7, 0x6f, 0x06, 0x27, + 0x95, 0xcc, 0xbe, 0xc8, 0x91, 0x28, 0xe5, 0x2b, + 0x02, 0xf3, 0xe5, 0x96, 0x39, 0xf1, 0x3c, 0x46 }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0xc8, 0xef, 0x79, 0xb5, 0x14, 0xd7, 0x68, 0x26, + 0x77, 0xbc, 0x79, 0x31, 0xe0, 0x6e, 0xe5, 0xc2, + 0x7c, 0x9b, 0x39, 0x2b, 0x4a, 0xe9, 0x48, 0x44, + 0x73, 0xf5, 0x54, 0xe6, 0x67, 0x8e, 0xcc, 0x2e }, + .result = { 0x64, 0x7a, 0x46, 0xb6, 0xfc, 0x3f, 0x40, 0xd6, + 0x21, 0x41, 0xee, 0x3c, 0xee, 0x70, 0x6b, 0x4d, + 0x7a, 0x92, 0x71, 0x59, 0x3a, 0x7b, 0x14, 0x3e, + 0x8e, 0x2e, 0x22, 0x79, 0x88, 0x3e, 0x45, 0x50 }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0x64, 0xae, 0xac, 0x25, 0x04, 0x14, 0x48, 0x61, + 0x53, 0x2b, 0x7b, 0xbc, 0xb6, 0xc8, 0x7d, 0x67, + 0xdd, 0x4c, 0x1f, 0x07, 0xeb, 0xc2, 0xe0, 0x6e, + 0xff, 0xb9, 0x5a, 0xec, 0xc6, 0x17, 0x0b, 0x2c }, + .result = { 0x4f, 0xf0, 0x3d, 0x5f, 0xb4, 0x3c, 0xd8, 0x65, + 0x7a, 0x3c, 0xf3, 0x7c, 0x13, 0x8c, 0xad, 0xce, + 0xcc, 0xe5, 0x09, 0xe4, 0xeb, 0xa0, 0x89, 0xd0, + 0xef, 0x40, 0xb4, 0xe4, 0xfb, 0x94, 0x61, 0x55 }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0xbf, 0x68, 0xe3, 0x5e, 0x9b, 0xdb, 0x7e, 0xee, + 0x1b, 0x50, 0x57, 0x02, 0x21, 0x86, 0x0f, 0x5d, + 0xcd, 0xad, 0x8a, 0xcb, 0xab, 0x03, 0x1b, 0x14, + 0x97, 0x4c, 0xc4, 0x90, 0x13, 0xc4, 0x98, 0x31 }, + .result = { 0x21, 0xce, 0xe5, 0x2e, 0xfd, 0xbc, 0x81, 0x2e, + 0x1d, 0x02, 0x1a, 0x4a, 0xf1, 0xe1, 0xd8, 0xbc, + 0x4d, 0xb3, 0xc4, 0x00, 0xe4, 0xd2, 0xa2, 0xc5, + 0x6a, 0x39, 0x26, 0xdb, 0x4d, 0x99, 0xc6, 0x5b }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0x53, 0x47, 0xc4, 0x91, 0x33, 0x1a, 0x64, 0xb4, + 0x3d, 0xdc, 0x68, 0x30, 0x34, 0xe6, 0x77, 0xf5, + 0x3d, 0xc3, 0x2b, 0x52, 0xa5, 0x2a, 0x57, 0x7c, + 0x15, 0xa8, 0x3b, 0xf2, 0x98, 0xe9, 0x9f, 0x19 }, + .result = { 0x18, 0xcb, 0x89, 0xe4, 0xe2, 0x0c, 0x0c, 0x2b, + 0xd3, 0x24, 0x30, 0x52, 0x45, 0x26, 0x6c, 0x93, + 0x27, 0x69, 0x0b, 0xbe, 0x79, 0xac, 0xb8, 0x8f, + 0x5b, 0x8f, 0xb3, 0xf7, 0x4e, 0xca, 0x3e, 0x52 }, + .valid = true + }, + /* wycheproof - private key == -1 (mod order) */ + { + .private = { 0xa0, 0x23, 0xcd, 0xd0, 0x83, 0xef, 0x5b, 0xb8, + 0x2f, 0x10, 0xd6, 0x2e, 0x59, 0xe1, 0x5a, 0x68, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50 }, + .public = { 0x25, 0x8e, 0x04, 0x52, 0x3b, 0x8d, 0x25, 0x3e, + 0xe6, 0x57, 0x19, 0xfc, 0x69, 0x06, 0xc6, 0x57, + 0x19, 0x2d, 0x80, 0x71, 0x7e, 0xdc, 0x82, 0x8f, + 0xa0, 0xaf, 0x21, 0x68, 0x6e, 0x2f, 0xaa, 0x75 }, + .result = { 0x25, 0x8e, 0x04, 0x52, 0x3b, 0x8d, 0x25, 0x3e, + 0xe6, 0x57, 0x19, 0xfc, 0x69, 0x06, 0xc6, 0x57, + 0x19, 0x2d, 0x80, 0x71, 0x7e, 0xdc, 0x82, 0x8f, + 0xa0, 0xaf, 0x21, 0x68, 0x6e, 0x2f, 0xaa, 0x75 }, + .valid = true + }, + /* wycheproof - private key == 1 (mod order) on twist */ + { + .private = { 0x58, 0x08, 0x3d, 0xd2, 0x61, 0xad, 0x91, 0xef, + 0xf9, 0x52, 0x32, 0x2e, 0xc8, 0x24, 0xc6, 0x82, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x5f }, + .public = { 0x2e, 0xae, 0x5e, 0xc3, 0xdd, 0x49, 0x4e, 0x9f, + 0x2d, 0x37, 0xd2, 0x58, 0xf8, 0x73, 0xa8, 0xe6, + 0xe9, 0xd0, 0xdb, 0xd1, 0xe3, 0x83, 0xef, 0x64, + 0xd9, 0x8b, 0xb9, 0x1b, 0x3e, 0x0b, 0xe0, 0x35 }, + .result = { 0x2e, 0xae, 0x5e, 0xc3, 0xdd, 0x49, 0x4e, 0x9f, + 0x2d, 0x37, 0xd2, 0x58, 0xf8, 0x73, 0xa8, 0xe6, + 0xe9, 0xd0, 0xdb, 0xd1, 0xe3, 0x83, 0xef, 0x64, + 0xd9, 0x8b, 0xb9, 0x1b, 0x3e, 0x0b, 0xe0, 0x35 }, + .valid = true + } +}; + +bool __init curve25519_selftest(void) +{ + bool success = true, ret, ret2; + size_t i = 0, j; + u8 in[CURVE25519_KEY_SIZE]; + u8 out[CURVE25519_KEY_SIZE], out2[CURVE25519_KEY_SIZE], + out3[CURVE25519_KEY_SIZE]; + + for (i = 0; i < ARRAY_SIZE(curve25519_test_vectors); ++i) { + memset(out, 0, CURVE25519_KEY_SIZE); + ret = curve25519(out, curve25519_test_vectors[i].private, + curve25519_test_vectors[i].public); + if (ret != curve25519_test_vectors[i].valid || + memcmp(out, curve25519_test_vectors[i].result, + CURVE25519_KEY_SIZE)) { + pr_err("curve25519 self-test %zu: FAIL\n", i + 1); + success = false; + } + } + + for (i = 0; i < 5; ++i) { + get_random_bytes(in, sizeof(in)); + ret = curve25519_generate_public(out, in); + ret2 = curve25519(out2, in, (u8[CURVE25519_KEY_SIZE]){ 9 }); + curve25519_generic(out3, in, (u8[CURVE25519_KEY_SIZE]){ 9 }); + if (ret != ret2 || + memcmp(out, out2, CURVE25519_KEY_SIZE) || + memcmp(out, out3, CURVE25519_KEY_SIZE)) { + pr_err("curve25519 basepoint self-test %zu: FAIL: input - 0x", + i + 1); + for (j = CURVE25519_KEY_SIZE; j-- > 0;) + printk(KERN_CONT "%02x", in[j]); + printk(KERN_CONT "\n"); + success = false; + } + } + + return success; +} diff --git a/lib/crypto/curve25519.c b/lib/crypto/curve25519.c index 0106bebe6900..c03ccdb99434 100644 --- a/lib/crypto/curve25519.c +++ b/lib/crypto/curve25519.c @@ -13,6 +13,8 @@ #include #include +bool curve25519_selftest(void); + const u8 curve25519_null_point[CURVE25519_KEY_SIZE] __aligned(32) = { 0 }; const u8 curve25519_base_point[CURVE25519_KEY_SIZE] __aligned(32) = { 9 }; @@ -20,6 +22,21 @@ EXPORT_SYMBOL(curve25519_null_point); EXPORT_SYMBOL(curve25519_base_point); EXPORT_SYMBOL(curve25519_generic); +static int __init mod_init(void) +{ + if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && + WARN_ON(!curve25519_selftest())) + return -ENODEV; + return 0; +} + +static void __exit mod_exit(void) +{ +} + +module_init(mod_init); +module_exit(mod_exit); + MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("Curve25519 scalar multiplication"); MODULE_AUTHOR("Jason A. Donenfeld "); -- cgit v1.2.3 From 4bdc0d676a643140bdf17dbf7eafedee3d496a3c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 6 Jan 2020 09:43:50 +0100 Subject: remove ioremap_nocache and devm_ioremap_nocache ioremap has provided non-cached semantics by default since the Linux 2.6 days, so remove the additional ioremap_nocache interface. Signed-off-by: Christoph Hellwig Acked-by: Arnd Bergmann --- Documentation/driver-api/driver-model/devres.rst | 1 - .../sound/kernel-api/writing-an-alsa-driver.rst | 2 +- Documentation/x86/pat.rst | 2 -- arch/alpha/include/asm/io.h | 10 ++-------- arch/arm/include/asm/io.h | 8 -------- arch/arm/mach-bcm/platsmp.c | 4 ++-- arch/arm/mach-davinci/devices.c | 2 +- arch/arm/mach-pxa/magician.c | 2 +- arch/arm/mach-shmobile/platsmp-apmu.c | 2 +- arch/arm/mach-shmobile/pm-rcar-gen2.c | 2 +- arch/arm/mach-shmobile/setup-r8a7740.c | 8 ++++---- arch/arm/mach-shmobile/setup-r8a7778.c | 2 +- arch/hexagon/include/asm/io.h | 1 - arch/ia64/include/asm/vga.h | 2 +- arch/ia64/kernel/cyclone.c | 8 ++++---- arch/m68k/include/asm/kmap.h | 1 - arch/mips/ar7/clock.c | 8 ++++---- arch/mips/ar7/gpio.c | 2 +- arch/mips/ar7/platform.c | 2 +- arch/mips/ath25/ar2315.c | 4 ++-- arch/mips/ath25/ar5312.c | 6 +++--- arch/mips/ath25/board.c | 2 +- arch/mips/ath79/common.c | 2 +- arch/mips/ath79/setup.c | 4 ++-- arch/mips/cavium-octeon/setup.c | 2 +- arch/mips/generic/board-ocelot.c | 2 +- arch/mips/include/asm/io.h | 1 - arch/mips/kernel/mips-cm.c | 4 ++-- arch/mips/kernel/mips-cpc.c | 2 +- arch/mips/lantiq/falcon/sysctrl.c | 6 +++--- arch/mips/lantiq/irq.c | 4 ++-- arch/mips/lantiq/xway/sysctrl.c | 6 +++--- arch/mips/loongson2ef/common/reset.c | 4 ++-- arch/mips/loongson32/common/prom.c | 8 ++++---- arch/mips/loongson32/common/reset.c | 2 +- arch/mips/loongson32/common/time.c | 2 +- arch/mips/loongson64/reset.c | 2 +- arch/mips/mti-malta/malta-dtshim.c | 2 +- arch/mips/pci/pci-alchemy.c | 2 +- arch/mips/pci/pci-ar2315.c | 2 +- arch/mips/pci/pci-bcm63xx.c | 2 +- arch/mips/pci/pci-rt2880.c | 2 +- arch/mips/pic32/pic32mzda/early_console.c | 2 +- arch/mips/pic32/pic32mzda/early_pin.c | 4 ++-- arch/mips/pmcs-msp71xx/msp_serial.c | 4 ++-- arch/mips/ralink/irq.c | 2 +- arch/mips/ralink/of.c | 2 +- arch/mips/rb532/devices.c | 2 +- arch/mips/rb532/gpio.c | 2 +- arch/mips/rb532/prom.c | 2 +- arch/mips/rb532/setup.c | 2 +- arch/mips/sni/rm200.c | 4 ++-- arch/parisc/include/asm/io.h | 5 ++--- arch/parisc/kernel/perf.c | 2 +- arch/powerpc/include/asm/io.h | 3 --- arch/sh/boards/board-sh7785lcr.c | 2 +- arch/sh/boards/mach-cayman/irq.c | 2 +- arch/sh/boards/mach-cayman/setup.c | 2 +- arch/sh/boards/mach-sdk7786/fpga.c | 2 +- arch/sh/drivers/heartbeat.c | 2 +- arch/sh/drivers/pci/pci-sh5.c | 4 ++-- arch/sh/include/asm/io.h | 1 - arch/sh/kernel/cpu/irq/intc-sh5.c | 2 +- arch/sh/kernel/cpu/sh2/smp-j2.c | 4 ++-- arch/sh/kernel/cpu/sh5/clock-sh5.c | 2 +- arch/sh/kernel/dma-coherent.c | 2 +- arch/sparc/include/asm/io_64.h | 1 - arch/unicore32/include/asm/io.h | 1 - arch/x86/kernel/apb_timer.c | 2 +- arch/x86/kernel/hpet.c | 2 +- arch/x86/kernel/quirks.c | 2 +- arch/x86/kernel/tboot.c | 2 +- arch/x86/mm/testmmiotrace.c | 4 ++-- arch/x86/pci/mmconfig_64.c | 2 +- drivers/acpi/acpi_lpit.c | 2 +- drivers/ata/pata_arasan_cf.c | 2 +- drivers/ata/pata_octeon_cf.c | 6 +++--- drivers/ata/pata_rb532_cf.c | 2 +- drivers/atm/eni.c | 8 +------- drivers/bcma/driver_chipcommon_b.c | 2 +- drivers/bcma/driver_pci_host.c | 6 +++--- drivers/bcma/host_soc.c | 2 +- drivers/bcma/scan.c | 6 +++--- drivers/block/umem.c | 2 +- drivers/bus/fsl-mc/mc-io.c | 4 ++-- drivers/char/agp/generic.c | 2 +- drivers/char/agp/intel-gtt.c | 2 +- drivers/char/applicom.c | 4 ++-- drivers/char/hw_random/intel-rng.c | 2 +- drivers/char/hw_random/octeon-rng.c | 4 ++-- drivers/clk/renesas/clk-rz.c | 4 ++-- drivers/clocksource/sh_cmt.c | 2 +- drivers/clocksource/sh_mtu2.c | 2 +- drivers/clocksource/sh_tmu.c | 2 +- drivers/cpufreq/pcc-cpufreq.c | 2 +- drivers/crypto/hifn_795x.c | 2 +- drivers/dma/altera-msgdma.c | 4 ++-- drivers/edac/i3000_edac.c | 2 +- drivers/edac/i3200_edac.c | 2 +- drivers/edac/i82975x_edac.c | 2 +- drivers/edac/ie31200_edac.c | 2 +- drivers/edac/x38_edac.c | 2 +- drivers/firewire/nosy.c | 2 +- drivers/firmware/broadcom/bcm47xx_nvram.c | 2 +- drivers/gpu/drm/gma500/gtt.c | 2 +- drivers/gpu/drm/gma500/psb_drv.c | 2 +- drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c | 2 +- drivers/gpu/drm/i915/i915_gem_gtt.c | 2 +- drivers/gpu/drm/msm/msm_drv.c | 2 +- drivers/gpu/drm/radeon/radeon_ttm.c | 2 +- drivers/gpu/drm/sti/sti_dvo.c | 2 +- drivers/gpu/drm/sti/sti_hda.c | 4 ++-- drivers/gpu/drm/sti/sti_hdmi.c | 2 +- drivers/gpu/drm/sti/sti_tvout.c | 2 +- drivers/gpu/drm/sti/sti_vtg.c | 2 +- drivers/gpu/drm/tilcdc/tilcdc_drv.c | 2 +- drivers/gpu/drm/ttm/ttm_bo_util.c | 4 ++-- drivers/hwmon/i5k_amb.c | 2 +- drivers/i2c/busses/i2c-highlander.c | 2 +- drivers/i2c/busses/i2c-pmcmsp.c | 2 +- drivers/infiniband/hw/bnxt_re/qplib_fp.c | 2 +- drivers/infiniband/hw/bnxt_re/qplib_rcfw.c | 4 ++-- drivers/infiniband/hw/bnxt_re/qplib_res.c | 2 +- drivers/infiniband/hw/hfi1/pcie.c | 4 ++-- drivers/infiniband/hw/qib/qib_iba7322.c | 2 +- drivers/infiniband/hw/qib/qib_init.c | 4 ++-- drivers/infiniband/hw/qib/qib_pcie.c | 2 +- drivers/input/keyboard/pxa930_rotary.c | 2 +- drivers/input/keyboard/sh_keysc.c | 2 +- drivers/input/mouse/pxa930_trkball.c | 2 +- drivers/input/serio/gscps2.c | 2 +- drivers/iommu/amd_iommu_init.c | 2 +- drivers/ipack/carriers/tpci200.c | 4 ++-- drivers/ipack/devices/ipoctal.c | 6 +++--- drivers/irqchip/irq-mips-gic.c | 2 +- drivers/irqchip/irq-renesas-intc-irqpin.c | 2 +- drivers/media/common/videobuf2/videobuf2-vmalloc.c | 2 +- drivers/media/pci/cx18/cx18-driver.c | 2 +- drivers/media/pci/ivtv/ivtv-driver.c | 6 +++--- drivers/media/platform/davinci/dm355_ccdc.c | 2 +- drivers/media/platform/davinci/dm644x_ccdc.c | 2 +- drivers/media/platform/davinci/isif.c | 2 +- drivers/media/platform/tegra-cec/tegra_cec.c | 2 +- drivers/misc/cardreader/rtsx_pcr.c | 2 +- drivers/misc/mic/scif/scif_nodeqp.c | 2 +- drivers/misc/pti.c | 2 +- drivers/mmc/host/sdhci-acpi.c | 2 +- drivers/mmc/host/via-sdmmc.c | 2 +- drivers/mtd/devices/bcm47xxsflash.c | 2 +- drivers/mtd/maps/amd76xrom.c | 2 +- drivers/mtd/maps/ck804xrom.c | 2 +- drivers/mtd/maps/esb2rom.c | 2 +- drivers/mtd/maps/ichxrom.c | 2 +- drivers/mtd/maps/intel_vr_nor.c | 4 ++-- drivers/mtd/maps/l440gx.c | 2 +- drivers/mtd/maps/netsc520.c | 4 ++-- drivers/mtd/maps/nettel.c | 8 ++++---- drivers/mtd/maps/pci.c | 4 ++-- drivers/mtd/maps/sc520cdp.c | 8 ++++---- drivers/mtd/maps/scb2_flash.c | 2 +- drivers/mtd/maps/ts5500_flash.c | 4 ++-- drivers/mtd/nand/raw/au1550nd.c | 2 +- drivers/mtd/nand/raw/denali_pci.c | 6 +++--- drivers/mtd/nand/raw/fsl_upm.c | 2 +- drivers/net/can/at91_can.c | 2 +- drivers/net/can/cc770/cc770_isa.c | 2 +- drivers/net/can/sja1000/sja1000_isa.c | 2 +- drivers/net/can/sja1000/sja1000_platform.c | 2 +- drivers/net/can/softing/softing_main.c | 2 +- drivers/net/ethernet/alacritech/slicoss.c | 2 +- drivers/net/ethernet/altera/altera_tse_main.c | 4 ++-- drivers/net/ethernet/amd/au1000_eth.c | 6 +++--- drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c | 2 +- drivers/net/ethernet/atheros/ag71xx.c | 3 +-- drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c | 2 +- drivers/net/ethernet/broadcom/sb1250-mac.c | 2 +- drivers/net/ethernet/brocade/bna/bnad.c | 2 +- drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c | 2 +- drivers/net/ethernet/dec/tulip/de2104x.c | 2 +- drivers/net/ethernet/ethoc.c | 4 ++-- drivers/net/ethernet/i825xx/sni_82596.c | 6 +++--- drivers/net/ethernet/korina.c | 6 +++--- drivers/net/ethernet/lantiq_etop.c | 2 +- drivers/net/ethernet/marvell/skge.c | 2 +- drivers/net/ethernet/marvell/sky2.c | 2 +- drivers/net/ethernet/natsemi/ns83820.c | 2 +- drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c | 8 ++++---- .../ethernet/netronome/nfp/nfpcore/nfp6000_pcie.c | 6 +++--- drivers/net/ethernet/sfc/ef10.c | 2 +- drivers/net/ethernet/sfc/efx.c | 2 +- drivers/net/ethernet/sfc/falcon/efx.c | 2 +- drivers/net/ethernet/smsc/smsc911x.c | 2 +- drivers/net/ethernet/ti/netcp_core.c | 2 +- drivers/net/ethernet/xilinx/ll_temac_main.c | 4 ++-- drivers/net/fddi/defxx.c | 2 +- drivers/net/fddi/defza.c | 2 +- drivers/net/fjes/fjes_hw.c | 2 +- drivers/net/wan/wanxl.c | 4 ++-- drivers/net/wireless/ath/ath10k/ahb.c | 4 ++-- drivers/net/wireless/ath/ath5k/ahb.c | 2 +- drivers/net/wireless/ath/ath9k/ahb.c | 2 +- .../net/wireless/broadcom/brcm80211/brcmfmac/pcie.c | 4 ++-- drivers/net/wireless/intel/ipw2x00/ipw2100.c | 2 +- drivers/opp/ti-opp-supply.c | 2 +- drivers/parisc/ccio-dma.c | 2 +- drivers/parisc/dino.c | 2 +- drivers/parisc/eisa.c | 4 ++-- drivers/parisc/iosapic.c | 2 +- drivers/parisc/lba_pci.c | 8 ++++---- drivers/parisc/sba_iommu.c | 4 ++-- drivers/pci/controller/dwc/pci-dra7xx.c | 2 +- drivers/pci/controller/dwc/pcie-designware-ep.c | 2 +- drivers/pci/msi.c | 2 +- drivers/pci/pci.c | 2 +- drivers/pci/quirks.c | 4 ++-- drivers/pinctrl/bcm/pinctrl-ns2-mux.c | 2 +- drivers/pinctrl/bcm/pinctrl-nsp-mux.c | 2 +- drivers/pinctrl/freescale/pinctrl-imx1-core.c | 2 +- drivers/pinctrl/pinctrl-amd.c | 2 +- drivers/platform/x86/intel_scu_ipc.c | 2 +- drivers/platform/x86/intel_telemetry_pltdrv.c | 4 ++-- drivers/platform/x86/pmc_atom.c | 2 +- drivers/platform/x86/samsung-laptop.c | 4 ++-- drivers/regulator/ti-abb-regulator.c | 4 ++-- drivers/rtc/rtc-sh.c | 2 +- drivers/scsi/aic7xxx/aic79xx_osm_pci.c | 2 +- drivers/scsi/aic7xxx/aic7xxx_osm_pci.c | 2 +- drivers/scsi/arcmsr/arcmsr_hba.c | 2 +- drivers/scsi/be2iscsi/be_main.c | 6 +++--- drivers/scsi/bnx2fc/bnx2fc_hwi.c | 2 +- drivers/scsi/bnx2i/bnx2i_hwi.c | 4 ++-- drivers/scsi/csiostor/csio_init.c | 2 +- drivers/scsi/hpsa.c | 2 +- drivers/scsi/lasi700.c | 2 +- drivers/scsi/megaraid/megaraid_mbox.c | 2 +- drivers/scsi/megaraid/megaraid_sas_base.c | 2 +- drivers/scsi/myrb.c | 2 +- drivers/scsi/myrs.c | 2 +- drivers/scsi/pcmcia/nsp_cs.c | 2 +- drivers/scsi/qla2xxx/qla_mr.c | 4 ++-- drivers/scsi/smartpqi/smartpqi_init.c | 2 +- drivers/scsi/sni_53c710.c | 2 +- drivers/scsi/sun3x_esp.c | 4 ++-- drivers/scsi/zalon.c | 2 +- drivers/scsi/zorro_esp.c | 6 +++--- drivers/sh/clk/core.c | 2 +- drivers/sh/intc/core.c | 2 +- drivers/sh/intc/userimask.c | 2 +- drivers/soc/tegra/flowctrl.c | 2 +- drivers/soc/tegra/fuse/fuse-tegra.c | 2 +- drivers/soc/tegra/fuse/tegra-apbmisc.c | 4 ++-- drivers/soc/tegra/pmc.c | 4 ++-- drivers/soc/xilinx/xlnx_vcu.c | 4 ++-- drivers/spi/spi-dw-mid.c | 2 +- drivers/spi/spi-jcore.c | 2 +- drivers/spi/spi-npcm-fiu.c | 2 +- drivers/ssb/driver_extif.c | 2 +- drivers/ssb/driver_pcicore.c | 6 +++--- drivers/staging/gasket/gasket_core.c | 2 +- drivers/staging/kpc2000/kpc2000/core.c | 4 ++-- drivers/staging/kpc2000/kpc2000_i2c.c | 2 +- drivers/staging/kpc2000/kpc2000_spi.c | 2 +- drivers/staging/kpc2000/kpc_dma/kpc_dma_driver.c | 2 +- drivers/staging/media/allegro-dvt/allegro-core.c | 4 ++-- drivers/staging/qlge/qlge_main.c | 4 ++-- drivers/staging/rtl8192e/rtl8192e/rtl_core.c | 2 +- drivers/staging/rts5208/rtsx.c | 2 +- drivers/staging/sm750fb/sm750_hw.c | 2 +- drivers/staging/uwb/whc-rc.c | 6 +++--- drivers/tc/tc.c | 2 +- drivers/tty/cyclades.c | 10 +++++----- drivers/tty/mips_ejtag_fdc.c | 2 +- drivers/tty/moxa.c | 4 ++-- drivers/tty/serial/8250/8250_gsc.c | 2 +- drivers/tty/serial/8250/8250_omap.c | 2 +- drivers/tty/serial/8250/8250_pci.c | 6 +++--- drivers/tty/serial/8250/8250_port.c | 2 +- drivers/tty/serial/dz.c | 2 +- drivers/tty/serial/lantiq.c | 2 +- drivers/tty/serial/meson_uart.c | 2 +- drivers/tty/serial/mux.c | 2 +- drivers/tty/serial/owl-uart.c | 2 +- drivers/tty/serial/pic32_uart.c | 2 +- drivers/tty/serial/rda-uart.c | 2 +- drivers/tty/serial/sb1250-duart.c | 4 ++-- drivers/tty/serial/sh-sci.c | 2 +- drivers/tty/serial/zs.c | 2 +- drivers/tty/synclink.c | 4 ++-- drivers/tty/synclink_gt.c | 2 +- drivers/tty/synclinkmp.c | 8 ++++---- drivers/usb/core/hcd-pci.c | 2 +- drivers/usb/early/xhci-dbc.c | 2 +- drivers/usb/gadget/udc/amd5536udc_pci.c | 2 +- drivers/usb/gadget/udc/goku_udc.c | 2 +- drivers/usb/gadget/udc/net2272.c | 6 +++--- drivers/usb/gadget/udc/net2280.c | 2 +- drivers/usb/host/ehci-pmcmsp.c | 6 +++--- drivers/usb/host/pci-quirks.c | 2 +- drivers/usb/isp1760/isp1760-if.c | 4 ++-- drivers/usb/roles/intel-xhci-usb-role-switch.c | 2 +- drivers/usb/typec/ucsi/ucsi_acpi.c | 2 +- drivers/vfio/pci/vfio_pci_rdwr.c | 2 +- drivers/vfio/platform/reset/vfio_platform_amdxgbe.c | 4 ++-- .../vfio/platform/reset/vfio_platform_bcmflexrm.c | 2 +- .../vfio/platform/reset/vfio_platform_calxedaxgmac.c | 2 +- drivers/vfio/platform/vfio_platform_common.c | 4 ++-- drivers/video/fbdev/carminefb.c | 4 ++-- drivers/video/fbdev/i810/i810_main.c | 2 +- drivers/video/fbdev/intelfb/intelfbdrv.c | 2 +- drivers/video/fbdev/kyro/fbdev.c | 2 +- drivers/video/fbdev/matrox/matroxfb_base.c | 2 +- drivers/video/fbdev/mbx/mbxfb.c | 4 ++-- drivers/video/fbdev/mmp/hw/mmp_ctrl.c | 2 +- drivers/video/fbdev/pm2fb.c | 2 +- drivers/video/fbdev/pm3fb.c | 4 ++-- drivers/video/fbdev/pmag-aa-fb.c | 4 ++-- drivers/video/fbdev/pmag-ba-fb.c | 4 ++-- drivers/video/fbdev/pmagb-b-fb.c | 4 ++-- drivers/video/fbdev/pvr2fb.c | 4 ++-- drivers/video/fbdev/pxa168fb.c | 2 +- drivers/video/fbdev/s1d13xxxfb.c | 4 ++-- drivers/video/fbdev/sh7760fb.c | 2 +- drivers/video/fbdev/sh_mobile_lcdcfb.c | 2 +- drivers/video/fbdev/sstfb.c | 4 ++-- drivers/video/fbdev/stifb.c | 4 ++-- drivers/video/fbdev/tdfxfb.c | 2 +- drivers/video/fbdev/tgafb.c | 2 +- drivers/video/fbdev/tridentfb.c | 4 ++-- drivers/video/fbdev/valkyriefb.c | 2 +- drivers/video/fbdev/vermilion/cr_pll.c | 2 +- drivers/video/fbdev/vermilion/vermilion.c | 4 ++-- drivers/video/fbdev/via/via-core.c | 2 +- drivers/video/fbdev/w100fb.c | 6 +++--- drivers/vme/boards/vme_vmivme7805.c | 2 +- drivers/vme/bridges/vme_ca91cx42.c | 4 ++-- drivers/vme/bridges/vme_tsi148.c | 4 ++-- drivers/w1/masters/matrox_w1.c | 2 +- drivers/watchdog/bcm63xx_wdt.c | 2 +- drivers/watchdog/intel_scu_watchdog.c | 2 +- drivers/watchdog/rc32434_wdt.c | 4 ++-- include/asm-generic/io.h | 4 ---- include/asm-generic/iomap.h | 4 ++-- include/linux/io.h | 6 ++---- lib/devres.c | 20 -------------------- scripts/coccinelle/free/devm_free.cocci | 4 ---- scripts/coccinelle/free/iounmap.cocci | 2 +- sound/drivers/ml403-ac97cr.c | 2 +- sound/isa/msnd/msnd_pinnacle.c | 2 +- sound/parisc/harmony.c | 2 +- sound/pci/aw2/aw2-alsa.c | 2 +- sound/pci/cs46xx/cs46xx_lib.c | 2 +- sound/pci/echoaudio/echoaudio.c | 2 +- sound/pci/nm256/nm256.c | 6 +++--- sound/pci/rme32.c | 2 +- sound/pci/rme96.c | 2 +- sound/pci/rme9652/hdsp.c | 2 +- sound/pci/rme9652/hdspm.c | 2 +- sound/pci/rme9652/rme9652.c | 2 +- sound/pci/sis7019.c | 2 +- sound/pci/ymfpci/ymfpci_main.c | 2 +- sound/soc/au1x/ac97c.c | 2 +- sound/soc/au1x/i2sc.c | 2 +- sound/soc/intel/atom/sst/sst_acpi.c | 10 +++++----- sound/soc/sh/fsi.c | 2 +- sound/x86/intel_hdmi_audio.c | 2 +- tools/testing/nvdimm/Kbuild | 3 +-- tools/testing/nvdimm/test/iomap.c | 12 +++--------- tools/testing/nvdimm/test/nfit_test.h | 2 -- 368 files changed, 508 insertions(+), 581 deletions(-) (limited to 'lib') diff --git a/Documentation/driver-api/driver-model/devres.rst b/Documentation/driver-api/driver-model/devres.rst index 13046fcf0a5d..20e07e40be02 100644 --- a/Documentation/driver-api/driver-model/devres.rst +++ b/Documentation/driver-api/driver-model/devres.rst @@ -313,7 +313,6 @@ IOMAP devm_ioport_map() devm_ioport_unmap() devm_ioremap() - devm_ioremap_nocache() devm_ioremap_uc() devm_ioremap_wc() devm_ioremap_resource() : checks resource, requests memory region, ioremaps diff --git a/Documentation/sound/kernel-api/writing-an-alsa-driver.rst b/Documentation/sound/kernel-api/writing-an-alsa-driver.rst index f169d58ca019..ddef812ddf8f 100644 --- a/Documentation/sound/kernel-api/writing-an-alsa-driver.rst +++ b/Documentation/sound/kernel-api/writing-an-alsa-driver.rst @@ -1058,7 +1058,7 @@ and the allocation would be like below: return err; } chip->iobase_phys = pci_resource_start(pci, 0); - chip->iobase_virt = ioremap_nocache(chip->iobase_phys, + chip->iobase_virt = ioremap(chip->iobase_phys, pci_resource_len(pci, 0)); and the corresponding destructor would be: diff --git a/Documentation/x86/pat.rst b/Documentation/x86/pat.rst index 9a298fd97d74..5d901771016d 100644 --- a/Documentation/x86/pat.rst +++ b/Documentation/x86/pat.rst @@ -44,8 +44,6 @@ address range to avoid any aliasing. +------------------------+----------+--------------+------------------+ | ioremap_uc | -- | UC | UC | +------------------------+----------+--------------+------------------+ -| ioremap_nocache | -- | UC- | UC- | -+------------------------+----------+--------------+------------------+ | ioremap_wc | -- | -- | WC | +------------------------+----------+--------------+------------------+ | ioremap_wt | -- | -- | WT | diff --git a/arch/alpha/include/asm/io.h b/arch/alpha/include/asm/io.h index 1989b946a28d..d1ed5a8133c5 100644 --- a/arch/alpha/include/asm/io.h +++ b/arch/alpha/include/asm/io.h @@ -283,14 +283,8 @@ static inline void __iomem *ioremap(unsigned long port, unsigned long size) return IO_CONCAT(__IO_PREFIX,ioremap) (port, size); } -static inline void __iomem * ioremap_nocache(unsigned long offset, - unsigned long size) -{ - return ioremap(offset, size); -} - -#define ioremap_wc ioremap_nocache -#define ioremap_uc ioremap_nocache +#define ioremap_wc ioremap +#define ioremap_uc ioremap static inline void iounmap(volatile void __iomem *addr) { diff --git a/arch/arm/include/asm/io.h b/arch/arm/include/asm/io.h index aefdabdbeb84..ab2b654084fa 100644 --- a/arch/arm/include/asm/io.h +++ b/arch/arm/include/asm/io.h @@ -356,7 +356,6 @@ static inline void memcpy_toio(volatile void __iomem *to, const void *from, * * Function Memory type Cacheability Cache hint * ioremap() Device n/a n/a - * ioremap_nocache() Device n/a n/a * ioremap_cache() Normal Writeback Read allocate * ioremap_wc() Normal Non-cacheable n/a * ioremap_wt() Normal Non-cacheable n/a @@ -368,13 +367,6 @@ static inline void memcpy_toio(volatile void __iomem *to, const void *from, * - unaligned accesses are "unpredictable" * - writes may be delayed before they hit the endpoint device * - * ioremap_nocache() is the same as ioremap() as there are too many device - * drivers using this for device registers, and documentation which tells - * people to use it for such for this to be any different. This is not a - * safe fallback for memory-like mappings, or memory regions where the - * compiler may generate unaligned accesses - eg, via inlining its own - * memcpy. - * * All normal memory mappings have the following properties: * - reads can be repeated with no side effects * - repeated reads return the last value written diff --git a/arch/arm/mach-bcm/platsmp.c b/arch/arm/mach-bcm/platsmp.c index 21400b3fa5fe..c9db2a9006d9 100644 --- a/arch/arm/mach-bcm/platsmp.c +++ b/arch/arm/mach-bcm/platsmp.c @@ -105,7 +105,7 @@ static int nsp_write_lut(unsigned int cpu) if (!secondary_boot_addr) return -EINVAL; - sku_rom_lut = ioremap_nocache((phys_addr_t)secondary_boot_addr, + sku_rom_lut = ioremap((phys_addr_t)secondary_boot_addr, sizeof(phys_addr_t)); if (!sku_rom_lut) { pr_warn("unable to ioremap SKU-ROM LUT register for cpu %u\n", cpu); @@ -174,7 +174,7 @@ static int kona_boot_secondary(unsigned int cpu, struct task_struct *idle) if (!secondary_boot_addr) return -EINVAL; - boot_reg = ioremap_nocache((phys_addr_t)secondary_boot_addr, + boot_reg = ioremap((phys_addr_t)secondary_boot_addr, sizeof(phys_addr_t)); if (!boot_reg) { pr_err("unable to map boot register for cpu %u\n", cpu_id); diff --git a/arch/arm/mach-davinci/devices.c b/arch/arm/mach-davinci/devices.c index 3e447d468845..e650131ee88f 100644 --- a/arch/arm/mach-davinci/devices.c +++ b/arch/arm/mach-davinci/devices.c @@ -34,7 +34,7 @@ void __iomem *davinci_sysmod_base; void davinci_map_sysmod(void) { - davinci_sysmod_base = ioremap_nocache(DAVINCI_SYSTEM_MODULE_BASE, + davinci_sysmod_base = ioremap(DAVINCI_SYSTEM_MODULE_BASE, 0x800); /* * Throw a bug since a lot of board initialization code depends diff --git a/arch/arm/mach-pxa/magician.c b/arch/arm/mach-pxa/magician.c index e1a394ac3eea..868dc0cf4859 100644 --- a/arch/arm/mach-pxa/magician.c +++ b/arch/arm/mach-pxa/magician.c @@ -1008,7 +1008,7 @@ static void __init magician_init(void) pxa_set_udc_info(&magician_udc_info); /* Check LCD type we have */ - cpld = ioremap_nocache(PXA_CS3_PHYS, 0x1000); + cpld = ioremap(PXA_CS3_PHYS, 0x1000); if (cpld) { u8 board_id = __raw_readb(cpld + 0x14); diff --git a/arch/arm/mach-shmobile/platsmp-apmu.c b/arch/arm/mach-shmobile/platsmp-apmu.c index 96330ef25641..e771ce70e132 100644 --- a/arch/arm/mach-shmobile/platsmp-apmu.c +++ b/arch/arm/mach-shmobile/platsmp-apmu.c @@ -189,7 +189,7 @@ static void apmu_init_cpu(struct resource *res, int cpu, int bit) if ((cpu >= ARRAY_SIZE(apmu_cpus)) || apmu_cpus[cpu].iomem) return; - apmu_cpus[cpu].iomem = ioremap_nocache(res->start, resource_size(res)); + apmu_cpus[cpu].iomem = ioremap(res->start, resource_size(res)); apmu_cpus[cpu].bit = bit; pr_debug("apmu ioremap %d %d %pr\n", cpu, bit, res); diff --git a/arch/arm/mach-shmobile/pm-rcar-gen2.c b/arch/arm/mach-shmobile/pm-rcar-gen2.c index e84599dd96f1..672081405a7e 100644 --- a/arch/arm/mach-shmobile/pm-rcar-gen2.c +++ b/arch/arm/mach-shmobile/pm-rcar-gen2.c @@ -103,7 +103,7 @@ map: iounmap(p); /* setup reset vectors */ - p = ioremap_nocache(RST, 0x63); + p = ioremap(RST, 0x63); bar = phys_to_sbar(res.start); if (has_a15) { writel_relaxed(bar, p + CA15BAR); diff --git a/arch/arm/mach-shmobile/setup-r8a7740.c b/arch/arm/mach-shmobile/setup-r8a7740.c index 787d039b5a07..f760c27c9907 100644 --- a/arch/arm/mach-shmobile/setup-r8a7740.c +++ b/arch/arm/mach-shmobile/setup-r8a7740.c @@ -28,7 +28,7 @@ static void __init r8a7740_meram_workaround(void) { void __iomem *reg; - reg = ioremap_nocache(MEBUFCNTR, 4); + reg = ioremap(MEBUFCNTR, 4); if (reg) { iowrite32(0x01600164, reg); iounmap(reg); @@ -37,9 +37,9 @@ static void __init r8a7740_meram_workaround(void) static void __init r8a7740_init_irq_of(void) { - void __iomem *intc_prio_base = ioremap_nocache(0xe6900010, 0x10); - void __iomem *intc_msk_base = ioremap_nocache(0xe6900040, 0x10); - void __iomem *pfc_inta_ctrl = ioremap_nocache(0xe605807c, 0x4); + void __iomem *intc_prio_base = ioremap(0xe6900010, 0x10); + void __iomem *intc_msk_base = ioremap(0xe6900040, 0x10); + void __iomem *pfc_inta_ctrl = ioremap(0xe605807c, 0x4); irqchip_init(); diff --git a/arch/arm/mach-shmobile/setup-r8a7778.c b/arch/arm/mach-shmobile/setup-r8a7778.c index ce51794f64c7..2bc93f391bcf 100644 --- a/arch/arm/mach-shmobile/setup-r8a7778.c +++ b/arch/arm/mach-shmobile/setup-r8a7778.c @@ -22,7 +22,7 @@ static void __init r8a7778_init_irq_dt(void) { - void __iomem *base = ioremap_nocache(0xfe700000, 0x00100000); + void __iomem *base = ioremap(0xfe700000, 0x00100000); BUG_ON(!base); diff --git a/arch/hexagon/include/asm/io.h b/arch/hexagon/include/asm/io.h index b0dbc3473172..bda2a9c2df78 100644 --- a/arch/hexagon/include/asm/io.h +++ b/arch/hexagon/include/asm/io.h @@ -172,7 +172,6 @@ static inline void writel(u32 data, volatile void __iomem *addr) #define writel_relaxed __raw_writel void __iomem *ioremap(unsigned long phys_addr, unsigned long size); -#define ioremap_nocache ioremap #define ioremap_uc(X, Y) ioremap((X), (Y)) diff --git a/arch/ia64/include/asm/vga.h b/arch/ia64/include/asm/vga.h index 30cb373f3de8..64ce0b971a0a 100644 --- a/arch/ia64/include/asm/vga.h +++ b/arch/ia64/include/asm/vga.h @@ -18,7 +18,7 @@ extern unsigned long vga_console_iobase; extern unsigned long vga_console_membase; -#define VGA_MAP_MEM(x,s) ((unsigned long) ioremap_nocache(vga_console_membase + (x), s)) +#define VGA_MAP_MEM(x,s) ((unsigned long) ioremap(vga_console_membase + (x), s)) #define vga_readb(x) (*(x)) #define vga_writeb(x,y) (*(y) = (x)) diff --git a/arch/ia64/kernel/cyclone.c b/arch/ia64/kernel/cyclone.c index f80eb7fb544d..258d7b70c0f3 100644 --- a/arch/ia64/kernel/cyclone.c +++ b/arch/ia64/kernel/cyclone.c @@ -50,7 +50,7 @@ int __init init_cyclone_clock(void) /* find base address */ offset = (CYCLONE_CBAR_ADDR); - reg = ioremap_nocache(offset, sizeof(u64)); + reg = ioremap(offset, sizeof(u64)); if(!reg){ printk(KERN_ERR "Summit chipset: Could not find valid CBAR" " register.\n"); @@ -68,7 +68,7 @@ int __init init_cyclone_clock(void) /* setup PMCC */ offset = (base + CYCLONE_PMCC_OFFSET); - reg = ioremap_nocache(offset, sizeof(u64)); + reg = ioremap(offset, sizeof(u64)); if(!reg){ printk(KERN_ERR "Summit chipset: Could not find valid PMCC" " register.\n"); @@ -80,7 +80,7 @@ int __init init_cyclone_clock(void) /* setup MPCS */ offset = (base + CYCLONE_MPCS_OFFSET); - reg = ioremap_nocache(offset, sizeof(u64)); + reg = ioremap(offset, sizeof(u64)); if(!reg){ printk(KERN_ERR "Summit chipset: Could not find valid MPCS" " register.\n"); @@ -92,7 +92,7 @@ int __init init_cyclone_clock(void) /* map in cyclone_timer */ offset = (base + CYCLONE_MPMC_OFFSET); - cyclone_timer = ioremap_nocache(offset, sizeof(u32)); + cyclone_timer = ioremap(offset, sizeof(u32)); if(!cyclone_timer){ printk(KERN_ERR "Summit chipset: Could not find valid MPMC" " register.\n"); diff --git a/arch/m68k/include/asm/kmap.h b/arch/m68k/include/asm/kmap.h index 559cb91bede1..dec05743d426 100644 --- a/arch/m68k/include/asm/kmap.h +++ b/arch/m68k/include/asm/kmap.h @@ -27,7 +27,6 @@ static inline void __iomem *ioremap(unsigned long physaddr, unsigned long size) return __ioremap(physaddr, size, IOMAP_NOCACHE_SER); } -#define ioremap_nocache ioremap #define ioremap_uc ioremap #define ioremap_wt ioremap_wt static inline void __iomem *ioremap_wt(unsigned long physaddr, diff --git a/arch/mips/ar7/clock.c b/arch/mips/ar7/clock.c index 7de162432d7f..95def949c971 100644 --- a/arch/mips/ar7/clock.c +++ b/arch/mips/ar7/clock.c @@ -236,9 +236,9 @@ static void tnetd7300_set_clock(u32 shift, struct tnetd7300_clock *clock, static void __init tnetd7300_init_clocks(void) { - u32 *bootcr = (u32 *)ioremap_nocache(AR7_REGS_DCL, 4); + u32 *bootcr = (u32 *)ioremap(AR7_REGS_DCL, 4); struct tnetd7300_clocks *clocks = - ioremap_nocache(UR8_REGS_CLOCKS, + ioremap(UR8_REGS_CLOCKS, sizeof(struct tnetd7300_clocks)); bus_clk.rate = tnetd7300_get_clock(BUS_PLL_SOURCE_SHIFT, @@ -320,9 +320,9 @@ static int tnetd7200_get_clock_base(int clock_id, u32 *bootcr) static void __init tnetd7200_init_clocks(void) { - u32 *bootcr = (u32 *)ioremap_nocache(AR7_REGS_DCL, 4); + u32 *bootcr = (u32 *)ioremap(AR7_REGS_DCL, 4); struct tnetd7200_clocks *clocks = - ioremap_nocache(AR7_REGS_CLOCKS, + ioremap(AR7_REGS_CLOCKS, sizeof(struct tnetd7200_clocks)); int cpu_base, cpu_mul, cpu_prediv, cpu_postdiv; int dsp_base, dsp_mul, dsp_prediv, dsp_postdiv; diff --git a/arch/mips/ar7/gpio.c b/arch/mips/ar7/gpio.c index 2292e55c12e2..8b006addd6ba 100644 --- a/arch/mips/ar7/gpio.c +++ b/arch/mips/ar7/gpio.c @@ -308,7 +308,7 @@ int __init ar7_gpio_init(void) size = 0x1f; } - gpch->regs = ioremap_nocache(AR7_REGS_GPIO, size); + gpch->regs = ioremap(AR7_REGS_GPIO, size); if (!gpch->regs) { printk(KERN_ERR "%s: failed to ioremap regs\n", gpch->chip.label); diff --git a/arch/mips/ar7/platform.c b/arch/mips/ar7/platform.c index 1f2028266493..215149a85d83 100644 --- a/arch/mips/ar7/platform.c +++ b/arch/mips/ar7/platform.c @@ -702,7 +702,7 @@ static int __init ar7_register_devices(void) pr_warn("unable to register usb slave: %d\n", res); /* Register watchdog only if enabled in hardware */ - bootcr = ioremap_nocache(AR7_REGS_DCL, 4); + bootcr = ioremap(AR7_REGS_DCL, 4); val = readl(bootcr); iounmap(bootcr); if (val & AR7_WDT_HW_ENA) { diff --git a/arch/mips/ath25/ar2315.c b/arch/mips/ath25/ar2315.c index 8da996142d6a..24f619199ee7 100644 --- a/arch/mips/ath25/ar2315.c +++ b/arch/mips/ath25/ar2315.c @@ -262,7 +262,7 @@ void __init ar2315_plat_mem_setup(void) u32 config; /* Detect memory size */ - sdram_base = ioremap_nocache(AR2315_SDRAMCTL_BASE, + sdram_base = ioremap(AR2315_SDRAMCTL_BASE, AR2315_SDRAMCTL_SIZE); memcfg = __raw_readl(sdram_base + AR2315_MEM_CFG); memsize = 1 + ATH25_REG_MS(memcfg, AR2315_MEM_CFG_DATA_WIDTH); @@ -272,7 +272,7 @@ void __init ar2315_plat_mem_setup(void) add_memory_region(0, memsize, BOOT_MEM_RAM); iounmap(sdram_base); - ar2315_rst_base = ioremap_nocache(AR2315_RST_BASE, AR2315_RST_SIZE); + ar2315_rst_base = ioremap(AR2315_RST_BASE, AR2315_RST_SIZE); /* Detect the hardware based on the device ID */ devid = ar2315_rst_reg_read(AR2315_SREV) & AR2315_REV_CHIP; diff --git a/arch/mips/ath25/ar5312.c b/arch/mips/ath25/ar5312.c index acd55a9cffe3..47f3e98974fc 100644 --- a/arch/mips/ath25/ar5312.c +++ b/arch/mips/ath25/ar5312.c @@ -185,7 +185,7 @@ static void __init ar5312_flash_init(void) void __iomem *flashctl_base; u32 ctl; - flashctl_base = ioremap_nocache(AR5312_FLASHCTL_BASE, + flashctl_base = ioremap(AR5312_FLASHCTL_BASE, AR5312_FLASHCTL_SIZE); ctl = __raw_readl(flashctl_base + AR5312_FLASHCTL0); @@ -358,7 +358,7 @@ void __init ar5312_plat_mem_setup(void) u32 devid; /* Detect memory size */ - sdram_base = ioremap_nocache(AR5312_SDRAMCTL_BASE, + sdram_base = ioremap(AR5312_SDRAMCTL_BASE, AR5312_SDRAMCTL_SIZE); memcfg = __raw_readl(sdram_base + AR5312_MEM_CFG1); bank0_ac = ATH25_REG_MS(memcfg, AR5312_MEM_CFG1_AC0); @@ -369,7 +369,7 @@ void __init ar5312_plat_mem_setup(void) add_memory_region(0, memsize, BOOT_MEM_RAM); iounmap(sdram_base); - ar5312_rst_base = ioremap_nocache(AR5312_RST_BASE, AR5312_RST_SIZE); + ar5312_rst_base = ioremap(AR5312_RST_BASE, AR5312_RST_SIZE); devid = ar5312_rst_reg_read(AR5312_REV); devid >>= AR5312_REV_WMAC_MIN_S; diff --git a/arch/mips/ath25/board.c b/arch/mips/ath25/board.c index 989e71015ee6..cb99f9739910 100644 --- a/arch/mips/ath25/board.c +++ b/arch/mips/ath25/board.c @@ -111,7 +111,7 @@ int __init ath25_find_config(phys_addr_t base, unsigned long size) u8 *mac_addr; u32 offset; - flash_base = ioremap_nocache(base, size); + flash_base = ioremap(base, size); flash_limit = flash_base + size; ath25_board.config = NULL; diff --git a/arch/mips/ath79/common.c b/arch/mips/ath79/common.c index 63eacb8b0eb5..137abbc65c60 100644 --- a/arch/mips/ath79/common.c +++ b/arch/mips/ath79/common.c @@ -41,7 +41,7 @@ static void __iomem *ath79_ddr_pci_win_base; void ath79_ddr_ctrl_init(void) { - ath79_ddr_base = ioremap_nocache(AR71XX_DDR_CTRL_BASE, + ath79_ddr_base = ioremap(AR71XX_DDR_CTRL_BASE, AR71XX_DDR_CTRL_SIZE); if (soc_is_ar913x() || soc_is_ar724x() || soc_is_ar933x()) { ath79_ddr_wb_flush_base = ath79_ddr_base + 0x7c; diff --git a/arch/mips/ath79/setup.c b/arch/mips/ath79/setup.c index ea385a865781..484ee28922a9 100644 --- a/arch/mips/ath79/setup.c +++ b/arch/mips/ath79/setup.c @@ -226,9 +226,9 @@ void __init plat_mem_setup(void) else if (fw_passed_dtb) __dt_setup_arch((void *)KSEG0ADDR(fw_passed_dtb)); - ath79_reset_base = ioremap_nocache(AR71XX_RESET_BASE, + ath79_reset_base = ioremap(AR71XX_RESET_BASE, AR71XX_RESET_SIZE); - ath79_pll_base = ioremap_nocache(AR71XX_PLL_BASE, + ath79_pll_base = ioremap(AR71XX_PLL_BASE, AR71XX_PLL_SIZE); ath79_detect_sys_type(); ath79_ddr_ctrl_init(); diff --git a/arch/mips/cavium-octeon/setup.c b/arch/mips/cavium-octeon/setup.c index 1f742c32a883..4f34d92b52f9 100644 --- a/arch/mips/cavium-octeon/setup.c +++ b/arch/mips/cavium-octeon/setup.c @@ -357,7 +357,7 @@ static void octeon_write_lcd(const char *s) { if (octeon_bootinfo->led_display_base_addr) { void __iomem *lcd_address = - ioremap_nocache(octeon_bootinfo->led_display_base_addr, + ioremap(octeon_bootinfo->led_display_base_addr, 8); int i; for (i = 0; i < 8; i++, s++) { diff --git a/arch/mips/generic/board-ocelot.c b/arch/mips/generic/board-ocelot.c index 06d92fb37769..c238e95190ac 100644 --- a/arch/mips/generic/board-ocelot.c +++ b/arch/mips/generic/board-ocelot.c @@ -51,7 +51,7 @@ static void __init ocelot_earlyprintk_init(void) { void __iomem *uart_base; - uart_base = ioremap_nocache(UART_UART, 0x20); + uart_base = ioremap(UART_UART, 0x20); setup_8250_early_printk_port((unsigned long)uart_base, 2, 50000); } diff --git a/arch/mips/include/asm/io.h b/arch/mips/include/asm/io.h index d9caa811a2fa..cf1f2a4a2418 100644 --- a/arch/mips/include/asm/io.h +++ b/arch/mips/include/asm/io.h @@ -227,7 +227,6 @@ static inline void __iomem *ioremap_prot(phys_addr_t offset, */ #define ioremap(offset, size) \ __ioremap_mode((offset), (size), _CACHE_UNCACHED) -#define ioremap_nocache ioremap #define ioremap_uc ioremap /* diff --git a/arch/mips/kernel/mips-cm.c b/arch/mips/kernel/mips-cm.c index e5ea3db23d6b..cdb93ed91cde 100644 --- a/arch/mips/kernel/mips-cm.c +++ b/arch/mips/kernel/mips-cm.c @@ -194,7 +194,7 @@ static void mips_cm_probe_l2sync(void) write_gcr_l2_only_sync_base(addr | CM_GCR_L2_ONLY_SYNC_BASE_SYNCEN); /* Map the region */ - mips_cm_l2sync_base = ioremap_nocache(addr, MIPS_CM_L2SYNC_SIZE); + mips_cm_l2sync_base = ioremap(addr, MIPS_CM_L2SYNC_SIZE); } int mips_cm_probe(void) @@ -215,7 +215,7 @@ int mips_cm_probe(void) if (!addr) return -ENODEV; - mips_gcr_base = ioremap_nocache(addr, MIPS_CM_GCR_SIZE); + mips_gcr_base = ioremap(addr, MIPS_CM_GCR_SIZE); if (!mips_gcr_base) return -ENXIO; diff --git a/arch/mips/kernel/mips-cpc.c b/arch/mips/kernel/mips-cpc.c index 69e3e0b556bf..8d2535123f11 100644 --- a/arch/mips/kernel/mips-cpc.c +++ b/arch/mips/kernel/mips-cpc.c @@ -78,7 +78,7 @@ int mips_cpc_probe(void) if (!addr) return -ENODEV; - mips_cpc_base = ioremap_nocache(addr, 0x8000); + mips_cpc_base = ioremap(addr, 0x8000); if (!mips_cpc_base) return -ENXIO; diff --git a/arch/mips/lantiq/falcon/sysctrl.c b/arch/mips/lantiq/falcon/sysctrl.c index 037b08f3257e..42222f849bd2 100644 --- a/arch/mips/lantiq/falcon/sysctrl.c +++ b/arch/mips/lantiq/falcon/sysctrl.c @@ -221,16 +221,16 @@ void __init ltq_soc_init(void) res_sys[2].name) < 0)) pr_err("Failed to request core resources"); - status_membase = ioremap_nocache(res_status.start, + status_membase = ioremap(res_status.start, resource_size(&res_status)); - ltq_ebu_membase = ioremap_nocache(res_ebu.start, + ltq_ebu_membase = ioremap(res_ebu.start, resource_size(&res_ebu)); if (!status_membase || !ltq_ebu_membase) panic("Failed to remap core resources"); for (i = 0; i < 3; i++) { - sysctl_membase[i] = ioremap_nocache(res_sys[i].start, + sysctl_membase[i] = ioremap(res_sys[i].start, resource_size(&res_sys[i])); if (!sysctl_membase[i]) panic("Failed to remap sysctrl resources"); diff --git a/arch/mips/lantiq/irq.c b/arch/mips/lantiq/irq.c index 115b417dfb8e..df8eed3875f6 100644 --- a/arch/mips/lantiq/irq.c +++ b/arch/mips/lantiq/irq.c @@ -349,7 +349,7 @@ int __init icu_of_init(struct device_node *node, struct device_node *parent) res.name)) pr_err("Failed to request icu%i memory\n", vpe); - ltq_icu_membase[vpe] = ioremap_nocache(res.start, + ltq_icu_membase[vpe] = ioremap(res.start, resource_size(&res)); if (!ltq_icu_membase[vpe]) @@ -402,7 +402,7 @@ int __init icu_of_init(struct device_node *node, struct device_node *parent) res.name)) pr_err("Failed to request eiu memory"); - ltq_eiu_membase = ioremap_nocache(res.start, + ltq_eiu_membase = ioremap(res.start, resource_size(&res)); if (!ltq_eiu_membase) panic("Failed to remap eiu memory"); diff --git a/arch/mips/lantiq/xway/sysctrl.c b/arch/mips/lantiq/xway/sysctrl.c index 156a95ac5c72..aa37545ebe8f 100644 --- a/arch/mips/lantiq/xway/sysctrl.c +++ b/arch/mips/lantiq/xway/sysctrl.c @@ -431,10 +431,10 @@ void __init ltq_soc_init(void) res_ebu.name)) pr_err("Failed to request core resources"); - pmu_membase = ioremap_nocache(res_pmu.start, resource_size(&res_pmu)); - ltq_cgu_membase = ioremap_nocache(res_cgu.start, + pmu_membase = ioremap(res_pmu.start, resource_size(&res_pmu)); + ltq_cgu_membase = ioremap(res_cgu.start, resource_size(&res_cgu)); - ltq_ebu_membase = ioremap_nocache(res_ebu.start, + ltq_ebu_membase = ioremap(res_ebu.start, resource_size(&res_ebu)); if (!pmu_membase || !ltq_cgu_membase || !ltq_ebu_membase) panic("Failed to remap core resources"); diff --git a/arch/mips/loongson2ef/common/reset.c b/arch/mips/loongson2ef/common/reset.c index e7c87161ce00..e49c40646995 100644 --- a/arch/mips/loongson2ef/common/reset.c +++ b/arch/mips/loongson2ef/common/reset.c @@ -17,11 +17,11 @@ static inline void loongson_reboot(void) { #ifndef CONFIG_CPU_JUMP_WORKAROUNDS - ((void (*)(void))ioremap_nocache(LOONGSON_BOOT_BASE, 4)) (); + ((void (*)(void))ioremap(LOONGSON_BOOT_BASE, 4)) (); #else void (*func)(void); - func = (void *)ioremap_nocache(LOONGSON_BOOT_BASE, 4); + func = (void *)ioremap(LOONGSON_BOOT_BASE, 4); __asm__ __volatile__( " .set noat \n" diff --git a/arch/mips/loongson32/common/prom.c b/arch/mips/loongson32/common/prom.c index 73dd25142484..fd76114fa3b0 100644 --- a/arch/mips/loongson32/common/prom.c +++ b/arch/mips/loongson32/common/prom.c @@ -26,13 +26,13 @@ void __init prom_init(void) memsize = DEFAULT_MEMSIZE; if (strstr(arcs_cmdline, "console=ttyS3")) - uart_base = ioremap_nocache(LS1X_UART3_BASE, 0x0f); + uart_base = ioremap(LS1X_UART3_BASE, 0x0f); else if (strstr(arcs_cmdline, "console=ttyS2")) - uart_base = ioremap_nocache(LS1X_UART2_BASE, 0x0f); + uart_base = ioremap(LS1X_UART2_BASE, 0x0f); else if (strstr(arcs_cmdline, "console=ttyS1")) - uart_base = ioremap_nocache(LS1X_UART1_BASE, 0x0f); + uart_base = ioremap(LS1X_UART1_BASE, 0x0f); else - uart_base = ioremap_nocache(LS1X_UART0_BASE, 0x0f); + uart_base = ioremap(LS1X_UART0_BASE, 0x0f); setup_8250_early_printk_port((unsigned long)uart_base, 0, 0); } diff --git a/arch/mips/loongson32/common/reset.c b/arch/mips/loongson32/common/reset.c index 6c36a414dde7..0c7399b303fb 100644 --- a/arch/mips/loongson32/common/reset.c +++ b/arch/mips/loongson32/common/reset.c @@ -37,7 +37,7 @@ static void ls1x_power_off(void) static int __init ls1x_reboot_setup(void) { - wdt_reg_base = ioremap_nocache(LS1X_WDT_BASE, (SZ_4 + SZ_8)); + wdt_reg_base = ioremap(LS1X_WDT_BASE, (SZ_4 + SZ_8)); if (!wdt_reg_base) panic("Failed to remap watchdog registers"); diff --git a/arch/mips/loongson32/common/time.c b/arch/mips/loongson32/common/time.c index f97662045c73..4cc73f7ac0d4 100644 --- a/arch/mips/loongson32/common/time.c +++ b/arch/mips/loongson32/common/time.c @@ -49,7 +49,7 @@ static inline void ls1x_pwmtimer_restart(void) void __init ls1x_pwmtimer_init(void) { - timer_reg_base = ioremap_nocache(LS1X_TIMER_BASE, SZ_16); + timer_reg_base = ioremap(LS1X_TIMER_BASE, SZ_16); if (!timer_reg_base) panic("Failed to remap timer registers"); diff --git a/arch/mips/loongson64/reset.c b/arch/mips/loongson64/reset.c index 88b3bd5fed25..bc7671079f0c 100644 --- a/arch/mips/loongson64/reset.c +++ b/arch/mips/loongson64/reset.c @@ -17,7 +17,7 @@ static inline void loongson_reboot(void) { - ((void (*)(void))ioremap_nocache(LOONGSON_BOOT_BASE, 4)) (); + ((void (*)(void))ioremap(LOONGSON_BOOT_BASE, 4)) (); } static void loongson_restart(char *command) diff --git a/arch/mips/mti-malta/malta-dtshim.c b/arch/mips/mti-malta/malta-dtshim.c index 98a063093b69..0ddf03df6268 100644 --- a/arch/mips/mti-malta/malta-dtshim.c +++ b/arch/mips/mti-malta/malta-dtshim.c @@ -240,7 +240,7 @@ static void __init remove_gic(void *fdt) * On systems using the RocIT system controller a GIC may be * present without a CM. Detect whether that is the case. */ - biu_base = ioremap_nocache(MSC01_BIU_REG_BASE, + biu_base = ioremap(MSC01_BIU_REG_BASE, MSC01_BIU_ADDRSPACE_SZ); sc_cfg = __raw_readl(biu_base + MSC01_SC_CFG_OFS); if (sc_cfg & MSC01_SC_CFG_GICPRES_MSK) { diff --git a/arch/mips/pci/pci-alchemy.c b/arch/mips/pci/pci-alchemy.c index 4f2411f489af..01a2af8215c8 100644 --- a/arch/mips/pci/pci-alchemy.c +++ b/arch/mips/pci/pci-alchemy.c @@ -409,7 +409,7 @@ static int alchemy_pci_probe(struct platform_device *pdev) goto out6; } - ctx->regs = ioremap_nocache(r->start, resource_size(r)); + ctx->regs = ioremap(r->start, resource_size(r)); if (!ctx->regs) { dev_err(&pdev->dev, "cannot map pci regs\n"); ret = -ENODEV; diff --git a/arch/mips/pci/pci-ar2315.c b/arch/mips/pci/pci-ar2315.c index 0fed6fc17fe4..490953f51528 100644 --- a/arch/mips/pci/pci-ar2315.c +++ b/arch/mips/pci/pci-ar2315.c @@ -441,7 +441,7 @@ static int ar2315_pci_probe(struct platform_device *pdev) apc->mem_res.flags = IORESOURCE_MEM; /* Remap PCI config space */ - apc->cfg_mem = devm_ioremap_nocache(dev, res->start, + apc->cfg_mem = devm_ioremap(dev, res->start, AR2315_PCI_CFG_SIZE); if (!apc->cfg_mem) { dev_err(dev, "failed to remap PCI config space\n"); diff --git a/arch/mips/pci/pci-bcm63xx.c b/arch/mips/pci/pci-bcm63xx.c index 151d9b5870bb..5548365605c0 100644 --- a/arch/mips/pci/pci-bcm63xx.c +++ b/arch/mips/pci/pci-bcm63xx.c @@ -221,7 +221,7 @@ static int __init bcm63xx_register_pci(void) * a spinlock for each io access, so this is currently kind of * broken on SMP. */ - pci_iospace_start = ioremap_nocache(BCM_PCI_IO_BASE_PA, 4); + pci_iospace_start = ioremap(BCM_PCI_IO_BASE_PA, 4); if (!pci_iospace_start) return -ENOMEM; diff --git a/arch/mips/pci/pci-rt2880.c b/arch/mips/pci/pci-rt2880.c index c9f4d4ba058a..e1f12e398136 100644 --- a/arch/mips/pci/pci-rt2880.c +++ b/arch/mips/pci/pci-rt2880.c @@ -218,7 +218,7 @@ static int rt288x_pci_probe(struct platform_device *pdev) { void __iomem *io_map_base; - rt2880_pci_base = ioremap_nocache(RT2880_PCI_BASE, PAGE_SIZE); + rt2880_pci_base = ioremap(RT2880_PCI_BASE, PAGE_SIZE); io_map_base = ioremap(RT2880_PCI_IO_BASE, RT2880_PCI_IO_SIZE); rt2880_pci_controller.io_map_base = (unsigned long) io_map_base; diff --git a/arch/mips/pic32/pic32mzda/early_console.c b/arch/mips/pic32/pic32mzda/early_console.c index 8c236738b5ee..25372e62783b 100644 --- a/arch/mips/pic32/pic32mzda/early_console.c +++ b/arch/mips/pic32/pic32mzda/early_console.c @@ -135,7 +135,7 @@ void __init fw_init_early_console(char port) char *arch_cmdline = pic32_getcmdline(); int baud = -1; - uart_base = ioremap_nocache(PIC32_BASE_UART, 0xc00); + uart_base = ioremap(PIC32_BASE_UART, 0xc00); baud = get_baud_from_cmdline(arch_cmdline); if (port == -1) diff --git a/arch/mips/pic32/pic32mzda/early_pin.c b/arch/mips/pic32/pic32mzda/early_pin.c index 504e6ab399b5..f2822632b017 100644 --- a/arch/mips/pic32/pic32mzda/early_pin.c +++ b/arch/mips/pic32/pic32mzda/early_pin.c @@ -122,7 +122,7 @@ static const struct void pic32_pps_input(int function, int pin) { - void __iomem *pps_base = ioremap_nocache(PPS_BASE, 0xF4); + void __iomem *pps_base = ioremap(PPS_BASE, 0xF4); int i; for (i = 0; i < ARRAY_SIZE(input_pin_reg); i++) { @@ -252,7 +252,7 @@ static const struct void pic32_pps_output(int function, int pin) { - void __iomem *pps_base = ioremap_nocache(PPS_BASE, 0x170); + void __iomem *pps_base = ioremap(PPS_BASE, 0x170); int i; for (i = 0; i < ARRAY_SIZE(output_pin_reg); i++) { diff --git a/arch/mips/pmcs-msp71xx/msp_serial.c b/arch/mips/pmcs-msp71xx/msp_serial.c index 8e6e8db8dd5f..940c684f6921 100644 --- a/arch/mips/pmcs-msp71xx/msp_serial.c +++ b/arch/mips/pmcs-msp71xx/msp_serial.c @@ -105,7 +105,7 @@ void __init msp_serial_setup(void) /* Initialize first serial port */ up.mapbase = MSP_UART0_BASE; - up.membase = ioremap_nocache(up.mapbase, MSP_UART_REG_LEN); + up.membase = ioremap(up.mapbase, MSP_UART_REG_LEN); up.irq = MSP_INT_UART0; up.uartclk = uartclk; up.regshift = 2; @@ -143,7 +143,7 @@ void __init msp_serial_setup(void) } up.mapbase = MSP_UART1_BASE; - up.membase = ioremap_nocache(up.mapbase, MSP_UART_REG_LEN); + up.membase = ioremap(up.mapbase, MSP_UART_REG_LEN); up.irq = MSP_INT_UART1; up.line = 1; up.private_data = (void*)UART1_STATUS_REG; diff --git a/arch/mips/ralink/irq.c b/arch/mips/ralink/irq.c index c945d76cfce5..220ca0cd7945 100644 --- a/arch/mips/ralink/irq.c +++ b/arch/mips/ralink/irq.c @@ -165,7 +165,7 @@ static int __init intc_of_init(struct device_node *node, res.name)) pr_err("Failed to request intc memory"); - rt_intc_membase = ioremap_nocache(res.start, + rt_intc_membase = ioremap(res.start, resource_size(&res)); if (!rt_intc_membase) panic("Failed to remap intc memory"); diff --git a/arch/mips/ralink/of.c b/arch/mips/ralink/of.c index 59b23095bfbb..90c6d4a11c5d 100644 --- a/arch/mips/ralink/of.c +++ b/arch/mips/ralink/of.c @@ -43,7 +43,7 @@ __iomem void *plat_of_remap_node(const char *node) res.name)) panic("Failed to request resources for %s", node); - return ioremap_nocache(res.start, resource_size(&res)); + return ioremap(res.start, resource_size(&res)); } void __init device_tree_init(void) diff --git a/arch/mips/rb532/devices.c b/arch/mips/rb532/devices.c index c9ecf17f8660..dd34f1b32b79 100644 --- a/arch/mips/rb532/devices.c +++ b/arch/mips/rb532/devices.c @@ -286,7 +286,7 @@ static int __init plat_setup_devices(void) nand_slot0_res[0].end = nand_slot0_res[0].start + 0x1000; /* Read and map device controller 3 */ - dev3.base = ioremap_nocache(readl(IDT434_REG_BASE + DEV3BASE), 1); + dev3.base = ioremap(readl(IDT434_REG_BASE + DEV3BASE), 1); if (!dev3.base) { printk(KERN_ERR "rb532: cannot remap device controller 3\n"); diff --git a/arch/mips/rb532/gpio.c b/arch/mips/rb532/gpio.c index fdc704abc8d4..94f02ada4082 100644 --- a/arch/mips/rb532/gpio.c +++ b/arch/mips/rb532/gpio.c @@ -192,7 +192,7 @@ int __init rb532_gpio_init(void) struct resource *r; r = rb532_gpio_reg0_res; - rb532_gpio_chip->regbase = ioremap_nocache(r->start, resource_size(r)); + rb532_gpio_chip->regbase = ioremap(r->start, resource_size(r)); if (!rb532_gpio_chip->regbase) { printk(KERN_ERR "rb532: cannot remap GPIO register 0\n"); diff --git a/arch/mips/rb532/prom.c b/arch/mips/rb532/prom.c index 26e957b21fbf..303cc3dc1749 100644 --- a/arch/mips/rb532/prom.c +++ b/arch/mips/rb532/prom.c @@ -110,7 +110,7 @@ void __init prom_init(void) phys_addr_t memsize; phys_addr_t ddrbase; - ddr = ioremap_nocache(ddr_reg[0].start, + ddr = ioremap(ddr_reg[0].start, ddr_reg[0].end - ddr_reg[0].start); if (!ddr) { diff --git a/arch/mips/rb532/setup.c b/arch/mips/rb532/setup.c index 1aa4df1385cb..51af9d374d66 100644 --- a/arch/mips/rb532/setup.c +++ b/arch/mips/rb532/setup.c @@ -49,7 +49,7 @@ void __init plat_mem_setup(void) set_io_port_base(KSEG1); - pci_reg = ioremap_nocache(pci0_res[0].start, + pci_reg = ioremap(pci0_res[0].start, pci0_res[0].end - pci0_res[0].start); if (!pci_reg) { printk(KERN_ERR "Could not remap PCI registers\n"); diff --git a/arch/mips/sni/rm200.c b/arch/mips/sni/rm200.c index 160b88000b4b..f6fa9afcbfd3 100644 --- a/arch/mips/sni/rm200.c +++ b/arch/mips/sni/rm200.c @@ -399,10 +399,10 @@ void __init sni_rm200_i8259_irqs(void) { int i; - rm200_pic_master = ioremap_nocache(0x16000020, 4); + rm200_pic_master = ioremap(0x16000020, 4); if (!rm200_pic_master) return; - rm200_pic_slave = ioremap_nocache(0x160000a0, 4); + rm200_pic_slave = ioremap(0x160000a0, 4); if (!rm200_pic_slave) { iounmap(rm200_pic_master); return; diff --git a/arch/parisc/include/asm/io.h b/arch/parisc/include/asm/io.h index 46212b52c23e..cab8f64ca4a2 100644 --- a/arch/parisc/include/asm/io.h +++ b/arch/parisc/include/asm/io.h @@ -128,9 +128,8 @@ static inline void gsc_writeq(unsigned long long val, unsigned long addr) * The standard PCI ioremap interfaces */ void __iomem *ioremap(unsigned long offset, unsigned long size); -#define ioremap_nocache(off, sz) ioremap((off), (sz)) -#define ioremap_wc ioremap_nocache -#define ioremap_uc ioremap_nocache +#define ioremap_wc ioremap +#define ioremap_uc ioremap extern void iounmap(const volatile void __iomem *addr); diff --git a/arch/parisc/kernel/perf.c b/arch/parisc/kernel/perf.c index 676683641d00..e1a8fee3ad49 100644 --- a/arch/parisc/kernel/perf.c +++ b/arch/parisc/kernel/perf.c @@ -792,7 +792,7 @@ static int perf_write_image(uint64_t *memaddr) return -1; } - runway = ioremap_nocache(cpu_device->hpa.start, 4096); + runway = ioremap(cpu_device->hpa.start, 4096); if (!runway) { pr_err("perf_write_image: ioremap failed!\n"); return -ENOMEM; diff --git a/arch/powerpc/include/asm/io.h b/arch/powerpc/include/asm/io.h index a63ec938636d..635969b5b58e 100644 --- a/arch/powerpc/include/asm/io.h +++ b/arch/powerpc/include/asm/io.h @@ -691,8 +691,6 @@ static inline void iosync(void) * * ioremap_prot allows to specify the page flags as an argument and can * also be hooked by the platform via ppc_md. * - * * ioremap_nocache is identical to ioremap - * * * ioremap_wc enables write combining * * * ioremap_wt enables write through @@ -715,7 +713,6 @@ extern void __iomem *ioremap_prot(phys_addr_t address, unsigned long size, extern void __iomem *ioremap_wc(phys_addr_t address, unsigned long size); void __iomem *ioremap_wt(phys_addr_t address, unsigned long size); void __iomem *ioremap_coherent(phys_addr_t address, unsigned long size); -#define ioremap_nocache(addr, size) ioremap((addr), (size)) #define ioremap_uc(addr, size) ioremap((addr), (size)) #define ioremap_cache(addr, size) \ ioremap_prot((addr), (size), pgprot_val(PAGE_KERNEL)) diff --git a/arch/sh/boards/board-sh7785lcr.c b/arch/sh/boards/board-sh7785lcr.c index d964c4d6b139..77dad1e511b4 100644 --- a/arch/sh/boards/board-sh7785lcr.c +++ b/arch/sh/boards/board-sh7785lcr.c @@ -341,7 +341,7 @@ static void __init sh7785lcr_setup(char **cmdline_p) pm_power_off = sh7785lcr_power_off; /* sm501 DRAM configuration */ - sm501_reg = ioremap_nocache(SM107_REG_ADDR, SM501_DRAM_CONTROL); + sm501_reg = ioremap(SM107_REG_ADDR, SM501_DRAM_CONTROL); if (!sm501_reg) { printk(KERN_ERR "%s: ioremap error.\n", __func__); return; diff --git a/arch/sh/boards/mach-cayman/irq.c b/arch/sh/boards/mach-cayman/irq.c index 9108789fafef..3b6ea2d99013 100644 --- a/arch/sh/boards/mach-cayman/irq.c +++ b/arch/sh/boards/mach-cayman/irq.c @@ -137,7 +137,7 @@ void init_cayman_irq(void) { int i; - epld_virt = (unsigned long)ioremap_nocache(EPLD_BASE, 1024); + epld_virt = (unsigned long)ioremap(EPLD_BASE, 1024); if (!epld_virt) { printk(KERN_ERR "Cayman IRQ: Unable to remap EPLD\n"); return; diff --git a/arch/sh/boards/mach-cayman/setup.c b/arch/sh/boards/mach-cayman/setup.c index 4cec14700adc..8ef76e288da0 100644 --- a/arch/sh/boards/mach-cayman/setup.c +++ b/arch/sh/boards/mach-cayman/setup.c @@ -99,7 +99,7 @@ static int __init smsc_superio_setup(void) { unsigned char devid, devrev; - smsc_superio_virt = (unsigned long)ioremap_nocache(SMSC_SUPERIO_BASE, 1024); + smsc_superio_virt = (unsigned long)ioremap(SMSC_SUPERIO_BASE, 1024); if (!smsc_superio_virt) { panic("Unable to remap SMSC SuperIO\n"); } diff --git a/arch/sh/boards/mach-sdk7786/fpga.c b/arch/sh/boards/mach-sdk7786/fpga.c index 895576ff8376..a37e1e88c6b1 100644 --- a/arch/sh/boards/mach-sdk7786/fpga.c +++ b/arch/sh/boards/mach-sdk7786/fpga.c @@ -32,7 +32,7 @@ static void __iomem *sdk7786_fpga_probe(void) * is reserved. */ for (area = PA_AREA0; area < PA_AREA7; area += SZ_64M) { - base = ioremap_nocache(area + FPGA_REGS_OFFSET, FPGA_REGS_SIZE); + base = ioremap(area + FPGA_REGS_OFFSET, FPGA_REGS_SIZE); if (!base) { /* Failed to remap this area, move along. */ continue; diff --git a/arch/sh/drivers/heartbeat.c b/arch/sh/drivers/heartbeat.c index cf2fcccca812..24391b444b28 100644 --- a/arch/sh/drivers/heartbeat.c +++ b/arch/sh/drivers/heartbeat.c @@ -96,7 +96,7 @@ static int heartbeat_drv_probe(struct platform_device *pdev) return -ENOMEM; } - hd->base = ioremap_nocache(res->start, resource_size(res)); + hd->base = ioremap(res->start, resource_size(res)); if (unlikely(!hd->base)) { dev_err(&pdev->dev, "ioremap failed\n"); diff --git a/arch/sh/drivers/pci/pci-sh5.c b/arch/sh/drivers/pci/pci-sh5.c index 49303fab187b..03225d27770b 100644 --- a/arch/sh/drivers/pci/pci-sh5.c +++ b/arch/sh/drivers/pci/pci-sh5.c @@ -115,12 +115,12 @@ static int __init sh5pci_init(void) return -EINVAL; } - pcicr_virt = (unsigned long)ioremap_nocache(SH5PCI_ICR_BASE, 1024); + pcicr_virt = (unsigned long)ioremap(SH5PCI_ICR_BASE, 1024); if (!pcicr_virt) { panic("Unable to remap PCICR\n"); } - PCI_IO_AREA = (unsigned long)ioremap_nocache(SH5PCI_IO_BASE, 0x10000); + PCI_IO_AREA = (unsigned long)ioremap(SH5PCI_IO_BASE, 0x10000); if (!PCI_IO_AREA) { panic("Unable to remap PCIIO\n"); } diff --git a/arch/sh/include/asm/io.h b/arch/sh/include/asm/io.h index 1495489225ac..39c9ead489e5 100644 --- a/arch/sh/include/asm/io.h +++ b/arch/sh/include/asm/io.h @@ -367,7 +367,6 @@ static inline void ioremap_fixed_init(void) { } static inline int iounmap_fixed(void __iomem *addr) { return -EINVAL; } #endif -#define ioremap_nocache ioremap #define ioremap_uc ioremap /* diff --git a/arch/sh/kernel/cpu/irq/intc-sh5.c b/arch/sh/kernel/cpu/irq/intc-sh5.c index 744f903b4df3..1b3050facda8 100644 --- a/arch/sh/kernel/cpu/irq/intc-sh5.c +++ b/arch/sh/kernel/cpu/irq/intc-sh5.c @@ -124,7 +124,7 @@ void __init plat_irq_setup(void) unsigned long reg; int i; - intc_virt = (unsigned long)ioremap_nocache(INTC_BASE, 1024); + intc_virt = (unsigned long)ioremap(INTC_BASE, 1024); if (!intc_virt) { panic("Unable to remap INTC\n"); } diff --git a/arch/sh/kernel/cpu/sh2/smp-j2.c b/arch/sh/kernel/cpu/sh2/smp-j2.c index ae44dc24c455..d0d5d81455ae 100644 --- a/arch/sh/kernel/cpu/sh2/smp-j2.c +++ b/arch/sh/kernel/cpu/sh2/smp-j2.c @@ -88,8 +88,8 @@ static void j2_start_cpu(unsigned int cpu, unsigned long entry_point) if (!np) return; if (of_property_read_u32_array(np, "cpu-release-addr", regs, 2)) return; - release = ioremap_nocache(regs[0], sizeof(u32)); - initpc = ioremap_nocache(regs[1], sizeof(u32)); + release = ioremap(regs[0], sizeof(u32)); + initpc = ioremap(regs[1], sizeof(u32)); __raw_writel(entry_point, initpc); __raw_writel(1, release); diff --git a/arch/sh/kernel/cpu/sh5/clock-sh5.c b/arch/sh/kernel/cpu/sh5/clock-sh5.c index 43763c26a752..dee6be2c2344 100644 --- a/arch/sh/kernel/cpu/sh5/clock-sh5.c +++ b/arch/sh/kernel/cpu/sh5/clock-sh5.c @@ -68,7 +68,7 @@ static struct sh_clk_ops *sh5_clk_ops[] = { void __init arch_init_clk_ops(struct sh_clk_ops **ops, int idx) { - cprc_base = (unsigned long)ioremap_nocache(CPRC_BASE, 1024); + cprc_base = (unsigned long)ioremap(CPRC_BASE, 1024); BUG_ON(!cprc_base); if (idx < ARRAY_SIZE(sh5_clk_ops)) diff --git a/arch/sh/kernel/dma-coherent.c b/arch/sh/kernel/dma-coherent.c index eeb25a4fa55f..d4811691b93c 100644 --- a/arch/sh/kernel/dma-coherent.c +++ b/arch/sh/kernel/dma-coherent.c @@ -28,7 +28,7 @@ void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, arch_sync_dma_for_device(virt_to_phys(ret), size, DMA_BIDIRECTIONAL); - ret_nocache = (void __force *)ioremap_nocache(virt_to_phys(ret), size); + ret_nocache = (void __force *)ioremap(virt_to_phys(ret), size); if (!ret_nocache) { free_pages((unsigned long)ret, order); return NULL; diff --git a/arch/sparc/include/asm/io_64.h b/arch/sparc/include/asm/io_64.h index f4afa301954a..9bb27e5c22f1 100644 --- a/arch/sparc/include/asm/io_64.h +++ b/arch/sparc/include/asm/io_64.h @@ -406,7 +406,6 @@ static inline void __iomem *ioremap(unsigned long offset, unsigned long size) return (void __iomem *)offset; } -#define ioremap_nocache(X,Y) ioremap((X),(Y)) #define ioremap_uc(X,Y) ioremap((X),(Y)) #define ioremap_wc(X,Y) ioremap((X),(Y)) #define ioremap_wt(X,Y) ioremap((X),(Y)) diff --git a/arch/unicore32/include/asm/io.h b/arch/unicore32/include/asm/io.h index 4b460e01acfa..3ca74e1cde7d 100644 --- a/arch/unicore32/include/asm/io.h +++ b/arch/unicore32/include/asm/io.h @@ -31,7 +31,6 @@ extern void __uc32_iounmap(volatile void __iomem *addr); * */ #define ioremap(cookie, size) __uc32_ioremap(cookie, size) -#define ioremap_nocache(cookie, size) __uc32_ioremap(cookie, size) #define iounmap(cookie) __uc32_iounmap(cookie) #define readb_relaxed readb diff --git a/arch/x86/kernel/apb_timer.c b/arch/x86/kernel/apb_timer.c index 5da106f84e84..fe698f96617c 100644 --- a/arch/x86/kernel/apb_timer.c +++ b/arch/x86/kernel/apb_timer.c @@ -95,7 +95,7 @@ static inline void apbt_set_mapping(void) printk(KERN_WARNING "No timer base from SFI, use default\n"); apbt_address = APBT_DEFAULT_BASE; } - apbt_virt_address = ioremap_nocache(apbt_address, APBT_MMAP_SIZE); + apbt_virt_address = ioremap(apbt_address, APBT_MMAP_SIZE); if (!apbt_virt_address) { pr_debug("Failed mapping APBT phy address at %lu\n",\ (unsigned long)apbt_address); diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c index c6f791bc481e..7a50f0b62a70 100644 --- a/arch/x86/kernel/hpet.c +++ b/arch/x86/kernel/hpet.c @@ -84,7 +84,7 @@ static inline void hpet_writel(unsigned int d, unsigned int a) static inline void hpet_set_mapping(void) { - hpet_virt_address = ioremap_nocache(hpet_address, HPET_MMAP_SIZE); + hpet_virt_address = ioremap(hpet_address, HPET_MMAP_SIZE); } static inline void hpet_clear_mapping(void) diff --git a/arch/x86/kernel/quirks.c b/arch/x86/kernel/quirks.c index 1daf8f2aa21f..896d74cb5081 100644 --- a/arch/x86/kernel/quirks.c +++ b/arch/x86/kernel/quirks.c @@ -110,7 +110,7 @@ static void ich_force_enable_hpet(struct pci_dev *dev) } /* use bits 31:14, 16 kB aligned */ - rcba_base = ioremap_nocache(rcba, 0x4000); + rcba_base = ioremap(rcba, 0x4000); if (rcba_base == NULL) { dev_printk(KERN_DEBUG, &dev->dev, "ioremap failed; " "cannot force enable HPET\n"); diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c index 4c61f0713832..b89f6ac6a0c0 100644 --- a/arch/x86/kernel/tboot.c +++ b/arch/x86/kernel/tboot.c @@ -354,7 +354,7 @@ static ssize_t tboot_log_read(struct file *file, char __user *user_buf, size_t c void *kbuf; int ret = -EFAULT; - log_base = ioremap_nocache(TBOOT_SERIAL_LOG_ADDR, TBOOT_SERIAL_LOG_SIZE); + log_base = ioremap(TBOOT_SERIAL_LOG_ADDR, TBOOT_SERIAL_LOG_SIZE); if (!log_base) return ret; diff --git a/arch/x86/mm/testmmiotrace.c b/arch/x86/mm/testmmiotrace.c index 92153d054d6c..bda73cb7a044 100644 --- a/arch/x86/mm/testmmiotrace.c +++ b/arch/x86/mm/testmmiotrace.c @@ -79,7 +79,7 @@ static void do_read_far_test(void __iomem *p) static void do_test(unsigned long size) { - void __iomem *p = ioremap_nocache(mmio_address, size); + void __iomem *p = ioremap(mmio_address, size); if (!p) { pr_err("could not ioremap, aborting.\n"); return; @@ -104,7 +104,7 @@ static void do_test_bulk_ioremapping(void) int i; for (i = 0; i < 10; ++i) { - p = ioremap_nocache(mmio_address, PAGE_SIZE); + p = ioremap(mmio_address, PAGE_SIZE); if (p) iounmap(p); } diff --git a/arch/x86/pci/mmconfig_64.c b/arch/x86/pci/mmconfig_64.c index 887d181b769b..0c7b6e66c644 100644 --- a/arch/x86/pci/mmconfig_64.c +++ b/arch/x86/pci/mmconfig_64.c @@ -105,7 +105,7 @@ static void __iomem *mcfg_ioremap(struct pci_mmcfg_region *cfg) start = cfg->address + PCI_MMCFG_BUS_OFFSET(cfg->start_bus); num_buses = cfg->end_bus - cfg->start_bus + 1; size = PCI_MMCFG_BUS_OFFSET(num_buses); - addr = ioremap_nocache(start, size); + addr = ioremap(start, size); if (addr) addr -= PCI_MMCFG_BUS_OFFSET(cfg->start_bus); return addr; diff --git a/drivers/acpi/acpi_lpit.c b/drivers/acpi/acpi_lpit.c index 433376e819bb..953437a216f6 100644 --- a/drivers/acpi/acpi_lpit.c +++ b/drivers/acpi/acpi_lpit.c @@ -104,7 +104,7 @@ static void lpit_update_residency(struct lpit_residency_info *info, info->gaddr = lpit_native->residency_counter; if (info->gaddr.space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) { - info->iomem_addr = ioremap_nocache(info->gaddr.address, + info->iomem_addr = ioremap(info->gaddr.address, info->gaddr.bit_width / 8); if (!info->iomem_addr) return; diff --git a/drivers/ata/pata_arasan_cf.c b/drivers/ata/pata_arasan_cf.c index 135173c8d138..391dff0f25a2 100644 --- a/drivers/ata/pata_arasan_cf.c +++ b/drivers/ata/pata_arasan_cf.c @@ -824,7 +824,7 @@ static int arasan_cf_probe(struct platform_device *pdev) quirk |= CF_BROKEN_MWDMA | CF_BROKEN_UDMA; acdev->pbase = res->start; - acdev->vbase = devm_ioremap_nocache(&pdev->dev, res->start, + acdev->vbase = devm_ioremap(&pdev->dev, res->start, resource_size(res)); if (!acdev->vbase) { dev_warn(&pdev->dev, "ioremap fail\n"); diff --git a/drivers/ata/pata_octeon_cf.c b/drivers/ata/pata_octeon_cf.c index d3d851b014a3..bd87476ab481 100644 --- a/drivers/ata/pata_octeon_cf.c +++ b/drivers/ata/pata_octeon_cf.c @@ -891,7 +891,7 @@ static int octeon_cf_probe(struct platform_device *pdev) of_node_put(dma_node); return -EINVAL; } - cf_port->dma_base = (u64)devm_ioremap_nocache(&pdev->dev, res_dma->start, + cf_port->dma_base = (u64)devm_ioremap(&pdev->dev, res_dma->start, resource_size(res_dma)); if (!cf_port->dma_base) { of_node_put(dma_node); @@ -909,7 +909,7 @@ static int octeon_cf_probe(struct platform_device *pdev) if (!res_cs1) return -EINVAL; - cs1 = devm_ioremap_nocache(&pdev->dev, res_cs1->start, + cs1 = devm_ioremap(&pdev->dev, res_cs1->start, resource_size(res_cs1)); if (!cs1) return rv; @@ -925,7 +925,7 @@ static int octeon_cf_probe(struct platform_device *pdev) if (!res_cs0) return -EINVAL; - cs0 = devm_ioremap_nocache(&pdev->dev, res_cs0->start, + cs0 = devm_ioremap(&pdev->dev, res_cs0->start, resource_size(res_cs0)); if (!cs0) return rv; diff --git a/drivers/ata/pata_rb532_cf.c b/drivers/ata/pata_rb532_cf.c index deae466395de..479c4b29b856 100644 --- a/drivers/ata/pata_rb532_cf.c +++ b/drivers/ata/pata_rb532_cf.c @@ -140,7 +140,7 @@ static int rb532_pata_driver_probe(struct platform_device *pdev) info->gpio_line = gpiod; info->irq = irq; - info->iobase = devm_ioremap_nocache(&pdev->dev, res->start, + info->iobase = devm_ioremap(&pdev->dev, res->start, resource_size(res)); if (!info->iobase) return -ENOMEM; diff --git a/drivers/atm/eni.c b/drivers/atm/eni.c index b23d1e4bad33..8fad56f185ba 100644 --- a/drivers/atm/eni.c +++ b/drivers/atm/eni.c @@ -31,12 +31,6 @@ #include "suni.h" #include "eni.h" -#if !defined(__i386__) && !defined(__x86_64__) -#ifndef ioremap_nocache -#define ioremap_nocache(X,Y) ioremap(X,Y) -#endif -#endif - /* * TODO: * @@ -1725,7 +1719,7 @@ static int eni_do_init(struct atm_dev *dev) } printk(KERN_NOTICE DEV_LABEL "(itf %d): rev.%d,base=0x%lx,irq=%d,", dev->number,pci_dev->revision,real_base,eni_dev->irq); - if (!(base = ioremap_nocache(real_base,MAP_MAX_SIZE))) { + if (!(base = ioremap(real_base,MAP_MAX_SIZE))) { printk("\n"); printk(KERN_ERR DEV_LABEL "(itf %d): can't set up page " "mapping\n",dev->number); diff --git a/drivers/bcma/driver_chipcommon_b.c b/drivers/bcma/driver_chipcommon_b.c index 57f10b58b47c..c153c96a6145 100644 --- a/drivers/bcma/driver_chipcommon_b.c +++ b/drivers/bcma/driver_chipcommon_b.c @@ -48,7 +48,7 @@ int bcma_core_chipcommon_b_init(struct bcma_drv_cc_b *ccb) return 0; ccb->setup_done = 1; - ccb->mii = ioremap_nocache(ccb->core->addr_s[1], BCMA_CORE_SIZE); + ccb->mii = ioremap(ccb->core->addr_s[1], BCMA_CORE_SIZE); if (!ccb->mii) return -ENOMEM; diff --git a/drivers/bcma/driver_pci_host.c b/drivers/bcma/driver_pci_host.c index c42cec7c7ecc..88a93c266c19 100644 --- a/drivers/bcma/driver_pci_host.c +++ b/drivers/bcma/driver_pci_host.c @@ -115,7 +115,7 @@ static int bcma_extpci_read_config(struct bcma_drv_pci *pc, unsigned int dev, if (unlikely(!addr)) goto out; err = -ENOMEM; - mmio = ioremap_nocache(addr, sizeof(val)); + mmio = ioremap(addr, sizeof(val)); if (!mmio) goto out; @@ -180,7 +180,7 @@ static int bcma_extpci_write_config(struct bcma_drv_pci *pc, unsigned int dev, if (unlikely(!addr)) goto out; err = -ENOMEM; - mmio = ioremap_nocache(addr, sizeof(val)); + mmio = ioremap(addr, sizeof(val)); if (!mmio) goto out; @@ -515,7 +515,7 @@ void bcma_core_pci_hostmode_init(struct bcma_drv_pci *pc) /* Ok, ready to run, register it to the system. * The following needs change, if we want to port hostmode * to non-MIPS platform. */ - io_map_base = (unsigned long)ioremap_nocache(pc_host->mem_resource.start, + io_map_base = (unsigned long)ioremap(pc_host->mem_resource.start, resource_size(&pc_host->mem_resource)); pc_host->pci_controller.io_map_base = io_map_base; set_io_port_base(pc_host->pci_controller.io_map_base); diff --git a/drivers/bcma/host_soc.c b/drivers/bcma/host_soc.c index c8073b509a2b..90d5bdc12e03 100644 --- a/drivers/bcma/host_soc.c +++ b/drivers/bcma/host_soc.c @@ -172,7 +172,7 @@ int __init bcma_host_soc_register(struct bcma_soc *soc) /* iomap only first core. We have to read some register on this core * to scan the bus. */ - bus->mmio = ioremap_nocache(BCMA_ADDR_BASE, BCMA_CORE_SIZE * 1); + bus->mmio = ioremap(BCMA_ADDR_BASE, BCMA_CORE_SIZE * 1); if (!bus->mmio) return -ENOMEM; diff --git a/drivers/bcma/scan.c b/drivers/bcma/scan.c index 4a2d1b235fb5..fd546c51b076 100644 --- a/drivers/bcma/scan.c +++ b/drivers/bcma/scan.c @@ -425,11 +425,11 @@ static int bcma_get_next_core(struct bcma_bus *bus, u32 __iomem **eromptr, } } if (bus->hosttype == BCMA_HOSTTYPE_SOC) { - core->io_addr = ioremap_nocache(core->addr, BCMA_CORE_SIZE); + core->io_addr = ioremap(core->addr, BCMA_CORE_SIZE); if (!core->io_addr) return -ENOMEM; if (core->wrap) { - core->io_wrap = ioremap_nocache(core->wrap, + core->io_wrap = ioremap(core->wrap, BCMA_CORE_SIZE); if (!core->io_wrap) { iounmap(core->io_addr); @@ -472,7 +472,7 @@ int bcma_bus_scan(struct bcma_bus *bus) erombase = bcma_scan_read32(bus, 0, BCMA_CC_EROM); if (bus->hosttype == BCMA_HOSTTYPE_SOC) { - eromptr = ioremap_nocache(erombase, BCMA_CORE_SIZE); + eromptr = ioremap(erombase, BCMA_CORE_SIZE); if (!eromptr) return -ENOMEM; } else { diff --git a/drivers/block/umem.c b/drivers/block/umem.c index 1f3f9e0f02a8..4eaf97d7a170 100644 --- a/drivers/block/umem.c +++ b/drivers/block/umem.c @@ -827,7 +827,7 @@ static int mm_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) goto failed_req_csr; } - card->csr_remap = ioremap_nocache(csr_base, csr_len); + card->csr_remap = ioremap(csr_base, csr_len); if (!card->csr_remap) { dev_printk(KERN_ERR, &card->dev->dev, "Unable to remap memory region\n"); diff --git a/drivers/bus/fsl-mc/mc-io.c b/drivers/bus/fsl-mc/mc-io.c index d9629fc13a15..6ae48ad80409 100644 --- a/drivers/bus/fsl-mc/mc-io.c +++ b/drivers/bus/fsl-mc/mc-io.c @@ -97,12 +97,12 @@ int __must_check fsl_create_mc_io(struct device *dev, return -EBUSY; } - mc_portal_virt_addr = devm_ioremap_nocache(dev, + mc_portal_virt_addr = devm_ioremap(dev, mc_portal_phys_addr, mc_portal_size); if (!mc_portal_virt_addr) { dev_err(dev, - "devm_ioremap_nocache failed for MC portal %pa\n", + "devm_ioremap failed for MC portal %pa\n", &mc_portal_phys_addr); return -ENXIO; } diff --git a/drivers/char/agp/generic.c b/drivers/char/agp/generic.c index ab154a75acf0..9e84239f88d4 100644 --- a/drivers/char/agp/generic.c +++ b/drivers/char/agp/generic.c @@ -941,7 +941,7 @@ int agp_generic_create_gatt_table(struct agp_bridge_data *bridge) bridge->gatt_table = (u32 __iomem *)table; #else - bridge->gatt_table = ioremap_nocache(virt_to_phys(table), + bridge->gatt_table = ioremap(virt_to_phys(table), (PAGE_SIZE * (1 << page_order))); bridge->driver->cache_flush(); #endif diff --git a/drivers/char/agp/intel-gtt.c b/drivers/char/agp/intel-gtt.c index c6271ce250b3..66a62d17a3f5 100644 --- a/drivers/char/agp/intel-gtt.c +++ b/drivers/char/agp/intel-gtt.c @@ -1087,7 +1087,7 @@ static void intel_i9xx_setup_flush(void) } if (intel_private.ifp_resource.start) - intel_private.i9xx_flush_page = ioremap_nocache(intel_private.ifp_resource.start, PAGE_SIZE); + intel_private.i9xx_flush_page = ioremap(intel_private.ifp_resource.start, PAGE_SIZE); if (!intel_private.i9xx_flush_page) dev_err(&intel_private.pcidev->dev, "can't ioremap flush page - no chipset flushing\n"); diff --git a/drivers/char/applicom.c b/drivers/char/applicom.c index eb108b3c619a..51121a4b82c7 100644 --- a/drivers/char/applicom.c +++ b/drivers/char/applicom.c @@ -204,7 +204,7 @@ static int __init applicom_init(void) if (pci_enable_device(dev)) return -EIO; - RamIO = ioremap_nocache(pci_resource_start(dev, 0), LEN_RAM_IO); + RamIO = ioremap(pci_resource_start(dev, 0), LEN_RAM_IO); if (!RamIO) { printk(KERN_INFO "ac.o: Failed to ioremap PCI memory " @@ -259,7 +259,7 @@ static int __init applicom_init(void) /* Now try the specified ISA cards */ for (i = 0; i < MAX_ISA_BOARD; i++) { - RamIO = ioremap_nocache(mem + (LEN_RAM_IO * i), LEN_RAM_IO); + RamIO = ioremap(mem + (LEN_RAM_IO * i), LEN_RAM_IO); if (!RamIO) { printk(KERN_INFO "ac.o: Failed to ioremap the ISA card's memory space (slot #%d)\n", i + 1); diff --git a/drivers/char/hw_random/intel-rng.c b/drivers/char/hw_random/intel-rng.c index 290c880266bf..9f205bd1acc0 100644 --- a/drivers/char/hw_random/intel-rng.c +++ b/drivers/char/hw_random/intel-rng.c @@ -317,7 +317,7 @@ PFX "RNG, try using the 'no_fwh_detect' option.\n"; return -EBUSY; } - intel_rng_hw->mem = ioremap_nocache(INTEL_FWH_ADDR, INTEL_FWH_ADDR_LEN); + intel_rng_hw->mem = ioremap(INTEL_FWH_ADDR, INTEL_FWH_ADDR_LEN); if (intel_rng_hw->mem == NULL) return -EBUSY; diff --git a/drivers/char/hw_random/octeon-rng.c b/drivers/char/hw_random/octeon-rng.c index 8c78aa090492..7be8067ac4e8 100644 --- a/drivers/char/hw_random/octeon-rng.c +++ b/drivers/char/hw_random/octeon-rng.c @@ -81,13 +81,13 @@ static int octeon_rng_probe(struct platform_device *pdev) return -ENOENT; - rng->control_status = devm_ioremap_nocache(&pdev->dev, + rng->control_status = devm_ioremap(&pdev->dev, res_ports->start, sizeof(u64)); if (!rng->control_status) return -ENOENT; - rng->result = devm_ioremap_nocache(&pdev->dev, + rng->result = devm_ioremap(&pdev->dev, res_result->start, sizeof(u64)); if (!rng->result) diff --git a/drivers/clk/renesas/clk-rz.c b/drivers/clk/renesas/clk-rz.c index fbc34beafc78..7b703f14e20b 100644 --- a/drivers/clk/renesas/clk-rz.c +++ b/drivers/clk/renesas/clk-rz.c @@ -37,8 +37,8 @@ static u16 __init rz_cpg_read_mode_pins(void) void __iomem *ppr0, *pibc0; u16 modes; - ppr0 = ioremap_nocache(PPR0, 2); - pibc0 = ioremap_nocache(PIBC0, 2); + ppr0 = ioremap(PPR0, 2); + pibc0 = ioremap(PIBC0, 2); BUG_ON(!ppr0 || !pibc0); iowrite16(4, pibc0); /* enable input buffer */ modes = ioread16(ppr0); diff --git a/drivers/clocksource/sh_cmt.c b/drivers/clocksource/sh_cmt.c index 9cde50cb3220..12ac75f7571f 100644 --- a/drivers/clocksource/sh_cmt.c +++ b/drivers/clocksource/sh_cmt.c @@ -905,7 +905,7 @@ static int sh_cmt_map_memory(struct sh_cmt_device *cmt) return -ENXIO; } - cmt->mapbase = ioremap_nocache(mem->start, resource_size(mem)); + cmt->mapbase = ioremap(mem->start, resource_size(mem)); if (cmt->mapbase == NULL) { dev_err(&cmt->pdev->dev, "failed to remap I/O memory\n"); return -ENXIO; diff --git a/drivers/clocksource/sh_mtu2.c b/drivers/clocksource/sh_mtu2.c index 64526e50d471..bfccb31e94ad 100644 --- a/drivers/clocksource/sh_mtu2.c +++ b/drivers/clocksource/sh_mtu2.c @@ -377,7 +377,7 @@ static int sh_mtu2_map_memory(struct sh_mtu2_device *mtu) return -ENXIO; } - mtu->mapbase = ioremap_nocache(res->start, resource_size(res)); + mtu->mapbase = ioremap(res->start, resource_size(res)); if (mtu->mapbase == NULL) return -ENXIO; diff --git a/drivers/clocksource/sh_tmu.c b/drivers/clocksource/sh_tmu.c index d49690d15536..d41df9ba3725 100644 --- a/drivers/clocksource/sh_tmu.c +++ b/drivers/clocksource/sh_tmu.c @@ -486,7 +486,7 @@ static int sh_tmu_map_memory(struct sh_tmu_device *tmu) return -ENXIO; } - tmu->mapbase = ioremap_nocache(res->start, resource_size(res)); + tmu->mapbase = ioremap(res->start, resource_size(res)); if (tmu->mapbase == NULL) return -ENXIO; diff --git a/drivers/cpufreq/pcc-cpufreq.c b/drivers/cpufreq/pcc-cpufreq.c index fdc767fdbe6a..89d4fa8b65e9 100644 --- a/drivers/cpufreq/pcc-cpufreq.c +++ b/drivers/cpufreq/pcc-cpufreq.c @@ -445,7 +445,7 @@ static int __init pcc_cpufreq_probe(void) goto out_free; } - pcch_virt_addr = ioremap_nocache(mem_resource->minimum, + pcch_virt_addr = ioremap(mem_resource->minimum, mem_resource->address_length); if (pcch_virt_addr == NULL) { pr_debug("probe: could not map shared mem region\n"); diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index 4e7323884ae3..354836468c5d 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -2507,7 +2507,7 @@ static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) addr = pci_resource_start(pdev, i); size = pci_resource_len(pdev, i); - dev->bar[i] = ioremap_nocache(addr, size); + dev->bar[i] = ioremap(addr, size); if (!dev->bar[i]) { err = -ENOMEM; goto err_out_unmap_bars; diff --git a/drivers/dma/altera-msgdma.c b/drivers/dma/altera-msgdma.c index 832aefbe7af9..539e785039ca 100644 --- a/drivers/dma/altera-msgdma.c +++ b/drivers/dma/altera-msgdma.c @@ -772,10 +772,10 @@ static int request_and_map(struct platform_device *pdev, const char *name, return -EBUSY; } - *ptr = devm_ioremap_nocache(device, region->start, + *ptr = devm_ioremap(device, region->start, resource_size(region)); if (*ptr == NULL) { - dev_err(device, "ioremap_nocache of %s failed!", name); + dev_err(device, "ioremap of %s failed!", name); return -ENOMEM; } diff --git a/drivers/edac/i3000_edac.c b/drivers/edac/i3000_edac.c index f564a4a8a4ae..5c1eea96230c 100644 --- a/drivers/edac/i3000_edac.c +++ b/drivers/edac/i3000_edac.c @@ -324,7 +324,7 @@ static int i3000_probe1(struct pci_dev *pdev, int dev_idx) pci_read_config_dword(pdev, I3000_MCHBAR, (u32 *) & mchbar); mchbar &= I3000_MCHBAR_MASK; - window = ioremap_nocache(mchbar, I3000_MMR_WINDOW_SIZE); + window = ioremap(mchbar, I3000_MMR_WINDOW_SIZE); if (!window) { printk(KERN_ERR "i3000: cannot map mmio space at 0x%lx\n", mchbar); diff --git a/drivers/edac/i3200_edac.c b/drivers/edac/i3200_edac.c index 432b375a4075..a8988db6d423 100644 --- a/drivers/edac/i3200_edac.c +++ b/drivers/edac/i3200_edac.c @@ -280,7 +280,7 @@ static void __iomem *i3200_map_mchbar(struct pci_dev *pdev) return NULL; } - window = ioremap_nocache(u.mchbar, I3200_MMR_WINDOW_SIZE); + window = ioremap(u.mchbar, I3200_MMR_WINDOW_SIZE); if (!window) printk(KERN_ERR "i3200: cannot map mmio space at 0x%llx\n", (unsigned long long)u.mchbar); diff --git a/drivers/edac/i82975x_edac.c b/drivers/edac/i82975x_edac.c index 7c6a2d4d2360..6be99e0d850d 100644 --- a/drivers/edac/i82975x_edac.c +++ b/drivers/edac/i82975x_edac.c @@ -485,7 +485,7 @@ static int i82975x_probe1(struct pci_dev *pdev, int dev_idx) goto fail0; } mchbar &= 0xffffc000; /* bits 31:14 used for 16K window */ - mch_window = ioremap_nocache(mchbar, 0x1000); + mch_window = ioremap(mchbar, 0x1000); if (!mch_window) { edac_dbg(3, "error ioremapping MCHBAR!\n"); goto fail0; diff --git a/drivers/edac/ie31200_edac.c b/drivers/edac/ie31200_edac.c index 4f65073f230b..d68346a8e141 100644 --- a/drivers/edac/ie31200_edac.c +++ b/drivers/edac/ie31200_edac.c @@ -357,7 +357,7 @@ static void __iomem *ie31200_map_mchbar(struct pci_dev *pdev) return NULL; } - window = ioremap_nocache(u.mchbar, IE31200_MMR_WINDOW_SIZE); + window = ioremap(u.mchbar, IE31200_MMR_WINDOW_SIZE); if (!window) ie31200_printk(KERN_ERR, "Cannot map mmio space at 0x%llx\n", (unsigned long long)u.mchbar); diff --git a/drivers/edac/x38_edac.c b/drivers/edac/x38_edac.c index cc779f3f9e2d..a65e2f78a402 100644 --- a/drivers/edac/x38_edac.c +++ b/drivers/edac/x38_edac.c @@ -266,7 +266,7 @@ static void __iomem *x38_map_mchbar(struct pci_dev *pdev) return NULL; } - window = ioremap_nocache(u.mchbar, X38_MMR_WINDOW_SIZE); + window = ioremap(u.mchbar, X38_MMR_WINDOW_SIZE); if (!window) printk(KERN_ERR "x38: cannot map mmio space at 0x%llx\n", (unsigned long long)u.mchbar); diff --git a/drivers/firewire/nosy.c b/drivers/firewire/nosy.c index 0cc746673677..6ca2f5ab6c57 100644 --- a/drivers/firewire/nosy.c +++ b/drivers/firewire/nosy.c @@ -551,7 +551,7 @@ add_card(struct pci_dev *dev, const struct pci_device_id *unused) INIT_LIST_HEAD(&lynx->client_list); kref_init(&lynx->kref); - lynx->registers = ioremap_nocache(pci_resource_start(dev, 0), + lynx->registers = ioremap(pci_resource_start(dev, 0), PCILYNX_MAX_REGISTER); if (lynx->registers == NULL) { dev_err(&dev->dev, "Failed to map registers\n"); diff --git a/drivers/firmware/broadcom/bcm47xx_nvram.c b/drivers/firmware/broadcom/bcm47xx_nvram.c index da04fdae62a1..835ece9c00f1 100644 --- a/drivers/firmware/broadcom/bcm47xx_nvram.c +++ b/drivers/firmware/broadcom/bcm47xx_nvram.c @@ -120,7 +120,7 @@ int bcm47xx_nvram_init_from_mem(u32 base, u32 lim) void __iomem *iobase; int err; - iobase = ioremap_nocache(base, lim); + iobase = ioremap(base, lim); if (!iobase) return -ENOMEM; diff --git a/drivers/gpu/drm/gma500/gtt.c b/drivers/gpu/drm/gma500/gtt.c index afaf4bea21cf..9278bcfad1bf 100644 --- a/drivers/gpu/drm/gma500/gtt.c +++ b/drivers/gpu/drm/gma500/gtt.c @@ -503,7 +503,7 @@ int psb_gtt_init(struct drm_device *dev, int resume) * Map the GTT and the stolen memory area */ if (!resume) - dev_priv->gtt_map = ioremap_nocache(pg->gtt_phys_start, + dev_priv->gtt_map = ioremap(pg->gtt_phys_start, gtt_pages << PAGE_SHIFT); if (!dev_priv->gtt_map) { dev_err(dev->dev, "Failure to map gtt.\n"); diff --git a/drivers/gpu/drm/gma500/psb_drv.c b/drivers/gpu/drm/gma500/psb_drv.c index 7005f8f69c68..0900052fc484 100644 --- a/drivers/gpu/drm/gma500/psb_drv.c +++ b/drivers/gpu/drm/gma500/psb_drv.c @@ -256,7 +256,7 @@ static int psb_driver_load(struct drm_device *dev, unsigned long flags) PSB_AUX_RESOURCE); resource_len = pci_resource_len(dev_priv->aux_pdev, PSB_AUX_RESOURCE); - dev_priv->aux_reg = ioremap_nocache(resource_start, + dev_priv->aux_reg = ioremap(resource_start, resource_len); if (!dev_priv->aux_reg) goto out_err; diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c index 2fd4ca91a62d..8dd5a43e5486 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c +++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c @@ -211,7 +211,7 @@ static int hibmc_hw_map(struct hibmc_drm_private *priv) ioaddr = pci_resource_start(pdev, 1); iosize = pci_resource_len(pdev, 1); - priv->mmio = devm_ioremap_nocache(dev->dev, ioaddr, iosize); + priv->mmio = devm_ioremap(dev->dev, ioaddr, iosize); if (!priv->mmio) { DRM_ERROR("Cannot map mmio region\n"); return -ENOMEM; diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index 6239a9adbf14..f24064893b30 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -2847,7 +2847,7 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) * readback check when writing GTT PTE entries. */ if (IS_GEN9_LP(dev_priv) || INTEL_GEN(dev_priv) >= 10) - ggtt->gsm = ioremap_nocache(phys_addr, size); + ggtt->gsm = ioremap(phys_addr, size); else ggtt->gsm = ioremap_wc(phys_addr, size); if (!ggtt->gsm) { diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c index c84f0a8b3f2c..ac678ace09a3 100644 --- a/drivers/gpu/drm/msm/msm_drv.c +++ b/drivers/gpu/drm/msm/msm_drv.c @@ -138,7 +138,7 @@ void __iomem *msm_ioremap(struct platform_device *pdev, const char *name, size = resource_size(res); - ptr = devm_ioremap_nocache(&pdev->dev, res->start, size); + ptr = devm_ioremap(&pdev->dev, res->start, size); if (!ptr) { DRM_DEV_ERROR(&pdev->dev, "failed to ioremap: %s\n", name); return ERR_PTR(-ENOMEM); diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c index 098bc9f40b98..15bf8a207cb0 100644 --- a/drivers/gpu/drm/radeon/radeon_ttm.c +++ b/drivers/gpu/drm/radeon/radeon_ttm.c @@ -443,7 +443,7 @@ static int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_ mem->bus.size); else mem->bus.addr = - ioremap_nocache(mem->bus.base + mem->bus.offset, + ioremap(mem->bus.base + mem->bus.offset, mem->bus.size); if (!mem->bus.addr) return -ENOMEM; diff --git a/drivers/gpu/drm/sti/sti_dvo.c b/drivers/gpu/drm/sti/sti_dvo.c index 68289b0b063a..68261c7f8c5f 100644 --- a/drivers/gpu/drm/sti/sti_dvo.c +++ b/drivers/gpu/drm/sti/sti_dvo.c @@ -534,7 +534,7 @@ static int sti_dvo_probe(struct platform_device *pdev) DRM_ERROR("Invalid dvo resource\n"); return -ENOMEM; } - dvo->regs = devm_ioremap_nocache(dev, res->start, + dvo->regs = devm_ioremap(dev, res->start, resource_size(res)); if (!dvo->regs) return -ENOMEM; diff --git a/drivers/gpu/drm/sti/sti_hda.c b/drivers/gpu/drm/sti/sti_hda.c index 8f7bf33815fd..2bb32009d117 100644 --- a/drivers/gpu/drm/sti/sti_hda.c +++ b/drivers/gpu/drm/sti/sti_hda.c @@ -759,14 +759,14 @@ static int sti_hda_probe(struct platform_device *pdev) DRM_ERROR("Invalid hda resource\n"); return -ENOMEM; } - hda->regs = devm_ioremap_nocache(dev, res->start, resource_size(res)); + hda->regs = devm_ioremap(dev, res->start, resource_size(res)); if (!hda->regs) return -ENOMEM; res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "video-dacs-ctrl"); if (res) { - hda->video_dacs_ctrl = devm_ioremap_nocache(dev, res->start, + hda->video_dacs_ctrl = devm_ioremap(dev, res->start, resource_size(res)); if (!hda->video_dacs_ctrl) return -ENOMEM; diff --git a/drivers/gpu/drm/sti/sti_hdmi.c b/drivers/gpu/drm/sti/sti_hdmi.c index 814560ead4e1..64ed102033c8 100644 --- a/drivers/gpu/drm/sti/sti_hdmi.c +++ b/drivers/gpu/drm/sti/sti_hdmi.c @@ -1393,7 +1393,7 @@ static int sti_hdmi_probe(struct platform_device *pdev) ret = -ENOMEM; goto release_adapter; } - hdmi->regs = devm_ioremap_nocache(dev, res->start, resource_size(res)); + hdmi->regs = devm_ioremap(dev, res->start, resource_size(res)); if (!hdmi->regs) { ret = -ENOMEM; goto release_adapter; diff --git a/drivers/gpu/drm/sti/sti_tvout.c b/drivers/gpu/drm/sti/sti_tvout.c index 5767e93dd1cd..c36a8da373cb 100644 --- a/drivers/gpu/drm/sti/sti_tvout.c +++ b/drivers/gpu/drm/sti/sti_tvout.c @@ -860,7 +860,7 @@ static int sti_tvout_probe(struct platform_device *pdev) DRM_ERROR("Invalid glue resource\n"); return -ENOMEM; } - tvout->regs = devm_ioremap_nocache(dev, res->start, resource_size(res)); + tvout->regs = devm_ioremap(dev, res->start, resource_size(res)); if (!tvout->regs) return -ENOMEM; diff --git a/drivers/gpu/drm/sti/sti_vtg.c b/drivers/gpu/drm/sti/sti_vtg.c index 0b17ac8a3faa..5e5f82b6a5d9 100644 --- a/drivers/gpu/drm/sti/sti_vtg.c +++ b/drivers/gpu/drm/sti/sti_vtg.c @@ -393,7 +393,7 @@ static int vtg_probe(struct platform_device *pdev) DRM_ERROR("Get memory resource failed\n"); return -ENOMEM; } - vtg->regs = devm_ioremap_nocache(dev, res->start, resource_size(res)); + vtg->regs = devm_ioremap(dev, res->start, resource_size(res)); if (!vtg->regs) { DRM_ERROR("failed to remap I/O memory\n"); return -ENOMEM; diff --git a/drivers/gpu/drm/tilcdc/tilcdc_drv.c b/drivers/gpu/drm/tilcdc/tilcdc_drv.c index 2a9e67597375..a3612369750f 100644 --- a/drivers/gpu/drm/tilcdc/tilcdc_drv.c +++ b/drivers/gpu/drm/tilcdc/tilcdc_drv.c @@ -256,7 +256,7 @@ static int tilcdc_init(struct drm_driver *ddrv, struct device *dev) goto init_failed; } - priv->mmio = ioremap_nocache(res->start, resource_size(res)); + priv->mmio = ioremap(res->start, resource_size(res)); if (!priv->mmio) { dev_err(dev, "failed to ioremap\n"); ret = -ENOMEM; diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 6b0883a1776e..97fd1dafc3e8 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -218,7 +218,7 @@ static int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *m if (mem->placement & TTM_PL_FLAG_WC) addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size); else - addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size); + addr = ioremap(mem->bus.base + mem->bus.offset, mem->bus.size); if (!addr) { (void) ttm_mem_io_lock(man, false); ttm_mem_io_free(bdev, mem); @@ -565,7 +565,7 @@ static int ttm_bo_ioremap(struct ttm_buffer_object *bo, map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset, size); else - map->virtual = ioremap_nocache(bo->mem.bus.base + bo->mem.bus.offset + offset, + map->virtual = ioremap(bo->mem.bus.base + bo->mem.bus.offset + offset, size); } return (!map->virtual) ? -ENOMEM : 0; diff --git a/drivers/hwmon/i5k_amb.c b/drivers/hwmon/i5k_amb.c index b09c39abd3a8..eeac4b04df27 100644 --- a/drivers/hwmon/i5k_amb.c +++ b/drivers/hwmon/i5k_amb.c @@ -528,7 +528,7 @@ static int i5k_amb_probe(struct platform_device *pdev) goto err; } - data->amb_mmio = ioremap_nocache(data->amb_base, data->amb_len); + data->amb_mmio = ioremap(data->amb_base, data->amb_len); if (!data->amb_mmio) { res = -EBUSY; goto err_map_failed; diff --git a/drivers/i2c/busses/i2c-highlander.c b/drivers/i2c/busses/i2c-highlander.c index ff340d7ae2e5..abfe3094c047 100644 --- a/drivers/i2c/busses/i2c-highlander.c +++ b/drivers/i2c/busses/i2c-highlander.c @@ -369,7 +369,7 @@ static int highlander_i2c_probe(struct platform_device *pdev) if (unlikely(!dev)) return -ENOMEM; - dev->base = ioremap_nocache(res->start, resource_size(res)); + dev->base = ioremap(res->start, resource_size(res)); if (unlikely(!dev->base)) { ret = -ENXIO; goto err; diff --git a/drivers/i2c/busses/i2c-pmcmsp.c b/drivers/i2c/busses/i2c-pmcmsp.c index 0829cb696d9d..4fde74eb34a7 100644 --- a/drivers/i2c/busses/i2c-pmcmsp.c +++ b/drivers/i2c/busses/i2c-pmcmsp.c @@ -281,7 +281,7 @@ static int pmcmsptwi_probe(struct platform_device *pldev) } /* remap the memory */ - pmcmsptwi_data.iobase = ioremap_nocache(res->start, + pmcmsptwi_data.iobase = ioremap(res->start, resource_size(res)); if (!pmcmsptwi_data.iobase) { dev_err(&pldev->dev, diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c index 958c1ff9c515..0441c7eea36b 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c @@ -442,7 +442,7 @@ int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq, goto fail; } /* Unconditionally map 8 bytes to support 57500 series */ - nq->bar_reg_iomem = ioremap_nocache(nq_base + nq->bar_reg_off, 8); + nq->bar_reg_iomem = ioremap(nq_base + nq->bar_reg_off, 8); if (!nq->bar_reg_iomem) { rc = -ENOMEM; goto fail; diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c index 5cdfa84faf85..1291b12287a5 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c @@ -717,7 +717,7 @@ int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev, if (!res_base) return -ENOMEM; - rcfw->cmdq_bar_reg_iomem = ioremap_nocache(res_base + + rcfw->cmdq_bar_reg_iomem = ioremap(res_base + RCFW_COMM_BASE_OFFSET, RCFW_COMM_SIZE); if (!rcfw->cmdq_bar_reg_iomem) { @@ -739,7 +739,7 @@ int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev, "CREQ BAR region %d resc start is 0!\n", rcfw->creq_bar_reg); /* Unconditionally map 8 bytes to support 57500 series */ - rcfw->creq_bar_reg_iomem = ioremap_nocache(res_base + cp_bar_reg_off, + rcfw->creq_bar_reg_iomem = ioremap(res_base + cp_bar_reg_off, 8); if (!rcfw->creq_bar_reg_iomem) { dev_err(&rcfw->pdev->dev, "CREQ BAR region %d mapping failed\n", diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c index bdbde8e22420..60ea1b924b67 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c @@ -704,7 +704,7 @@ static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res, return -ENOMEM; } - dpit->dbr_bar_reg_iomem = ioremap_nocache(bar_reg_base + dbr_offset, + dpit->dbr_bar_reg_iomem = ioremap(bar_reg_base + dbr_offset, dbr_len); if (!dpit->dbr_bar_reg_iomem) { dev_err(&res->pdev->dev, diff --git a/drivers/infiniband/hw/hfi1/pcie.c b/drivers/infiniband/hw/hfi1/pcie.c index 61362bd6d3ce..1a6268d61977 100644 --- a/drivers/infiniband/hw/hfi1/pcie.c +++ b/drivers/infiniband/hw/hfi1/pcie.c @@ -161,7 +161,7 @@ int hfi1_pcie_ddinit(struct hfi1_devdata *dd, struct pci_dev *pdev) return -EINVAL; } - dd->kregbase1 = ioremap_nocache(addr, RCV_ARRAY); + dd->kregbase1 = ioremap(addr, RCV_ARRAY); if (!dd->kregbase1) { dd_dev_err(dd, "UC mapping of kregbase1 failed\n"); return -ENOMEM; @@ -179,7 +179,7 @@ int hfi1_pcie_ddinit(struct hfi1_devdata *dd, struct pci_dev *pdev) dd_dev_info(dd, "RcvArray count: %u\n", rcv_array_count); dd->base2_start = RCV_ARRAY + rcv_array_count * 8; - dd->kregbase2 = ioremap_nocache( + dd->kregbase2 = ioremap( addr + dd->base2_start, TXE_PIO_SEND - dd->base2_start); if (!dd->kregbase2) { diff --git a/drivers/infiniband/hw/qib/qib_iba7322.c b/drivers/infiniband/hw/qib/qib_iba7322.c index dd4843379f51..91d64dd71a8a 100644 --- a/drivers/infiniband/hw/qib/qib_iba7322.c +++ b/drivers/infiniband/hw/qib/qib_iba7322.c @@ -6630,7 +6630,7 @@ static int qib_init_7322_variables(struct qib_devdata *dd) /* vl15 buffers start just after the 4k buffers */ vl15off = dd->physaddr + (dd->piobufbase >> 32) + dd->piobcnt4k * dd->align4k; - dd->piovl15base = ioremap_nocache(vl15off, + dd->piovl15base = ioremap(vl15off, NUM_VL15_BUFS * dd->align4k); if (!dd->piovl15base) { ret = -ENOMEM; diff --git a/drivers/infiniband/hw/qib/qib_init.c b/drivers/infiniband/hw/qib/qib_init.c index d4fd8a6cff7b..43c8ee1f46e0 100644 --- a/drivers/infiniband/hw/qib/qib_init.c +++ b/drivers/infiniband/hw/qib/qib_init.c @@ -1759,7 +1759,7 @@ int init_chip_wc_pat(struct qib_devdata *dd, u32 vl15buflen) qib_userlen = dd->ureg_align * dd->cfgctxts; /* Sanity checks passed, now create the new mappings */ - qib_kregbase = ioremap_nocache(qib_physaddr, qib_kreglen); + qib_kregbase = ioremap(qib_physaddr, qib_kreglen); if (!qib_kregbase) goto bail; @@ -1768,7 +1768,7 @@ int init_chip_wc_pat(struct qib_devdata *dd, u32 vl15buflen) goto bail_kregbase; if (qib_userlen) { - qib_userbase = ioremap_nocache(qib_physaddr + dd->uregbase, + qib_userbase = ioremap(qib_physaddr + dd->uregbase, qib_userlen); if (!qib_userbase) goto bail_piobase; diff --git a/drivers/infiniband/hw/qib/qib_pcie.c b/drivers/infiniband/hw/qib/qib_pcie.c index 864f2af171f7..3dc6ce033319 100644 --- a/drivers/infiniband/hw/qib/qib_pcie.c +++ b/drivers/infiniband/hw/qib/qib_pcie.c @@ -145,7 +145,7 @@ int qib_pcie_ddinit(struct qib_devdata *dd, struct pci_dev *pdev, addr = pci_resource_start(pdev, 0); len = pci_resource_len(pdev, 0); - dd->kregbase = ioremap_nocache(addr, len); + dd->kregbase = ioremap(addr, len); if (!dd->kregbase) return -ENOMEM; diff --git a/drivers/input/keyboard/pxa930_rotary.c b/drivers/input/keyboard/pxa930_rotary.c index f7414091d94e..2fe9dcfe0a6f 100644 --- a/drivers/input/keyboard/pxa930_rotary.c +++ b/drivers/input/keyboard/pxa930_rotary.c @@ -107,7 +107,7 @@ static int pxa930_rotary_probe(struct platform_device *pdev) if (!r) return -ENOMEM; - r->mmio_base = ioremap_nocache(res->start, resource_size(res)); + r->mmio_base = ioremap(res->start, resource_size(res)); if (r->mmio_base == NULL) { dev_err(&pdev->dev, "failed to remap IO memory\n"); err = -ENXIO; diff --git a/drivers/input/keyboard/sh_keysc.c b/drivers/input/keyboard/sh_keysc.c index 27ad73f43451..c155adebf96e 100644 --- a/drivers/input/keyboard/sh_keysc.c +++ b/drivers/input/keyboard/sh_keysc.c @@ -195,7 +195,7 @@ static int sh_keysc_probe(struct platform_device *pdev) memcpy(&priv->pdata, dev_get_platdata(&pdev->dev), sizeof(priv->pdata)); pdata = &priv->pdata; - priv->iomem_base = ioremap_nocache(res->start, resource_size(res)); + priv->iomem_base = ioremap(res->start, resource_size(res)); if (priv->iomem_base == NULL) { dev_err(&pdev->dev, "failed to remap I/O memory\n"); error = -ENXIO; diff --git a/drivers/input/mouse/pxa930_trkball.c b/drivers/input/mouse/pxa930_trkball.c index 41acde60b60f..3332b77eef2a 100644 --- a/drivers/input/mouse/pxa930_trkball.c +++ b/drivers/input/mouse/pxa930_trkball.c @@ -167,7 +167,7 @@ static int pxa930_trkball_probe(struct platform_device *pdev) goto failed; } - trkball->mmio_base = ioremap_nocache(res->start, resource_size(res)); + trkball->mmio_base = ioremap(res->start, resource_size(res)); if (!trkball->mmio_base) { dev_err(&pdev->dev, "failed to ioremap registers\n"); error = -ENXIO; diff --git a/drivers/input/serio/gscps2.c b/drivers/input/serio/gscps2.c index 96f9b5397367..2f9775de3c5b 100644 --- a/drivers/input/serio/gscps2.c +++ b/drivers/input/serio/gscps2.c @@ -349,7 +349,7 @@ static int __init gscps2_probe(struct parisc_device *dev) ps2port->port = serio; ps2port->padev = dev; - ps2port->addr = ioremap_nocache(hpa, GSC_STATUS + 4); + ps2port->addr = ioremap(hpa, GSC_STATUS + 4); spin_lock_init(&ps2port->lock); gscps2_reset(ps2port); diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index 568c52317757..27bde309e6f7 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -440,7 +440,7 @@ static u8 __iomem * __init iommu_map_mmio_space(u64 address, u64 end) return NULL; } - return (u8 __iomem *)ioremap_nocache(address, end); + return (u8 __iomem *)ioremap(address, end); } static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu) diff --git a/drivers/ipack/carriers/tpci200.c b/drivers/ipack/carriers/tpci200.c index d246d74ec3a5..23445ebfda5c 100644 --- a/drivers/ipack/carriers/tpci200.c +++ b/drivers/ipack/carriers/tpci200.c @@ -298,7 +298,7 @@ static int tpci200_register(struct tpci200_board *tpci200) /* Map internal tpci200 driver user space */ tpci200->info->interface_regs = - ioremap_nocache(pci_resource_start(tpci200->info->pdev, + ioremap(pci_resource_start(tpci200->info->pdev, TPCI200_IP_INTERFACE_BAR), TPCI200_IFACE_SIZE); if (!tpci200->info->interface_regs) { @@ -541,7 +541,7 @@ static int tpci200_pci_probe(struct pci_dev *pdev, ret = -EBUSY; goto out_err_pci_request; } - tpci200->info->cfg_regs = ioremap_nocache( + tpci200->info->cfg_regs = ioremap( pci_resource_start(pdev, TPCI200_CFG_MEM_BAR), pci_resource_len(pdev, TPCI200_CFG_MEM_BAR)); if (!tpci200->info->cfg_regs) { diff --git a/drivers/ipack/devices/ipoctal.c b/drivers/ipack/devices/ipoctal.c index 9c2a4b5d30cf..d480a514c983 100644 --- a/drivers/ipack/devices/ipoctal.c +++ b/drivers/ipack/devices/ipoctal.c @@ -276,7 +276,7 @@ static int ipoctal_inst_slot(struct ipoctal *ipoctal, unsigned int bus_nr, ipoctal->board_id = ipoctal->dev->id_device; region = &ipoctal->dev->region[IPACK_IO_SPACE]; - addr = devm_ioremap_nocache(&ipoctal->dev->dev, + addr = devm_ioremap(&ipoctal->dev->dev, region->start, region->size); if (!addr) { dev_err(&ipoctal->dev->dev, @@ -292,7 +292,7 @@ static int ipoctal_inst_slot(struct ipoctal *ipoctal, unsigned int bus_nr, region = &ipoctal->dev->region[IPACK_INT_SPACE]; ipoctal->int_space = - devm_ioremap_nocache(&ipoctal->dev->dev, + devm_ioremap(&ipoctal->dev->dev, region->start, region->size); if (!ipoctal->int_space) { dev_err(&ipoctal->dev->dev, @@ -303,7 +303,7 @@ static int ipoctal_inst_slot(struct ipoctal *ipoctal, unsigned int bus_nr, region = &ipoctal->dev->region[IPACK_MEM8_SPACE]; ipoctal->mem8_space = - devm_ioremap_nocache(&ipoctal->dev->dev, + devm_ioremap(&ipoctal->dev->dev, region->start, 0x8000); if (!ipoctal->mem8_space) { dev_err(&ipoctal->dev->dev, diff --git a/drivers/irqchip/irq-mips-gic.c b/drivers/irqchip/irq-mips-gic.c index f3985469c221..d70507133c1d 100644 --- a/drivers/irqchip/irq-mips-gic.c +++ b/drivers/irqchip/irq-mips-gic.c @@ -716,7 +716,7 @@ static int __init gic_of_init(struct device_node *node, __sync(); } - mips_gic_base = ioremap_nocache(gic_base, gic_len); + mips_gic_base = ioremap(gic_base, gic_len); gicconfig = read_gic_config(); gic_shared_intrs = gicconfig & GIC_CONFIG_NUMINTERRUPTS; diff --git a/drivers/irqchip/irq-renesas-intc-irqpin.c b/drivers/irqchip/irq-renesas-intc-irqpin.c index f82bc60a6793..6e5e3172796b 100644 --- a/drivers/irqchip/irq-renesas-intc-irqpin.c +++ b/drivers/irqchip/irq-renesas-intc-irqpin.c @@ -460,7 +460,7 @@ static int intc_irqpin_probe(struct platform_device *pdev) goto err0; } - i->iomem = devm_ioremap_nocache(dev, io[k]->start, + i->iomem = devm_ioremap(dev, io[k]->start, resource_size(io[k])); if (!i->iomem) { dev_err(dev, "failed to remap IOMEM\n"); diff --git a/drivers/media/common/videobuf2/videobuf2-vmalloc.c b/drivers/media/common/videobuf2/videobuf2-vmalloc.c index 04d51ca63223..4c8c96a35282 100644 --- a/drivers/media/common/videobuf2/videobuf2-vmalloc.c +++ b/drivers/media/common/videobuf2/videobuf2-vmalloc.c @@ -105,7 +105,7 @@ static void *vb2_vmalloc_get_userptr(struct device *dev, unsigned long vaddr, if (nums[i-1] + 1 != nums[i]) goto fail_map; buf->vaddr = (__force void *) - ioremap_nocache(__pfn_to_phys(nums[0]), size + offset); + ioremap(__pfn_to_phys(nums[0]), size + offset); } else { buf->vaddr = vm_map_ram(frame_vector_pages(vec), n_pages, -1, PAGE_KERNEL); diff --git a/drivers/media/pci/cx18/cx18-driver.c b/drivers/media/pci/cx18/cx18-driver.c index fd47bd07ffd8..2f1eeeb6e7c7 100644 --- a/drivers/media/pci/cx18/cx18-driver.c +++ b/drivers/media/pci/cx18/cx18-driver.c @@ -938,7 +938,7 @@ static int cx18_probe(struct pci_dev *pci_dev, /* map io memory */ CX18_DEBUG_INFO("attempting ioremap at 0x%llx len 0x%08x\n", (u64)cx->base_addr + CX18_MEM_OFFSET, CX18_MEM_SIZE); - cx->enc_mem = ioremap_nocache(cx->base_addr + CX18_MEM_OFFSET, + cx->enc_mem = ioremap(cx->base_addr + CX18_MEM_OFFSET, CX18_MEM_SIZE); if (!cx->enc_mem) { CX18_ERR("ioremap failed. Can't get a window into CX23418 memory and register space\n"); diff --git a/drivers/media/pci/ivtv/ivtv-driver.c b/drivers/media/pci/ivtv/ivtv-driver.c index 3f3f40ea890b..1f79700a6307 100644 --- a/drivers/media/pci/ivtv/ivtv-driver.c +++ b/drivers/media/pci/ivtv/ivtv-driver.c @@ -1042,7 +1042,7 @@ static int ivtv_probe(struct pci_dev *pdev, const struct pci_device_id *pci_id) /* map io memory */ IVTV_DEBUG_INFO("attempting ioremap at 0x%llx len 0x%08x\n", (u64)itv->base_addr + IVTV_ENCODER_OFFSET, IVTV_ENCODER_SIZE); - itv->enc_mem = ioremap_nocache(itv->base_addr + IVTV_ENCODER_OFFSET, + itv->enc_mem = ioremap(itv->base_addr + IVTV_ENCODER_OFFSET, IVTV_ENCODER_SIZE); if (!itv->enc_mem) { IVTV_ERR("ioremap failed. Can't get a window into CX23415/6 encoder memory\n"); @@ -1056,7 +1056,7 @@ static int ivtv_probe(struct pci_dev *pdev, const struct pci_device_id *pci_id) if (itv->has_cx23415) { IVTV_DEBUG_INFO("attempting ioremap at 0x%llx len 0x%08x\n", (u64)itv->base_addr + IVTV_DECODER_OFFSET, IVTV_DECODER_SIZE); - itv->dec_mem = ioremap_nocache(itv->base_addr + IVTV_DECODER_OFFSET, + itv->dec_mem = ioremap(itv->base_addr + IVTV_DECODER_OFFSET, IVTV_DECODER_SIZE); if (!itv->dec_mem) { IVTV_ERR("ioremap failed. Can't get a window into CX23415 decoder memory\n"); @@ -1075,7 +1075,7 @@ static int ivtv_probe(struct pci_dev *pdev, const struct pci_device_id *pci_id) IVTV_DEBUG_INFO("attempting ioremap at 0x%llx len 0x%08x\n", (u64)itv->base_addr + IVTV_REG_OFFSET, IVTV_REG_SIZE); itv->reg_mem = - ioremap_nocache(itv->base_addr + IVTV_REG_OFFSET, IVTV_REG_SIZE); + ioremap(itv->base_addr + IVTV_REG_OFFSET, IVTV_REG_SIZE); if (!itv->reg_mem) { IVTV_ERR("ioremap failed. Can't get a window into CX23415/6 register space\n"); IVTV_ERR("Each capture card with a CX23415/6 needs 64 kB of vmalloc address space for this window\n"); diff --git a/drivers/media/platform/davinci/dm355_ccdc.c b/drivers/media/platform/davinci/dm355_ccdc.c index f299baf7cbe0..e06d113dfe96 100644 --- a/drivers/media/platform/davinci/dm355_ccdc.c +++ b/drivers/media/platform/davinci/dm355_ccdc.c @@ -883,7 +883,7 @@ static int dm355_ccdc_probe(struct platform_device *pdev) goto fail_nores; } - ccdc_cfg.base_addr = ioremap_nocache(res->start, resource_size(res)); + ccdc_cfg.base_addr = ioremap(res->start, resource_size(res)); if (!ccdc_cfg.base_addr) { status = -ENOMEM; goto fail_nomem; diff --git a/drivers/media/platform/davinci/dm644x_ccdc.c b/drivers/media/platform/davinci/dm644x_ccdc.c index 2fc6c9c38f9c..c6378c4e0074 100644 --- a/drivers/media/platform/davinci/dm644x_ccdc.c +++ b/drivers/media/platform/davinci/dm644x_ccdc.c @@ -817,7 +817,7 @@ static int dm644x_ccdc_probe(struct platform_device *pdev) goto fail_nores; } - ccdc_cfg.base_addr = ioremap_nocache(res->start, resource_size(res)); + ccdc_cfg.base_addr = ioremap(res->start, resource_size(res)); if (!ccdc_cfg.base_addr) { status = -ENOMEM; goto fail_nomem; diff --git a/drivers/media/platform/davinci/isif.c b/drivers/media/platform/davinci/isif.c index e2e7ab7b7f45..b49378b18e5d 100644 --- a/drivers/media/platform/davinci/isif.c +++ b/drivers/media/platform/davinci/isif.c @@ -1045,7 +1045,7 @@ static int isif_probe(struct platform_device *pdev) status = -EBUSY; goto fail_nobase_res; } - addr = ioremap_nocache(res->start, resource_size(res)); + addr = ioremap(res->start, resource_size(res)); if (!addr) { status = -ENOMEM; goto fail_base_iomap; diff --git a/drivers/media/platform/tegra-cec/tegra_cec.c b/drivers/media/platform/tegra-cec/tegra_cec.c index a99caac59f44..1ac0c70a5981 100644 --- a/drivers/media/platform/tegra-cec/tegra_cec.c +++ b/drivers/media/platform/tegra-cec/tegra_cec.c @@ -351,7 +351,7 @@ static int tegra_cec_probe(struct platform_device *pdev) if (cec->tegra_cec_irq <= 0) return -EBUSY; - cec->cec_base = devm_ioremap_nocache(&pdev->dev, res->start, + cec->cec_base = devm_ioremap(&pdev->dev, res->start, resource_size(res)); if (!cec->cec_base) { diff --git a/drivers/misc/cardreader/rtsx_pcr.c b/drivers/misc/cardreader/rtsx_pcr.c index fd7b2167103d..06038b325b02 100644 --- a/drivers/misc/cardreader/rtsx_pcr.c +++ b/drivers/misc/cardreader/rtsx_pcr.c @@ -1512,7 +1512,7 @@ static int rtsx_pci_probe(struct pci_dev *pcidev, bar = 1; len = pci_resource_len(pcidev, bar); base = pci_resource_start(pcidev, bar); - pcr->remap_addr = ioremap_nocache(base, len); + pcr->remap_addr = ioremap(base, len); if (!pcr->remap_addr) { ret = -ENOMEM; goto free_handle; diff --git a/drivers/misc/mic/scif/scif_nodeqp.c b/drivers/misc/mic/scif/scif_nodeqp.c index c25fd40f3bd0..fcd999f50d14 100644 --- a/drivers/misc/mic/scif/scif_nodeqp.c +++ b/drivers/misc/mic/scif/scif_nodeqp.c @@ -788,7 +788,7 @@ scif_node_add(struct scif_dev *scifdev, struct scifmsg *msg) "failed to setup interrupts for %d\n", msg->src.node); goto interrupt_setup_error; } - newdev->mmio.va = ioremap_nocache(msg->payload[1], sdev->mmio->len); + newdev->mmio.va = ioremap(msg->payload[1], sdev->mmio->len); if (!newdev->mmio.va) { dev_err(&scifdev->sdev->dev, "failed to map mmio for %d\n", msg->src.node); diff --git a/drivers/misc/pti.c b/drivers/misc/pti.c index 359c5bab45ac..063e4419cd7e 100644 --- a/drivers/misc/pti.c +++ b/drivers/misc/pti.c @@ -834,7 +834,7 @@ static int pti_pci_probe(struct pci_dev *pdev, } drv_data->aperture_base = drv_data->pti_addr+APERTURE_14; drv_data->pti_ioaddr = - ioremap_nocache((u32)drv_data->aperture_base, + ioremap((u32)drv_data->aperture_base, APERTURE_LEN); if (!drv_data->pti_ioaddr) { retval = -ENOMEM; diff --git a/drivers/mmc/host/sdhci-acpi.c b/drivers/mmc/host/sdhci-acpi.c index 105e73d4a3b9..674f9e084cd2 100644 --- a/drivers/mmc/host/sdhci-acpi.c +++ b/drivers/mmc/host/sdhci-acpi.c @@ -719,7 +719,7 @@ static int sdhci_acpi_probe(struct platform_device *pdev) goto err_free; } - host->ioaddr = devm_ioremap_nocache(dev, iomem->start, + host->ioaddr = devm_ioremap(dev, iomem->start, resource_size(iomem)); if (host->ioaddr == NULL) { err = -ENOMEM; diff --git a/drivers/mmc/host/via-sdmmc.c b/drivers/mmc/host/via-sdmmc.c index f4ac064ff471..e48bddd95ce6 100644 --- a/drivers/mmc/host/via-sdmmc.c +++ b/drivers/mmc/host/via-sdmmc.c @@ -1106,7 +1106,7 @@ static int via_sd_probe(struct pci_dev *pcidev, len = pci_resource_len(pcidev, 0); base = pci_resource_start(pcidev, 0); - sdhost->mmiobase = ioremap_nocache(base, len); + sdhost->mmiobase = ioremap(base, len); if (!sdhost->mmiobase) { ret = -ENOMEM; goto free_mmc_host; diff --git a/drivers/mtd/devices/bcm47xxsflash.c b/drivers/mtd/devices/bcm47xxsflash.c index eccf2e5d905e..3af50db8b21b 100644 --- a/drivers/mtd/devices/bcm47xxsflash.c +++ b/drivers/mtd/devices/bcm47xxsflash.c @@ -320,7 +320,7 @@ static int bcm47xxsflash_bcma_probe(struct platform_device *pdev) * ChipCommon revision. */ if (b47s->bcma_cc->core->id.rev == 54) - b47s->window = ioremap_nocache(res->start, resource_size(res)); + b47s->window = ioremap(res->start, resource_size(res)); else b47s->window = ioremap_cache(res->start, resource_size(res)); if (!b47s->window) { diff --git a/drivers/mtd/maps/amd76xrom.c b/drivers/mtd/maps/amd76xrom.c index 462fadb56bdb..42a95ba40f2c 100644 --- a/drivers/mtd/maps/amd76xrom.c +++ b/drivers/mtd/maps/amd76xrom.c @@ -163,7 +163,7 @@ static int amd76xrom_init_one(struct pci_dev *pdev, /* FIXME handle registers 0x80 - 0x8C the bios region locks */ /* For write accesses caches are useless */ - window->virt = ioremap_nocache(window->phys, window->size); + window->virt = ioremap(window->phys, window->size); if (!window->virt) { printk(KERN_ERR MOD_NAME ": ioremap(%08lx, %08lx) failed\n", window->phys, window->size); diff --git a/drivers/mtd/maps/ck804xrom.c b/drivers/mtd/maps/ck804xrom.c index c9b7b4d5a923..460494212f6a 100644 --- a/drivers/mtd/maps/ck804xrom.c +++ b/drivers/mtd/maps/ck804xrom.c @@ -191,7 +191,7 @@ static int __init ck804xrom_init_one(struct pci_dev *pdev, /* FIXME handle registers 0x80 - 0x8C the bios region locks */ /* For write accesses caches are useless */ - window->virt = ioremap_nocache(window->phys, window->size); + window->virt = ioremap(window->phys, window->size); if (!window->virt) { printk(KERN_ERR MOD_NAME ": ioremap(%08lx, %08lx) failed\n", window->phys, window->size); diff --git a/drivers/mtd/maps/esb2rom.c b/drivers/mtd/maps/esb2rom.c index 5c27c6994896..85e14150a073 100644 --- a/drivers/mtd/maps/esb2rom.c +++ b/drivers/mtd/maps/esb2rom.c @@ -249,7 +249,7 @@ static int __init esb2rom_init_one(struct pci_dev *pdev, } /* Map the firmware hub into my address space. */ - window->virt = ioremap_nocache(window->phys, window->size); + window->virt = ioremap(window->phys, window->size); if (!window->virt) { printk(KERN_ERR MOD_NAME ": ioremap(%08lx, %08lx) failed\n", window->phys, window->size); diff --git a/drivers/mtd/maps/ichxrom.c b/drivers/mtd/maps/ichxrom.c index 6b989f391baa..fda72c5fd8f9 100644 --- a/drivers/mtd/maps/ichxrom.c +++ b/drivers/mtd/maps/ichxrom.c @@ -184,7 +184,7 @@ static int __init ichxrom_init_one(struct pci_dev *pdev, } /* Map the firmware hub into my address space. */ - window->virt = ioremap_nocache(window->phys, window->size); + window->virt = ioremap(window->phys, window->size); if (!window->virt) { printk(KERN_ERR MOD_NAME ": ioremap(%08lx, %08lx) failed\n", window->phys, window->size); diff --git a/drivers/mtd/maps/intel_vr_nor.c b/drivers/mtd/maps/intel_vr_nor.c index 69503aef981e..d67b845b0e89 100644 --- a/drivers/mtd/maps/intel_vr_nor.c +++ b/drivers/mtd/maps/intel_vr_nor.c @@ -133,7 +133,7 @@ static int vr_nor_init_maps(struct vr_nor_mtd *p) if (win_len < (CS0_START + CS0_SIZE)) return -ENXIO; - p->csr_base = ioremap_nocache(csr_phys, csr_len); + p->csr_base = ioremap(csr_phys, csr_len); if (!p->csr_base) return -ENOMEM; @@ -152,7 +152,7 @@ static int vr_nor_init_maps(struct vr_nor_mtd *p) p->map.bankwidth = (exp_timing_cs0 & TIMING_BYTE_EN) ? 1 : 2; p->map.phys = win_phys + CS0_START; p->map.size = CS0_SIZE; - p->map.virt = ioremap_nocache(p->map.phys, p->map.size); + p->map.virt = ioremap(p->map.phys, p->map.size); if (!p->map.virt) { err = -ENOMEM; goto release; diff --git a/drivers/mtd/maps/l440gx.c b/drivers/mtd/maps/l440gx.c index 0eeadfeb620d..832b880d1aaf 100644 --- a/drivers/mtd/maps/l440gx.c +++ b/drivers/mtd/maps/l440gx.c @@ -78,7 +78,7 @@ static int __init init_l440gx(void) return -ENODEV; } - l440gx_map.virt = ioremap_nocache(WINDOW_ADDR, WINDOW_SIZE); + l440gx_map.virt = ioremap(WINDOW_ADDR, WINDOW_SIZE); if (!l440gx_map.virt) { printk(KERN_WARNING "Failed to ioremap L440GX flash region\n"); diff --git a/drivers/mtd/maps/netsc520.c b/drivers/mtd/maps/netsc520.c index abc52b70bb00..0bb651624f05 100644 --- a/drivers/mtd/maps/netsc520.c +++ b/drivers/mtd/maps/netsc520.c @@ -82,10 +82,10 @@ static int __init init_netsc520(void) printk(KERN_NOTICE "NetSc520 flash device: 0x%Lx at 0x%Lx\n", (unsigned long long)netsc520_map.size, (unsigned long long)netsc520_map.phys); - netsc520_map.virt = ioremap_nocache(netsc520_map.phys, netsc520_map.size); + netsc520_map.virt = ioremap(netsc520_map.phys, netsc520_map.size); if (!netsc520_map.virt) { - printk("Failed to ioremap_nocache\n"); + printk("Failed to ioremap\n"); return -EIO; } diff --git a/drivers/mtd/maps/nettel.c b/drivers/mtd/maps/nettel.c index 50046d497398..7d349874ffeb 100644 --- a/drivers/mtd/maps/nettel.c +++ b/drivers/mtd/maps/nettel.c @@ -176,7 +176,7 @@ static int __init nettel_init(void) #endif int rc = 0; - nettel_mmcrp = (void *) ioremap_nocache(0xfffef000, 4096); + nettel_mmcrp = (void *) ioremap(0xfffef000, 4096); if (nettel_mmcrp == NULL) { printk("SNAPGEAR: failed to disable MMCR cache??\n"); return(-EIO); @@ -217,7 +217,7 @@ static int __init nettel_init(void) __asm__ ("wbinvd"); nettel_amd_map.phys = amdaddr; - nettel_amd_map.virt = ioremap_nocache(amdaddr, maxsize); + nettel_amd_map.virt = ioremap(amdaddr, maxsize); if (!nettel_amd_map.virt) { printk("SNAPGEAR: failed to ioremap() BOOTCS\n"); iounmap(nettel_mmcrp); @@ -303,7 +303,7 @@ static int __init nettel_init(void) /* Probe for the size of the first Intel flash */ nettel_intel_map.size = maxsize; nettel_intel_map.phys = intel0addr; - nettel_intel_map.virt = ioremap_nocache(intel0addr, maxsize); + nettel_intel_map.virt = ioremap(intel0addr, maxsize); if (!nettel_intel_map.virt) { printk("SNAPGEAR: failed to ioremap() ROMCS1\n"); rc = -EIO; @@ -337,7 +337,7 @@ static int __init nettel_init(void) iounmap(nettel_intel_map.virt); nettel_intel_map.size = maxsize; - nettel_intel_map.virt = ioremap_nocache(intel0addr, maxsize); + nettel_intel_map.virt = ioremap(intel0addr, maxsize); if (!nettel_intel_map.virt) { printk("SNAPGEAR: failed to ioremap() ROMCS1/2\n"); rc = -EIO; diff --git a/drivers/mtd/maps/pci.c b/drivers/mtd/maps/pci.c index 9a49f8a06fb8..377ef0fc4e3e 100644 --- a/drivers/mtd/maps/pci.c +++ b/drivers/mtd/maps/pci.c @@ -94,7 +94,7 @@ intel_iq80310_init(struct pci_dev *dev, struct map_pci_info *map) map->map.write = mtd_pci_write8, map->map.size = 0x00800000; - map->base = ioremap_nocache(pci_resource_start(dev, 0), + map->base = ioremap(pci_resource_start(dev, 0), pci_resource_len(dev, 0)); if (!map->base) @@ -188,7 +188,7 @@ intel_dc21285_init(struct pci_dev *dev, struct map_pci_info *map) map->map.read = mtd_pci_read32, map->map.write = mtd_pci_write32, map->map.size = len; - map->base = ioremap_nocache(base, len); + map->base = ioremap(base, len); if (!map->base) return -ENOMEM; diff --git a/drivers/mtd/maps/sc520cdp.c b/drivers/mtd/maps/sc520cdp.c index 03af2df90d47..9902b37e18b4 100644 --- a/drivers/mtd/maps/sc520cdp.c +++ b/drivers/mtd/maps/sc520cdp.c @@ -174,8 +174,8 @@ static void sc520cdp_setup_par(void) int i, j; /* map in SC520's MMCR area */ - mmcr = ioremap_nocache(SC520_MMCR_BASE, SC520_MMCR_EXTENT); - if(!mmcr) { /* ioremap_nocache failed: skip the PAR reprogramming */ + mmcr = ioremap(SC520_MMCR_BASE, SC520_MMCR_EXTENT); + if(!mmcr) { /* ioremap failed: skip the PAR reprogramming */ /* force physical address fields to BIOS defaults: */ for(i = 0; i < NUM_FLASH_BANKS; i++) sc520cdp_map[i].phys = par_table[i].default_address; @@ -225,10 +225,10 @@ static int __init init_sc520cdp(void) (unsigned long long)sc520cdp_map[i].size, (unsigned long long)sc520cdp_map[i].phys); - sc520cdp_map[i].virt = ioremap_nocache(sc520cdp_map[i].phys, sc520cdp_map[i].size); + sc520cdp_map[i].virt = ioremap(sc520cdp_map[i].phys, sc520cdp_map[i].size); if (!sc520cdp_map[i].virt) { - printk("Failed to ioremap_nocache\n"); + printk("Failed to ioremap\n"); for (j = 0; j < i; j++) { if (mymtd[j]) { map_destroy(mymtd[j]); diff --git a/drivers/mtd/maps/scb2_flash.c b/drivers/mtd/maps/scb2_flash.c index 2afb253bf456..57303f904bc1 100644 --- a/drivers/mtd/maps/scb2_flash.c +++ b/drivers/mtd/maps/scb2_flash.c @@ -152,7 +152,7 @@ static int scb2_flash_probe(struct pci_dev *dev, } /* remap the IO window (w/o caching) */ - scb2_ioaddr = ioremap_nocache(SCB2_ADDR, SCB2_WINDOW); + scb2_ioaddr = ioremap(SCB2_ADDR, SCB2_WINDOW); if (!scb2_ioaddr) { printk(KERN_ERR MODNAME ": Failed to ioremap window!\n"); if (!region_fail) diff --git a/drivers/mtd/maps/ts5500_flash.c b/drivers/mtd/maps/ts5500_flash.c index 6cfc8783c0e5..70d6e865f555 100644 --- a/drivers/mtd/maps/ts5500_flash.c +++ b/drivers/mtd/maps/ts5500_flash.c @@ -56,10 +56,10 @@ static int __init init_ts5500_map(void) { int rc = 0; - ts5500_map.virt = ioremap_nocache(ts5500_map.phys, ts5500_map.size); + ts5500_map.virt = ioremap(ts5500_map.phys, ts5500_map.size); if (!ts5500_map.virt) { - printk(KERN_ERR "Failed to ioremap_nocache\n"); + printk(KERN_ERR "Failed to ioremap\n"); rc = -EIO; goto err2; } diff --git a/drivers/mtd/nand/raw/au1550nd.c b/drivers/mtd/nand/raw/au1550nd.c index e10b76089048..75eb3e97fae3 100644 --- a/drivers/mtd/nand/raw/au1550nd.c +++ b/drivers/mtd/nand/raw/au1550nd.c @@ -404,7 +404,7 @@ static int au1550nd_probe(struct platform_device *pdev) goto out1; } - ctx->base = ioremap_nocache(r->start, 0x1000); + ctx->base = ioremap(r->start, 0x1000); if (!ctx->base) { dev_err(&pdev->dev, "cannot remap NAND memory area\n"); ret = -ENODEV; diff --git a/drivers/mtd/nand/raw/denali_pci.c b/drivers/mtd/nand/raw/denali_pci.c index d62aa5271753..2f77ee55e1bf 100644 --- a/drivers/mtd/nand/raw/denali_pci.c +++ b/drivers/mtd/nand/raw/denali_pci.c @@ -74,15 +74,15 @@ static int denali_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) return ret; } - denali->reg = ioremap_nocache(csr_base, csr_len); + denali->reg = ioremap(csr_base, csr_len); if (!denali->reg) { dev_err(&dev->dev, "Spectra: Unable to remap memory region\n"); return -ENOMEM; } - denali->host = ioremap_nocache(mem_base, mem_len); + denali->host = ioremap(mem_base, mem_len); if (!denali->host) { - dev_err(&dev->dev, "Spectra: ioremap_nocache failed!"); + dev_err(&dev->dev, "Spectra: ioremap failed!"); ret = -ENOMEM; goto out_unmap_reg; } diff --git a/drivers/mtd/nand/raw/fsl_upm.c b/drivers/mtd/nand/raw/fsl_upm.c index 1054cc070747..f31fae3a4c68 100644 --- a/drivers/mtd/nand/raw/fsl_upm.c +++ b/drivers/mtd/nand/raw/fsl_upm.c @@ -285,7 +285,7 @@ static int fun_probe(struct platform_device *ofdev) fun->wait_flags = FSL_UPM_WAIT_RUN_PATTERN | FSL_UPM_WAIT_WRITE_BYTE; - fun->io_base = devm_ioremap_nocache(&ofdev->dev, io_res.start, + fun->io_base = devm_ioremap(&ofdev->dev, io_res.start, resource_size(&io_res)); if (!fun->io_base) { ret = -ENOMEM; diff --git a/drivers/net/can/at91_can.c b/drivers/net/can/at91_can.c index c8e1a04ba384..9df2007b5e56 100644 --- a/drivers/net/can/at91_can.c +++ b/drivers/net/can/at91_can.c @@ -1302,7 +1302,7 @@ static int at91_can_probe(struct platform_device *pdev) goto exit_put; } - addr = ioremap_nocache(res->start, resource_size(res)); + addr = ioremap(res->start, resource_size(res)); if (!addr) { err = -ENOMEM; goto exit_release; diff --git a/drivers/net/can/cc770/cc770_isa.c b/drivers/net/can/cc770/cc770_isa.c index b9047d8110d5..194c86e0f340 100644 --- a/drivers/net/can/cc770/cc770_isa.c +++ b/drivers/net/can/cc770/cc770_isa.c @@ -175,7 +175,7 @@ static int cc770_isa_probe(struct platform_device *pdev) err = -EBUSY; goto exit; } - base = ioremap_nocache(mem[idx], iosize); + base = ioremap(mem[idx], iosize); if (!base) { err = -ENOMEM; goto exit_release; diff --git a/drivers/net/can/sja1000/sja1000_isa.c b/drivers/net/can/sja1000/sja1000_isa.c index 1c4d32d1a542..d513fac50718 100644 --- a/drivers/net/can/sja1000/sja1000_isa.c +++ b/drivers/net/can/sja1000/sja1000_isa.c @@ -130,7 +130,7 @@ static int sja1000_isa_probe(struct platform_device *pdev) err = -EBUSY; goto exit; } - base = ioremap_nocache(mem[idx], iosize); + base = ioremap(mem[idx], iosize); if (!base) { err = -ENOMEM; goto exit_release; diff --git a/drivers/net/can/sja1000/sja1000_platform.c b/drivers/net/can/sja1000/sja1000_platform.c index ff5a96f34085..d7222ba46622 100644 --- a/drivers/net/can/sja1000/sja1000_platform.c +++ b/drivers/net/can/sja1000/sja1000_platform.c @@ -229,7 +229,7 @@ static int sp_probe(struct platform_device *pdev) resource_size(res_mem), DRV_NAME)) return -EBUSY; - addr = devm_ioremap_nocache(&pdev->dev, res_mem->start, + addr = devm_ioremap(&pdev->dev, res_mem->start, resource_size(res_mem)); if (!addr) return -ENOMEM; diff --git a/drivers/net/can/softing/softing_main.c b/drivers/net/can/softing/softing_main.c index 8242fb287cbb..d1ddf763b188 100644 --- a/drivers/net/can/softing/softing_main.c +++ b/drivers/net/can/softing/softing_main.c @@ -777,7 +777,7 @@ static int softing_pdev_probe(struct platform_device *pdev) goto platform_resource_failed; card->dpram_phys = pres->start; card->dpram_size = resource_size(pres); - card->dpram = ioremap_nocache(card->dpram_phys, card->dpram_size); + card->dpram = ioremap(card->dpram_phys, card->dpram_size); if (!card->dpram) { dev_alert(&card->pdev->dev, "dpram ioremap failed\n"); goto ioremap_failed; diff --git a/drivers/net/ethernet/alacritech/slicoss.c b/drivers/net/ethernet/alacritech/slicoss.c index 80ef3e15bd22..9daef4c8feef 100644 --- a/drivers/net/ethernet/alacritech/slicoss.c +++ b/drivers/net/ethernet/alacritech/slicoss.c @@ -1791,7 +1791,7 @@ static int slic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) sdev->is_fiber = slic_is_fiber(pdev->subsystem_device); sdev->pdev = pdev; sdev->netdev = dev; - sdev->regs = ioremap_nocache(pci_resource_start(pdev, 0), + sdev->regs = ioremap(pci_resource_start(pdev, 0), pci_resource_len(pdev, 0)); if (!sdev->regs) { dev_err(&pdev->dev, "failed to map registers\n"); diff --git a/drivers/net/ethernet/altera/altera_tse_main.c b/drivers/net/ethernet/altera/altera_tse_main.c index 4cd53fc338b5..1671c1f36691 100644 --- a/drivers/net/ethernet/altera/altera_tse_main.c +++ b/drivers/net/ethernet/altera/altera_tse_main.c @@ -1332,10 +1332,10 @@ static int request_and_map(struct platform_device *pdev, const char *name, return -EBUSY; } - *ptr = devm_ioremap_nocache(device, region->start, + *ptr = devm_ioremap(device, region->start, resource_size(region)); if (*ptr == NULL) { - dev_err(device, "ioremap_nocache of %s failed!", name); + dev_err(device, "ioremap of %s failed!", name); return -ENOMEM; } diff --git a/drivers/net/ethernet/amd/au1000_eth.c b/drivers/net/ethernet/amd/au1000_eth.c index 1793950f0582..307e402db8c9 100644 --- a/drivers/net/ethernet/amd/au1000_eth.c +++ b/drivers/net/ethernet/amd/au1000_eth.c @@ -1161,7 +1161,7 @@ static int au1000_probe(struct platform_device *pdev) /* aup->mac is the base address of the MAC's registers */ aup->mac = (struct mac_reg *) - ioremap_nocache(base->start, resource_size(base)); + ioremap(base->start, resource_size(base)); if (!aup->mac) { dev_err(&pdev->dev, "failed to ioremap MAC registers\n"); err = -ENXIO; @@ -1169,7 +1169,7 @@ static int au1000_probe(struct platform_device *pdev) } /* Setup some variables for quick register address access */ - aup->enable = (u32 *)ioremap_nocache(macen->start, + aup->enable = (u32 *)ioremap(macen->start, resource_size(macen)); if (!aup->enable) { dev_err(&pdev->dev, "failed to ioremap MAC enable register\n"); @@ -1178,7 +1178,7 @@ static int au1000_probe(struct platform_device *pdev) } aup->mac_id = pdev->id; - aup->macdma = ioremap_nocache(macdma->start, resource_size(macdma)); + aup->macdma = ioremap(macdma->start, resource_size(macdma)); if (!aup->macdma) { dev_err(&pdev->dev, "failed to ioremap MACDMA registers\n"); err = -ENXIO; diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c b/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c index 2bb329606794..6b27af0db499 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c @@ -253,7 +253,7 @@ static int aq_pci_probe(struct pci_dev *pdev, goto err_free_aq_hw; } - self->aq_hw->mmio = ioremap_nocache(mmio_pa, reg_sz); + self->aq_hw->mmio = ioremap(mmio_pa, reg_sz); if (!self->aq_hw->mmio) { err = -EIO; goto err_free_aq_hw; diff --git a/drivers/net/ethernet/atheros/ag71xx.c b/drivers/net/ethernet/atheros/ag71xx.c index 61a334d1b5e6..60ba69db48c6 100644 --- a/drivers/net/ethernet/atheros/ag71xx.c +++ b/drivers/net/ethernet/atheros/ag71xx.c @@ -1687,8 +1687,7 @@ static int ag71xx_probe(struct platform_device *pdev) goto err_free; } - ag->mac_base = devm_ioremap_nocache(&pdev->dev, res->start, - resource_size(res)); + ag->mac_base = devm_ioremap(&pdev->dev, res->start, resource_size(res)); if (!ag->mac_base) { err = -ENOMEM; goto err_free; diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c index cff64e43bdd8..61fa32cdd3e3 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c @@ -14053,7 +14053,7 @@ static int bnx2x_init_one(struct pci_dev *pdev, rc = -ENOMEM; goto init_one_freemem; } - bp->doorbells = ioremap_nocache(pci_resource_start(pdev, 2), + bp->doorbells = ioremap(pci_resource_start(pdev, 2), doorbell_size); } if (!bp->doorbells) { diff --git a/drivers/net/ethernet/broadcom/sb1250-mac.c b/drivers/net/ethernet/broadcom/sb1250-mac.c index 1604ad32e920..f991537818fe 100644 --- a/drivers/net/ethernet/broadcom/sb1250-mac.c +++ b/drivers/net/ethernet/broadcom/sb1250-mac.c @@ -2537,7 +2537,7 @@ static int sbmac_probe(struct platform_device *pldev) res = platform_get_resource(pldev, IORESOURCE_MEM, 0); BUG_ON(!res); - sbm_base = ioremap_nocache(res->start, resource_size(res)); + sbm_base = ioremap(res->start, resource_size(res)); if (!sbm_base) { printk(KERN_ERR "%s: unable to map device registers\n", dev_name(&pldev->dev)); diff --git a/drivers/net/ethernet/brocade/bna/bnad.c b/drivers/net/ethernet/brocade/bna/bnad.c index e338272931d1..01a50a4b2113 100644 --- a/drivers/net/ethernet/brocade/bna/bnad.c +++ b/drivers/net/ethernet/brocade/bna/bnad.c @@ -3477,7 +3477,7 @@ bnad_init(struct bnad *bnad, bnad->pcidev = pdev; bnad->mmio_start = pci_resource_start(pdev, 0); bnad->mmio_len = pci_resource_len(pdev, 0); - bnad->bar0 = ioremap_nocache(bnad->mmio_start, bnad->mmio_len); + bnad->bar0 = ioremap(bnad->mmio_start, bnad->mmio_len); if (!bnad->bar0) { dev_err(&pdev->dev, "ioremap for bar0 failed\n"); return -ENOMEM; diff --git a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c index 58f89f6a040f..2824b5ed4ced 100644 --- a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c +++ b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c @@ -3265,7 +3265,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_free_adapter; } - adapter->regs = ioremap_nocache(mmio_start, mmio_len); + adapter->regs = ioremap(mmio_start, mmio_len); if (!adapter->regs) { dev_err(&pdev->dev, "cannot map device registers\n"); err = -ENOMEM; diff --git a/drivers/net/ethernet/dec/tulip/de2104x.c b/drivers/net/ethernet/dec/tulip/de2104x.c index f1a2da15dd0a..7852a4308194 100644 --- a/drivers/net/ethernet/dec/tulip/de2104x.c +++ b/drivers/net/ethernet/dec/tulip/de2104x.c @@ -2039,7 +2039,7 @@ static int de_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) } /* remap CSR registers */ - regs = ioremap_nocache(pciaddr, DE_REGS_SIZE); + regs = ioremap(pciaddr, DE_REGS_SIZE); if (!regs) { rc = -EIO; pr_err("Cannot map PCI MMIO (%llx@%lx) on pci dev %s\n", diff --git a/drivers/net/ethernet/ethoc.c b/drivers/net/ethernet/ethoc.c index ea4f17f5cce7..c6e74ae0ff0d 100644 --- a/drivers/net/ethernet/ethoc.c +++ b/drivers/net/ethernet/ethoc.c @@ -1087,7 +1087,7 @@ static int ethoc_probe(struct platform_device *pdev) priv = netdev_priv(netdev); priv->netdev = netdev; - priv->iobase = devm_ioremap_nocache(&pdev->dev, netdev->base_addr, + priv->iobase = devm_ioremap(&pdev->dev, netdev->base_addr, resource_size(mmio)); if (!priv->iobase) { dev_err(&pdev->dev, "cannot remap I/O memory space\n"); @@ -1096,7 +1096,7 @@ static int ethoc_probe(struct platform_device *pdev) } if (netdev->mem_end) { - priv->membase = devm_ioremap_nocache(&pdev->dev, + priv->membase = devm_ioremap(&pdev->dev, netdev->mem_start, resource_size(mem)); if (!priv->membase) { dev_err(&pdev->dev, "cannot remap memory space\n"); diff --git a/drivers/net/ethernet/i825xx/sni_82596.c b/drivers/net/ethernet/i825xx/sni_82596.c index 6436a98c5953..22f5887578b2 100644 --- a/drivers/net/ethernet/i825xx/sni_82596.c +++ b/drivers/net/ethernet/i825xx/sni_82596.c @@ -91,10 +91,10 @@ static int sni_82596_probe(struct platform_device *dev) idprom = platform_get_resource(dev, IORESOURCE_MEM, 2); if (!res || !ca || !options || !idprom) return -ENODEV; - mpu_addr = ioremap_nocache(res->start, 4); + mpu_addr = ioremap(res->start, 4); if (!mpu_addr) return -ENOMEM; - ca_addr = ioremap_nocache(ca->start, 4); + ca_addr = ioremap(ca->start, 4); if (!ca_addr) goto probe_failed_free_mpu; @@ -110,7 +110,7 @@ static int sni_82596_probe(struct platform_device *dev) netdevice->base_addr = res->start; netdevice->irq = platform_get_irq(dev, 0); - eth_addr = ioremap_nocache(idprom->start, 0x10); + eth_addr = ioremap(idprom->start, 0x10); if (!eth_addr) goto probe_failed; diff --git a/drivers/net/ethernet/korina.c b/drivers/net/ethernet/korina.c index ae195f8adff5..d3164537b694 100644 --- a/drivers/net/ethernet/korina.c +++ b/drivers/net/ethernet/korina.c @@ -1043,7 +1043,7 @@ static int korina_probe(struct platform_device *pdev) r = platform_get_resource_byname(pdev, IORESOURCE_MEM, "korina_regs"); dev->base_addr = r->start; - lp->eth_regs = ioremap_nocache(r->start, resource_size(r)); + lp->eth_regs = ioremap(r->start, resource_size(r)); if (!lp->eth_regs) { printk(KERN_ERR DRV_NAME ": cannot remap registers\n"); rc = -ENXIO; @@ -1051,7 +1051,7 @@ static int korina_probe(struct platform_device *pdev) } r = platform_get_resource_byname(pdev, IORESOURCE_MEM, "korina_dma_rx"); - lp->rx_dma_regs = ioremap_nocache(r->start, resource_size(r)); + lp->rx_dma_regs = ioremap(r->start, resource_size(r)); if (!lp->rx_dma_regs) { printk(KERN_ERR DRV_NAME ": cannot remap Rx DMA registers\n"); rc = -ENXIO; @@ -1059,7 +1059,7 @@ static int korina_probe(struct platform_device *pdev) } r = platform_get_resource_byname(pdev, IORESOURCE_MEM, "korina_dma_tx"); - lp->tx_dma_regs = ioremap_nocache(r->start, resource_size(r)); + lp->tx_dma_regs = ioremap(r->start, resource_size(r)); if (!lp->tx_dma_regs) { printk(KERN_ERR DRV_NAME ": cannot remap Tx DMA registers\n"); rc = -ENXIO; diff --git a/drivers/net/ethernet/lantiq_etop.c b/drivers/net/ethernet/lantiq_etop.c index 6e73ffe6f928..41f2f5480741 100644 --- a/drivers/net/ethernet/lantiq_etop.c +++ b/drivers/net/ethernet/lantiq_etop.c @@ -649,7 +649,7 @@ ltq_etop_probe(struct platform_device *pdev) goto err_out; } - ltq_etop_membase = devm_ioremap_nocache(&pdev->dev, + ltq_etop_membase = devm_ioremap(&pdev->dev, res->start, resource_size(res)); if (!ltq_etop_membase) { dev_err(&pdev->dev, "failed to remap etop engine %d\n", diff --git a/drivers/net/ethernet/marvell/skge.c b/drivers/net/ethernet/marvell/skge.c index 095f6c71b4fa..7515d079c600 100644 --- a/drivers/net/ethernet/marvell/skge.c +++ b/drivers/net/ethernet/marvell/skge.c @@ -3932,7 +3932,7 @@ static int skge_probe(struct pci_dev *pdev, const struct pci_device_id *ent) spin_lock_init(&hw->phy_lock); tasklet_init(&hw->phy_task, skge_extirq, (unsigned long) hw); - hw->regs = ioremap_nocache(pci_resource_start(pdev, 0), 0x4000); + hw->regs = ioremap(pci_resource_start(pdev, 0), 0x4000); if (!hw->regs) { dev_err(&pdev->dev, "cannot map device registers\n"); goto err_out_free_hw; diff --git a/drivers/net/ethernet/marvell/sky2.c b/drivers/net/ethernet/marvell/sky2.c index 5f56ee83e3b1..535dee35e04e 100644 --- a/drivers/net/ethernet/marvell/sky2.c +++ b/drivers/net/ethernet/marvell/sky2.c @@ -5022,7 +5022,7 @@ static int sky2_probe(struct pci_dev *pdev, const struct pci_device_id *ent) hw->pdev = pdev; sprintf(hw->irq_name, DRV_NAME "@pci:%s", pci_name(pdev)); - hw->regs = ioremap_nocache(pci_resource_start(pdev, 0), 0x4000); + hw->regs = ioremap(pci_resource_start(pdev, 0), 0x4000); if (!hw->regs) { dev_err(&pdev->dev, "cannot map device registers\n"); goto err_out_free_hw; diff --git a/drivers/net/ethernet/natsemi/ns83820.c b/drivers/net/ethernet/natsemi/ns83820.c index 6af9a7eee114..091254061052 100644 --- a/drivers/net/ethernet/natsemi/ns83820.c +++ b/drivers/net/ethernet/natsemi/ns83820.c @@ -1937,7 +1937,7 @@ static int ns83820_init_one(struct pci_dev *pci_dev, pci_set_master(pci_dev); addr = pci_resource_start(pci_dev, 1); - dev->base = ioremap_nocache(addr, PAGE_SIZE); + dev->base = ioremap(addr, PAGE_SIZE); dev->tx_descs = pci_alloc_consistent(pci_dev, 4 * DESC_SIZE * NR_TX_DESC, &dev->tx_phy_descs); dev->rx_info.descs = pci_alloc_consistent(pci_dev, diff --git a/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c b/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c index e4977cdf7678..c0e2f4394aef 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c @@ -106,7 +106,7 @@ static int nfp_netvf_pci_probe(struct pci_dev *pdev, * first NFP_NET_CFG_BAR_SZ of the BAR. This keeps the code * the identical for PF and VF drivers. */ - ctrl_bar = ioremap_nocache(pci_resource_start(pdev, NFP_NET_CTRL_BAR), + ctrl_bar = ioremap(pci_resource_start(pdev, NFP_NET_CTRL_BAR), NFP_NET_CFG_BAR_SZ); if (!ctrl_bar) { dev_err(&pdev->dev, @@ -200,7 +200,7 @@ static int nfp_netvf_pci_probe(struct pci_dev *pdev, bar_sz = (rx_bar_off + rx_bar_sz) - bar_off; map_addr = pci_resource_start(pdev, tx_bar_no) + bar_off; - vf->q_bar = ioremap_nocache(map_addr, bar_sz); + vf->q_bar = ioremap(map_addr, bar_sz); if (!vf->q_bar) { nn_err(nn, "Failed to map resource %d\n", tx_bar_no); err = -EIO; @@ -216,7 +216,7 @@ static int nfp_netvf_pci_probe(struct pci_dev *pdev, /* TX queues */ map_addr = pci_resource_start(pdev, tx_bar_no) + tx_bar_off; - nn->tx_bar = ioremap_nocache(map_addr, tx_bar_sz); + nn->tx_bar = ioremap(map_addr, tx_bar_sz); if (!nn->tx_bar) { nn_err(nn, "Failed to map resource %d\n", tx_bar_no); err = -EIO; @@ -225,7 +225,7 @@ static int nfp_netvf_pci_probe(struct pci_dev *pdev, /* RX queues */ map_addr = pci_resource_start(pdev, rx_bar_no) + rx_bar_off; - nn->rx_bar = ioremap_nocache(map_addr, rx_bar_sz); + nn->rx_bar = ioremap(map_addr, rx_bar_sz); if (!nn->rx_bar) { nn_err(nn, "Failed to map resource %d\n", rx_bar_no); err = -EIO; diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp6000_pcie.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp6000_pcie.c index 85d46f206b3c..b454db283aef 100644 --- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp6000_pcie.c +++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp6000_pcie.c @@ -611,7 +611,7 @@ static int enable_bars(struct nfp6000_pcie *nfp, u16 interface) /* Configure, and lock, BAR0.0 for General Target use (MSI-X SRAM) */ bar = &nfp->bar[0]; if (nfp_bar_resource_len(bar) >= NFP_PCI_MIN_MAP_SIZE) - bar->iomem = ioremap_nocache(nfp_bar_resource_start(bar), + bar->iomem = ioremap(nfp_bar_resource_start(bar), nfp_bar_resource_len(bar)); if (bar->iomem) { int pf; @@ -677,7 +677,7 @@ static int enable_bars(struct nfp6000_pcie *nfp, u16 interface) } bar = &nfp->bar[4 + i]; - bar->iomem = ioremap_nocache(nfp_bar_resource_start(bar), + bar->iomem = ioremap(nfp_bar_resource_start(bar), nfp_bar_resource_len(bar)); if (bar->iomem) { msg += snprintf(msg, end - msg, @@ -858,7 +858,7 @@ static int nfp6000_area_acquire(struct nfp_cpp_area *area) priv->iomem = priv->bar->iomem + priv->bar_offset; else /* Must have been too big. Sub-allocate. */ - priv->iomem = ioremap_nocache(priv->phys, priv->size); + priv->iomem = ioremap(priv->phys, priv->size); if (IS_ERR_OR_NULL(priv->iomem)) { dev_err(nfp->dev, "Can't ioremap() a %d byte region of BAR %d\n", diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c index 4d9bbccc6f89..a6ae2cdc1986 100644 --- a/drivers/net/ethernet/sfc/ef10.c +++ b/drivers/net/ethernet/sfc/ef10.c @@ -1401,7 +1401,7 @@ static int efx_ef10_dimension_resources(struct efx_nic *efx) } /* Shrink the original UC mapping of the memory BAR */ - membase = ioremap_nocache(efx->membase_phys, uc_mem_map_size); + membase = ioremap(efx->membase_phys, uc_mem_map_size); if (!membase) { netif_err(efx, probe, efx->net_dev, "could not shrink memory BAR to %x\n", diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c index 7a38d7f282a1..6891df471538 100644 --- a/drivers/net/ethernet/sfc/efx.c +++ b/drivers/net/ethernet/sfc/efx.c @@ -1338,7 +1338,7 @@ static int efx_init_io(struct efx_nic *efx) rc = -EIO; goto fail3; } - efx->membase = ioremap_nocache(efx->membase_phys, mem_map_size); + efx->membase = ioremap(efx->membase_phys, mem_map_size); if (!efx->membase) { netif_err(efx, probe, efx->net_dev, "could not map memory BAR at %llx+%x\n", diff --git a/drivers/net/ethernet/sfc/falcon/efx.c b/drivers/net/ethernet/sfc/falcon/efx.c index eecc348b1c32..53ae9faeb4c3 100644 --- a/drivers/net/ethernet/sfc/falcon/efx.c +++ b/drivers/net/ethernet/sfc/falcon/efx.c @@ -1265,7 +1265,7 @@ static int ef4_init_io(struct ef4_nic *efx) rc = -EIO; goto fail3; } - efx->membase = ioremap_nocache(efx->membase_phys, mem_map_size); + efx->membase = ioremap(efx->membase_phys, mem_map_size); if (!efx->membase) { netif_err(efx, probe, efx->net_dev, "could not map memory BAR at %llx+%x\n", diff --git a/drivers/net/ethernet/smsc/smsc911x.c b/drivers/net/ethernet/smsc/smsc911x.c index 38068fc34141..6d90a097ce4e 100644 --- a/drivers/net/ethernet/smsc/smsc911x.c +++ b/drivers/net/ethernet/smsc/smsc911x.c @@ -2454,7 +2454,7 @@ static int smsc911x_drv_probe(struct platform_device *pdev) pdata = netdev_priv(dev); dev->irq = irq; - pdata->ioaddr = ioremap_nocache(res->start, res_size); + pdata->ioaddr = ioremap(res->start, res_size); if (!pdata->ioaddr) { retval = -ENOMEM; goto out_ioremap_fail; diff --git a/drivers/net/ethernet/ti/netcp_core.c b/drivers/net/ethernet/ti/netcp_core.c index 1b2702f74455..675f31de59dd 100644 --- a/drivers/net/ethernet/ti/netcp_core.c +++ b/drivers/net/ethernet/ti/netcp_core.c @@ -2019,7 +2019,7 @@ static int netcp_create_interface(struct netcp_device *netcp_device, goto quit; } - efuse = devm_ioremap_nocache(dev, res.start, size); + efuse = devm_ioremap(dev, res.start, size); if (!efuse) { dev_err(dev, "could not map resource\n"); devm_release_mem_region(dev, res.start, size); diff --git a/drivers/net/ethernet/xilinx/ll_temac_main.c b/drivers/net/ethernet/xilinx/ll_temac_main.c index 21c1b4322ea7..c66aab78dcac 100644 --- a/drivers/net/ethernet/xilinx/ll_temac_main.c +++ b/drivers/net/ethernet/xilinx/ll_temac_main.c @@ -1202,7 +1202,7 @@ static int temac_probe(struct platform_device *pdev) /* map device registers */ res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - lp->regs = devm_ioremap_nocache(&pdev->dev, res->start, + lp->regs = devm_ioremap(&pdev->dev, res->start, resource_size(res)); if (IS_ERR(lp->regs)) { dev_err(&pdev->dev, "could not map TEMAC registers\n"); @@ -1296,7 +1296,7 @@ static int temac_probe(struct platform_device *pdev) } else if (pdata) { /* 2nd memory resource specifies DMA registers */ res = platform_get_resource(pdev, IORESOURCE_MEM, 1); - lp->sdma_regs = devm_ioremap_nocache(&pdev->dev, res->start, + lp->sdma_regs = devm_ioremap(&pdev->dev, res->start, resource_size(res)); if (IS_ERR(lp->sdma_regs)) { dev_err(&pdev->dev, diff --git a/drivers/net/fddi/defxx.c b/drivers/net/fddi/defxx.c index 56b7791911bf..077c68498f04 100644 --- a/drivers/net/fddi/defxx.c +++ b/drivers/net/fddi/defxx.c @@ -614,7 +614,7 @@ static int dfx_register(struct device *bdev) /* Set up I/O base address. */ if (dfx_use_mmio) { - bp->base.mem = ioremap_nocache(bar_start[0], bar_len[0]); + bp->base.mem = ioremap(bar_start[0], bar_len[0]); if (!bp->base.mem) { printk(KERN_ERR "%s: Cannot map MMIO\n", print_name); err = -ENOMEM; diff --git a/drivers/net/fddi/defza.c b/drivers/net/fddi/defza.c index 060712c666bf..eaf85db53a5e 100644 --- a/drivers/net/fddi/defza.c +++ b/drivers/net/fddi/defza.c @@ -1318,7 +1318,7 @@ static int fza_probe(struct device *bdev) } /* MMIO mapping setup. */ - mmio = ioremap_nocache(start, len); + mmio = ioremap(start, len); if (!mmio) { pr_err("%s: cannot map MMIO\n", fp->name); ret = -ENOMEM; diff --git a/drivers/net/fjes/fjes_hw.c b/drivers/net/fjes/fjes_hw.c index 8a4fbfacad7e..065bb0a40b1d 100644 --- a/drivers/net/fjes/fjes_hw.c +++ b/drivers/net/fjes/fjes_hw.c @@ -40,7 +40,7 @@ static u8 *fjes_hw_iomap(struct fjes_hw *hw) return NULL; } - base = (u8 *)ioremap_nocache(hw->hw_res.start, hw->hw_res.size); + base = (u8 *)ioremap(hw->hw_res.start, hw->hw_res.size); return base; } diff --git a/drivers/net/wan/wanxl.c b/drivers/net/wan/wanxl.c index 34e94ee806d6..23f93f1c815d 100644 --- a/drivers/net/wan/wanxl.c +++ b/drivers/net/wan/wanxl.c @@ -635,7 +635,7 @@ static int wanxl_pci_init_one(struct pci_dev *pdev, /* set up PLX mapping */ plx_phy = pci_resource_start(pdev, 0); - card->plx = ioremap_nocache(plx_phy, 0x70); + card->plx = ioremap(plx_phy, 0x70); if (!card->plx) { pr_err("ioremap() failed\n"); wanxl_pci_remove_one(pdev); @@ -704,7 +704,7 @@ static int wanxl_pci_init_one(struct pci_dev *pdev, PCI_DMA_FROMDEVICE); } - mem = ioremap_nocache(mem_phy, PDM_OFFSET + sizeof(firmware)); + mem = ioremap(mem_phy, PDM_OFFSET + sizeof(firmware)); if (!mem) { pr_err("ioremap() failed\n"); wanxl_pci_remove_one(pdev); diff --git a/drivers/net/wireless/ath/ath10k/ahb.c b/drivers/net/wireless/ath/ath10k/ahb.c index f80854180e21..ed87bc00f2aa 100644 --- a/drivers/net/wireless/ath/ath10k/ahb.c +++ b/drivers/net/wireless/ath/ath10k/ahb.c @@ -458,7 +458,7 @@ static int ath10k_ahb_resource_init(struct ath10k *ar) ar_ahb->mem_len = resource_size(res); - ar_ahb->gcc_mem = ioremap_nocache(ATH10K_GCC_REG_BASE, + ar_ahb->gcc_mem = ioremap(ATH10K_GCC_REG_BASE, ATH10K_GCC_REG_SIZE); if (!ar_ahb->gcc_mem) { ath10k_err(ar, "gcc mem ioremap error\n"); @@ -466,7 +466,7 @@ static int ath10k_ahb_resource_init(struct ath10k *ar) goto err_mem_unmap; } - ar_ahb->tcsr_mem = ioremap_nocache(ATH10K_TCSR_REG_BASE, + ar_ahb->tcsr_mem = ioremap(ATH10K_TCSR_REG_BASE, ATH10K_TCSR_REG_SIZE); if (!ar_ahb->tcsr_mem) { ath10k_err(ar, "tcsr mem ioremap error\n"); diff --git a/drivers/net/wireless/ath/ath5k/ahb.c b/drivers/net/wireless/ath/ath5k/ahb.c index c0794f5988b3..2c9cec8b53d9 100644 --- a/drivers/net/wireless/ath/ath5k/ahb.c +++ b/drivers/net/wireless/ath/ath5k/ahb.c @@ -106,7 +106,7 @@ static int ath_ahb_probe(struct platform_device *pdev) goto err_out; } - mem = ioremap_nocache(res->start, resource_size(res)); + mem = ioremap(res->start, resource_size(res)); if (mem == NULL) { dev_err(&pdev->dev, "ioremap failed\n"); ret = -ENOMEM; diff --git a/drivers/net/wireless/ath/ath9k/ahb.c b/drivers/net/wireless/ath/ath9k/ahb.c index 63019c3de034..cdefb8e2daf1 100644 --- a/drivers/net/wireless/ath/ath9k/ahb.c +++ b/drivers/net/wireless/ath/ath9k/ahb.c @@ -92,7 +92,7 @@ static int ath_ahb_probe(struct platform_device *pdev) return -ENXIO; } - mem = devm_ioremap_nocache(&pdev->dev, res->start, resource_size(res)); + mem = devm_ioremap(&pdev->dev, res->start, resource_size(res)); if (mem == NULL) { dev_err(&pdev->dev, "ioremap failed\n"); return -ENOMEM; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index f64ce5074a55..c85840cabebe 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -1643,8 +1643,8 @@ static int brcmf_pcie_get_resource(struct brcmf_pciedev_info *devinfo) return -EINVAL; } - devinfo->regs = ioremap_nocache(bar0_addr, BRCMF_PCIE_REG_MAP_SIZE); - devinfo->tcm = ioremap_nocache(bar1_addr, bar1_size); + devinfo->regs = ioremap(bar0_addr, BRCMF_PCIE_REG_MAP_SIZE); + devinfo->tcm = ioremap(bar1_addr, bar1_size); if (!devinfo->regs || !devinfo->tcm) { brcmf_err(bus, "ioremap() failed (%p,%p)\n", devinfo->regs, diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2100.c b/drivers/net/wireless/intel/ipw2x00/ipw2100.c index c4c83ab60cbc..e85858eec8ff 100644 --- a/drivers/net/wireless/intel/ipw2x00/ipw2100.c +++ b/drivers/net/wireless/intel/ipw2x00/ipw2100.c @@ -6167,7 +6167,7 @@ static int ipw2100_pci_init_one(struct pci_dev *pci_dev, ioaddr = pci_iomap(pci_dev, 0, 0); if (!ioaddr) { printk(KERN_WARNING DRV_NAME - "Error calling ioremap_nocache.\n"); + "Error calling ioremap.\n"); err = -EIO; goto fail; } diff --git a/drivers/opp/ti-opp-supply.c b/drivers/opp/ti-opp-supply.c index 1c69c404df11..e3357e91decb 100644 --- a/drivers/opp/ti-opp-supply.c +++ b/drivers/opp/ti-opp-supply.c @@ -90,7 +90,7 @@ static int _store_optimized_voltages(struct device *dev, goto out_map; } - base = ioremap_nocache(res->start, resource_size(res)); + base = ioremap(res->start, resource_size(res)); if (!base) { dev_err(dev, "Unable to map Efuse registers\n"); ret = -ENOMEM; diff --git a/drivers/parisc/ccio-dma.c b/drivers/parisc/ccio-dma.c index ad290f79983b..a5507f75b524 100644 --- a/drivers/parisc/ccio-dma.c +++ b/drivers/parisc/ccio-dma.c @@ -1534,7 +1534,7 @@ static int __init ccio_probe(struct parisc_device *dev) *ioc_p = ioc; ioc->hw_path = dev->hw_path; - ioc->ioc_regs = ioremap_nocache(dev->hpa.start, 4096); + ioc->ioc_regs = ioremap(dev->hpa.start, 4096); if (!ioc->ioc_regs) { kfree(ioc); return -ENOMEM; diff --git a/drivers/parisc/dino.c b/drivers/parisc/dino.c index 2f1cac89ddf5..889d7ce282eb 100644 --- a/drivers/parisc/dino.c +++ b/drivers/parisc/dino.c @@ -974,7 +974,7 @@ static int __init dino_probe(struct parisc_device *dev) } dino_dev->hba.dev = dev; - dino_dev->hba.base_addr = ioremap_nocache(hpa, 4096); + dino_dev->hba.base_addr = ioremap(hpa, 4096); dino_dev->hba.lmmio_space_offset = PCI_F_EXTEND; spin_lock_init(&dino_dev->dinosaur_pen); dino_dev->hba.iommu = ccio_get_iommu(dev); diff --git a/drivers/parisc/eisa.c b/drivers/parisc/eisa.c index 37a2c5db761d..9d00a24277aa 100644 --- a/drivers/parisc/eisa.c +++ b/drivers/parisc/eisa.c @@ -354,10 +354,10 @@ static int __init eisa_probe(struct parisc_device *dev) eisa_dev.eeprom_addr = MIRAGE_EEPROM_BASE_ADDR; } } - eisa_eeprom_addr = ioremap_nocache(eisa_dev.eeprom_addr, HPEE_MAX_LENGTH); + eisa_eeprom_addr = ioremap(eisa_dev.eeprom_addr, HPEE_MAX_LENGTH); if (!eisa_eeprom_addr) { result = -ENOMEM; - printk(KERN_ERR "EISA: ioremap_nocache failed!\n"); + printk(KERN_ERR "EISA: ioremap failed!\n"); goto error_free_irq; } result = eisa_enumerator(eisa_dev.eeprom_addr, &eisa_dev.hba.io_space, diff --git a/drivers/parisc/iosapic.c b/drivers/parisc/iosapic.c index 32f506f00c89..8a3b0c3a1e92 100644 --- a/drivers/parisc/iosapic.c +++ b/drivers/parisc/iosapic.c @@ -927,7 +927,7 @@ void *iosapic_register(unsigned long hpa) return NULL; } - isi->addr = ioremap_nocache(hpa, 4096); + isi->addr = ioremap(hpa, 4096); isi->isi_hpa = hpa; isi->isi_version = iosapic_rd_version(isi); isi->isi_num_vectors = IOSAPIC_IRDT_MAX_ENTRY(isi->isi_version) + 1; diff --git a/drivers/parisc/lba_pci.c b/drivers/parisc/lba_pci.c index a99e385c68bd..732b516c7bf8 100644 --- a/drivers/parisc/lba_pci.c +++ b/drivers/parisc/lba_pci.c @@ -1134,7 +1134,7 @@ lba_pat_resources(struct parisc_device *pa_dev, struct lba_device *lba_dev) ** Postable I/O port space is per PCI host adapter. ** base of 64MB PIOP region */ - lba_dev->iop_base = ioremap_nocache(p->start, 64 * 1024 * 1024); + lba_dev->iop_base = ioremap(p->start, 64 * 1024 * 1024); sprintf(lba_dev->hba.io_name, "PCI%02x Ports", (int)lba_dev->hba.bus_num.start); @@ -1476,7 +1476,7 @@ lba_driver_probe(struct parisc_device *dev) u32 func_class; void *tmp_obj; char *version; - void __iomem *addr = ioremap_nocache(dev->hpa.start, 4096); + void __iomem *addr = ioremap(dev->hpa.start, 4096); int max; /* Read HW Rev First */ @@ -1575,7 +1575,7 @@ lba_driver_probe(struct parisc_device *dev) } else { if (!astro_iop_base) { /* Sprockets PDC uses NPIOP region */ - astro_iop_base = ioremap_nocache(LBA_PORT_BASE, 64 * 1024); + astro_iop_base = ioremap(LBA_PORT_BASE, 64 * 1024); pci_port = &lba_astro_port_ops; } @@ -1693,7 +1693,7 @@ void __init lba_init(void) */ void lba_set_iregs(struct parisc_device *lba, u32 ibase, u32 imask) { - void __iomem * base_addr = ioremap_nocache(lba->hpa.start, 4096); + void __iomem * base_addr = ioremap(lba->hpa.start, 4096); imask <<= 2; /* adjust for hints - 2 more bits */ diff --git a/drivers/parisc/sba_iommu.c b/drivers/parisc/sba_iommu.c index de8e4e347249..7e112829d250 100644 --- a/drivers/parisc/sba_iommu.c +++ b/drivers/parisc/sba_iommu.c @@ -1513,7 +1513,7 @@ sba_ioc_init(struct parisc_device *sba, struct ioc *ioc, int ioc_num) static void __iomem *ioc_remap(struct sba_device *sba_dev, unsigned int offset) { - return ioremap_nocache(sba_dev->dev->hpa.start + offset, SBA_FUNC_SIZE); + return ioremap(sba_dev->dev->hpa.start + offset, SBA_FUNC_SIZE); } static void sba_hw_init(struct sba_device *sba_dev) @@ -1883,7 +1883,7 @@ static int __init sba_driver_callback(struct parisc_device *dev) u32 func_class; int i; char *version; - void __iomem *sba_addr = ioremap_nocache(dev->hpa.start, SBA_FUNC_SIZE); + void __iomem *sba_addr = ioremap(dev->hpa.start, SBA_FUNC_SIZE); #ifdef CONFIG_PROC_FS struct proc_dir_entry *root; #endif diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c index b20651cea09f..9bf7fa99b103 100644 --- a/drivers/pci/controller/dwc/pci-dra7xx.c +++ b/drivers/pci/controller/dwc/pci-dra7xx.c @@ -719,7 +719,7 @@ static int __init dra7xx_pcie_probe(struct platform_device *pdev) } res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ti_conf"); - base = devm_ioremap_nocache(dev, res->start, resource_size(res)); + base = devm_ioremap(dev, res->start, resource_size(res)); if (!base) return -ENOMEM; diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index 3dd2e2697294..cfeccd7e9fff 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -434,7 +434,7 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, tbl_addr += (tbl_offset + ((interrupt_num - 1) * PCI_MSIX_ENTRY_SIZE)); tbl_addr &= PCI_BASE_ADDRESS_MEM_MASK; - msix_tbl = ioremap_nocache(ep->phys_base + tbl_addr, + msix_tbl = ioremap(ep->phys_base + tbl_addr, PCI_MSIX_ENTRY_SIZE); if (!msix_tbl) return -EINVAL; diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c index c7709e49f0e4..6b43a5455c7a 100644 --- a/drivers/pci/msi.c +++ b/drivers/pci/msi.c @@ -688,7 +688,7 @@ static void __iomem *msix_map_region(struct pci_dev *dev, unsigned nr_entries) table_offset &= PCI_MSIX_TABLE_OFFSET; phys_addr = pci_resource_start(dev, bir) + table_offset; - return ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE); + return ioremap(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE); } static int msix_setup_entries(struct pci_dev *dev, void __iomem *base, diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index e87196cc1a7f..df21e3227b57 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -184,7 +184,7 @@ void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar) pci_warn(pdev, "can't ioremap BAR %d: %pR\n", bar, res); return NULL; } - return ioremap_nocache(res->start, resource_size(res)); + return ioremap(res->start, resource_size(res)); } EXPORT_SYMBOL_GPL(pci_ioremap_bar); diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 4937a088d7d8..0d92374b0a11 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -1571,7 +1571,7 @@ static void asus_hides_smbus_lpc_ich6_suspend(struct pci_dev *dev) pci_read_config_dword(dev, 0xF0, &rcba); /* use bits 31:14, 16 kB aligned */ - asus_rcba_base = ioremap_nocache(rcba & 0xFFFFC000, 0x4000); + asus_rcba_base = ioremap(rcba & 0xFFFFC000, 0x4000); if (asus_rcba_base == NULL) return; } @@ -4784,7 +4784,7 @@ static int pci_quirk_enable_intel_lpc_acs(struct pci_dev *dev) if (!(rcba & INTEL_LPC_RCBA_ENABLE)) return -EINVAL; - rcba_mem = ioremap_nocache(rcba & INTEL_LPC_RCBA_MASK, + rcba_mem = ioremap(rcba & INTEL_LPC_RCBA_MASK, PAGE_ALIGN(INTEL_UPDCR_REG)); if (!rcba_mem) return -ENOMEM; diff --git a/drivers/pinctrl/bcm/pinctrl-ns2-mux.c b/drivers/pinctrl/bcm/pinctrl-ns2-mux.c index 32f268f173d1..57044ab376d3 100644 --- a/drivers/pinctrl/bcm/pinctrl-ns2-mux.c +++ b/drivers/pinctrl/bcm/pinctrl-ns2-mux.c @@ -1049,7 +1049,7 @@ static int ns2_pinmux_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_MEM, 1); if (!res) return -EINVAL; - pinctrl->base1 = devm_ioremap_nocache(&pdev->dev, res->start, + pinctrl->base1 = devm_ioremap(&pdev->dev, res->start, resource_size(res)); if (!pinctrl->base1) { dev_err(&pdev->dev, "unable to map I/O space\n"); diff --git a/drivers/pinctrl/bcm/pinctrl-nsp-mux.c b/drivers/pinctrl/bcm/pinctrl-nsp-mux.c index 3756fc9d5826..f1d60a708815 100644 --- a/drivers/pinctrl/bcm/pinctrl-nsp-mux.c +++ b/drivers/pinctrl/bcm/pinctrl-nsp-mux.c @@ -578,7 +578,7 @@ static int nsp_pinmux_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_MEM, 1); if (!res) return -EINVAL; - pinctrl->base1 = devm_ioremap_nocache(&pdev->dev, res->start, + pinctrl->base1 = devm_ioremap(&pdev->dev, res->start, resource_size(res)); if (!pinctrl->base1) { dev_err(&pdev->dev, "unable to map I/O space\n"); diff --git a/drivers/pinctrl/freescale/pinctrl-imx1-core.c b/drivers/pinctrl/freescale/pinctrl-imx1-core.c index 7e29e3fecdb2..c00d0022d311 100644 --- a/drivers/pinctrl/freescale/pinctrl-imx1-core.c +++ b/drivers/pinctrl/freescale/pinctrl-imx1-core.c @@ -611,7 +611,7 @@ int imx1_pinctrl_core_probe(struct platform_device *pdev, if (!res) return -ENOENT; - ipctl->base = devm_ioremap_nocache(&pdev->dev, res->start, + ipctl->base = devm_ioremap(&pdev->dev, res->start, resource_size(res)); if (!ipctl->base) return -ENOMEM; diff --git a/drivers/pinctrl/pinctrl-amd.c b/drivers/pinctrl/pinctrl-amd.c index eab078244a4c..73aff6591de2 100644 --- a/drivers/pinctrl/pinctrl-amd.c +++ b/drivers/pinctrl/pinctrl-amd.c @@ -866,7 +866,7 @@ static int amd_gpio_probe(struct platform_device *pdev) return -EINVAL; } - gpio_dev->base = devm_ioremap_nocache(&pdev->dev, res->start, + gpio_dev->base = devm_ioremap(&pdev->dev, res->start, resource_size(res)); if (!gpio_dev->base) return -ENOMEM; diff --git a/drivers/platform/x86/intel_scu_ipc.c b/drivers/platform/x86/intel_scu_ipc.c index cdab916fbf92..5ea5245b8b15 100644 --- a/drivers/platform/x86/intel_scu_ipc.c +++ b/drivers/platform/x86/intel_scu_ipc.c @@ -652,7 +652,7 @@ static int ipc_probe(struct pci_dev *pdev, const struct pci_device_id *id) scu->ipc_base = pcim_iomap_table(pdev)[0]; - scu->i2c_base = ioremap_nocache(pdata->i2c_base, pdata->i2c_len); + scu->i2c_base = ioremap(pdata->i2c_base, pdata->i2c_len); if (!scu->i2c_base) return -ENOMEM; diff --git a/drivers/platform/x86/intel_telemetry_pltdrv.c b/drivers/platform/x86/intel_telemetry_pltdrv.c index df8565bad595..ebfcfead23b4 100644 --- a/drivers/platform/x86/intel_telemetry_pltdrv.c +++ b/drivers/platform/x86/intel_telemetry_pltdrv.c @@ -1156,7 +1156,7 @@ static int telemetry_pltdrv_probe(struct platform_device *pdev) telm_conf->ioss_config.ssram_base_addr = res1->start; telm_conf->ioss_config.ssram_size = size; - telm_conf->pss_config.regmap = ioremap_nocache( + telm_conf->pss_config.regmap = ioremap( telm_conf->pss_config.ssram_base_addr, telm_conf->pss_config.ssram_size); if (!telm_conf->pss_config.regmap) { @@ -1164,7 +1164,7 @@ static int telemetry_pltdrv_probe(struct platform_device *pdev) goto out; } - telm_conf->ioss_config.regmap = ioremap_nocache( + telm_conf->ioss_config.regmap = ioremap( telm_conf->ioss_config.ssram_base_addr, telm_conf->ioss_config.ssram_size); if (!telm_conf->ioss_config.regmap) { diff --git a/drivers/platform/x86/pmc_atom.c b/drivers/platform/x86/pmc_atom.c index 52ef1419b671..3e3c66dfec2e 100644 --- a/drivers/platform/x86/pmc_atom.c +++ b/drivers/platform/x86/pmc_atom.c @@ -489,7 +489,7 @@ static int pmc_setup_dev(struct pci_dev *pdev, const struct pci_device_id *ent) pci_read_config_dword(pdev, PMC_BASE_ADDR_OFFSET, &pmc->base_addr); pmc->base_addr &= PMC_BASE_ADDR_MASK; - pmc->regmap = ioremap_nocache(pmc->base_addr, PMC_MMIO_REG_LEN); + pmc->regmap = ioremap(pmc->base_addr, PMC_MMIO_REG_LEN); if (!pmc->regmap) { dev_err(&pdev->dev, "error: ioremap failed\n"); return -ENOMEM; diff --git a/drivers/platform/x86/samsung-laptop.c b/drivers/platform/x86/samsung-laptop.c index 9b6a93ff41ff..23e40aa2176e 100644 --- a/drivers/platform/x86/samsung-laptop.c +++ b/drivers/platform/x86/samsung-laptop.c @@ -1394,7 +1394,7 @@ static int __init samsung_sabi_init(struct samsung_laptop *samsung) int ret = 0; int i; - samsung->f0000_segment = ioremap_nocache(0xf0000, 0xffff); + samsung->f0000_segment = ioremap(0xf0000, 0xffff); if (!samsung->f0000_segment) { if (debug || force) pr_err("Can't map the segment at 0xf0000\n"); @@ -1434,7 +1434,7 @@ static int __init samsung_sabi_init(struct samsung_laptop *samsung) if (debug) samsung_sabi_infos(samsung, loca, ifaceP); - samsung->sabi_iface = ioremap_nocache(ifaceP, 16); + samsung->sabi_iface = ioremap(ifaceP, 16); if (!samsung->sabi_iface) { pr_err("Can't remap %x\n", ifaceP); ret = -EINVAL; diff --git a/drivers/regulator/ti-abb-regulator.c b/drivers/regulator/ti-abb-regulator.c index 89b9314d64c9..af9abcd9c166 100644 --- a/drivers/regulator/ti-abb-regulator.c +++ b/drivers/regulator/ti-abb-regulator.c @@ -748,7 +748,7 @@ static int ti_abb_probe(struct platform_device *pdev) * We may have shared interrupt register offsets which are * write-1-to-clear between domains ensuring exclusivity. */ - abb->int_base = devm_ioremap_nocache(dev, res->start, + abb->int_base = devm_ioremap(dev, res->start, resource_size(res)); if (!abb->int_base) { dev_err(dev, "Unable to map '%s'\n", pname); @@ -768,7 +768,7 @@ static int ti_abb_probe(struct platform_device *pdev) * We may have shared efuse register offsets which are read-only * between domains */ - abb->efuse_base = devm_ioremap_nocache(dev, res->start, + abb->efuse_base = devm_ioremap(dev, res->start, resource_size(res)); if (!abb->efuse_base) { dev_err(dev, "Unable to map '%s'\n", pname); diff --git a/drivers/rtc/rtc-sh.c b/drivers/rtc/rtc-sh.c index 579b3ff5c644..feb1f8e52c00 100644 --- a/drivers/rtc/rtc-sh.c +++ b/drivers/rtc/rtc-sh.c @@ -504,7 +504,7 @@ static int __init sh_rtc_probe(struct platform_device *pdev) if (unlikely(!rtc->res)) return -EBUSY; - rtc->regbase = devm_ioremap_nocache(&pdev->dev, rtc->res->start, + rtc->regbase = devm_ioremap(&pdev->dev, rtc->res->start, rtc->regsize); if (unlikely(!rtc->regbase)) return -EINVAL; diff --git a/drivers/scsi/aic7xxx/aic79xx_osm_pci.c b/drivers/scsi/aic7xxx/aic79xx_osm_pci.c index 8466aa784ec1..8b891a05d9e7 100644 --- a/drivers/scsi/aic7xxx/aic79xx_osm_pci.c +++ b/drivers/scsi/aic7xxx/aic79xx_osm_pci.c @@ -293,7 +293,7 @@ ahd_linux_pci_reserve_mem_region(struct ahd_softc *ahd, if (!request_mem_region(start, 0x1000, "aic79xx")) error = ENOMEM; if (!error) { - *maddr = ioremap_nocache(base_page, base_offset + 512); + *maddr = ioremap(base_page, base_offset + 512); if (*maddr == NULL) { error = ENOMEM; release_mem_region(start, 0x1000); diff --git a/drivers/scsi/aic7xxx/aic7xxx_osm_pci.c b/drivers/scsi/aic7xxx/aic7xxx_osm_pci.c index 717d8d1082ce..9b293b1f0b71 100644 --- a/drivers/scsi/aic7xxx/aic7xxx_osm_pci.c +++ b/drivers/scsi/aic7xxx/aic7xxx_osm_pci.c @@ -372,7 +372,7 @@ ahc_linux_pci_reserve_mem_region(struct ahc_softc *ahc, if (!request_mem_region(start, 0x1000, "aic7xxx")) error = ENOMEM; if (error == 0) { - *maddr = ioremap_nocache(start, 256); + *maddr = ioremap(start, 256); if (*maddr == NULL) { error = ENOMEM; release_mem_region(start, 0x1000); diff --git a/drivers/scsi/arcmsr/arcmsr_hba.c b/drivers/scsi/arcmsr/arcmsr_hba.c index db687ef8a99e..40dc8eac0e3a 100644 --- a/drivers/scsi/arcmsr/arcmsr_hba.c +++ b/drivers/scsi/arcmsr/arcmsr_hba.c @@ -270,7 +270,7 @@ static bool arcmsr_remap_pciregion(struct AdapterControlBlock *acb) break; } case ACB_ADAPTER_TYPE_C:{ - acb->pmuC = ioremap_nocache(pci_resource_start(pdev, 1), pci_resource_len(pdev, 1)); + acb->pmuC = ioremap(pci_resource_start(pdev, 1), pci_resource_len(pdev, 1)); if (!acb->pmuC) { printk(KERN_NOTICE "arcmsr%d: memory mapping region fail \n", acb->host->host_no); return false; diff --git a/drivers/scsi/be2iscsi/be_main.c b/drivers/scsi/be2iscsi/be_main.c index 0760d0bd8a10..9b81cfbbc5c5 100644 --- a/drivers/scsi/be2iscsi/be_main.c +++ b/drivers/scsi/be2iscsi/be_main.c @@ -453,14 +453,14 @@ static int beiscsi_map_pci_bars(struct beiscsi_hba *phba, u8 __iomem *addr; int pcicfg_reg; - addr = ioremap_nocache(pci_resource_start(pcidev, 2), + addr = ioremap(pci_resource_start(pcidev, 2), pci_resource_len(pcidev, 2)); if (addr == NULL) return -ENOMEM; phba->ctrl.csr = addr; phba->csr_va = addr; - addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024); + addr = ioremap(pci_resource_start(pcidev, 4), 128 * 1024); if (addr == NULL) goto pci_map_err; phba->ctrl.db = addr; @@ -471,7 +471,7 @@ static int beiscsi_map_pci_bars(struct beiscsi_hba *phba, else pcicfg_reg = 0; - addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg), + addr = ioremap(pci_resource_start(pcidev, pcicfg_reg), pci_resource_len(pcidev, pcicfg_reg)); if (addr == NULL) diff --git a/drivers/scsi/bnx2fc/bnx2fc_hwi.c b/drivers/scsi/bnx2fc/bnx2fc_hwi.c index f069e09beb10..6f8335ddb1f2 100644 --- a/drivers/scsi/bnx2fc/bnx2fc_hwi.c +++ b/drivers/scsi/bnx2fc/bnx2fc_hwi.c @@ -1414,7 +1414,7 @@ int bnx2fc_map_doorbell(struct bnx2fc_rport *tgt) reg_base = pci_resource_start(hba->pcidev, BNX2X_DOORBELL_PCI_BAR); reg_off = (1 << BNX2X_DB_SHIFT) * (context_id & 0x1FFFF); - tgt->ctx_base = ioremap_nocache(reg_base + reg_off, 4); + tgt->ctx_base = ioremap(reg_base + reg_off, 4); if (!tgt->ctx_base) return -ENOMEM; return 0; diff --git a/drivers/scsi/bnx2i/bnx2i_hwi.c b/drivers/scsi/bnx2i/bnx2i_hwi.c index 12666313b937..e53ebc5eff85 100644 --- a/drivers/scsi/bnx2i/bnx2i_hwi.c +++ b/drivers/scsi/bnx2i/bnx2i_hwi.c @@ -2715,7 +2715,7 @@ int bnx2i_map_ep_dbell_regs(struct bnx2i_endpoint *ep) reg_base = pci_resource_start(ep->hba->pcidev, BNX2X_DOORBELL_PCI_BAR); reg_off = (1 << BNX2X_DB_SHIFT) * (cid_num & 0x1FFFF); - ep->qp.ctx_base = ioremap_nocache(reg_base + reg_off, 4); + ep->qp.ctx_base = ioremap(reg_base + reg_off, 4); if (!ep->qp.ctx_base) return -ENOMEM; goto arm_cq; @@ -2736,7 +2736,7 @@ int bnx2i_map_ep_dbell_regs(struct bnx2i_endpoint *ep) /* 5709 device in normal node and 5706/5708 devices */ reg_off = CTX_OFFSET + (MB_KERNEL_CTX_SIZE * cid_num); - ep->qp.ctx_base = ioremap_nocache(ep->hba->reg_base + reg_off, + ep->qp.ctx_base = ioremap(ep->hba->reg_base + reg_off, MB_KERNEL_CTX_SIZE); if (!ep->qp.ctx_base) return -ENOMEM; diff --git a/drivers/scsi/csiostor/csio_init.c b/drivers/scsi/csiostor/csio_init.c index 2e8a3ac575cb..8dea7d53788a 100644 --- a/drivers/scsi/csiostor/csio_init.c +++ b/drivers/scsi/csiostor/csio_init.c @@ -529,7 +529,7 @@ static struct csio_hw *csio_hw_alloc(struct pci_dev *pdev) goto err_free_hw; /* Get the start address of registers from BAR 0 */ - hw->regstart = ioremap_nocache(pci_resource_start(pdev, 0), + hw->regstart = ioremap(pci_resource_start(pdev, 0), pci_resource_len(pdev, 0)); if (!hw->regstart) { csio_err(hw, "Could not map BAR 0, regstart = %p\n", diff --git a/drivers/scsi/hpsa.c b/drivers/scsi/hpsa.c index 216e557f703e..1a4ddfacb458 100644 --- a/drivers/scsi/hpsa.c +++ b/drivers/scsi/hpsa.c @@ -6876,7 +6876,7 @@ static void __iomem *remap_pci_mem(ulong base, ulong size) { ulong page_base = ((ulong) base) & PAGE_MASK; ulong page_offs = ((ulong) base) - page_base; - void __iomem *page_remapped = ioremap_nocache(page_base, + void __iomem *page_remapped = ioremap(page_base, page_offs + size); return page_remapped ? (page_remapped + page_offs) : NULL; diff --git a/drivers/scsi/lasi700.c b/drivers/scsi/lasi700.c index abac2f350aee..c48a73a0f517 100644 --- a/drivers/scsi/lasi700.c +++ b/drivers/scsi/lasi700.c @@ -98,7 +98,7 @@ lasi700_probe(struct parisc_device *dev) hostdata->dev = &dev->dev; dma_set_mask(&dev->dev, DMA_BIT_MASK(32)); - hostdata->base = ioremap_nocache(base, 0x100); + hostdata->base = ioremap(base, 0x100); hostdata->differential = 0; if (dev->id.sversion == LASI_700_SVERSION) { diff --git a/drivers/scsi/megaraid/megaraid_mbox.c b/drivers/scsi/megaraid/megaraid_mbox.c index f6ac819e6e96..8443f2f35be2 100644 --- a/drivers/scsi/megaraid/megaraid_mbox.c +++ b/drivers/scsi/megaraid/megaraid_mbox.c @@ -731,7 +731,7 @@ megaraid_init_mbox(adapter_t *adapter) goto out_free_raid_dev; } - raid_dev->baseaddr = ioremap_nocache(raid_dev->baseport, 128); + raid_dev->baseaddr = ioremap(raid_dev->baseport, 128); if (!raid_dev->baseaddr) { diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c index a4bc81479284..c60cd9fc4240 100644 --- a/drivers/scsi/megaraid/megaraid_sas_base.c +++ b/drivers/scsi/megaraid/megaraid_sas_base.c @@ -5875,7 +5875,7 @@ static int megasas_init_fw(struct megasas_instance *instance) } base_addr = pci_resource_start(instance->pdev, instance->bar); - instance->reg_set = ioremap_nocache(base_addr, 8192); + instance->reg_set = ioremap(base_addr, 8192); if (!instance->reg_set) { dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to map IO mem\n"); diff --git a/drivers/scsi/myrb.c b/drivers/scsi/myrb.c index 539ac8ce4fcd..d4bd31a75b9d 100644 --- a/drivers/scsi/myrb.c +++ b/drivers/scsi/myrb.c @@ -3531,7 +3531,7 @@ static struct myrb_hba *myrb_detect(struct pci_dev *pdev, spin_lock_init(&cb->queue_lock); if (mmio_size < PAGE_SIZE) mmio_size = PAGE_SIZE; - cb->mmio_base = ioremap_nocache(cb->pci_addr & PAGE_MASK, mmio_size); + cb->mmio_base = ioremap(cb->pci_addr & PAGE_MASK, mmio_size); if (cb->mmio_base == NULL) { dev_err(&pdev->dev, "Unable to map Controller Register Window\n"); diff --git a/drivers/scsi/myrs.c b/drivers/scsi/myrs.c index eb0dd566330a..5c5666491c2e 100644 --- a/drivers/scsi/myrs.c +++ b/drivers/scsi/myrs.c @@ -2311,7 +2311,7 @@ static struct myrs_hba *myrs_detect(struct pci_dev *pdev, /* Map the Controller Register Window. */ if (mmio_size < PAGE_SIZE) mmio_size = PAGE_SIZE; - cs->mmio_base = ioremap_nocache(cs->pci_addr & PAGE_MASK, mmio_size); + cs->mmio_base = ioremap(cs->pci_addr & PAGE_MASK, mmio_size); if (cs->mmio_base == NULL) { dev_err(&pdev->dev, "Unable to map Controller Register Window\n"); diff --git a/drivers/scsi/pcmcia/nsp_cs.c b/drivers/scsi/pcmcia/nsp_cs.c index 93616f9fd6d7..d79ce97a04bd 100644 --- a/drivers/scsi/pcmcia/nsp_cs.c +++ b/drivers/scsi/pcmcia/nsp_cs.c @@ -1560,7 +1560,7 @@ static int nsp_cs_config_check(struct pcmcia_device *p_dev, void *priv_data) goto next_entry; data->MmioAddress = (unsigned long) - ioremap_nocache(p_dev->resource[2]->start, + ioremap(p_dev->resource[2]->start, resource_size(p_dev->resource[2])); data->MmioLength = resource_size(p_dev->resource[2]); } diff --git a/drivers/scsi/qla2xxx/qla_mr.c b/drivers/scsi/qla2xxx/qla_mr.c index 605b59c76c90..a3a44d4ace1e 100644 --- a/drivers/scsi/qla2xxx/qla_mr.c +++ b/drivers/scsi/qla2xxx/qla_mr.c @@ -789,7 +789,7 @@ qlafx00_iospace_config(struct qla_hw_data *ha) } ha->cregbase = - ioremap_nocache(pci_resource_start(ha->pdev, 0), BAR0_LEN_FX00); + ioremap(pci_resource_start(ha->pdev, 0), BAR0_LEN_FX00); if (!ha->cregbase) { ql_log_pci(ql_log_fatal, ha->pdev, 0x0128, "cannot remap MMIO (%s), aborting\n", pci_name(ha->pdev)); @@ -810,7 +810,7 @@ qlafx00_iospace_config(struct qla_hw_data *ha) } ha->iobase = - ioremap_nocache(pci_resource_start(ha->pdev, 2), BAR2_LEN_FX00); + ioremap(pci_resource_start(ha->pdev, 2), BAR2_LEN_FX00); if (!ha->iobase) { ql_log_pci(ql_log_fatal, ha->pdev, 0x012b, "cannot remap MMIO (%s), aborting\n", pci_name(ha->pdev)); diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c index 412ac56ecd60..b7492568e02f 100644 --- a/drivers/scsi/smartpqi/smartpqi_init.c +++ b/drivers/scsi/smartpqi/smartpqi_init.c @@ -7457,7 +7457,7 @@ static int pqi_pci_init(struct pqi_ctrl_info *ctrl_info) goto disable_device; } - ctrl_info->iomem_base = ioremap_nocache(pci_resource_start( + ctrl_info->iomem_base = ioremap(pci_resource_start( ctrl_info->pci_dev, 0), sizeof(struct pqi_ctrl_registers)); if (!ctrl_info->iomem_base) { diff --git a/drivers/scsi/sni_53c710.c b/drivers/scsi/sni_53c710.c index a85d52b5dc32..f8397978f8ab 100644 --- a/drivers/scsi/sni_53c710.c +++ b/drivers/scsi/sni_53c710.c @@ -71,7 +71,7 @@ static int snirm710_probe(struct platform_device *dev) hostdata->dev = &dev->dev; dma_set_mask(&dev->dev, DMA_BIT_MASK(32)); - hostdata->base = ioremap_nocache(base, 0x100); + hostdata->base = ioremap(base, 0x100); hostdata->differential = 0; hostdata->clock = SNIRM710_CLOCK; diff --git a/drivers/scsi/sun3x_esp.c b/drivers/scsi/sun3x_esp.c index 440a73eae647..f37df79e37e1 100644 --- a/drivers/scsi/sun3x_esp.c +++ b/drivers/scsi/sun3x_esp.c @@ -190,7 +190,7 @@ static int esp_sun3x_probe(struct platform_device *dev) if (!res || !res->start) goto fail_unlink; - esp->regs = ioremap_nocache(res->start, 0x20); + esp->regs = ioremap(res->start, 0x20); if (!esp->regs) goto fail_unmap_regs; @@ -198,7 +198,7 @@ static int esp_sun3x_probe(struct platform_device *dev) if (!res || !res->start) goto fail_unmap_regs; - esp->dma_regs = ioremap_nocache(res->start, 0x10); + esp->dma_regs = ioremap(res->start, 0x10); esp->command_block = dma_alloc_coherent(esp->dev, 16, &esp->command_block_dma, diff --git a/drivers/scsi/zalon.c b/drivers/scsi/zalon.c index 77bce208210e..7eac76cccc4c 100644 --- a/drivers/scsi/zalon.c +++ b/drivers/scsi/zalon.c @@ -89,7 +89,7 @@ zalon_probe(struct parisc_device *dev) struct gsc_irq gsc_irq; u32 zalon_vers; int error = -ENODEV; - void __iomem *zalon = ioremap_nocache(dev->hpa.start, 4096); + void __iomem *zalon = ioremap(dev->hpa.start, 4096); void __iomem *io_port = zalon + GSC_SCSI_ZALON_OFFSET; static int unit = 0; struct Scsi_Host *host; diff --git a/drivers/scsi/zorro_esp.c b/drivers/scsi/zorro_esp.c index a23a8e5794f5..bdd82e497d5f 100644 --- a/drivers/scsi/zorro_esp.c +++ b/drivers/scsi/zorro_esp.c @@ -801,7 +801,7 @@ static int zorro_esp_probe(struct zorro_dev *z, /* additional setup required for Fastlane */ if (zep->zorro3 && ent->driver_data == ZORRO_BLZ1230II) { /* map full address space up to ESP base for DMA */ - zep->board_base = ioremap_nocache(board, + zep->board_base = ioremap(board, FASTLANE_ESP_ADDR-1); if (!zep->board_base) { pr_err("Cannot allocate board address space\n"); @@ -816,7 +816,7 @@ static int zorro_esp_probe(struct zorro_dev *z, esp->ops = zdd->esp_ops; if (ioaddr > 0xffffff) - esp->regs = ioremap_nocache(ioaddr, 0x20); + esp->regs = ioremap(ioaddr, 0x20); else /* ZorroII address space remapped nocache by early startup */ esp->regs = ZTWO_VADDR(ioaddr); @@ -842,7 +842,7 @@ static int zorro_esp_probe(struct zorro_dev *z, * Only Fastlane Z3 for now - add switch for correct struct * dma_registers size if adding any more */ - esp->dma_regs = ioremap_nocache(dmaaddr, + esp->dma_regs = ioremap(dmaaddr, sizeof(struct fastlane_dma_registers)); } else /* ZorroII address space remapped nocache by early startup */ diff --git a/drivers/sh/clk/core.c b/drivers/sh/clk/core.c index 9475353f49d6..d996782a7106 100644 --- a/drivers/sh/clk/core.c +++ b/drivers/sh/clk/core.c @@ -368,7 +368,7 @@ static int clk_establish_mapping(struct clk *clk) if (!mapping->base && mapping->phys) { kref_init(&mapping->ref); - mapping->base = ioremap_nocache(mapping->phys, mapping->len); + mapping->base = ioremap(mapping->phys, mapping->len); if (unlikely(!mapping->base)) return -ENXIO; } else if (mapping->base) { diff --git a/drivers/sh/intc/core.c b/drivers/sh/intc/core.c index 8485e812d9b2..f8e070d67fa3 100644 --- a/drivers/sh/intc/core.c +++ b/drivers/sh/intc/core.c @@ -213,7 +213,7 @@ int __init register_intc_controller(struct intc_desc *desc) WARN_ON(resource_type(res) != IORESOURCE_MEM); d->window[k].phys = res->start; d->window[k].size = resource_size(res); - d->window[k].virt = ioremap_nocache(res->start, + d->window[k].virt = ioremap(res->start, resource_size(res)); if (!d->window[k].virt) goto err2; diff --git a/drivers/sh/intc/userimask.c b/drivers/sh/intc/userimask.c index 87d69e7471f9..f9f043a3d90a 100644 --- a/drivers/sh/intc/userimask.c +++ b/drivers/sh/intc/userimask.c @@ -73,7 +73,7 @@ int register_intc_userimask(unsigned long addr) if (unlikely(uimask)) return -EBUSY; - uimask = ioremap_nocache(addr, SZ_4K); + uimask = ioremap(addr, SZ_4K); if (unlikely(!uimask)) return -ENOMEM; diff --git a/drivers/soc/tegra/flowctrl.c b/drivers/soc/tegra/flowctrl.c index eb96a3086d6d..5db919d96aba 100644 --- a/drivers/soc/tegra/flowctrl.c +++ b/drivers/soc/tegra/flowctrl.c @@ -219,7 +219,7 @@ static int __init tegra_flowctrl_init(void) return 0; } - tegra_flowctrl_base = ioremap_nocache(res.start, resource_size(&res)); + tegra_flowctrl_base = ioremap(res.start, resource_size(&res)); if (!tegra_flowctrl_base) return -ENXIO; diff --git a/drivers/soc/tegra/fuse/fuse-tegra.c b/drivers/soc/tegra/fuse/fuse-tegra.c index 4d719d4b8d5a..606abbe55bba 100644 --- a/drivers/soc/tegra/fuse/fuse-tegra.c +++ b/drivers/soc/tegra/fuse/fuse-tegra.c @@ -408,7 +408,7 @@ static int __init tegra_init_fuse(void) } } - fuse->base = ioremap_nocache(regs.start, resource_size(®s)); + fuse->base = ioremap(regs.start, resource_size(®s)); if (!fuse->base) { pr_err("failed to map FUSE registers\n"); return -ENXIO; diff --git a/drivers/soc/tegra/fuse/tegra-apbmisc.c b/drivers/soc/tegra/fuse/tegra-apbmisc.c index df76778af601..a2fd6ccd48f9 100644 --- a/drivers/soc/tegra/fuse/tegra-apbmisc.c +++ b/drivers/soc/tegra/fuse/tegra-apbmisc.c @@ -159,11 +159,11 @@ void __init tegra_init_apbmisc(void) } } - apbmisc_base = ioremap_nocache(apbmisc.start, resource_size(&apbmisc)); + apbmisc_base = ioremap(apbmisc.start, resource_size(&apbmisc)); if (!apbmisc_base) pr_err("failed to map APBMISC registers\n"); - strapping_base = ioremap_nocache(straps.start, resource_size(&straps)); + strapping_base = ioremap(straps.start, resource_size(&straps)); if (!strapping_base) pr_err("failed to map strapping options registers\n"); diff --git a/drivers/soc/tegra/pmc.c b/drivers/soc/tegra/pmc.c index ea0e11a09c12..1699dda6b393 100644 --- a/drivers/soc/tegra/pmc.c +++ b/drivers/soc/tegra/pmc.c @@ -2826,7 +2826,7 @@ static void tegra186_pmc_setup_irq_polarity(struct tegra_pmc *pmc, of_address_to_resource(np, index, ®s); - wake = ioremap_nocache(regs.start, resource_size(®s)); + wake = ioremap(regs.start, resource_size(®s)); if (!wake) { dev_err(pmc->dev, "failed to map PMC wake registers\n"); return; @@ -3097,7 +3097,7 @@ static int __init tegra_pmc_early_init(void) } } - pmc->base = ioremap_nocache(regs.start, resource_size(®s)); + pmc->base = ioremap(regs.start, resource_size(®s)); if (!pmc->base) { pr_err("failed to map PMC registers\n"); of_node_put(np); diff --git a/drivers/soc/xilinx/xlnx_vcu.c b/drivers/soc/xilinx/xlnx_vcu.c index a840c0272135..a3aa40996f13 100644 --- a/drivers/soc/xilinx/xlnx_vcu.c +++ b/drivers/soc/xilinx/xlnx_vcu.c @@ -511,7 +511,7 @@ static int xvcu_probe(struct platform_device *pdev) return -ENODEV; } - xvcu->vcu_slcr_ba = devm_ioremap_nocache(&pdev->dev, res->start, + xvcu->vcu_slcr_ba = devm_ioremap(&pdev->dev, res->start, resource_size(res)); if (!xvcu->vcu_slcr_ba) { dev_err(&pdev->dev, "vcu_slcr register mapping failed.\n"); @@ -524,7 +524,7 @@ static int xvcu_probe(struct platform_device *pdev) return -ENODEV; } - xvcu->logicore_reg_ba = devm_ioremap_nocache(&pdev->dev, res->start, + xvcu->logicore_reg_ba = devm_ioremap(&pdev->dev, res->start, resource_size(res)); if (!xvcu->logicore_reg_ba) { dev_err(&pdev->dev, "logicore register mapping failed.\n"); diff --git a/drivers/spi/spi-dw-mid.c b/drivers/spi/spi-dw-mid.c index 2663bb12d9ce..0d86c37e0aeb 100644 --- a/drivers/spi/spi-dw-mid.c +++ b/drivers/spi/spi-dw-mid.c @@ -301,7 +301,7 @@ int dw_spi_mid_init(struct dw_spi *dws) void __iomem *clk_reg; u32 clk_cdiv; - clk_reg = ioremap_nocache(MRST_CLK_SPI_REG, 16); + clk_reg = ioremap(MRST_CLK_SPI_REG, 16); if (!clk_reg) return -ENOMEM; diff --git a/drivers/spi/spi-jcore.c b/drivers/spi/spi-jcore.c index cc49fa41fbab..bba10f030e33 100644 --- a/drivers/spi/spi-jcore.c +++ b/drivers/spi/spi-jcore.c @@ -170,7 +170,7 @@ static int jcore_spi_probe(struct platform_device *pdev) if (!devm_request_mem_region(&pdev->dev, res->start, resource_size(res), pdev->name)) goto exit_busy; - hw->base = devm_ioremap_nocache(&pdev->dev, res->start, + hw->base = devm_ioremap(&pdev->dev, res->start, resource_size(res)); if (!hw->base) goto exit_busy; diff --git a/drivers/spi/spi-npcm-fiu.c b/drivers/spi/spi-npcm-fiu.c index cb52fd8008d0..d25ee32862e0 100644 --- a/drivers/spi/spi-npcm-fiu.c +++ b/drivers/spi/spi-npcm-fiu.c @@ -603,7 +603,7 @@ static int npcm_fiu_dirmap_create(struct spi_mem_dirmap_desc *desc) if (!chip->flash_region_mapped_ptr) { chip->flash_region_mapped_ptr = - devm_ioremap_nocache(fiu->dev, (fiu->res_mem->start + + devm_ioremap(fiu->dev, (fiu->res_mem->start + (fiu->info->max_map_size * desc->mem->spi->chip_select)), (u32)desc->info.length); diff --git a/drivers/ssb/driver_extif.c b/drivers/ssb/driver_extif.c index 06b68dd6e022..bc275968fcc6 100644 --- a/drivers/ssb/driver_extif.c +++ b/drivers/ssb/driver_extif.c @@ -63,7 +63,7 @@ int ssb_extif_serial_init(struct ssb_extif *extif, struct ssb_serial_port *ports for (i = 0; i < 2; i++) { void __iomem *uart_regs; - uart_regs = ioremap_nocache(SSB_EUART, 16); + uart_regs = ioremap(SSB_EUART, 16); if (uart_regs) { uart_regs += (i * 8); diff --git a/drivers/ssb/driver_pcicore.c b/drivers/ssb/driver_pcicore.c index 6a5622e0ded5..c1186415896b 100644 --- a/drivers/ssb/driver_pcicore.c +++ b/drivers/ssb/driver_pcicore.c @@ -122,7 +122,7 @@ static int ssb_extpci_read_config(struct ssb_pcicore *pc, if (unlikely(!addr)) goto out; err = -ENOMEM; - mmio = ioremap_nocache(addr, len); + mmio = ioremap(addr, len); if (!mmio) goto out; @@ -168,7 +168,7 @@ static int ssb_extpci_write_config(struct ssb_pcicore *pc, if (unlikely(!addr)) goto out; err = -ENOMEM; - mmio = ioremap_nocache(addr, len); + mmio = ioremap(addr, len); if (!mmio) goto out; @@ -382,7 +382,7 @@ static void ssb_pcicore_init_hostmode(struct ssb_pcicore *pc) /* Ok, ready to run, register it to the system. * The following needs change, if we want to port hostmode * to non-MIPS platform. */ - ssb_pcicore_controller.io_map_base = (unsigned long)ioremap_nocache(SSB_PCI_MEM, 0x04000000); + ssb_pcicore_controller.io_map_base = (unsigned long)ioremap(SSB_PCI_MEM, 0x04000000); set_io_port_base(ssb_pcicore_controller.io_map_base); /* Give some time to the PCI controller to configure itself with the new * values. Not waiting at this point causes crashes of the machine. */ diff --git a/drivers/staging/gasket/gasket_core.c b/drivers/staging/gasket/gasket_core.c index cd8be80d2076..be6b50f454b4 100644 --- a/drivers/staging/gasket/gasket_core.c +++ b/drivers/staging/gasket/gasket_core.c @@ -303,7 +303,7 @@ static int gasket_map_pci_bar(struct gasket_dev *gasket_dev, int bar_num) } gasket_dev->bar_data[bar_num].virt_base = - ioremap_nocache(gasket_dev->bar_data[bar_num].phys_base, + ioremap(gasket_dev->bar_data[bar_num].phys_base, gasket_dev->bar_data[bar_num].length_bytes); if (!gasket_dev->bar_data[bar_num].virt_base) { dev_err(gasket_dev->dev, diff --git a/drivers/staging/kpc2000/kpc2000/core.c b/drivers/staging/kpc2000/kpc2000/core.c index 0a23727d0dc3..93cf28febdf6 100644 --- a/drivers/staging/kpc2000/kpc2000/core.c +++ b/drivers/staging/kpc2000/kpc2000/core.c @@ -338,7 +338,7 @@ static int kp2000_pcie_probe(struct pci_dev *pdev, reg_bar_phys_addr = pci_resource_start(pcard->pdev, REG_BAR); reg_bar_phys_len = pci_resource_len(pcard->pdev, REG_BAR); - pcard->regs_bar_base = ioremap_nocache(reg_bar_phys_addr, PAGE_SIZE); + pcard->regs_bar_base = ioremap(reg_bar_phys_addr, PAGE_SIZE); if (!pcard->regs_bar_base) { dev_err(&pcard->pdev->dev, "probe: REG_BAR could not remap memory to virtual space\n"); @@ -367,7 +367,7 @@ static int kp2000_pcie_probe(struct pci_dev *pdev, dma_bar_phys_addr = pci_resource_start(pcard->pdev, DMA_BAR); dma_bar_phys_len = pci_resource_len(pcard->pdev, DMA_BAR); - pcard->dma_bar_base = ioremap_nocache(dma_bar_phys_addr, + pcard->dma_bar_base = ioremap(dma_bar_phys_addr, dma_bar_phys_len); if (!pcard->dma_bar_base) { dev_err(&pcard->pdev->dev, diff --git a/drivers/staging/kpc2000/kpc2000_i2c.c b/drivers/staging/kpc2000/kpc2000_i2c.c index 5460bf973c9c..592099a1fca5 100644 --- a/drivers/staging/kpc2000/kpc2000_i2c.c +++ b/drivers/staging/kpc2000/kpc2000_i2c.c @@ -659,7 +659,7 @@ static int pi2c_probe(struct platform_device *pldev) if (!res) return -ENXIO; - priv->smba = (unsigned long)devm_ioremap_nocache(&pldev->dev, + priv->smba = (unsigned long)devm_ioremap(&pldev->dev, res->start, resource_size(res)); if (!priv->smba) diff --git a/drivers/staging/kpc2000/kpc2000_spi.c b/drivers/staging/kpc2000/kpc2000_spi.c index 8becf972af9c..1c360daa703d 100644 --- a/drivers/staging/kpc2000/kpc2000_spi.c +++ b/drivers/staging/kpc2000/kpc2000_spi.c @@ -464,7 +464,7 @@ kp_spi_probe(struct platform_device *pldev) goto free_master; } - kpspi->base = devm_ioremap_nocache(&pldev->dev, r->start, + kpspi->base = devm_ioremap(&pldev->dev, r->start, resource_size(r)); status = spi_register_master(master); diff --git a/drivers/staging/kpc2000/kpc_dma/kpc_dma_driver.c b/drivers/staging/kpc2000/kpc_dma/kpc_dma_driver.c index a05ae6d40db9..ec79a8500caf 100644 --- a/drivers/staging/kpc2000/kpc_dma/kpc_dma_driver.c +++ b/drivers/staging/kpc2000/kpc_dma/kpc_dma_driver.c @@ -122,7 +122,7 @@ int kpc_dma_probe(struct platform_device *pldev) rv = -ENXIO; goto err_kfree; } - ldev->eng_regs = ioremap_nocache(r->start, resource_size(r)); + ldev->eng_regs = ioremap(r->start, resource_size(r)); if (!ldev->eng_regs) { dev_err(&ldev->pldev->dev, "%s: failed to ioremap engine regs!\n", __func__); rv = -ENXIO; diff --git a/drivers/staging/media/allegro-dvt/allegro-core.c b/drivers/staging/media/allegro-dvt/allegro-core.c index 6f0cd0784786..3be41698df4c 100644 --- a/drivers/staging/media/allegro-dvt/allegro-core.c +++ b/drivers/staging/media/allegro-dvt/allegro-core.c @@ -2914,7 +2914,7 @@ static int allegro_probe(struct platform_device *pdev) "regs resource missing from device tree\n"); return -EINVAL; } - regs = devm_ioremap_nocache(&pdev->dev, res->start, resource_size(res)); + regs = devm_ioremap(&pdev->dev, res->start, resource_size(res)); if (IS_ERR(regs)) { dev_err(&pdev->dev, "failed to map registers\n"); return PTR_ERR(regs); @@ -2932,7 +2932,7 @@ static int allegro_probe(struct platform_device *pdev) "sram resource missing from device tree\n"); return -EINVAL; } - sram_regs = devm_ioremap_nocache(&pdev->dev, + sram_regs = devm_ioremap(&pdev->dev, sram_res->start, resource_size(sram_res)); if (IS_ERR(sram_regs)) { diff --git a/drivers/staging/qlge/qlge_main.c b/drivers/staging/qlge/qlge_main.c index 6ad4515311f7..d890d38a1d29 100644 --- a/drivers/staging/qlge/qlge_main.c +++ b/drivers/staging/qlge/qlge_main.c @@ -4455,7 +4455,7 @@ static int ql_init_device(struct pci_dev *pdev, struct net_device *ndev, pdev->needs_freset = 1; pci_save_state(pdev); qdev->reg_base = - ioremap_nocache(pci_resource_start(pdev, 1), + ioremap(pci_resource_start(pdev, 1), pci_resource_len(pdev, 1)); if (!qdev->reg_base) { dev_err(&pdev->dev, "Register mapping failed.\n"); @@ -4465,7 +4465,7 @@ static int ql_init_device(struct pci_dev *pdev, struct net_device *ndev, qdev->doorbell_area_size = pci_resource_len(pdev, 3); qdev->doorbell_area = - ioremap_nocache(pci_resource_start(pdev, 3), + ioremap(pci_resource_start(pdev, 3), pci_resource_len(pdev, 3)); if (!qdev->doorbell_area) { dev_err(&pdev->dev, "Doorbell register mapping failed.\n"); diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_core.c b/drivers/staging/rtl8192e/rtl8192e/rtl_core.c index dace81a7d1ba..e895473fcfd7 100644 --- a/drivers/staging/rtl8192e/rtl8192e/rtl_core.c +++ b/drivers/staging/rtl8192e/rtl8192e/rtl_core.c @@ -2463,7 +2463,7 @@ static int _rtl92e_pci_probe(struct pci_dev *pdev, } - ioaddr = (unsigned long)ioremap_nocache(pmem_start, pmem_len); + ioaddr = (unsigned long)ioremap(pmem_start, pmem_len); if (ioaddr == (unsigned long)NULL) { netdev_err(dev, "ioremap failed!"); goto err_rel_mem; diff --git a/drivers/staging/rts5208/rtsx.c b/drivers/staging/rts5208/rtsx.c index cb95ad6fa4f9..fbb42e5258fd 100644 --- a/drivers/staging/rts5208/rtsx.c +++ b/drivers/staging/rts5208/rtsx.c @@ -858,7 +858,7 @@ static int rtsx_probe(struct pci_dev *pci, dev_info(&pci->dev, "Resource length: 0x%x\n", (unsigned int)pci_resource_len(pci, 0)); dev->addr = pci_resource_start(pci, 0); - dev->remap_addr = ioremap_nocache(dev->addr, pci_resource_len(pci, 0)); + dev->remap_addr = ioremap(dev->addr, pci_resource_len(pci, 0)); if (!dev->remap_addr) { dev_err(&pci->dev, "ioremap error\n"); err = -ENXIO; diff --git a/drivers/staging/sm750fb/sm750_hw.c b/drivers/staging/sm750fb/sm750_hw.c index ea1d3d4efbc2..b8d60701f898 100644 --- a/drivers/staging/sm750fb/sm750_hw.c +++ b/drivers/staging/sm750fb/sm750_hw.c @@ -50,7 +50,7 @@ int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev) } /* now map mmio and vidmem */ - sm750_dev->pvReg = ioremap_nocache(sm750_dev->vidreg_start, + sm750_dev->pvReg = ioremap(sm750_dev->vidreg_start, sm750_dev->vidreg_size); if (!sm750_dev->pvReg) { pr_err("mmio failed\n"); diff --git a/drivers/staging/uwb/whc-rc.c b/drivers/staging/uwb/whc-rc.c index 34020ed351ab..a5ab255d7d36 100644 --- a/drivers/staging/uwb/whc-rc.c +++ b/drivers/staging/uwb/whc-rc.c @@ -216,11 +216,11 @@ int whcrc_setup_rc_umc(struct whcrc *whcrc) goto error_request_region; } - whcrc->rc_base = ioremap_nocache(whcrc->area, whcrc->rc_len); + whcrc->rc_base = ioremap(whcrc->area, whcrc->rc_len); if (whcrc->rc_base == NULL) { dev_err(dev, "can't ioremap registers (%zu bytes @ 0x%lx): %d\n", whcrc->rc_len, whcrc->area, result); - goto error_ioremap_nocache; + goto error_ioremap; } result = request_irq(umc_dev->irq, whcrc_irq_cb, IRQF_SHARED, @@ -254,7 +254,7 @@ error_cmd_buffer: free_irq(umc_dev->irq, whcrc); error_request_irq: iounmap(whcrc->rc_base); -error_ioremap_nocache: +error_ioremap: release_mem_region(whcrc->area, whcrc->rc_len); error_request_region: return result; diff --git a/drivers/tc/tc.c b/drivers/tc/tc.c index cf3fad2cb871..c5b17dd8f587 100644 --- a/drivers/tc/tc.c +++ b/drivers/tc/tc.c @@ -47,7 +47,7 @@ static void __init tc_bus_add_devices(struct tc_bus *tbus) for (slot = 0; slot < tbus->num_tcslots; slot++) { slotaddr = tbus->slot_base + slot * slotsize; extslotaddr = tbus->ext_slot_base + slot * extslotsize; - module = ioremap_nocache(slotaddr, slotsize); + module = ioremap(slotaddr, slotsize); BUG_ON(!module); offset = TC_OLDCARD; diff --git a/drivers/tty/cyclades.c b/drivers/tty/cyclades.c index 4562c8060d09..a6aabfd6e2da 100644 --- a/drivers/tty/cyclades.c +++ b/drivers/tty/cyclades.c @@ -3256,7 +3256,7 @@ static int __init cy_detect_isa(void) return nboard; /* probe for CD1400... */ - cy_isa_address = ioremap_nocache(isa_address, CyISA_Ywin); + cy_isa_address = ioremap(isa_address, CyISA_Ywin); if (cy_isa_address == NULL) { printk(KERN_ERR "Cyclom-Y/ISA: can't remap base " "address\n"); @@ -3690,13 +3690,13 @@ static int cy_pci_probe(struct pci_dev *pdev, device_id == PCI_DEVICE_ID_CYCLOM_Y_Hi) { card_name = "Cyclom-Y"; - addr0 = ioremap_nocache(pci_resource_start(pdev, 0), + addr0 = ioremap(pci_resource_start(pdev, 0), CyPCI_Yctl); if (addr0 == NULL) { dev_err(&pdev->dev, "can't remap ctl region\n"); goto err_reg; } - addr2 = ioremap_nocache(pci_resource_start(pdev, 2), + addr2 = ioremap(pci_resource_start(pdev, 2), CyPCI_Ywin); if (addr2 == NULL) { dev_err(&pdev->dev, "can't remap base region\n"); @@ -3712,7 +3712,7 @@ static int cy_pci_probe(struct pci_dev *pdev, } else if (device_id == PCI_DEVICE_ID_CYCLOM_Z_Hi) { struct RUNTIME_9060 __iomem *ctl_addr; - ctl_addr = addr0 = ioremap_nocache(pci_resource_start(pdev, 0), + ctl_addr = addr0 = ioremap(pci_resource_start(pdev, 0), CyPCI_Zctl); if (addr0 == NULL) { dev_err(&pdev->dev, "can't remap ctl region\n"); @@ -3727,7 +3727,7 @@ static int cy_pci_probe(struct pci_dev *pdev, mailbox = readl(&ctl_addr->mail_box_0); - addr2 = ioremap_nocache(pci_resource_start(pdev, 2), + addr2 = ioremap(pci_resource_start(pdev, 2), mailbox == ZE_V1 ? CyPCI_Ze_win : CyPCI_Zwin); if (addr2 == NULL) { dev_err(&pdev->dev, "can't remap base region\n"); diff --git a/drivers/tty/mips_ejtag_fdc.c b/drivers/tty/mips_ejtag_fdc.c index 4c1cd49ae95b..620d8488b83e 100644 --- a/drivers/tty/mips_ejtag_fdc.c +++ b/drivers/tty/mips_ejtag_fdc.c @@ -898,7 +898,7 @@ static int mips_ejtag_fdc_tty_probe(struct mips_cdmm_device *dev) atomic_set(&priv->xmit_total, 0); raw_spin_lock_init(&priv->lock); - priv->reg = devm_ioremap_nocache(priv->dev, dev->res.start, + priv->reg = devm_ioremap(priv->dev, dev->res.start, resource_size(&dev->res)); if (!priv->reg) { dev_err(priv->dev, "ioremap failed for resource %pR\n", diff --git a/drivers/tty/moxa.c b/drivers/tty/moxa.c index 3a1a5e0ee93f..9f13f7d49dd7 100644 --- a/drivers/tty/moxa.c +++ b/drivers/tty/moxa.c @@ -961,7 +961,7 @@ static int moxa_pci_probe(struct pci_dev *pdev, goto err; } - board->basemem = ioremap_nocache(pci_resource_start(pdev, 2), 0x4000); + board->basemem = ioremap(pci_resource_start(pdev, 2), 0x4000); if (board->basemem == NULL) { dev_err(&pdev->dev, "can't remap io space 2\n"); retval = -ENOMEM; @@ -1071,7 +1071,7 @@ static int __init moxa_init(void) brd->numPorts = type[i] == MOXA_BOARD_C218_ISA ? 8 : numports[i]; brd->busType = MOXA_BUS_TYPE_ISA; - brd->basemem = ioremap_nocache(baseaddr[i], 0x4000); + brd->basemem = ioremap(baseaddr[i], 0x4000); if (!brd->basemem) { printk(KERN_ERR "MOXA: can't remap %lx\n", baseaddr[i]); diff --git a/drivers/tty/serial/8250/8250_gsc.c b/drivers/tty/serial/8250/8250_gsc.c index 0809ae2aa9b1..673cda3d011d 100644 --- a/drivers/tty/serial/8250/8250_gsc.c +++ b/drivers/tty/serial/8250/8250_gsc.c @@ -55,7 +55,7 @@ static int __init serial_init_chip(struct parisc_device *dev) uart.port.uartclk = (dev->id.sversion != 0xad) ? 7272727 : 1843200; uart.port.mapbase = address; - uart.port.membase = ioremap_nocache(address, 16); + uart.port.membase = ioremap(address, 16); if (!uart.port.membase) { dev_warn(&dev->dev, "Failed to map memory\n"); return -ENOMEM; diff --git a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c index 836e736ae188..e603c66d6cc4 100644 --- a/drivers/tty/serial/8250/8250_omap.c +++ b/drivers/tty/serial/8250/8250_omap.c @@ -1147,7 +1147,7 @@ static int omap8250_probe(struct platform_device *pdev) if (!priv) return -ENOMEM; - membase = devm_ioremap_nocache(&pdev->dev, regs->start, + membase = devm_ioremap(&pdev->dev, regs->start, resource_size(regs)); if (!membase) return -ENODEV; diff --git a/drivers/tty/serial/8250/8250_pci.c b/drivers/tty/serial/8250/8250_pci.c index 022924d5ad54..939685fed396 100644 --- a/drivers/tty/serial/8250/8250_pci.c +++ b/drivers/tty/serial/8250/8250_pci.c @@ -275,7 +275,7 @@ static int pci_plx9050_init(struct pci_dev *dev) /* * enable/disable interrupts */ - p = ioremap_nocache(pci_resource_start(dev, 0), 0x80); + p = ioremap(pci_resource_start(dev, 0), 0x80); if (p == NULL) return -ENOMEM; writel(irq_config, p + 0x4c); @@ -299,7 +299,7 @@ static void pci_plx9050_exit(struct pci_dev *dev) /* * disable interrupts */ - p = ioremap_nocache(pci_resource_start(dev, 0), 0x80); + p = ioremap(pci_resource_start(dev, 0), 0x80); if (p != NULL) { writel(0, p + 0x4c); @@ -475,7 +475,7 @@ static int pci_siig10x_init(struct pci_dev *dev) break; } - p = ioremap_nocache(pci_resource_start(dev, 0), 0x80); + p = ioremap(pci_resource_start(dev, 0), 0x80); if (p == NULL) return -ENOMEM; diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c index 90655910b0c7..9ff5dfad590a 100644 --- a/drivers/tty/serial/8250/8250_port.c +++ b/drivers/tty/serial/8250/8250_port.c @@ -2766,7 +2766,7 @@ static int serial8250_request_std_resource(struct uart_8250_port *up) } if (port->flags & UPF_IOREMAP) { - port->membase = ioremap_nocache(port->mapbase, size); + port->membase = ioremap(port->mapbase, size); if (!port->membase) { release_mem_region(port->mapbase, size); ret = -ENOMEM; diff --git a/drivers/tty/serial/dz.c b/drivers/tty/serial/dz.c index 7b57e840e255..730da413d8ed 100644 --- a/drivers/tty/serial/dz.c +++ b/drivers/tty/serial/dz.c @@ -677,7 +677,7 @@ static void dz_release_port(struct uart_port *uport) static int dz_map_port(struct uart_port *uport) { if (!uport->membase) - uport->membase = ioremap_nocache(uport->mapbase, + uport->membase = ioremap(uport->mapbase, dec_kn_slot_size); if (!uport->membase) { printk(KERN_ERR "dz: Cannot map MMIO\n"); diff --git a/drivers/tty/serial/lantiq.c b/drivers/tty/serial/lantiq.c index fcbea43dc334..f67226df30d4 100644 --- a/drivers/tty/serial/lantiq.c +++ b/drivers/tty/serial/lantiq.c @@ -549,7 +549,7 @@ lqasc_request_port(struct uart_port *port) } if (port->flags & UPF_IOREMAP) { - port->membase = devm_ioremap_nocache(&pdev->dev, + port->membase = devm_ioremap(&pdev->dev, port->mapbase, size); if (port->membase == NULL) return -ENOMEM; diff --git a/drivers/tty/serial/meson_uart.c b/drivers/tty/serial/meson_uart.c index fbc5bc022a39..164b18372c02 100644 --- a/drivers/tty/serial/meson_uart.c +++ b/drivers/tty/serial/meson_uart.c @@ -411,7 +411,7 @@ static int meson_uart_request_port(struct uart_port *port) return -EBUSY; } - port->membase = devm_ioremap_nocache(port->dev, port->mapbase, + port->membase = devm_ioremap(port->dev, port->mapbase, port->mapsize); if (!port->membase) return -ENOMEM; diff --git a/drivers/tty/serial/mux.c b/drivers/tty/serial/mux.c index 00ce31e8d19a..fc58a004bef4 100644 --- a/drivers/tty/serial/mux.c +++ b/drivers/tty/serial/mux.c @@ -474,7 +474,7 @@ static int __init mux_probe(struct parisc_device *dev) port->iobase = 0; port->mapbase = dev->hpa.start + MUX_OFFSET + (i * MUX_LINE_OFFSET); - port->membase = ioremap_nocache(port->mapbase, MUX_LINE_OFFSET); + port->membase = ioremap(port->mapbase, MUX_LINE_OFFSET); port->iotype = UPIO_MEM; port->type = PORT_MUX; port->irq = 0; diff --git a/drivers/tty/serial/owl-uart.c b/drivers/tty/serial/owl-uart.c index d2d8b3494685..42c8cc93b603 100644 --- a/drivers/tty/serial/owl-uart.c +++ b/drivers/tty/serial/owl-uart.c @@ -427,7 +427,7 @@ static int owl_uart_request_port(struct uart_port *port) return -EBUSY; if (port->flags & UPF_IOREMAP) { - port->membase = devm_ioremap_nocache(port->dev, port->mapbase, + port->membase = devm_ioremap(port->dev, port->mapbase, resource_size(res)); if (!port->membase) return -EBUSY; diff --git a/drivers/tty/serial/pic32_uart.c b/drivers/tty/serial/pic32_uart.c index 0bdf1687983f..484b7e8d5381 100644 --- a/drivers/tty/serial/pic32_uart.c +++ b/drivers/tty/serial/pic32_uart.c @@ -618,7 +618,7 @@ static int pic32_uart_request_port(struct uart_port *port) "pic32_uart_mem")) return -EBUSY; - port->membase = devm_ioremap_nocache(port->dev, port->mapbase, + port->membase = devm_ioremap(port->dev, port->mapbase, resource_size(res_mem)); if (!port->membase) { dev_err(port->dev, "Unable to map registers\n"); diff --git a/drivers/tty/serial/rda-uart.c b/drivers/tty/serial/rda-uart.c index ff9a27d48bca..b5ef86ae2746 100644 --- a/drivers/tty/serial/rda-uart.c +++ b/drivers/tty/serial/rda-uart.c @@ -498,7 +498,7 @@ static int rda_uart_request_port(struct uart_port *port) return -EBUSY; if (port->flags & UPF_IOREMAP) { - port->membase = devm_ioremap_nocache(port->dev, port->mapbase, + port->membase = devm_ioremap(port->dev, port->mapbase, resource_size(res)); if (!port->membase) return -EBUSY; diff --git a/drivers/tty/serial/sb1250-duart.c b/drivers/tty/serial/sb1250-duart.c index 329aced26bd8..7c99340a3d66 100644 --- a/drivers/tty/serial/sb1250-duart.c +++ b/drivers/tty/serial/sb1250-duart.c @@ -668,7 +668,7 @@ static int sbd_map_port(struct uart_port *uport) struct sbd_duart *duart = sport->duart; if (!uport->membase) - uport->membase = ioremap_nocache(uport->mapbase, + uport->membase = ioremap(uport->mapbase, DUART_CHANREG_SPACING); if (!uport->membase) { printk(err); @@ -676,7 +676,7 @@ static int sbd_map_port(struct uart_port *uport) } if (!sport->memctrl) - sport->memctrl = ioremap_nocache(duart->mapctrl, + sport->memctrl = ioremap(duart->mapctrl, DUART_CHANREG_SPACING); if (!sport->memctrl) { printk(err); diff --git a/drivers/tty/serial/sh-sci.c b/drivers/tty/serial/sh-sci.c index 58bf9d496ba5..87ca6294de0e 100644 --- a/drivers/tty/serial/sh-sci.c +++ b/drivers/tty/serial/sh-sci.c @@ -2680,7 +2680,7 @@ static int sci_remap_port(struct uart_port *port) return 0; if (port->dev->of_node || (port->flags & UPF_IOREMAP)) { - port->membase = ioremap_nocache(port->mapbase, sport->reg_size); + port->membase = ioremap(port->mapbase, sport->reg_size); if (unlikely(!port->membase)) { dev_err(port->dev, "can't remap port#%d\n", port->line); return -ENXIO; diff --git a/drivers/tty/serial/zs.c b/drivers/tty/serial/zs.c index b03d3e458ea2..89154ac4c577 100644 --- a/drivers/tty/serial/zs.c +++ b/drivers/tty/serial/zs.c @@ -992,7 +992,7 @@ static void zs_release_port(struct uart_port *uport) static int zs_map_port(struct uart_port *uport) { if (!uport->membase) - uport->membase = ioremap_nocache(uport->mapbase, + uport->membase = ioremap(uport->mapbase, ZS_CHAN_IO_SIZE); if (!uport->membase) { printk(KERN_ERR "zs: Cannot map MMIO\n"); diff --git a/drivers/tty/synclink.c b/drivers/tty/synclink.c index 84f26e43b229..0ca13f889d84 100644 --- a/drivers/tty/synclink.c +++ b/drivers/tty/synclink.c @@ -4054,7 +4054,7 @@ static int mgsl_claim_resources(struct mgsl_struct *info) } info->lcr_mem_requested = true; - info->memory_base = ioremap_nocache(info->phys_memory_base, + info->memory_base = ioremap(info->phys_memory_base, 0x40000); if (!info->memory_base) { printk( "%s(%d):Can't map shared memory on device %s MemAddr=%08X\n", @@ -4068,7 +4068,7 @@ static int mgsl_claim_resources(struct mgsl_struct *info) goto errout; } - info->lcr_base = ioremap_nocache(info->phys_lcr_base, + info->lcr_base = ioremap(info->phys_lcr_base, PAGE_SIZE); if (!info->lcr_base) { printk( "%s(%d):Can't map LCR memory on device %s MemAddr=%08X\n", diff --git a/drivers/tty/synclink_gt.c b/drivers/tty/synclink_gt.c index e8a9047de451..e506fc489d48 100644 --- a/drivers/tty/synclink_gt.c +++ b/drivers/tty/synclink_gt.c @@ -3450,7 +3450,7 @@ static int claim_resources(struct slgt_info *info) else info->reg_addr_requested = true; - info->reg_addr = ioremap_nocache(info->phys_reg_addr, SLGT_REG_SIZE); + info->reg_addr = ioremap(info->phys_reg_addr, SLGT_REG_SIZE); if (!info->reg_addr) { DBGERR(("%s can't map device registers, addr=%08X\n", info->device_name, info->phys_reg_addr)); diff --git a/drivers/tty/synclinkmp.c b/drivers/tty/synclinkmp.c index fcb91bf7a15b..b9d974474b64 100644 --- a/drivers/tty/synclinkmp.c +++ b/drivers/tty/synclinkmp.c @@ -3559,7 +3559,7 @@ static int claim_resources(SLMP_INFO *info) else info->sca_statctrl_requested = true; - info->memory_base = ioremap_nocache(info->phys_memory_base, + info->memory_base = ioremap(info->phys_memory_base, SCA_MEM_SIZE); if (!info->memory_base) { printk( "%s(%d):%s Can't map shared memory, MemAddr=%08X\n", @@ -3568,7 +3568,7 @@ static int claim_resources(SLMP_INFO *info) goto errout; } - info->lcr_base = ioremap_nocache(info->phys_lcr_base, PAGE_SIZE); + info->lcr_base = ioremap(info->phys_lcr_base, PAGE_SIZE); if (!info->lcr_base) { printk( "%s(%d):%s Can't map LCR memory, MemAddr=%08X\n", __FILE__,__LINE__,info->device_name, info->phys_lcr_base ); @@ -3577,7 +3577,7 @@ static int claim_resources(SLMP_INFO *info) } info->lcr_base += info->lcr_offset; - info->sca_base = ioremap_nocache(info->phys_sca_base, PAGE_SIZE); + info->sca_base = ioremap(info->phys_sca_base, PAGE_SIZE); if (!info->sca_base) { printk( "%s(%d):%s Can't map SCA memory, MemAddr=%08X\n", __FILE__,__LINE__,info->device_name, info->phys_sca_base ); @@ -3586,7 +3586,7 @@ static int claim_resources(SLMP_INFO *info) } info->sca_base += info->sca_offset; - info->statctrl_base = ioremap_nocache(info->phys_statctrl_base, + info->statctrl_base = ioremap(info->phys_statctrl_base, PAGE_SIZE); if (!info->statctrl_base) { printk( "%s(%d):%s Can't map SCA Status/Control memory, MemAddr=%08X\n", diff --git a/drivers/usb/core/hcd-pci.c b/drivers/usb/core/hcd-pci.c index 9ae2a7a93df2..f0a259937da8 100644 --- a/drivers/usb/core/hcd-pci.c +++ b/drivers/usb/core/hcd-pci.c @@ -222,7 +222,7 @@ int usb_hcd_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) retval = -EBUSY; goto put_hcd; } - hcd->regs = devm_ioremap_nocache(&dev->dev, hcd->rsrc_start, + hcd->regs = devm_ioremap(&dev->dev, hcd->rsrc_start, hcd->rsrc_len); if (hcd->regs == NULL) { dev_dbg(&dev->dev, "error mapping memory\n"); diff --git a/drivers/usb/early/xhci-dbc.c b/drivers/usb/early/xhci-dbc.c index cac991173ac0..971c6b92484a 100644 --- a/drivers/usb/early/xhci-dbc.c +++ b/drivers/usb/early/xhci-dbc.c @@ -971,7 +971,7 @@ static int __init xdbc_init(void) goto free_and_quit; } - base = ioremap_nocache(xdbc.xhci_start, xdbc.xhci_length); + base = ioremap(xdbc.xhci_start, xdbc.xhci_length); if (!base) { xdbc_trace("failed to remap the io address\n"); ret = -ENOMEM; diff --git a/drivers/usb/gadget/udc/amd5536udc_pci.c b/drivers/usb/gadget/udc/amd5536udc_pci.c index 57b6f66331cf..bfd1c9e80a1f 100644 --- a/drivers/usb/gadget/udc/amd5536udc_pci.c +++ b/drivers/usb/gadget/udc/amd5536udc_pci.c @@ -116,7 +116,7 @@ static int udc_pci_probe( goto err_memreg; } - dev->virt_addr = ioremap_nocache(resource, len); + dev->virt_addr = ioremap(resource, len); if (!dev->virt_addr) { dev_dbg(&pdev->dev, "start address cannot be mapped\n"); retval = -EFAULT; diff --git a/drivers/usb/gadget/udc/goku_udc.c b/drivers/usb/gadget/udc/goku_udc.c index c3721225b61e..4a46f661d0e4 100644 --- a/drivers/usb/gadget/udc/goku_udc.c +++ b/drivers/usb/gadget/udc/goku_udc.c @@ -1782,7 +1782,7 @@ static int goku_probe(struct pci_dev *pdev, const struct pci_device_id *id) } dev->got_region = 1; - base = ioremap_nocache(resource, len); + base = ioremap(resource, len); if (base == NULL) { DBG(dev, "can't map memory\n"); retval = -EFAULT; diff --git a/drivers/usb/gadget/udc/net2272.c b/drivers/usb/gadget/udc/net2272.c index 247de0faaeb7..a8273b589456 100644 --- a/drivers/usb/gadget/udc/net2272.c +++ b/drivers/usb/gadget/udc/net2272.c @@ -2323,7 +2323,7 @@ net2272_rdk1_probe(struct pci_dev *pdev, struct net2272 *dev) goto err; } - mem_mapped_addr[i] = ioremap_nocache(resource, len); + mem_mapped_addr[i] = ioremap(resource, len); if (mem_mapped_addr[i] == NULL) { release_mem_region(resource, len); dev_dbg(dev->dev, "can't map memory\n"); @@ -2401,7 +2401,7 @@ net2272_rdk2_probe(struct pci_dev *pdev, struct net2272 *dev) goto err; } - mem_mapped_addr[i] = ioremap_nocache(resource, len); + mem_mapped_addr[i] = ioremap(resource, len); if (mem_mapped_addr[i] == NULL) { release_mem_region(resource, len); dev_dbg(dev->dev, "can't map memory\n"); @@ -2625,7 +2625,7 @@ net2272_plat_probe(struct platform_device *pdev) ret = -EBUSY; goto err; } - dev->base_addr = ioremap_nocache(base, len); + dev->base_addr = ioremap(base, len); if (!dev->base_addr) { dev_dbg(dev->dev, "can't map memory\n"); ret = -EFAULT; diff --git a/drivers/usb/gadget/udc/net2280.c b/drivers/usb/gadget/udc/net2280.c index 51efee21915f..1fd1b9186e46 100644 --- a/drivers/usb/gadget/udc/net2280.c +++ b/drivers/usb/gadget/udc/net2280.c @@ -3659,7 +3659,7 @@ static int net2280_probe(struct pci_dev *pdev, const struct pci_device_id *id) * 8051 code into the chip, e.g. to turn on PCI PM. */ - base = ioremap_nocache(resource, len); + base = ioremap(resource, len); if (base == NULL) { ep_dbg(dev, "can't map memory\n"); retval = -EFAULT; diff --git a/drivers/usb/host/ehci-pmcmsp.c b/drivers/usb/host/ehci-pmcmsp.c index a2b610dbedfc..2d462fbbe0a6 100644 --- a/drivers/usb/host/ehci-pmcmsp.c +++ b/drivers/usb/host/ehci-pmcmsp.c @@ -107,7 +107,7 @@ static int usb_hcd_msp_map_regs(struct mspusb_device *dev) if (!request_mem_region(res->start, res_len, "mab regs")) return -EBUSY; - dev->mab_regs = ioremap_nocache(res->start, res_len); + dev->mab_regs = ioremap(res->start, res_len); if (dev->mab_regs == NULL) { retval = -ENOMEM; goto err1; @@ -124,7 +124,7 @@ static int usb_hcd_msp_map_regs(struct mspusb_device *dev) retval = -EBUSY; goto err2; } - dev->usbid_regs = ioremap_nocache(res->start, res_len); + dev->usbid_regs = ioremap(res->start, res_len); if (dev->usbid_regs == NULL) { retval = -ENOMEM; goto err3; @@ -178,7 +178,7 @@ int usb_hcd_msp_probe(const struct hc_driver *driver, retval = -EBUSY; goto err1; } - hcd->regs = ioremap_nocache(hcd->rsrc_start, hcd->rsrc_len); + hcd->regs = ioremap(hcd->rsrc_start, hcd->rsrc_len); if (!hcd->regs) { pr_debug("ioremap failed"); retval = -ENOMEM; diff --git a/drivers/usb/host/pci-quirks.c b/drivers/usb/host/pci-quirks.c index 6c7f0a876b96..beb2efa71341 100644 --- a/drivers/usb/host/pci-quirks.c +++ b/drivers/usb/host/pci-quirks.c @@ -1150,7 +1150,7 @@ static void quirk_usb_handoff_xhci(struct pci_dev *pdev) if (!mmio_resource_enabled(pdev, 0)) return; - base = ioremap_nocache(pci_resource_start(pdev, 0), len); + base = ioremap(pci_resource_start(pdev, 0), len); if (base == NULL) return; diff --git a/drivers/usb/isp1760/isp1760-if.c b/drivers/usb/isp1760/isp1760-if.c index 07cc82ff327c..ccd30f835888 100644 --- a/drivers/usb/isp1760/isp1760-if.c +++ b/drivers/usb/isp1760/isp1760-if.c @@ -50,7 +50,7 @@ static int isp1761_pci_init(struct pci_dev *dev) } /* map available memory */ - iobase = ioremap_nocache(mem_start, mem_length); + iobase = ioremap(mem_start, mem_length); if (!iobase) { printk(KERN_ERR "Error ioremap failed\n"); release_mem_region(mem_start, mem_length); @@ -101,7 +101,7 @@ static int isp1761_pci_init(struct pci_dev *dev) return -EBUSY; } - iobase = ioremap_nocache(mem_start, mem_length); + iobase = ioremap(mem_start, mem_length); if (!iobase) { printk(KERN_ERR "ioremap #1\n"); release_mem_region(mem_start, mem_length); diff --git a/drivers/usb/roles/intel-xhci-usb-role-switch.c b/drivers/usb/roles/intel-xhci-usb-role-switch.c index 409851306e99..80d6559bbcb2 100644 --- a/drivers/usb/roles/intel-xhci-usb-role-switch.c +++ b/drivers/usb/roles/intel-xhci-usb-role-switch.c @@ -161,7 +161,7 @@ static int intel_xhci_usb_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!res) return -EINVAL; - data->base = devm_ioremap_nocache(dev, res->start, resource_size(res)); + data->base = devm_ioremap(dev, res->start, resource_size(res)); if (!data->base) return -ENOMEM; diff --git a/drivers/usb/typec/ucsi/ucsi_acpi.c b/drivers/usb/typec/ucsi/ucsi_acpi.c index 3f1786170098..9fc4f338e870 100644 --- a/drivers/usb/typec/ucsi/ucsi_acpi.c +++ b/drivers/usb/typec/ucsi/ucsi_acpi.c @@ -127,7 +127,7 @@ static int ucsi_acpi_probe(struct platform_device *pdev) return -ENODEV; } - /* This will make sure we can use ioremap_nocache() */ + /* This will make sure we can use ioremap() */ status = acpi_release_memory(ACPI_HANDLE(&pdev->dev), res, 1); if (ACPI_FAILURE(status)) return -ENOMEM; diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c index 0120d8324a40..a87992892a9f 100644 --- a/drivers/vfio/pci/vfio_pci_rdwr.c +++ b/drivers/vfio/pci/vfio_pci_rdwr.c @@ -230,7 +230,7 @@ ssize_t vfio_pci_vga_rw(struct vfio_pci_device *vdev, char __user *buf, switch ((u32)pos) { case 0xa0000 ... 0xbffff: count = min(count, (size_t)(0xc0000 - pos)); - iomem = ioremap_nocache(0xa0000, 0xbffff - 0xa0000 + 1); + iomem = ioremap(0xa0000, 0xbffff - 0xa0000 + 1); off = pos - 0xa0000; rsrc = VGA_RSRC_LEGACY_MEM; is_ioport = false; diff --git a/drivers/vfio/platform/reset/vfio_platform_amdxgbe.c b/drivers/vfio/platform/reset/vfio_platform_amdxgbe.c index 2d2babe21b2f..40d4fb9276ba 100644 --- a/drivers/vfio/platform/reset/vfio_platform_amdxgbe.c +++ b/drivers/vfio/platform/reset/vfio_platform_amdxgbe.c @@ -54,13 +54,13 @@ static int vfio_platform_amdxgbe_reset(struct vfio_platform_device *vdev) if (!xgmac_regs->ioaddr) { xgmac_regs->ioaddr = - ioremap_nocache(xgmac_regs->addr, xgmac_regs->size); + ioremap(xgmac_regs->addr, xgmac_regs->size); if (!xgmac_regs->ioaddr) return -ENOMEM; } if (!xpcs_regs->ioaddr) { xpcs_regs->ioaddr = - ioremap_nocache(xpcs_regs->addr, xpcs_regs->size); + ioremap(xpcs_regs->addr, xpcs_regs->size); if (!xpcs_regs->ioaddr) return -ENOMEM; } diff --git a/drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c b/drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c index 16165a62b86d..96064ef8f629 100644 --- a/drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c +++ b/drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c @@ -82,7 +82,7 @@ static int vfio_platform_bcmflexrm_reset(struct vfio_platform_device *vdev) /* Map FlexRM ring registers if not mapped */ if (!reg->ioaddr) { - reg->ioaddr = ioremap_nocache(reg->addr, reg->size); + reg->ioaddr = ioremap(reg->addr, reg->size); if (!reg->ioaddr) return -ENOMEM; } diff --git a/drivers/vfio/platform/reset/vfio_platform_calxedaxgmac.c b/drivers/vfio/platform/reset/vfio_platform_calxedaxgmac.c index f67bab547501..09a9453b75c5 100644 --- a/drivers/vfio/platform/reset/vfio_platform_calxedaxgmac.c +++ b/drivers/vfio/platform/reset/vfio_platform_calxedaxgmac.c @@ -52,7 +52,7 @@ static int vfio_platform_calxedaxgmac_reset(struct vfio_platform_device *vdev) if (!reg->ioaddr) { reg->ioaddr = - ioremap_nocache(reg->addr, reg->size); + ioremap(reg->addr, reg->size); if (!reg->ioaddr) return -ENOMEM; } diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c index e8f2bdbe0542..c0771a9567fb 100644 --- a/drivers/vfio/platform/vfio_platform_common.c +++ b/drivers/vfio/platform/vfio_platform_common.c @@ -409,7 +409,7 @@ static ssize_t vfio_platform_read_mmio(struct vfio_platform_region *reg, if (!reg->ioaddr) { reg->ioaddr = - ioremap_nocache(reg->addr, reg->size); + ioremap(reg->addr, reg->size); if (!reg->ioaddr) return -ENOMEM; @@ -486,7 +486,7 @@ static ssize_t vfio_platform_write_mmio(struct vfio_platform_region *reg, if (!reg->ioaddr) { reg->ioaddr = - ioremap_nocache(reg->addr, reg->size); + ioremap(reg->addr, reg->size); if (!reg->ioaddr) return -ENOMEM; diff --git a/drivers/video/fbdev/carminefb.c b/drivers/video/fbdev/carminefb.c index 9f3be0258623..27ba2ed4138a 100644 --- a/drivers/video/fbdev/carminefb.c +++ b/drivers/video/fbdev/carminefb.c @@ -633,7 +633,7 @@ static int carminefb_probe(struct pci_dev *dev, const struct pci_device_id *ent) ret = -EBUSY; goto err_free_hw; } - hw->v_regs = ioremap_nocache(carminefb_fix.mmio_start, + hw->v_regs = ioremap(carminefb_fix.mmio_start, carminefb_fix.mmio_len); if (!hw->v_regs) { printk(KERN_ERR "carminefb: Can't remap %s register.\n", @@ -664,7 +664,7 @@ static int carminefb_probe(struct pci_dev *dev, const struct pci_device_id *ent) goto err_unmap_vregs; } - hw->screen_mem = ioremap_nocache(carminefb_fix.smem_start, + hw->screen_mem = ioremap(carminefb_fix.smem_start, carminefb_fix.smem_len); if (!hw->screen_mem) { printk(KERN_ERR "carmine: Can't ioremap smem area.\n"); diff --git a/drivers/video/fbdev/i810/i810_main.c b/drivers/video/fbdev/i810/i810_main.c index d18f7b31932c..aa7583d963ac 100644 --- a/drivers/video/fbdev/i810/i810_main.c +++ b/drivers/video/fbdev/i810/i810_main.c @@ -1883,7 +1883,7 @@ static int i810_allocate_pci_resource(struct i810fb_par *par, } par->res_flags |= MMIO_REQ; - par->mmio_start_virtual = ioremap_nocache(par->mmio_start_phys, + par->mmio_start_virtual = ioremap(par->mmio_start_phys, MMIO_SIZE); if (!par->mmio_start_virtual) { printk("i810fb_init: cannot remap mmio region\n"); diff --git a/drivers/video/fbdev/intelfb/intelfbdrv.c b/drivers/video/fbdev/intelfb/intelfbdrv.c index a76c61512c60..a09fc2eaa40d 100644 --- a/drivers/video/fbdev/intelfb/intelfbdrv.c +++ b/drivers/video/fbdev/intelfb/intelfbdrv.c @@ -654,7 +654,7 @@ static int intelfb_pci_register(struct pci_dev *pdev, } dinfo->mmio_base = - (u8 __iomem *)ioremap_nocache(dinfo->mmio_base_phys, + (u8 __iomem *)ioremap(dinfo->mmio_base_phys, INTEL_REG_SIZE); if (!dinfo->mmio_base) { ERR_MSG("Cannot remap MMIO region.\n"); diff --git a/drivers/video/fbdev/kyro/fbdev.c b/drivers/video/fbdev/kyro/fbdev.c index a7bd9f25911b..a8660926924b 100644 --- a/drivers/video/fbdev/kyro/fbdev.c +++ b/drivers/video/fbdev/kyro/fbdev.c @@ -683,7 +683,7 @@ static int kyrofb_probe(struct pci_dev *pdev, const struct pci_device_id *ent) kyro_fix.mmio_len = pci_resource_len(pdev, 1); currentpar->regbase = deviceInfo.pSTGReg = - ioremap_nocache(kyro_fix.mmio_start, kyro_fix.mmio_len); + ioremap(kyro_fix.mmio_start, kyro_fix.mmio_len); if (!currentpar->regbase) goto out_free_fb; diff --git a/drivers/video/fbdev/matrox/matroxfb_base.c b/drivers/video/fbdev/matrox/matroxfb_base.c index 1a555f70923a..36cc718b96ae 100644 --- a/drivers/video/fbdev/matrox/matroxfb_base.c +++ b/drivers/video/fbdev/matrox/matroxfb_base.c @@ -1710,7 +1710,7 @@ static int initMatrox2(struct matrox_fb_info *minfo, struct board *b) memsize = mem; err = -ENOMEM; - minfo->mmio.vbase.vaddr = ioremap_nocache(ctrlptr_phys, 16384); + minfo->mmio.vbase.vaddr = ioremap(ctrlptr_phys, 16384); if (!minfo->mmio.vbase.vaddr) { printk(KERN_ERR "matroxfb: cannot ioremap(%lX, 16384), matroxfb disabled\n", ctrlptr_phys); goto failVideoMR; diff --git a/drivers/video/fbdev/mbx/mbxfb.c b/drivers/video/fbdev/mbx/mbxfb.c index 50935252b50b..3de4b3ed990a 100644 --- a/drivers/video/fbdev/mbx/mbxfb.c +++ b/drivers/video/fbdev/mbx/mbxfb.c @@ -938,7 +938,7 @@ static int mbxfb_probe(struct platform_device *dev) } mfbi->reg_phys_addr = mfbi->reg_res->start; - mfbi->reg_virt_addr = devm_ioremap_nocache(&dev->dev, + mfbi->reg_virt_addr = devm_ioremap(&dev->dev, mfbi->reg_phys_addr, res_size(mfbi->reg_req)); if (!mfbi->reg_virt_addr) { @@ -948,7 +948,7 @@ static int mbxfb_probe(struct platform_device *dev) } virt_base_2700 = mfbi->reg_virt_addr; - mfbi->fb_virt_addr = devm_ioremap_nocache(&dev->dev, mfbi->fb_phys_addr, + mfbi->fb_virt_addr = devm_ioremap(&dev->dev, mfbi->fb_phys_addr, res_size(mfbi->fb_req)); if (!mfbi->fb_virt_addr) { dev_err(&dev->dev, "failed to ioremap frame buffer\n"); diff --git a/drivers/video/fbdev/mmp/hw/mmp_ctrl.c b/drivers/video/fbdev/mmp/hw/mmp_ctrl.c index 17174cd7a5bb..974e4c28b08b 100644 --- a/drivers/video/fbdev/mmp/hw/mmp_ctrl.c +++ b/drivers/video/fbdev/mmp/hw/mmp_ctrl.c @@ -485,7 +485,7 @@ static int mmphw_probe(struct platform_device *pdev) goto failed; } - ctrl->reg_base = devm_ioremap_nocache(ctrl->dev, + ctrl->reg_base = devm_ioremap(ctrl->dev, res->start, resource_size(res)); if (ctrl->reg_base == NULL) { dev_err(ctrl->dev, "%s: res %pR map failed\n", __func__, res); diff --git a/drivers/video/fbdev/pm2fb.c b/drivers/video/fbdev/pm2fb.c index 1dcf02e12af4..7cc1216b1389 100644 --- a/drivers/video/fbdev/pm2fb.c +++ b/drivers/video/fbdev/pm2fb.c @@ -1563,7 +1563,7 @@ static int pm2fb_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_exit_neither; } default_par->v_regs = - ioremap_nocache(pm2fb_fix.mmio_start, pm2fb_fix.mmio_len); + ioremap(pm2fb_fix.mmio_start, pm2fb_fix.mmio_len); if (!default_par->v_regs) { printk(KERN_WARNING "pm2fb: Can't remap %s register area.\n", pm2fb_fix.id); diff --git a/drivers/video/fbdev/pm3fb.c b/drivers/video/fbdev/pm3fb.c index 6130aa56a1e9..2fa46607e0fc 100644 --- a/drivers/video/fbdev/pm3fb.c +++ b/drivers/video/fbdev/pm3fb.c @@ -1236,7 +1236,7 @@ static unsigned long pm3fb_size_memory(struct pm3_par *par) return 0; } screen_mem = - ioremap_nocache(pm3fb_fix.smem_start, pm3fb_fix.smem_len); + ioremap(pm3fb_fix.smem_start, pm3fb_fix.smem_len); if (!screen_mem) { printk(KERN_WARNING "pm3fb: Can't ioremap smem area.\n"); release_mem_region(pm3fb_fix.smem_start, pm3fb_fix.smem_len); @@ -1347,7 +1347,7 @@ static int pm3fb_probe(struct pci_dev *dev, const struct pci_device_id *ent) goto err_exit_neither; } par->v_regs = - ioremap_nocache(pm3fb_fix.mmio_start, pm3fb_fix.mmio_len); + ioremap(pm3fb_fix.mmio_start, pm3fb_fix.mmio_len); if (!par->v_regs) { printk(KERN_WARNING "pm3fb: Can't remap %s register area.\n", pm3fb_fix.id); diff --git a/drivers/video/fbdev/pmag-aa-fb.c b/drivers/video/fbdev/pmag-aa-fb.c index d1e78ce3a9c2..d5bf185fc376 100644 --- a/drivers/video/fbdev/pmag-aa-fb.c +++ b/drivers/video/fbdev/pmag-aa-fb.c @@ -188,7 +188,7 @@ static int pmagaafb_probe(struct device *dev) /* MMIO mapping setup. */ info->fix.mmio_start = start + PMAG_AA_BT455_OFFSET; - par->mmio = ioremap_nocache(info->fix.mmio_start, info->fix.mmio_len); + par->mmio = ioremap(info->fix.mmio_start, info->fix.mmio_len); if (!par->mmio) { printk(KERN_ERR "%s: Cannot map MMIO\n", dev_name(dev)); err = -ENOMEM; @@ -199,7 +199,7 @@ static int pmagaafb_probe(struct device *dev) /* Frame buffer mapping setup. */ info->fix.smem_start = start + PMAG_AA_ONBOARD_FBMEM_OFFSET; - info->screen_base = ioremap_nocache(info->fix.smem_start, + info->screen_base = ioremap(info->fix.smem_start, info->fix.smem_len); if (!info->screen_base) { printk(KERN_ERR "%s: Cannot map FB\n", dev_name(dev)); diff --git a/drivers/video/fbdev/pmag-ba-fb.c b/drivers/video/fbdev/pmag-ba-fb.c index 56b912bb28de..2ddcdf7919a2 100644 --- a/drivers/video/fbdev/pmag-ba-fb.c +++ b/drivers/video/fbdev/pmag-ba-fb.c @@ -180,7 +180,7 @@ static int pmagbafb_probe(struct device *dev) /* MMIO mapping setup. */ info->fix.mmio_start = start; - par->mmio = ioremap_nocache(info->fix.mmio_start, info->fix.mmio_len); + par->mmio = ioremap(info->fix.mmio_start, info->fix.mmio_len); if (!par->mmio) { printk(KERN_ERR "%s: Cannot map MMIO\n", dev_name(dev)); err = -ENOMEM; @@ -190,7 +190,7 @@ static int pmagbafb_probe(struct device *dev) /* Frame buffer mapping setup. */ info->fix.smem_start = start + PMAG_BA_FBMEM; - info->screen_base = ioremap_nocache(info->fix.smem_start, + info->screen_base = ioremap(info->fix.smem_start, info->fix.smem_len); if (!info->screen_base) { printk(KERN_ERR "%s: Cannot map FB\n", dev_name(dev)); diff --git a/drivers/video/fbdev/pmagb-b-fb.c b/drivers/video/fbdev/pmagb-b-fb.c index 2822b2225924..90d2b04feb42 100644 --- a/drivers/video/fbdev/pmagb-b-fb.c +++ b/drivers/video/fbdev/pmagb-b-fb.c @@ -287,7 +287,7 @@ static int pmagbbfb_probe(struct device *dev) /* MMIO mapping setup. */ info->fix.mmio_start = start; - par->mmio = ioremap_nocache(info->fix.mmio_start, info->fix.mmio_len); + par->mmio = ioremap(info->fix.mmio_start, info->fix.mmio_len); if (!par->mmio) { printk(KERN_ERR "%s: Cannot map MMIO\n", dev_name(dev)); err = -ENOMEM; @@ -298,7 +298,7 @@ static int pmagbbfb_probe(struct device *dev) /* Frame buffer mapping setup. */ info->fix.smem_start = start + PMAGB_B_FBMEM; - par->smem = ioremap_nocache(info->fix.smem_start, info->fix.smem_len); + par->smem = ioremap(info->fix.smem_start, info->fix.smem_len); if (!par->smem) { printk(KERN_ERR "%s: Cannot map FB\n", dev_name(dev)); err = -ENOMEM; diff --git a/drivers/video/fbdev/pvr2fb.c b/drivers/video/fbdev/pvr2fb.c index 0a3b2b7c7891..c680b3e651cb 100644 --- a/drivers/video/fbdev/pvr2fb.c +++ b/drivers/video/fbdev/pvr2fb.c @@ -770,7 +770,7 @@ static int __maybe_unused pvr2fb_common_init(void) struct pvr2fb_par *par = currentpar; unsigned long modememused, rev; - fb_info->screen_base = ioremap_nocache(pvr2_fix.smem_start, + fb_info->screen_base = ioremap(pvr2_fix.smem_start, pvr2_fix.smem_len); if (!fb_info->screen_base) { @@ -778,7 +778,7 @@ static int __maybe_unused pvr2fb_common_init(void) goto out_err; } - par->mmio_base = ioremap_nocache(pvr2_fix.mmio_start, + par->mmio_base = ioremap(pvr2_fix.mmio_start, pvr2_fix.mmio_len); if (!par->mmio_base) { printk(KERN_ERR "pvr2fb: Failed to remap mmio space\n"); diff --git a/drivers/video/fbdev/pxa168fb.c b/drivers/video/fbdev/pxa168fb.c index 1410f476e135..5615054a0cad 100644 --- a/drivers/video/fbdev/pxa168fb.c +++ b/drivers/video/fbdev/pxa168fb.c @@ -665,7 +665,7 @@ static int pxa168fb_probe(struct platform_device *pdev) /* * Map LCD controller registers. */ - fbi->reg_base = devm_ioremap_nocache(&pdev->dev, res->start, + fbi->reg_base = devm_ioremap(&pdev->dev, res->start, resource_size(res)); if (fbi->reg_base == NULL) { ret = -ENOMEM; diff --git a/drivers/video/fbdev/s1d13xxxfb.c b/drivers/video/fbdev/s1d13xxxfb.c index e04efb567b5c..8048499e398d 100644 --- a/drivers/video/fbdev/s1d13xxxfb.c +++ b/drivers/video/fbdev/s1d13xxxfb.c @@ -809,7 +809,7 @@ static int s1d13xxxfb_probe(struct platform_device *pdev) platform_set_drvdata(pdev, info); default_par = info->par; - default_par->regs = ioremap_nocache(pdev->resource[1].start, + default_par->regs = ioremap(pdev->resource[1].start, pdev->resource[1].end - pdev->resource[1].start +1); if (!default_par->regs) { printk(KERN_ERR PFX "unable to map registers\n"); @@ -818,7 +818,7 @@ static int s1d13xxxfb_probe(struct platform_device *pdev) } info->pseudo_palette = default_par->pseudo_palette; - info->screen_base = ioremap_nocache(pdev->resource[0].start, + info->screen_base = ioremap(pdev->resource[0].start, pdev->resource[0].end - pdev->resource[0].start +1); if (!info->screen_base) { diff --git a/drivers/video/fbdev/sh7760fb.c b/drivers/video/fbdev/sh7760fb.c index ab8fe838c776..f72b03594719 100644 --- a/drivers/video/fbdev/sh7760fb.c +++ b/drivers/video/fbdev/sh7760fb.c @@ -463,7 +463,7 @@ static int sh7760fb_probe(struct platform_device *pdev) goto out_fb; } - par->base = ioremap_nocache(res->start, resource_size(res)); + par->base = ioremap(res->start, resource_size(res)); if (!par->base) { dev_err(&pdev->dev, "cannot remap\n"); ret = -ENODEV; diff --git a/drivers/video/fbdev/sh_mobile_lcdcfb.c b/drivers/video/fbdev/sh_mobile_lcdcfb.c index c249763dbf0b..54ee7e02a244 100644 --- a/drivers/video/fbdev/sh_mobile_lcdcfb.c +++ b/drivers/video/fbdev/sh_mobile_lcdcfb.c @@ -2588,7 +2588,7 @@ static int sh_mobile_lcdc_probe(struct platform_device *pdev) if (num_channels == 2) priv->forced_fourcc = pdata->ch[0].fourcc; - priv->base = ioremap_nocache(res->start, resource_size(res)); + priv->base = ioremap(res->start, resource_size(res)); if (!priv->base) { error = -ENOMEM; goto err1; diff --git a/drivers/video/fbdev/sstfb.c b/drivers/video/fbdev/sstfb.c index 4e22ae383c87..1f171a527174 100644 --- a/drivers/video/fbdev/sstfb.c +++ b/drivers/video/fbdev/sstfb.c @@ -1363,14 +1363,14 @@ static int sstfb_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto fail_fb_mem; } - par->mmio_vbase = ioremap_nocache(fix->mmio_start, + par->mmio_vbase = ioremap(fix->mmio_start, fix->mmio_len); if (!par->mmio_vbase) { printk(KERN_ERR "sstfb: cannot remap register area %#lx\n", fix->mmio_start); goto fail_mmio_remap; } - info->screen_base = ioremap_nocache(fix->smem_start, 0x400000); + info->screen_base = ioremap(fix->smem_start, 0x400000); if (!info->screen_base) { printk(KERN_ERR "sstfb: cannot remap framebuffer %#lx\n", fix->smem_start); diff --git a/drivers/video/fbdev/stifb.c b/drivers/video/fbdev/stifb.c index 9e88e3f594c2..46709443a82f 100644 --- a/drivers/video/fbdev/stifb.c +++ b/drivers/video/fbdev/stifb.c @@ -1198,7 +1198,7 @@ static int __init stifb_init_fb(struct sti_struct *sti, int bpp_pref) case S9000_ID_TOMCAT: /* Dual CRX, behaves else like a CRX */ /* FIXME: TomCat supports two heads: * fb.iobase = REGION_BASE(fb_info,3); - * fb.screen_base = ioremap_nocache(REGION_BASE(fb_info,2),xxx); + * fb.screen_base = ioremap(REGION_BASE(fb_info,2),xxx); * for now we only support the left one ! */ xres = fb->ngle_rom.x_size_visible; yres = fb->ngle_rom.y_size_visible; @@ -1291,7 +1291,7 @@ static int __init stifb_init_fb(struct sti_struct *sti, int bpp_pref) strcpy(fix->id, "stifb"); info->fbops = &stifb_ops; - info->screen_base = ioremap_nocache(REGION_BASE(fb,1), fix->smem_len); + info->screen_base = ioremap(REGION_BASE(fb,1), fix->smem_len); if (!info->screen_base) { printk(KERN_ERR "stifb: failed to map memory\n"); goto out_err0; diff --git a/drivers/video/fbdev/tdfxfb.c b/drivers/video/fbdev/tdfxfb.c index fdbb1ea66e6c..0337d1a1a70b 100644 --- a/drivers/video/fbdev/tdfxfb.c +++ b/drivers/video/fbdev/tdfxfb.c @@ -1417,7 +1417,7 @@ static int tdfxfb_probe(struct pci_dev *pdev, const struct pci_device_id *id) } default_par->regbase_virt = - ioremap_nocache(info->fix.mmio_start, info->fix.mmio_len); + ioremap(info->fix.mmio_start, info->fix.mmio_len); if (!default_par->regbase_virt) { printk(KERN_ERR "fb: Can't remap %s register area.\n", info->fix.id); diff --git a/drivers/video/fbdev/tgafb.c b/drivers/video/fbdev/tgafb.c index 286b2371c7dd..1966f1d70899 100644 --- a/drivers/video/fbdev/tgafb.c +++ b/drivers/video/fbdev/tgafb.c @@ -1438,7 +1438,7 @@ static int tgafb_register(struct device *dev) } /* Map the framebuffer. */ - mem_base = ioremap_nocache(bar0_start, bar0_len); + mem_base = ioremap(bar0_start, bar0_len); if (!mem_base) { printk(KERN_ERR "tgafb: Cannot map MMIO\n"); goto err1; diff --git a/drivers/video/fbdev/tridentfb.c b/drivers/video/fbdev/tridentfb.c index da74bf6c5996..91b2f6ca2607 100644 --- a/drivers/video/fbdev/tridentfb.c +++ b/drivers/video/fbdev/tridentfb.c @@ -1556,7 +1556,7 @@ static int trident_pci_probe(struct pci_dev *dev, return -1; } - default_par->io_virt = ioremap_nocache(tridentfb_fix.mmio_start, + default_par->io_virt = ioremap(tridentfb_fix.mmio_start, tridentfb_fix.mmio_len); if (!default_par->io_virt) { @@ -1579,7 +1579,7 @@ static int trident_pci_probe(struct pci_dev *dev, goto out_unmap1; } - info->screen_base = ioremap_nocache(tridentfb_fix.smem_start, + info->screen_base = ioremap(tridentfb_fix.smem_start, tridentfb_fix.smem_len); if (!info->screen_base) { diff --git a/drivers/video/fbdev/valkyriefb.c b/drivers/video/fbdev/valkyriefb.c index e04fde9c1fcd..97a59b5a4570 100644 --- a/drivers/video/fbdev/valkyriefb.c +++ b/drivers/video/fbdev/valkyriefb.c @@ -356,7 +356,7 @@ int __init valkyriefb_init(void) p->total_vram = 0x100000; p->frame_buffer_phys = frame_buffer_phys; #ifdef CONFIG_MAC - p->frame_buffer = ioremap_nocache(frame_buffer_phys, p->total_vram); + p->frame_buffer = ioremap(frame_buffer_phys, p->total_vram); #else p->frame_buffer = ioremap_wt(frame_buffer_phys, p->total_vram); #endif diff --git a/drivers/video/fbdev/vermilion/cr_pll.c b/drivers/video/fbdev/vermilion/cr_pll.c index c1e3738e6789..79d42b23d850 100644 --- a/drivers/video/fbdev/vermilion/cr_pll.c +++ b/drivers/video/fbdev/vermilion/cr_pll.c @@ -159,7 +159,7 @@ static int __init cr_pll_init(void) pci_read_config_dword(mch_dev, CRVML_REG_MCHBAR, &mch_bar); mch_regs_base = - ioremap_nocache(mch_bar, CRVML_MCHMAP_SIZE); + ioremap(mch_bar, CRVML_MCHMAP_SIZE); if (!mch_regs_base) { printk(KERN_ERR "Carillo Ranch MCH device was not enabled.\n"); diff --git a/drivers/video/fbdev/vermilion/vermilion.c b/drivers/video/fbdev/vermilion/vermilion.c index 498038a964ee..ff61605b8764 100644 --- a/drivers/video/fbdev/vermilion/vermilion.c +++ b/drivers/video/fbdev/vermilion/vermilion.c @@ -317,7 +317,7 @@ static int vmlfb_enable_mmio(struct vml_par *par) ": Could not claim display controller MMIO.\n"); return -EBUSY; } - par->vdc_mem = ioremap_nocache(par->vdc_mem_base, par->vdc_mem_size); + par->vdc_mem = ioremap(par->vdc_mem_base, par->vdc_mem_size); if (par->vdc_mem == NULL) { printk(KERN_ERR MODULE_NAME ": Could not map display controller MMIO.\n"); @@ -332,7 +332,7 @@ static int vmlfb_enable_mmio(struct vml_par *par) err = -EBUSY; goto out_err_1; } - par->gpu_mem = ioremap_nocache(par->gpu_mem_base, par->gpu_mem_size); + par->gpu_mem = ioremap(par->gpu_mem_base, par->gpu_mem_size); if (par->gpu_mem == NULL) { printk(KERN_ERR MODULE_NAME ": Could not map GPU MMIO.\n"); err = -ENOMEM; diff --git a/drivers/video/fbdev/via/via-core.c b/drivers/video/fbdev/via/via-core.c index ffa2ca2d3f5e..703ddee9a244 100644 --- a/drivers/video/fbdev/via/via-core.c +++ b/drivers/video/fbdev/via/via-core.c @@ -442,7 +442,7 @@ static int via_pci_setup_mmio(struct viafb_dev *vdev) */ vdev->engine_start = pci_resource_start(vdev->pdev, 1); vdev->engine_len = pci_resource_len(vdev->pdev, 1); - vdev->engine_mmio = ioremap_nocache(vdev->engine_start, + vdev->engine_mmio = ioremap(vdev->engine_start, vdev->engine_len); if (vdev->engine_mmio == NULL) dev_err(&vdev->pdev->dev, diff --git a/drivers/video/fbdev/w100fb.c b/drivers/video/fbdev/w100fb.c index 3be07807edcd..0796b1d90981 100644 --- a/drivers/video/fbdev/w100fb.c +++ b/drivers/video/fbdev/w100fb.c @@ -648,12 +648,12 @@ int w100fb_probe(struct platform_device *pdev) return -EINVAL; /* Remap the chip base address */ - remapped_base = ioremap_nocache(mem->start+W100_CFG_BASE, W100_CFG_LEN); + remapped_base = ioremap(mem->start+W100_CFG_BASE, W100_CFG_LEN); if (remapped_base == NULL) goto out; /* Map the register space */ - remapped_regs = ioremap_nocache(mem->start+W100_REG_BASE, W100_REG_LEN); + remapped_regs = ioremap(mem->start+W100_REG_BASE, W100_REG_LEN); if (remapped_regs == NULL) goto out; @@ -672,7 +672,7 @@ int w100fb_probe(struct platform_device *pdev) printk(" at 0x%08lx.\n", (unsigned long) mem->start+W100_CFG_BASE); /* Remap the framebuffer */ - remapped_fbuf = ioremap_nocache(mem->start+MEM_WINDOW_BASE, MEM_WINDOW_SIZE); + remapped_fbuf = ioremap(mem->start+MEM_WINDOW_BASE, MEM_WINDOW_SIZE); if (remapped_fbuf == NULL) goto out; diff --git a/drivers/vme/boards/vme_vmivme7805.c b/drivers/vme/boards/vme_vmivme7805.c index 1b6e42e5e8fd..51e056bae943 100644 --- a/drivers/vme/boards/vme_vmivme7805.c +++ b/drivers/vme/boards/vme_vmivme7805.c @@ -55,7 +55,7 @@ static int vmic_probe(struct pci_dev *pdev, const struct pci_device_id *id) } /* Map registers in BAR 0 */ - vmic_base = ioremap_nocache(pci_resource_start(pdev, 0), 16); + vmic_base = ioremap(pci_resource_start(pdev, 0), 16); if (!vmic_base) { dev_err(&pdev->dev, "Unable to remap CRG region\n"); retval = -EIO; diff --git a/drivers/vme/bridges/vme_ca91cx42.c b/drivers/vme/bridges/vme_ca91cx42.c index 1edb8a5de873..ea938dc29c5e 100644 --- a/drivers/vme/bridges/vme_ca91cx42.c +++ b/drivers/vme/bridges/vme_ca91cx42.c @@ -554,7 +554,7 @@ static int ca91cx42_alloc_resource(struct vme_master_resource *image, goto err_resource; } - image->kern_base = ioremap_nocache( + image->kern_base = ioremap( image->bus_resource.start, size); if (!image->kern_base) { dev_err(ca91cx42_bridge->parent, "Failed to remap resource\n"); @@ -1638,7 +1638,7 @@ static int ca91cx42_probe(struct pci_dev *pdev, const struct pci_device_id *id) } /* map registers in BAR 0 */ - ca91cx42_device->base = ioremap_nocache(pci_resource_start(pdev, 0), + ca91cx42_device->base = ioremap(pci_resource_start(pdev, 0), 4096); if (!ca91cx42_device->base) { dev_err(&pdev->dev, "Unable to remap CRG region\n"); diff --git a/drivers/vme/bridges/vme_tsi148.c b/drivers/vme/bridges/vme_tsi148.c index 7e079d39bd76..50ae26977a02 100644 --- a/drivers/vme/bridges/vme_tsi148.c +++ b/drivers/vme/bridges/vme_tsi148.c @@ -770,7 +770,7 @@ static int tsi148_alloc_resource(struct vme_master_resource *image, goto err_resource; } - image->kern_base = ioremap_nocache( + image->kern_base = ioremap( image->bus_resource.start, size); if (!image->kern_base) { dev_err(tsi148_bridge->parent, "Failed to remap resource\n"); @@ -2317,7 +2317,7 @@ static int tsi148_probe(struct pci_dev *pdev, const struct pci_device_id *id) } /* map registers in BAR 0 */ - tsi148_device->base = ioremap_nocache(pci_resource_start(pdev, 0), + tsi148_device->base = ioremap(pci_resource_start(pdev, 0), 4096); if (!tsi148_device->base) { dev_err(&pdev->dev, "Unable to remap CRG region\n"); diff --git a/drivers/w1/masters/matrox_w1.c b/drivers/w1/masters/matrox_w1.c index 3110791a2f1c..ee716c715710 100644 --- a/drivers/w1/masters/matrox_w1.c +++ b/drivers/w1/masters/matrox_w1.c @@ -139,7 +139,7 @@ static int matrox_w1_probe(struct pci_dev *pdev, const struct pci_device_id *ent dev->phys_addr = pci_resource_start(pdev, 1); - dev->virt_addr = ioremap_nocache(dev->phys_addr, 16384); + dev->virt_addr = ioremap(dev->phys_addr, 16384); if (!dev->virt_addr) { dev_err(&pdev->dev, "%s: failed to ioremap(0x%lx, %d).\n", __func__, dev->phys_addr, 16384); diff --git a/drivers/watchdog/bcm63xx_wdt.c b/drivers/watchdog/bcm63xx_wdt.c index 8a043b52aa2f..7cdb25363ea0 100644 --- a/drivers/watchdog/bcm63xx_wdt.c +++ b/drivers/watchdog/bcm63xx_wdt.c @@ -246,7 +246,7 @@ static int bcm63xx_wdt_probe(struct platform_device *pdev) return -ENODEV; } - bcm63xx_wdt_device.regs = devm_ioremap_nocache(&pdev->dev, r->start, + bcm63xx_wdt_device.regs = devm_ioremap(&pdev->dev, r->start, resource_size(r)); if (!bcm63xx_wdt_device.regs) { dev_err(&pdev->dev, "failed to remap I/O resources\n"); diff --git a/drivers/watchdog/intel_scu_watchdog.c b/drivers/watchdog/intel_scu_watchdog.c index 6ad5bf3451ec..804e35940983 100644 --- a/drivers/watchdog/intel_scu_watchdog.c +++ b/drivers/watchdog/intel_scu_watchdog.c @@ -463,7 +463,7 @@ static int __init intel_scu_watchdog_init(void) return -ENODEV; } - tmp_addr = ioremap_nocache(watchdog_device.timer_tbl_ptr->phys_addr, + tmp_addr = ioremap(watchdog_device.timer_tbl_ptr->phys_addr, 20); if (tmp_addr == NULL) { diff --git a/drivers/watchdog/rc32434_wdt.c b/drivers/watchdog/rc32434_wdt.c index 1dfede0abf18..aee3c2efd565 100644 --- a/drivers/watchdog/rc32434_wdt.c +++ b/drivers/watchdog/rc32434_wdt.c @@ -26,7 +26,7 @@ #include /* For platform_driver framework */ #include /* For spin_lock/spin_unlock/... */ #include /* For copy_to_user/put_user/... */ -#include /* For devm_ioremap_nocache */ +#include /* For devm_ioremap */ #include /* For the Watchdog registers */ @@ -267,7 +267,7 @@ static int rc32434_wdt_probe(struct platform_device *pdev) return -ENODEV; } - wdt_reg = devm_ioremap_nocache(&pdev->dev, r->start, resource_size(r)); + wdt_reg = devm_ioremap(&pdev->dev, r->start, resource_size(r)); if (!wdt_reg) { pr_err("failed to remap I/O resources\n"); return -ENXIO; diff --git a/include/asm-generic/io.h b/include/asm-generic/io.h index 325fc98cc9ff..d39ac997dda8 100644 --- a/include/asm-generic/io.h +++ b/include/asm-generic/io.h @@ -960,10 +960,6 @@ static inline void __iomem *ioremap(phys_addr_t addr, size_t size) } #endif /* !CONFIG_MMU || CONFIG_GENERIC_IOREMAP */ -#ifndef ioremap_nocache -#define ioremap_nocache ioremap -#endif - #ifndef ioremap_wc #define ioremap_wc ioremap #endif diff --git a/include/asm-generic/iomap.h b/include/asm-generic/iomap.h index a008f504a2d0..9d28a5e82f73 100644 --- a/include/asm-generic/iomap.h +++ b/include/asm-generic/iomap.h @@ -94,11 +94,11 @@ extern void ioport_unmap(void __iomem *); #endif #ifndef ARCH_HAS_IOREMAP_WC -#define ioremap_wc ioremap_nocache +#define ioremap_wc ioremap #endif #ifndef ARCH_HAS_IOREMAP_WT -#define ioremap_wt ioremap_nocache +#define ioremap_wt ioremap #endif #ifdef CONFIG_PCI diff --git a/include/linux/io.h b/include/linux/io.h index a59834bc0a11..b1c44bb4b2d7 100644 --- a/include/linux/io.h +++ b/include/linux/io.h @@ -66,8 +66,6 @@ void __iomem *devm_ioremap(struct device *dev, resource_size_t offset, resource_size_t size); void __iomem *devm_ioremap_uc(struct device *dev, resource_size_t offset, resource_size_t size); -void __iomem *devm_ioremap_nocache(struct device *dev, resource_size_t offset, - resource_size_t size); void __iomem *devm_ioremap_wc(struct device *dev, resource_size_t offset, resource_size_t size); void devm_iounmap(struct device *dev, void __iomem *addr); @@ -87,7 +85,7 @@ void *__devm_memremap_pages(struct device *dev, struct resource *res); * Posting") mandate non-posted configuration transactions. There is * no ioremap API in the kernel that can guarantee non-posted write * semantics across arches so provide a default implementation for - * mapping PCI config space that defaults to ioremap_nocache(); arches + * mapping PCI config space that defaults to ioremap(); arches * should override it if they have memory mapping implementations that * guarantee non-posted writes semantics to make the memory mapping * compliant with the PCI specification. @@ -97,7 +95,7 @@ void *__devm_memremap_pages(struct device *dev, struct resource *res); static inline void __iomem *pci_remap_cfgspace(phys_addr_t offset, size_t size) { - return ioremap_nocache(offset, size); + return ioremap(offset, size); } #endif #endif diff --git a/lib/devres.c b/lib/devres.c index f56070cf970b..6ef51f159c54 100644 --- a/lib/devres.c +++ b/lib/devres.c @@ -8,7 +8,6 @@ enum devm_ioremap_type { DEVM_IOREMAP = 0, - DEVM_IOREMAP_NC, DEVM_IOREMAP_UC, DEVM_IOREMAP_WC, }; @@ -37,9 +36,6 @@ static void __iomem *__devm_ioremap(struct device *dev, resource_size_t offset, case DEVM_IOREMAP: addr = ioremap(offset, size); break; - case DEVM_IOREMAP_NC: - addr = ioremap_nocache(offset, size); - break; case DEVM_IOREMAP_UC: addr = ioremap_uc(offset, size); break; @@ -87,22 +83,6 @@ void __iomem *devm_ioremap_uc(struct device *dev, resource_size_t offset, } EXPORT_SYMBOL_GPL(devm_ioremap_uc); -/** - * devm_ioremap_nocache - Managed ioremap_nocache() - * @dev: Generic device to remap IO address for - * @offset: Resource address to map - * @size: Size of map - * - * Managed ioremap_nocache(). Map is automatically unmapped on driver - * detach. - */ -void __iomem *devm_ioremap_nocache(struct device *dev, resource_size_t offset, - resource_size_t size) -{ - return __devm_ioremap(dev, offset, size, DEVM_IOREMAP_NC); -} -EXPORT_SYMBOL(devm_ioremap_nocache); - /** * devm_ioremap_wc - Managed ioremap_wc() * @dev: Generic device to remap IO address for diff --git a/scripts/coccinelle/free/devm_free.cocci b/scripts/coccinelle/free/devm_free.cocci index 441799b5359b..9330d4294b74 100644 --- a/scripts/coccinelle/free/devm_free.cocci +++ b/scripts/coccinelle/free/devm_free.cocci @@ -51,8 +51,6 @@ expression x; x = devm_request_irq(...) | x = devm_ioremap(...) -| - x = devm_ioremap_nocache(...) | x = devm_ioport_map(...) ) @@ -84,8 +82,6 @@ position p; x = request_irq(...) | x = ioremap(...) -| - x = ioremap_nocache(...) | x = ioport_map(...) ) diff --git a/scripts/coccinelle/free/iounmap.cocci b/scripts/coccinelle/free/iounmap.cocci index 0e60e1113a1d..63b81d0c97b6 100644 --- a/scripts/coccinelle/free/iounmap.cocci +++ b/scripts/coccinelle/free/iounmap.cocci @@ -23,7 +23,7 @@ int ret; position p1,p2,p3; @@ -e = \(ioremap@p1\|ioremap_nocache@p1\)(...) +e = \(ioremap@p1\)(...) ... when != iounmap(e) if (<+...e...+>) S ... when any diff --git a/sound/drivers/ml403-ac97cr.c b/sound/drivers/ml403-ac97cr.c index 70a6d1832698..db1295f840b7 100644 --- a/sound/drivers/ml403-ac97cr.c +++ b/sound/drivers/ml403-ac97cr.c @@ -1123,7 +1123,7 @@ snd_ml403_ac97cr_create(struct snd_card *card, struct platform_device *pfdev, PDEBUG(INIT_INFO, "Trying to reserve resources now ...\n"); resource = platform_get_resource(pfdev, IORESOURCE_MEM, 0); /* get "port" */ - ml403_ac97cr->port = ioremap_nocache(resource->start, + ml403_ac97cr->port = ioremap(resource->start, (resource->end) - (resource->start) + 1); if (ml403_ac97cr->port == NULL) { diff --git a/sound/isa/msnd/msnd_pinnacle.c b/sound/isa/msnd/msnd_pinnacle.c index e435ebd0ced4..f15fe597582c 100644 --- a/sound/isa/msnd/msnd_pinnacle.c +++ b/sound/isa/msnd/msnd_pinnacle.c @@ -551,7 +551,7 @@ static int snd_msnd_attach(struct snd_card *card) free_irq(chip->irq, chip); return -EBUSY; } - chip->mappedbase = ioremap_nocache(chip->base, 0x8000); + chip->mappedbase = ioremap(chip->base, 0x8000); if (!chip->mappedbase) { printk(KERN_ERR LOGNAME ": unable to map memory region 0x%lx-0x%lx\n", diff --git a/sound/parisc/harmony.c b/sound/parisc/harmony.c index 6acc59c25379..0d1eced95f33 100644 --- a/sound/parisc/harmony.c +++ b/sound/parisc/harmony.c @@ -907,7 +907,7 @@ snd_harmony_create(struct snd_card *card, h->card = card; h->dev = padev; h->irq = -1; - h->iobase = ioremap_nocache(padev->hpa.start, HARMONY_SIZE); + h->iobase = ioremap(padev->hpa.start, HARMONY_SIZE); if (h->iobase == NULL) { printk(KERN_ERR PFX "unable to remap hpa 0x%lx\n", (unsigned long)padev->hpa.start); diff --git a/sound/pci/aw2/aw2-alsa.c b/sound/pci/aw2/aw2-alsa.c index 1cbfae856a2a..459c1691bb0c 100644 --- a/sound/pci/aw2/aw2-alsa.c +++ b/sound/pci/aw2/aw2-alsa.c @@ -271,7 +271,7 @@ static int snd_aw2_create(struct snd_card *card, } chip->iobase_phys = pci_resource_start(pci, 0); chip->iobase_virt = - ioremap_nocache(chip->iobase_phys, + ioremap(chip->iobase_phys, pci_resource_len(pci, 0)); if (chip->iobase_virt == NULL) { diff --git a/sound/pci/cs46xx/cs46xx_lib.c b/sound/pci/cs46xx/cs46xx_lib.c index 102a62965ac1..1465d7a17f7f 100644 --- a/sound/pci/cs46xx/cs46xx_lib.c +++ b/sound/pci/cs46xx/cs46xx_lib.c @@ -3983,7 +3983,7 @@ int snd_cs46xx_create(struct snd_card *card, snd_cs46xx_free(chip); return -EBUSY; } - region->remap_addr = ioremap_nocache(region->base, region->size); + region->remap_addr = ioremap(region->base, region->size); if (region->remap_addr == NULL) { dev_err(chip->card->dev, "%s ioremap problem\n", region->name); diff --git a/sound/pci/echoaudio/echoaudio.c b/sound/pci/echoaudio/echoaudio.c index 1465813bf7c6..dfd55419d667 100644 --- a/sound/pci/echoaudio/echoaudio.c +++ b/sound/pci/echoaudio/echoaudio.c @@ -1929,7 +1929,7 @@ static int snd_echo_create(struct snd_card *card, return -EBUSY; } chip->dsp_registers = (volatile u32 __iomem *) - ioremap_nocache(chip->dsp_registers_phys, sz); + ioremap(chip->dsp_registers_phys, sz); if (!chip->dsp_registers) { dev_err(chip->card->dev, "ioremap failed\n"); snd_echo_free(chip); diff --git a/sound/pci/nm256/nm256.c b/sound/pci/nm256/nm256.c index 1201c9c95660..77c683d19fbf 100644 --- a/sound/pci/nm256/nm256.c +++ b/sound/pci/nm256/nm256.c @@ -1353,7 +1353,7 @@ snd_nm256_peek_for_sig(struct nm256 *chip) unsigned long pointer_found = chip->buffer_end - 0x1400; u32 sig; - temp = ioremap_nocache(chip->buffer_addr + chip->buffer_end - 0x400, 16); + temp = ioremap(chip->buffer_addr + chip->buffer_end - 0x400, 16); if (temp == NULL) { dev_err(chip->card->dev, "Unable to scan for card signature in video RAM\n"); @@ -1518,7 +1518,7 @@ snd_nm256_create(struct snd_card *card, struct pci_dev *pci, err = -EBUSY; goto __error; } - chip->cport = ioremap_nocache(chip->cport_addr, NM_PORT2_SIZE); + chip->cport = ioremap(chip->cport_addr, NM_PORT2_SIZE); if (chip->cport == NULL) { dev_err(card->dev, "unable to map control port %lx\n", chip->cport_addr); @@ -1589,7 +1589,7 @@ snd_nm256_create(struct snd_card *card, struct pci_dev *pci, err = -EBUSY; goto __error; } - chip->buffer = ioremap_nocache(chip->buffer_addr, chip->buffer_size); + chip->buffer = ioremap(chip->buffer_addr, chip->buffer_size); if (chip->buffer == NULL) { err = -ENOMEM; dev_err(card->dev, "unable to map ring buffer at %lx\n", diff --git a/sound/pci/rme32.c b/sound/pci/rme32.c index 58a4b8df25d4..4a238de5a77e 100644 --- a/sound/pci/rme32.c +++ b/sound/pci/rme32.c @@ -1344,7 +1344,7 @@ static int snd_rme32_create(struct rme32 *rme32) return err; rme32->port = pci_resource_start(rme32->pci, 0); - rme32->iobase = ioremap_nocache(rme32->port, RME32_IO_SIZE); + rme32->iobase = ioremap(rme32->port, RME32_IO_SIZE); if (!rme32->iobase) { dev_err(rme32->card->dev, "unable to remap memory region 0x%lx-0x%lx\n", diff --git a/sound/pci/rme96.c b/sound/pci/rme96.c index 64ab55772eae..db6033074ace 100644 --- a/sound/pci/rme96.c +++ b/sound/pci/rme96.c @@ -1619,7 +1619,7 @@ snd_rme96_create(struct rme96 *rme96) return err; rme96->port = pci_resource_start(rme96->pci, 0); - rme96->iobase = ioremap_nocache(rme96->port, RME96_IO_SIZE); + rme96->iobase = ioremap(rme96->port, RME96_IO_SIZE); if (!rme96->iobase) { dev_err(rme96->card->dev, "unable to remap memory region 0x%lx-0x%lx\n", diff --git a/sound/pci/rme9652/hdsp.c b/sound/pci/rme9652/hdsp.c index cd20af465d8e..dfb06546ff25 100644 --- a/sound/pci/rme9652/hdsp.c +++ b/sound/pci/rme9652/hdsp.c @@ -5220,7 +5220,7 @@ static int snd_hdsp_create(struct snd_card *card, if ((err = pci_request_regions(pci, "hdsp")) < 0) return err; hdsp->port = pci_resource_start(pci, 0); - if ((hdsp->iobase = ioremap_nocache(hdsp->port, HDSP_IO_EXTENT)) == NULL) { + if ((hdsp->iobase = ioremap(hdsp->port, HDSP_IO_EXTENT)) == NULL) { dev_err(hdsp->card->dev, "unable to remap region 0x%lx-0x%lx\n", hdsp->port, hdsp->port + HDSP_IO_EXTENT - 1); return -EBUSY; diff --git a/sound/pci/rme9652/hdspm.c b/sound/pci/rme9652/hdspm.c index 75c06a7cc779..e2214ba4a38d 100644 --- a/sound/pci/rme9652/hdspm.c +++ b/sound/pci/rme9652/hdspm.c @@ -6594,7 +6594,7 @@ static int snd_hdspm_create(struct snd_card *card, dev_dbg(card->dev, "grabbed memory region 0x%lx-0x%lx\n", hdspm->port, hdspm->port + io_extent - 1); - hdspm->iobase = ioremap_nocache(hdspm->port, io_extent); + hdspm->iobase = ioremap(hdspm->port, io_extent); if (!hdspm->iobase) { dev_err(card->dev, "unable to remap region 0x%lx-0x%lx\n", hdspm->port, hdspm->port + io_extent - 1); diff --git a/sound/pci/rme9652/rme9652.c b/sound/pci/rme9652/rme9652.c index ef5c2f8e17c7..bb9130747fbb 100644 --- a/sound/pci/rme9652/rme9652.c +++ b/sound/pci/rme9652/rme9652.c @@ -2466,7 +2466,7 @@ static int snd_rme9652_create(struct snd_card *card, if ((err = pci_request_regions(pci, "rme9652")) < 0) return err; rme9652->port = pci_resource_start(pci, 0); - rme9652->iobase = ioremap_nocache(rme9652->port, RME9652_IO_EXTENT); + rme9652->iobase = ioremap(rme9652->port, RME9652_IO_EXTENT); if (rme9652->iobase == NULL) { dev_err(card->dev, "unable to remap region 0x%lx-0x%lx\n", rme9652->port, rme9652->port + RME9652_IO_EXTENT - 1); diff --git a/sound/pci/sis7019.c b/sound/pci/sis7019.c index ef7dd290ae05..ce13dcde4c36 100644 --- a/sound/pci/sis7019.c +++ b/sound/pci/sis7019.c @@ -1334,7 +1334,7 @@ static int sis_chip_create(struct snd_card *card, } rc = -EIO; - sis->ioaddr = ioremap_nocache(pci_resource_start(pci, 1), 0x4000); + sis->ioaddr = ioremap(pci_resource_start(pci, 1), 0x4000); if (!sis->ioaddr) { dev_err(&pci->dev, "unable to remap MMIO, aborting\n"); goto error_out_cleanup; diff --git a/sound/pci/ymfpci/ymfpci_main.c b/sound/pci/ymfpci/ymfpci_main.c index 125c11ed5064..d3907811f698 100644 --- a/sound/pci/ymfpci/ymfpci_main.c +++ b/sound/pci/ymfpci/ymfpci_main.c @@ -2373,7 +2373,7 @@ int snd_ymfpci_create(struct snd_card *card, chip->device_id = pci->device; chip->rev = pci->revision; chip->reg_area_phys = pci_resource_start(pci, 0); - chip->reg_area_virt = ioremap_nocache(chip->reg_area_phys, 0x8000); + chip->reg_area_virt = ioremap(chip->reg_area_phys, 0x8000); pci_set_master(pci); chip->src441_used = -1; diff --git a/sound/soc/au1x/ac97c.c b/sound/soc/au1x/ac97c.c index 0792c40e6cc1..d28302153d74 100644 --- a/sound/soc/au1x/ac97c.c +++ b/sound/soc/au1x/ac97c.c @@ -248,7 +248,7 @@ static int au1xac97c_drvprobe(struct platform_device *pdev) pdev->name)) return -EBUSY; - ctx->mmio = devm_ioremap_nocache(&pdev->dev, iores->start, + ctx->mmio = devm_ioremap(&pdev->dev, iores->start, resource_size(iores)); if (!ctx->mmio) return -EBUSY; diff --git a/sound/soc/au1x/i2sc.c b/sound/soc/au1x/i2sc.c index 46f2b447ec9a..7fd08fafa490 100644 --- a/sound/soc/au1x/i2sc.c +++ b/sound/soc/au1x/i2sc.c @@ -248,7 +248,7 @@ static int au1xi2s_drvprobe(struct platform_device *pdev) pdev->name)) return -EBUSY; - ctx->mmio = devm_ioremap_nocache(&pdev->dev, iores->start, + ctx->mmio = devm_ioremap(&pdev->dev, iores->start, resource_size(iores)); if (!ctx->mmio) return -EBUSY; diff --git a/sound/soc/intel/atom/sst/sst_acpi.c b/sound/soc/intel/atom/sst/sst_acpi.c index b728fb56ea4d..f3cfe83b9ac6 100644 --- a/sound/soc/intel/atom/sst/sst_acpi.c +++ b/sound/soc/intel/atom/sst/sst_acpi.c @@ -165,7 +165,7 @@ static int sst_platform_get_resources(struct intel_sst_drv *ctx) ctx->iram_base = rsrc->start + ctx->pdata->res_info->iram_offset; ctx->iram_end = ctx->iram_base + ctx->pdata->res_info->iram_size - 1; dev_info(ctx->dev, "IRAM base: %#x", ctx->iram_base); - ctx->iram = devm_ioremap_nocache(ctx->dev, ctx->iram_base, + ctx->iram = devm_ioremap(ctx->dev, ctx->iram_base, ctx->pdata->res_info->iram_size); if (!ctx->iram) { dev_err(ctx->dev, "unable to map IRAM\n"); @@ -175,7 +175,7 @@ static int sst_platform_get_resources(struct intel_sst_drv *ctx) ctx->dram_base = rsrc->start + ctx->pdata->res_info->dram_offset; ctx->dram_end = ctx->dram_base + ctx->pdata->res_info->dram_size - 1; dev_info(ctx->dev, "DRAM base: %#x", ctx->dram_base); - ctx->dram = devm_ioremap_nocache(ctx->dev, ctx->dram_base, + ctx->dram = devm_ioremap(ctx->dev, ctx->dram_base, ctx->pdata->res_info->dram_size); if (!ctx->dram) { dev_err(ctx->dev, "unable to map DRAM\n"); @@ -184,7 +184,7 @@ static int sst_platform_get_resources(struct intel_sst_drv *ctx) ctx->shim_phy_add = rsrc->start + ctx->pdata->res_info->shim_offset; dev_info(ctx->dev, "SHIM base: %#x", ctx->shim_phy_add); - ctx->shim = devm_ioremap_nocache(ctx->dev, ctx->shim_phy_add, + ctx->shim = devm_ioremap(ctx->dev, ctx->shim_phy_add, ctx->pdata->res_info->shim_size); if (!ctx->shim) { dev_err(ctx->dev, "unable to map SHIM\n"); @@ -197,7 +197,7 @@ static int sst_platform_get_resources(struct intel_sst_drv *ctx) /* Get mailbox addr */ ctx->mailbox_add = rsrc->start + ctx->pdata->res_info->mbox_offset; dev_info(ctx->dev, "Mailbox base: %#x", ctx->mailbox_add); - ctx->mailbox = devm_ioremap_nocache(ctx->dev, ctx->mailbox_add, + ctx->mailbox = devm_ioremap(ctx->dev, ctx->mailbox_add, ctx->pdata->res_info->mbox_size); if (!ctx->mailbox) { dev_err(ctx->dev, "unable to map mailbox\n"); @@ -216,7 +216,7 @@ static int sst_platform_get_resources(struct intel_sst_drv *ctx) ctx->ddr_base = rsrc->start; ctx->ddr_end = rsrc->end; dev_info(ctx->dev, "DDR base: %#x", ctx->ddr_base); - ctx->ddr = devm_ioremap_nocache(ctx->dev, ctx->ddr_base, + ctx->ddr = devm_ioremap(ctx->dev, ctx->ddr_base, resource_size(rsrc)); if (!ctx->ddr) { dev_err(ctx->dev, "unable to map DDR\n"); diff --git a/sound/soc/sh/fsi.c b/sound/soc/sh/fsi.c index e384fdc8d60e..2ead52bdb8ec 100644 --- a/sound/soc/sh/fsi.c +++ b/sound/soc/sh/fsi.c @@ -1955,7 +1955,7 @@ static int fsi_probe(struct platform_device *pdev) if (!master) return -ENOMEM; - master->base = devm_ioremap_nocache(&pdev->dev, + master->base = devm_ioremap(&pdev->dev, res->start, resource_size(res)); if (!master->base) { dev_err(&pdev->dev, "Unable to ioremap FSI registers.\n"); diff --git a/sound/x86/intel_hdmi_audio.c b/sound/x86/intel_hdmi_audio.c index cd389d21219a..cf258dd13050 100644 --- a/sound/x86/intel_hdmi_audio.c +++ b/sound/x86/intel_hdmi_audio.c @@ -1755,7 +1755,7 @@ static int hdmi_lpe_audio_probe(struct platform_device *pdev) __func__, (unsigned int)res_mmio->start, (unsigned int)res_mmio->end); - card_ctx->mmio_start = ioremap_nocache(res_mmio->start, + card_ctx->mmio_start = ioremap(res_mmio->start, (size_t)(resource_size(res_mmio))); if (!card_ctx->mmio_start) { dev_err(&pdev->dev, "Could not get ioremap\n"); diff --git a/tools/testing/nvdimm/Kbuild b/tools/testing/nvdimm/Kbuild index 6aca8d5be159..dbebf05f5931 100644 --- a/tools/testing/nvdimm/Kbuild +++ b/tools/testing/nvdimm/Kbuild @@ -1,10 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 ldflags-y += --wrap=ioremap_wc ldflags-y += --wrap=memremap -ldflags-y += --wrap=devm_ioremap_nocache +ldflags-y += --wrap=devm_ioremap ldflags-y += --wrap=devm_memremap ldflags-y += --wrap=devm_memunmap -ldflags-y += --wrap=ioremap_nocache ldflags-y += --wrap=ioremap ldflags-y += --wrap=iounmap ldflags-y += --wrap=memunmap diff --git a/tools/testing/nvdimm/test/iomap.c b/tools/testing/nvdimm/test/iomap.c index 6271ac757a4b..03e40b3b0106 100644 --- a/tools/testing/nvdimm/test/iomap.c +++ b/tools/testing/nvdimm/test/iomap.c @@ -73,7 +73,7 @@ void __iomem *__nfit_test_ioremap(resource_size_t offset, unsigned long size, return fallback_fn(offset, size); } -void __iomem *__wrap_devm_ioremap_nocache(struct device *dev, +void __iomem *__wrap_devm_ioremap(struct device *dev, resource_size_t offset, unsigned long size) { struct nfit_test_resource *nfit_res = get_nfit_res(offset); @@ -81,9 +81,9 @@ void __iomem *__wrap_devm_ioremap_nocache(struct device *dev, if (nfit_res) return (void __iomem *) nfit_res->buf + offset - nfit_res->res.start; - return devm_ioremap_nocache(dev, offset, size); + return devm_ioremap(dev, offset, size); } -EXPORT_SYMBOL(__wrap_devm_ioremap_nocache); +EXPORT_SYMBOL(__wrap_devm_ioremap); void *__wrap_devm_memremap(struct device *dev, resource_size_t offset, size_t size, unsigned long flags) @@ -187,12 +187,6 @@ void __wrap_devm_memunmap(struct device *dev, void *addr) } EXPORT_SYMBOL(__wrap_devm_memunmap); -void __iomem *__wrap_ioremap_nocache(resource_size_t offset, unsigned long size) -{ - return __nfit_test_ioremap(offset, size, ioremap_nocache); -} -EXPORT_SYMBOL(__wrap_ioremap_nocache); - void __iomem *__wrap_ioremap(resource_size_t offset, unsigned long size) { return __nfit_test_ioremap(offset, size, ioremap); diff --git a/tools/testing/nvdimm/test/nfit_test.h b/tools/testing/nvdimm/test/nfit_test.h index 0bf5640f1f07..db3c07beb9d1 100644 --- a/tools/testing/nvdimm/test/nfit_test.h +++ b/tools/testing/nvdimm/test/nfit_test.h @@ -207,8 +207,6 @@ typedef struct nfit_test_resource *(*nfit_test_lookup_fn)(resource_size_t); typedef union acpi_object *(*nfit_test_evaluate_dsm_fn)(acpi_handle handle, const guid_t *guid, u64 rev, u64 func, union acpi_object *argv4); -void __iomem *__wrap_ioremap_nocache(resource_size_t offset, - unsigned long size); void __wrap_iounmap(volatile void __iomem *addr); void nfit_test_setup(nfit_test_lookup_fn lookup, nfit_test_evaluate_dsm_fn evaluate); -- cgit v1.2.3 From c273a2bd8aa81b72e48736c3aa51f7ffeae39925 Mon Sep 17 00:00:00 2001 From: AKASHI Takahiro Date: Mon, 9 Dec 2019 12:03:44 +0900 Subject: libfdt: include fdt_addresses.c In the implementation of kexec_file_loaded-based kdump for arm64, fdt_appendprop_addrrange() will be needed. So include fdt_addresses.c in making libfdt. Signed-off-by: AKASHI Takahiro Cc: Rob Herring Cc: Frank Rowand Signed-off-by: Will Deacon --- lib/Makefile | 2 +- lib/fdt_addresses.c | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 lib/fdt_addresses.c (limited to 'lib') diff --git a/lib/Makefile b/lib/Makefile index 93217d44237f..c20b1debe9b4 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -223,7 +223,7 @@ KASAN_SANITIZE_stackdepot.o := n KCOV_INSTRUMENT_stackdepot.o := n libfdt_files = fdt.o fdt_ro.o fdt_wip.o fdt_rw.o fdt_sw.o fdt_strerror.o \ - fdt_empty_tree.o + fdt_empty_tree.o fdt_addresses.o $(foreach file, $(libfdt_files), \ $(eval CFLAGS_$(file) = -I $(srctree)/scripts/dtc/libfdt)) lib-$(CONFIG_LIBFDT) += $(libfdt_files) diff --git a/lib/fdt_addresses.c b/lib/fdt_addresses.c new file mode 100644 index 000000000000..23610bcf390b --- /dev/null +++ b/lib/fdt_addresses.c @@ -0,0 +1,2 @@ +#include +#include "../scripts/dtc/libfdt/fdt_addresses.c" -- cgit v1.2.3 From 109fb06fdc6f6788df7dfbc235f7636a38e28fd4 Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Mon, 6 Jan 2020 22:28:18 +0000 Subject: kunit: move string-stream.h to lib/kunit string-stream interfaces are not intended for external use; move them from include/kunit to lib/kunit accordingly. Co-developed-by: Knut Omang Signed-off-by: Knut Omang Signed-off-by: Alan Maguire Reviewed-by: Brendan Higgins Tested-by: Brendan Higgins Signed-off-by: Shuah Khan --- include/kunit/assert.h | 3 ++- include/kunit/string-stream.h | 51 ------------------------------------------ lib/kunit/assert.c | 2 ++ lib/kunit/string-stream-test.c | 3 ++- lib/kunit/string-stream.c | 3 ++- lib/kunit/string-stream.h | 51 ++++++++++++++++++++++++++++++++++++++++++ lib/kunit/test.c | 2 ++ 7 files changed, 61 insertions(+), 54 deletions(-) delete mode 100644 include/kunit/string-stream.h create mode 100644 lib/kunit/string-stream.h (limited to 'lib') diff --git a/include/kunit/assert.h b/include/kunit/assert.h index db6a0fca09b4..ad889b539ab3 100644 --- a/include/kunit/assert.h +++ b/include/kunit/assert.h @@ -9,10 +9,11 @@ #ifndef _KUNIT_ASSERT_H #define _KUNIT_ASSERT_H -#include #include +#include struct kunit; +struct string_stream; /** * enum kunit_assert_type - Type of expectation/assertion. diff --git a/include/kunit/string-stream.h b/include/kunit/string-stream.h deleted file mode 100644 index fe98a00b75a9..000000000000 --- a/include/kunit/string-stream.h +++ /dev/null @@ -1,51 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * C++ stream style string builder used in KUnit for building messages. - * - * Copyright (C) 2019, Google LLC. - * Author: Brendan Higgins - */ - -#ifndef _KUNIT_STRING_STREAM_H -#define _KUNIT_STRING_STREAM_H - -#include -#include -#include - -struct string_stream_fragment { - struct kunit *test; - struct list_head node; - char *fragment; -}; - -struct string_stream { - size_t length; - struct list_head fragments; - /* length and fragments are protected by this lock */ - spinlock_t lock; - struct kunit *test; - gfp_t gfp; -}; - -struct kunit; - -struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp); - -int __printf(2, 3) string_stream_add(struct string_stream *stream, - const char *fmt, ...); - -int string_stream_vadd(struct string_stream *stream, - const char *fmt, - va_list args); - -char *string_stream_get_string(struct string_stream *stream); - -int string_stream_append(struct string_stream *stream, - struct string_stream *other); - -bool string_stream_is_empty(struct string_stream *stream); - -int string_stream_destroy(struct string_stream *stream); - -#endif /* _KUNIT_STRING_STREAM_H */ diff --git a/lib/kunit/assert.c b/lib/kunit/assert.c index 86013d4cf891..9aca71cee3df 100644 --- a/lib/kunit/assert.c +++ b/lib/kunit/assert.c @@ -7,6 +7,8 @@ */ #include +#include "string-stream.h" + void kunit_base_assert_format(const struct kunit_assert *assert, struct string_stream *stream) { diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c index 76cc05eb00ed..6c70dc87f676 100644 --- a/lib/kunit/string-stream-test.c +++ b/lib/kunit/string-stream-test.c @@ -6,10 +6,11 @@ * Author: Brendan Higgins */ -#include #include #include +#include "string-stream.h" + static void string_stream_test_empty_on_creation(struct kunit *test) { struct string_stream *stream = alloc_string_stream(test, GFP_KERNEL); diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c index e6d17aacca30..350392013c14 100644 --- a/lib/kunit/string-stream.c +++ b/lib/kunit/string-stream.c @@ -6,11 +6,12 @@ * Author: Brendan Higgins */ -#include #include #include #include +#include "string-stream.h" + struct string_stream_fragment_alloc_context { struct kunit *test; int len; diff --git a/lib/kunit/string-stream.h b/lib/kunit/string-stream.h new file mode 100644 index 000000000000..fe98a00b75a9 --- /dev/null +++ b/lib/kunit/string-stream.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * C++ stream style string builder used in KUnit for building messages. + * + * Copyright (C) 2019, Google LLC. + * Author: Brendan Higgins + */ + +#ifndef _KUNIT_STRING_STREAM_H +#define _KUNIT_STRING_STREAM_H + +#include +#include +#include + +struct string_stream_fragment { + struct kunit *test; + struct list_head node; + char *fragment; +}; + +struct string_stream { + size_t length; + struct list_head fragments; + /* length and fragments are protected by this lock */ + spinlock_t lock; + struct kunit *test; + gfp_t gfp; +}; + +struct kunit; + +struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp); + +int __printf(2, 3) string_stream_add(struct string_stream *stream, + const char *fmt, ...); + +int string_stream_vadd(struct string_stream *stream, + const char *fmt, + va_list args); + +char *string_stream_get_string(struct string_stream *stream); + +int string_stream_append(struct string_stream *stream, + struct string_stream *other); + +bool string_stream_is_empty(struct string_stream *stream); + +int string_stream_destroy(struct string_stream *stream); + +#endif /* _KUNIT_STRING_STREAM_H */ diff --git a/lib/kunit/test.c b/lib/kunit/test.c index c83c0fa59cbd..36ebf47240f8 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -11,6 +11,8 @@ #include #include +#include "string-stream.h" + static void kunit_set_failure(struct kunit *test) { WRITE_ONCE(test->success, false); -- cgit v1.2.3 From 9bbb11c6be4623e38fdef8af0178c174acded9a6 Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Mon, 6 Jan 2020 22:28:19 +0000 Subject: kunit: hide unexported try-catch interface in try-catch-impl.h Define function as static inline in try-catch-impl.h to allow it to be used in kunit itself and tests. Also remove unused kunit_generic_try_catch Co-developed-by: Knut Omang Signed-off-by: Knut Omang Signed-off-by: Alan Maguire Reviewed-by: Brendan Higgins Tested-by: Brendan Higgins Signed-off-by: Shuah Khan --- include/kunit/try-catch.h | 10 ---------- lib/kunit/test-test.c | 2 ++ lib/kunit/test.c | 2 +- lib/kunit/try-catch-impl.h | 27 +++++++++++++++++++++++++++ lib/kunit/try-catch.c | 13 ++----------- 5 files changed, 32 insertions(+), 22 deletions(-) create mode 100644 lib/kunit/try-catch-impl.h (limited to 'lib') diff --git a/include/kunit/try-catch.h b/include/kunit/try-catch.h index 404f336cbdc8..c507dd43119d 100644 --- a/include/kunit/try-catch.h +++ b/include/kunit/try-catch.h @@ -53,11 +53,6 @@ struct kunit_try_catch { void *context; }; -void kunit_try_catch_init(struct kunit_try_catch *try_catch, - struct kunit *test, - kunit_try_catch_func_t try, - kunit_try_catch_func_t catch); - void kunit_try_catch_run(struct kunit_try_catch *try_catch, void *context); void __noreturn kunit_try_catch_throw(struct kunit_try_catch *try_catch); @@ -67,9 +62,4 @@ static inline int kunit_try_catch_get_result(struct kunit_try_catch *try_catch) return try_catch->try_result; } -/* - * Exposed for testing only. - */ -void kunit_generic_try_catch_init(struct kunit_try_catch *try_catch); - #endif /* _KUNIT_TRY_CATCH_H */ diff --git a/lib/kunit/test-test.c b/lib/kunit/test-test.c index 5ebe059d16e2..5a6cc0484eda 100644 --- a/lib/kunit/test-test.c +++ b/lib/kunit/test-test.c @@ -7,6 +7,8 @@ */ #include +#include "try-catch-impl.h" + struct kunit_try_catch_test_context { struct kunit_try_catch *try_catch; bool function_called; diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 36ebf47240f8..58a6227bb12c 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -7,11 +7,11 @@ */ #include -#include #include #include #include "string-stream.h" +#include "try-catch-impl.h" static void kunit_set_failure(struct kunit *test) { diff --git a/lib/kunit/try-catch-impl.h b/lib/kunit/try-catch-impl.h new file mode 100644 index 000000000000..203ba6a5e740 --- /dev/null +++ b/lib/kunit/try-catch-impl.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Internal kunit try catch implementation to be shared with tests. + * + * Copyright (C) 2019, Google LLC. + * Author: Brendan Higgins + */ + +#ifndef _KUNIT_TRY_CATCH_IMPL_H +#define _KUNIT_TRY_CATCH_IMPL_H + +#include +#include + +struct kunit; + +static inline void kunit_try_catch_init(struct kunit_try_catch *try_catch, + struct kunit *test, + kunit_try_catch_func_t try, + kunit_try_catch_func_t catch) +{ + try_catch->test = test; + try_catch->try = try; + try_catch->catch = catch; +} + +#endif /* _KUNIT_TRY_CATCH_IMPL_H */ diff --git a/lib/kunit/try-catch.c b/lib/kunit/try-catch.c index 55686839eb61..4a66d1684b46 100644 --- a/lib/kunit/try-catch.c +++ b/lib/kunit/try-catch.c @@ -8,12 +8,13 @@ */ #include -#include #include #include #include #include +#include "try-catch-impl.h" + void __noreturn kunit_try_catch_throw(struct kunit_try_catch *try_catch) { try_catch->try_result = -EFAULT; @@ -106,13 +107,3 @@ void kunit_try_catch_run(struct kunit_try_catch *try_catch, void *context) try_catch->catch(try_catch->context); } - -void kunit_try_catch_init(struct kunit_try_catch *try_catch, - struct kunit *test, - kunit_try_catch_func_t try, - kunit_try_catch_func_t catch) -{ - try_catch->test = test; - try_catch->try = try; - try_catch->catch = catch; -} -- cgit v1.2.3 From c475c77d5b56398303e726969e81208196b3aab3 Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Mon, 6 Jan 2020 22:28:20 +0000 Subject: kunit: allow kunit tests to be loaded as a module As tests are added to kunit, it will become less feasible to execute all built tests together. By supporting modular tests we provide a simple way to do selective execution on a running system; specifying CONFIG_KUNIT=y CONFIG_KUNIT_EXAMPLE_TEST=m ...means we can simply "insmod example-test.ko" to run the tests. To achieve this we need to do the following: o export the required symbols in kunit o string-stream tests utilize non-exported symbols so for now we skip building them when CONFIG_KUNIT_TEST=m. o drivers/base/power/qos-test.c contains a few unexported interface references, namely freq_qos_read_value() and freq_constraints_init(). Both of these could be potentially defined as static inline functions in include/linux/pm_qos.h, but for now we simply avoid supporting module build for that test suite. o support a new way of declaring test suites. Because a module cannot do multiple late_initcall()s, we provide a kunit_test_suites() macro to declare multiple suites within the same module at once. o some test module names would have been too general ("test-test" and "example-test" for kunit tests, "inode-test" for ext4 tests); rename these as appropriate ("kunit-test", "kunit-example-test" and "ext4-inode-test" respectively). Also define kunit_test_suite() via kunit_test_suites() as callers in other trees may need the old definition. Co-developed-by: Knut Omang Signed-off-by: Knut Omang Signed-off-by: Alan Maguire Reviewed-by: Brendan Higgins Acked-by: Theodore Ts'o # for ext4 bits Acked-by: David Gow # For list-test Reported-by: kbuild test robot Signed-off-by: Shuah Khan --- drivers/base/power/qos-test.c | 2 +- fs/ext4/Kconfig | 2 +- fs/ext4/Makefile | 3 +- fs/ext4/inode-test.c | 4 +- include/kunit/test.h | 37 +++-- kernel/sysctl-test.c | 4 +- lib/Kconfig.debug | 4 +- lib/kunit/Kconfig | 4 +- lib/kunit/Makefile | 10 +- lib/kunit/assert.c | 8 + lib/kunit/example-test.c | 88 ----------- lib/kunit/kunit-example-test.c | 90 +++++++++++ lib/kunit/kunit-test.c | 334 +++++++++++++++++++++++++++++++++++++++++ lib/kunit/string-stream-test.c | 2 +- lib/kunit/test-test.c | 333 ---------------------------------------- lib/kunit/test.c | 8 + lib/kunit/try-catch.c | 2 + lib/list-test.c | 4 +- 18 files changed, 494 insertions(+), 445 deletions(-) delete mode 100644 lib/kunit/example-test.c create mode 100644 lib/kunit/kunit-example-test.c create mode 100644 lib/kunit/kunit-test.c delete mode 100644 lib/kunit/test-test.c (limited to 'lib') diff --git a/drivers/base/power/qos-test.c b/drivers/base/power/qos-test.c index 3115db08d56b..79fc6c4418da 100644 --- a/drivers/base/power/qos-test.c +++ b/drivers/base/power/qos-test.c @@ -114,4 +114,4 @@ static struct kunit_suite pm_qos_test_module = { .name = "qos-kunit-test", .test_cases = pm_qos_test_cases, }; -kunit_test_suite(pm_qos_test_module); +kunit_test_suites(&pm_qos_test_module); diff --git a/fs/ext4/Kconfig b/fs/ext4/Kconfig index ef42ab040905..435510f83297 100644 --- a/fs/ext4/Kconfig +++ b/fs/ext4/Kconfig @@ -108,7 +108,7 @@ config EXT4_DEBUG echo 1 > /sys/module/ext4/parameters/mballoc_debug config EXT4_KUNIT_TESTS - bool "KUnit tests for ext4" + tristate "KUnit tests for ext4" select EXT4_FS depends on KUNIT help diff --git a/fs/ext4/Makefile b/fs/ext4/Makefile index 840b91d040f1..4ccb3c9189d8 100644 --- a/fs/ext4/Makefile +++ b/fs/ext4/Makefile @@ -13,5 +13,6 @@ ext4-y := balloc.o bitmap.o block_validity.o dir.o ext4_jbd2.o extents.o \ ext4-$(CONFIG_EXT4_FS_POSIX_ACL) += acl.o ext4-$(CONFIG_EXT4_FS_SECURITY) += xattr_security.o -ext4-$(CONFIG_EXT4_KUNIT_TESTS) += inode-test.o +ext4-inode-test-objs += inode-test.o +obj-$(CONFIG_EXT4_KUNIT_TESTS) += ext4-inode-test.o ext4-$(CONFIG_FS_VERITY) += verity.o diff --git a/fs/ext4/inode-test.c b/fs/ext4/inode-test.c index bbce1c328d85..d62d802c9c12 100644 --- a/fs/ext4/inode-test.c +++ b/fs/ext4/inode-test.c @@ -269,4 +269,6 @@ static struct kunit_suite ext4_inode_test_suite = { .test_cases = ext4_inode_test_cases, }; -kunit_test_suite(ext4_inode_test_suite); +kunit_test_suites(&ext4_inode_test_suite); + +MODULE_LICENSE("GPL v2"); diff --git a/include/kunit/test.h b/include/kunit/test.h index dba48304b3bd..2dfb550c6723 100644 --- a/include/kunit/test.h +++ b/include/kunit/test.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -197,31 +198,47 @@ void kunit_init_test(struct kunit *test, const char *name); int kunit_run_tests(struct kunit_suite *suite); /** - * kunit_test_suite() - used to register a &struct kunit_suite with KUnit. + * kunit_test_suites() - used to register one or more &struct kunit_suite + * with KUnit. * - * @suite: a statically allocated &struct kunit_suite. + * @suites: a statically allocated list of &struct kunit_suite. * - * Registers @suite with the test framework. See &struct kunit_suite for + * Registers @suites with the test framework. See &struct kunit_suite for * more information. * - * NOTE: Currently KUnit tests are all run as late_initcalls; this means + * When builtin, KUnit tests are all run as late_initcalls; this means * that they cannot test anything where tests must run at a different init * phase. One significant restriction resulting from this is that KUnit * cannot reliably test anything that is initialize in the late_init phase; * another is that KUnit is useless to test things that need to be run in * an earlier init phase. * + * An alternative is to build the tests as a module. Because modules + * do not support multiple late_initcall()s, we need to initialize an + * array of suites for a module. + * * TODO(brendanhiggins@google.com): Don't run all KUnit tests as * late_initcalls. I have some future work planned to dispatch all KUnit * tests from the same place, and at the very least to do so after * everything else is definitely initialized. */ -#define kunit_test_suite(suite) \ - static int kunit_suite_init##suite(void) \ - { \ - return kunit_run_tests(&suite); \ - } \ - late_initcall(kunit_suite_init##suite) +#define kunit_test_suites(...) \ + static struct kunit_suite *suites[] = { __VA_ARGS__, NULL}; \ + static int kunit_test_suites_init(void) \ + { \ + unsigned int i; \ + for (i = 0; suites[i] != NULL; i++) \ + kunit_run_tests(suites[i]); \ + return 0; \ + } \ + late_initcall(kunit_test_suites_init); \ + static void __exit kunit_test_suites_exit(void) \ + { \ + return; \ + } \ + module_exit(kunit_test_suites_exit) + +#define kunit_test_suite(suite) kunit_test_suites(&suite) /* * Like kunit_alloc_resource() below, but returns the struct kunit_resource diff --git a/kernel/sysctl-test.c b/kernel/sysctl-test.c index 2a63241a8453..ccb78509f1a8 100644 --- a/kernel/sysctl-test.c +++ b/kernel/sysctl-test.c @@ -389,4 +389,6 @@ static struct kunit_suite sysctl_test_suite = { .test_cases = sysctl_test_cases, }; -kunit_test_suite(sysctl_test_suite); +kunit_test_suites(&sysctl_test_suite); + +MODULE_LICENSE("GPL v2"); diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 5ffe144c9794..4ef415fff308 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2025,7 +2025,7 @@ config TEST_SYSCTL If unsure, say N. config SYSCTL_KUNIT_TEST - bool "KUnit test for sysctl" + tristate "KUnit test for sysctl" depends on KUNIT help This builds the proc sysctl unit test, which runs on boot. @@ -2036,7 +2036,7 @@ config SYSCTL_KUNIT_TEST If unsure, say N. config LIST_KUNIT_TEST - bool "KUnit Test for Kernel Linked-list structures" + tristate "KUnit Test for Kernel Linked-list structures" depends on KUNIT help This builds the linked list KUnit test suite. diff --git a/lib/kunit/Kconfig b/lib/kunit/Kconfig index af37016bfdd4..9ebd5e6d15fd 100644 --- a/lib/kunit/Kconfig +++ b/lib/kunit/Kconfig @@ -15,7 +15,7 @@ menuconfig KUNIT if KUNIT config KUNIT_TEST - bool "KUnit test for KUnit" + tristate "KUnit test for KUnit" help Enables the unit tests for the KUnit test framework. These tests test the KUnit test framework itself; the tests are both written using @@ -24,7 +24,7 @@ config KUNIT_TEST expected. config KUNIT_EXAMPLE_TEST - bool "Example test for KUnit" + tristate "Example test for KUnit" help Enables an example unit test that illustrates some of the basic features of KUnit. This test only exists to help new users understand diff --git a/lib/kunit/Makefile b/lib/kunit/Makefile index 769d9402b5d3..bc6e5e549e84 100644 --- a/lib/kunit/Makefile +++ b/lib/kunit/Makefile @@ -3,7 +3,11 @@ obj-$(CONFIG_KUNIT) += test.o \ assert.o \ try-catch.o -obj-$(CONFIG_KUNIT_TEST) += test-test.o \ - string-stream-test.o +obj-$(CONFIG_KUNIT_TEST) += kunit-test.o -obj-$(CONFIG_KUNIT_EXAMPLE_TEST) += example-test.o +# string-stream-test compiles built-in only. +ifeq ($(CONFIG_KUNIT_TEST),y) +obj-$(CONFIG_KUNIT_TEST) += string-stream-test.o +endif + +obj-$(CONFIG_KUNIT_EXAMPLE_TEST) += kunit-example-test.o diff --git a/lib/kunit/assert.c b/lib/kunit/assert.c index 9aca71cee3df..b24bebca052d 100644 --- a/lib/kunit/assert.c +++ b/lib/kunit/assert.c @@ -26,6 +26,7 @@ void kunit_base_assert_format(const struct kunit_assert *assert, string_stream_add(stream, "%s FAILED at %s:%d\n", expect_or_assert, assert->file, assert->line); } +EXPORT_SYMBOL_GPL(kunit_base_assert_format); void kunit_assert_print_msg(const struct kunit_assert *assert, struct string_stream *stream) @@ -33,6 +34,7 @@ void kunit_assert_print_msg(const struct kunit_assert *assert, if (assert->message.fmt) string_stream_add(stream, "\n%pV", &assert->message); } +EXPORT_SYMBOL_GPL(kunit_assert_print_msg); void kunit_fail_assert_format(const struct kunit_assert *assert, struct string_stream *stream) @@ -40,6 +42,7 @@ void kunit_fail_assert_format(const struct kunit_assert *assert, kunit_base_assert_format(assert, stream); string_stream_add(stream, "%pV", &assert->message); } +EXPORT_SYMBOL_GPL(kunit_fail_assert_format); void kunit_unary_assert_format(const struct kunit_assert *assert, struct string_stream *stream) @@ -58,6 +61,7 @@ void kunit_unary_assert_format(const struct kunit_assert *assert, unary_assert->condition); kunit_assert_print_msg(assert, stream); } +EXPORT_SYMBOL_GPL(kunit_unary_assert_format); void kunit_ptr_not_err_assert_format(const struct kunit_assert *assert, struct string_stream *stream) @@ -78,6 +82,7 @@ void kunit_ptr_not_err_assert_format(const struct kunit_assert *assert, } kunit_assert_print_msg(assert, stream); } +EXPORT_SYMBOL_GPL(kunit_ptr_not_err_assert_format); void kunit_binary_assert_format(const struct kunit_assert *assert, struct string_stream *stream) @@ -99,6 +104,7 @@ void kunit_binary_assert_format(const struct kunit_assert *assert, binary_assert->right_value); kunit_assert_print_msg(assert, stream); } +EXPORT_SYMBOL_GPL(kunit_binary_assert_format); void kunit_binary_ptr_assert_format(const struct kunit_assert *assert, struct string_stream *stream) @@ -120,6 +126,7 @@ void kunit_binary_ptr_assert_format(const struct kunit_assert *assert, binary_assert->right_value); kunit_assert_print_msg(assert, stream); } +EXPORT_SYMBOL_GPL(kunit_binary_ptr_assert_format); void kunit_binary_str_assert_format(const struct kunit_assert *assert, struct string_stream *stream) @@ -141,3 +148,4 @@ void kunit_binary_str_assert_format(const struct kunit_assert *assert, binary_assert->right_value); kunit_assert_print_msg(assert, stream); } +EXPORT_SYMBOL_GPL(kunit_binary_str_assert_format); diff --git a/lib/kunit/example-test.c b/lib/kunit/example-test.c deleted file mode 100644 index f64a829aa441..000000000000 --- a/lib/kunit/example-test.c +++ /dev/null @@ -1,88 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Example KUnit test to show how to use KUnit. - * - * Copyright (C) 2019, Google LLC. - * Author: Brendan Higgins - */ - -#include - -/* - * This is the most fundamental element of KUnit, the test case. A test case - * makes a set EXPECTATIONs and ASSERTIONs about the behavior of some code; if - * any expectations or assertions are not met, the test fails; otherwise, the - * test passes. - * - * In KUnit, a test case is just a function with the signature - * `void (*)(struct kunit *)`. `struct kunit` is a context object that stores - * information about the current test. - */ -static void example_simple_test(struct kunit *test) -{ - /* - * This is an EXPECTATION; it is how KUnit tests things. When you want - * to test a piece of code, you set some expectations about what the - * code should do. KUnit then runs the test and verifies that the code's - * behavior matched what was expected. - */ - KUNIT_EXPECT_EQ(test, 1 + 1, 2); -} - -/* - * This is run once before each test case, see the comment on - * example_test_suite for more information. - */ -static int example_test_init(struct kunit *test) -{ - kunit_info(test, "initializing\n"); - - return 0; -} - -/* - * Here we make a list of all the test cases we want to add to the test suite - * below. - */ -static struct kunit_case example_test_cases[] = { - /* - * This is a helper to create a test case object from a test case - * function; its exact function is not important to understand how to - * use KUnit, just know that this is how you associate test cases with a - * test suite. - */ - KUNIT_CASE(example_simple_test), - {} -}; - -/* - * This defines a suite or grouping of tests. - * - * Test cases are defined as belonging to the suite by adding them to - * `kunit_cases`. - * - * Often it is desirable to run some function which will set up things which - * will be used by every test; this is accomplished with an `init` function - * which runs before each test case is invoked. Similarly, an `exit` function - * may be specified which runs after every test case and can be used to for - * cleanup. For clarity, running tests in a test suite would behave as follows: - * - * suite.init(test); - * suite.test_case[0](test); - * suite.exit(test); - * suite.init(test); - * suite.test_case[1](test); - * suite.exit(test); - * ...; - */ -static struct kunit_suite example_test_suite = { - .name = "example", - .init = example_test_init, - .test_cases = example_test_cases, -}; - -/* - * This registers the above test suite telling KUnit that this is a suite of - * tests that need to be run. - */ -kunit_test_suite(example_test_suite); diff --git a/lib/kunit/kunit-example-test.c b/lib/kunit/kunit-example-test.c new file mode 100644 index 000000000000..be1164ecc476 --- /dev/null +++ b/lib/kunit/kunit-example-test.c @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Example KUnit test to show how to use KUnit. + * + * Copyright (C) 2019, Google LLC. + * Author: Brendan Higgins + */ + +#include + +/* + * This is the most fundamental element of KUnit, the test case. A test case + * makes a set EXPECTATIONs and ASSERTIONs about the behavior of some code; if + * any expectations or assertions are not met, the test fails; otherwise, the + * test passes. + * + * In KUnit, a test case is just a function with the signature + * `void (*)(struct kunit *)`. `struct kunit` is a context object that stores + * information about the current test. + */ +static void example_simple_test(struct kunit *test) +{ + /* + * This is an EXPECTATION; it is how KUnit tests things. When you want + * to test a piece of code, you set some expectations about what the + * code should do. KUnit then runs the test and verifies that the code's + * behavior matched what was expected. + */ + KUNIT_EXPECT_EQ(test, 1 + 1, 2); +} + +/* + * This is run once before each test case, see the comment on + * example_test_suite for more information. + */ +static int example_test_init(struct kunit *test) +{ + kunit_info(test, "initializing\n"); + + return 0; +} + +/* + * Here we make a list of all the test cases we want to add to the test suite + * below. + */ +static struct kunit_case example_test_cases[] = { + /* + * This is a helper to create a test case object from a test case + * function; its exact function is not important to understand how to + * use KUnit, just know that this is how you associate test cases with a + * test suite. + */ + KUNIT_CASE(example_simple_test), + {} +}; + +/* + * This defines a suite or grouping of tests. + * + * Test cases are defined as belonging to the suite by adding them to + * `kunit_cases`. + * + * Often it is desirable to run some function which will set up things which + * will be used by every test; this is accomplished with an `init` function + * which runs before each test case is invoked. Similarly, an `exit` function + * may be specified which runs after every test case and can be used to for + * cleanup. For clarity, running tests in a test suite would behave as follows: + * + * suite.init(test); + * suite.test_case[0](test); + * suite.exit(test); + * suite.init(test); + * suite.test_case[1](test); + * suite.exit(test); + * ...; + */ +static struct kunit_suite example_test_suite = { + .name = "example", + .init = example_test_init, + .test_cases = example_test_cases, +}; + +/* + * This registers the above test suite telling KUnit that this is a suite of + * tests that need to be run. + */ +kunit_test_suites(&example_test_suite); + +MODULE_LICENSE("GPL v2"); diff --git a/lib/kunit/kunit-test.c b/lib/kunit/kunit-test.c new file mode 100644 index 000000000000..ccb8d2e332f7 --- /dev/null +++ b/lib/kunit/kunit-test.c @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit test for core test infrastructure. + * + * Copyright (C) 2019, Google LLC. + * Author: Brendan Higgins + */ +#include + +#include "try-catch-impl.h" + +struct kunit_try_catch_test_context { + struct kunit_try_catch *try_catch; + bool function_called; +}; + +static void kunit_test_successful_try(void *data) +{ + struct kunit *test = data; + struct kunit_try_catch_test_context *ctx = test->priv; + + ctx->function_called = true; +} + +static void kunit_test_no_catch(void *data) +{ + struct kunit *test = data; + + KUNIT_FAIL(test, "Catch should not be called\n"); +} + +static void kunit_test_try_catch_successful_try_no_catch(struct kunit *test) +{ + struct kunit_try_catch_test_context *ctx = test->priv; + struct kunit_try_catch *try_catch = ctx->try_catch; + + kunit_try_catch_init(try_catch, + test, + kunit_test_successful_try, + kunit_test_no_catch); + kunit_try_catch_run(try_catch, test); + + KUNIT_EXPECT_TRUE(test, ctx->function_called); +} + +static void kunit_test_unsuccessful_try(void *data) +{ + struct kunit *test = data; + struct kunit_try_catch_test_context *ctx = test->priv; + struct kunit_try_catch *try_catch = ctx->try_catch; + + kunit_try_catch_throw(try_catch); + KUNIT_FAIL(test, "This line should never be reached\n"); +} + +static void kunit_test_catch(void *data) +{ + struct kunit *test = data; + struct kunit_try_catch_test_context *ctx = test->priv; + + ctx->function_called = true; +} + +static void kunit_test_try_catch_unsuccessful_try_does_catch(struct kunit *test) +{ + struct kunit_try_catch_test_context *ctx = test->priv; + struct kunit_try_catch *try_catch = ctx->try_catch; + + kunit_try_catch_init(try_catch, + test, + kunit_test_unsuccessful_try, + kunit_test_catch); + kunit_try_catch_run(try_catch, test); + + KUNIT_EXPECT_TRUE(test, ctx->function_called); +} + +static int kunit_try_catch_test_init(struct kunit *test) +{ + struct kunit_try_catch_test_context *ctx; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx); + test->priv = ctx; + + ctx->try_catch = kunit_kmalloc(test, + sizeof(*ctx->try_catch), + GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx->try_catch); + + return 0; +} + +static struct kunit_case kunit_try_catch_test_cases[] = { + KUNIT_CASE(kunit_test_try_catch_successful_try_no_catch), + KUNIT_CASE(kunit_test_try_catch_unsuccessful_try_does_catch), + {} +}; + +static struct kunit_suite kunit_try_catch_test_suite = { + .name = "kunit-try-catch-test", + .init = kunit_try_catch_test_init, + .test_cases = kunit_try_catch_test_cases, +}; + +/* + * Context for testing test managed resources + * is_resource_initialized is used to test arbitrary resources + */ +struct kunit_test_resource_context { + struct kunit test; + bool is_resource_initialized; + int allocate_order[2]; + int free_order[2]; +}; + +static int fake_resource_init(struct kunit_resource *res, void *context) +{ + struct kunit_test_resource_context *ctx = context; + + res->allocation = &ctx->is_resource_initialized; + ctx->is_resource_initialized = true; + return 0; +} + +static void fake_resource_free(struct kunit_resource *res) +{ + bool *is_resource_initialized = res->allocation; + + *is_resource_initialized = false; +} + +static void kunit_resource_test_init_resources(struct kunit *test) +{ + struct kunit_test_resource_context *ctx = test->priv; + + kunit_init_test(&ctx->test, "testing_test_init_test"); + + KUNIT_EXPECT_TRUE(test, list_empty(&ctx->test.resources)); +} + +static void kunit_resource_test_alloc_resource(struct kunit *test) +{ + struct kunit_test_resource_context *ctx = test->priv; + struct kunit_resource *res; + kunit_resource_free_t free = fake_resource_free; + + res = kunit_alloc_and_get_resource(&ctx->test, + fake_resource_init, + fake_resource_free, + GFP_KERNEL, + ctx); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, res); + KUNIT_EXPECT_PTR_EQ(test, + &ctx->is_resource_initialized, + (bool *) res->allocation); + KUNIT_EXPECT_TRUE(test, list_is_last(&res->node, &ctx->test.resources)); + KUNIT_EXPECT_PTR_EQ(test, free, res->free); +} + +static void kunit_resource_test_destroy_resource(struct kunit *test) +{ + struct kunit_test_resource_context *ctx = test->priv; + struct kunit_resource *res = kunit_alloc_and_get_resource( + &ctx->test, + fake_resource_init, + fake_resource_free, + GFP_KERNEL, + ctx); + + KUNIT_ASSERT_FALSE(test, + kunit_resource_destroy(&ctx->test, + kunit_resource_instance_match, + res->free, + res->allocation)); + + KUNIT_EXPECT_FALSE(test, ctx->is_resource_initialized); + KUNIT_EXPECT_TRUE(test, list_empty(&ctx->test.resources)); +} + +static void kunit_resource_test_cleanup_resources(struct kunit *test) +{ + int i; + struct kunit_test_resource_context *ctx = test->priv; + struct kunit_resource *resources[5]; + + for (i = 0; i < ARRAY_SIZE(resources); i++) { + resources[i] = kunit_alloc_and_get_resource(&ctx->test, + fake_resource_init, + fake_resource_free, + GFP_KERNEL, + ctx); + } + + kunit_cleanup(&ctx->test); + + KUNIT_EXPECT_TRUE(test, list_empty(&ctx->test.resources)); +} + +static void kunit_resource_test_mark_order(int order_array[], + size_t order_size, + int key) +{ + int i; + + for (i = 0; i < order_size && order_array[i]; i++) + ; + + order_array[i] = key; +} + +#define KUNIT_RESOURCE_TEST_MARK_ORDER(ctx, order_field, key) \ + kunit_resource_test_mark_order(ctx->order_field, \ + ARRAY_SIZE(ctx->order_field), \ + key) + +static int fake_resource_2_init(struct kunit_resource *res, void *context) +{ + struct kunit_test_resource_context *ctx = context; + + KUNIT_RESOURCE_TEST_MARK_ORDER(ctx, allocate_order, 2); + + res->allocation = ctx; + + return 0; +} + +static void fake_resource_2_free(struct kunit_resource *res) +{ + struct kunit_test_resource_context *ctx = res->allocation; + + KUNIT_RESOURCE_TEST_MARK_ORDER(ctx, free_order, 2); +} + +static int fake_resource_1_init(struct kunit_resource *res, void *context) +{ + struct kunit_test_resource_context *ctx = context; + + kunit_alloc_and_get_resource(&ctx->test, + fake_resource_2_init, + fake_resource_2_free, + GFP_KERNEL, + ctx); + + KUNIT_RESOURCE_TEST_MARK_ORDER(ctx, allocate_order, 1); + + res->allocation = ctx; + + return 0; +} + +static void fake_resource_1_free(struct kunit_resource *res) +{ + struct kunit_test_resource_context *ctx = res->allocation; + + KUNIT_RESOURCE_TEST_MARK_ORDER(ctx, free_order, 1); +} + +/* + * TODO(brendanhiggins@google.com): replace the arrays that keep track of the + * order of allocation and freeing with strict mocks using the IN_SEQUENCE macro + * to assert allocation and freeing order when the feature becomes available. + */ +static void kunit_resource_test_proper_free_ordering(struct kunit *test) +{ + struct kunit_test_resource_context *ctx = test->priv; + + /* fake_resource_1 allocates a fake_resource_2 in its init. */ + kunit_alloc_and_get_resource(&ctx->test, + fake_resource_1_init, + fake_resource_1_free, + GFP_KERNEL, + ctx); + + /* + * Since fake_resource_2_init calls KUNIT_RESOURCE_TEST_MARK_ORDER + * before returning to fake_resource_1_init, it should be the first to + * put its key in the allocate_order array. + */ + KUNIT_EXPECT_EQ(test, ctx->allocate_order[0], 2); + KUNIT_EXPECT_EQ(test, ctx->allocate_order[1], 1); + + kunit_cleanup(&ctx->test); + + /* + * Because fake_resource_2 finishes allocation before fake_resource_1, + * fake_resource_1 should be freed first since it could depend on + * fake_resource_2. + */ + KUNIT_EXPECT_EQ(test, ctx->free_order[0], 1); + KUNIT_EXPECT_EQ(test, ctx->free_order[1], 2); +} + +static int kunit_resource_test_init(struct kunit *test) +{ + struct kunit_test_resource_context *ctx = + kzalloc(sizeof(*ctx), GFP_KERNEL); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx); + + test->priv = ctx; + + kunit_init_test(&ctx->test, "test_test_context"); + + return 0; +} + +static void kunit_resource_test_exit(struct kunit *test) +{ + struct kunit_test_resource_context *ctx = test->priv; + + kunit_cleanup(&ctx->test); + kfree(ctx); +} + +static struct kunit_case kunit_resource_test_cases[] = { + KUNIT_CASE(kunit_resource_test_init_resources), + KUNIT_CASE(kunit_resource_test_alloc_resource), + KUNIT_CASE(kunit_resource_test_destroy_resource), + KUNIT_CASE(kunit_resource_test_cleanup_resources), + KUNIT_CASE(kunit_resource_test_proper_free_ordering), + {} +}; + +static struct kunit_suite kunit_resource_test_suite = { + .name = "kunit-resource-test", + .init = kunit_resource_test_init, + .exit = kunit_resource_test_exit, + .test_cases = kunit_resource_test_cases, +}; +kunit_test_suites(&kunit_try_catch_test_suite, &kunit_resource_test_suite); + +MODULE_LICENSE("GPL v2"); diff --git a/lib/kunit/string-stream-test.c b/lib/kunit/string-stream-test.c index 6c70dc87f676..110f3a993250 100644 --- a/lib/kunit/string-stream-test.c +++ b/lib/kunit/string-stream-test.c @@ -50,4 +50,4 @@ static struct kunit_suite string_stream_test_suite = { .name = "string-stream-test", .test_cases = string_stream_test_cases }; -kunit_test_suite(string_stream_test_suite); +kunit_test_suites(&string_stream_test_suite); diff --git a/lib/kunit/test-test.c b/lib/kunit/test-test.c deleted file mode 100644 index 5a6cc0484eda..000000000000 --- a/lib/kunit/test-test.c +++ /dev/null @@ -1,333 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * KUnit test for core test infrastructure. - * - * Copyright (C) 2019, Google LLC. - * Author: Brendan Higgins - */ -#include - -#include "try-catch-impl.h" - -struct kunit_try_catch_test_context { - struct kunit_try_catch *try_catch; - bool function_called; -}; - -static void kunit_test_successful_try(void *data) -{ - struct kunit *test = data; - struct kunit_try_catch_test_context *ctx = test->priv; - - ctx->function_called = true; -} - -static void kunit_test_no_catch(void *data) -{ - struct kunit *test = data; - - KUNIT_FAIL(test, "Catch should not be called\n"); -} - -static void kunit_test_try_catch_successful_try_no_catch(struct kunit *test) -{ - struct kunit_try_catch_test_context *ctx = test->priv; - struct kunit_try_catch *try_catch = ctx->try_catch; - - kunit_try_catch_init(try_catch, - test, - kunit_test_successful_try, - kunit_test_no_catch); - kunit_try_catch_run(try_catch, test); - - KUNIT_EXPECT_TRUE(test, ctx->function_called); -} - -static void kunit_test_unsuccessful_try(void *data) -{ - struct kunit *test = data; - struct kunit_try_catch_test_context *ctx = test->priv; - struct kunit_try_catch *try_catch = ctx->try_catch; - - kunit_try_catch_throw(try_catch); - KUNIT_FAIL(test, "This line should never be reached\n"); -} - -static void kunit_test_catch(void *data) -{ - struct kunit *test = data; - struct kunit_try_catch_test_context *ctx = test->priv; - - ctx->function_called = true; -} - -static void kunit_test_try_catch_unsuccessful_try_does_catch(struct kunit *test) -{ - struct kunit_try_catch_test_context *ctx = test->priv; - struct kunit_try_catch *try_catch = ctx->try_catch; - - kunit_try_catch_init(try_catch, - test, - kunit_test_unsuccessful_try, - kunit_test_catch); - kunit_try_catch_run(try_catch, test); - - KUNIT_EXPECT_TRUE(test, ctx->function_called); -} - -static int kunit_try_catch_test_init(struct kunit *test) -{ - struct kunit_try_catch_test_context *ctx; - - ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx); - test->priv = ctx; - - ctx->try_catch = kunit_kmalloc(test, - sizeof(*ctx->try_catch), - GFP_KERNEL); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx->try_catch); - - return 0; -} - -static struct kunit_case kunit_try_catch_test_cases[] = { - KUNIT_CASE(kunit_test_try_catch_successful_try_no_catch), - KUNIT_CASE(kunit_test_try_catch_unsuccessful_try_does_catch), - {} -}; - -static struct kunit_suite kunit_try_catch_test_suite = { - .name = "kunit-try-catch-test", - .init = kunit_try_catch_test_init, - .test_cases = kunit_try_catch_test_cases, -}; -kunit_test_suite(kunit_try_catch_test_suite); - -/* - * Context for testing test managed resources - * is_resource_initialized is used to test arbitrary resources - */ -struct kunit_test_resource_context { - struct kunit test; - bool is_resource_initialized; - int allocate_order[2]; - int free_order[2]; -}; - -static int fake_resource_init(struct kunit_resource *res, void *context) -{ - struct kunit_test_resource_context *ctx = context; - - res->allocation = &ctx->is_resource_initialized; - ctx->is_resource_initialized = true; - return 0; -} - -static void fake_resource_free(struct kunit_resource *res) -{ - bool *is_resource_initialized = res->allocation; - - *is_resource_initialized = false; -} - -static void kunit_resource_test_init_resources(struct kunit *test) -{ - struct kunit_test_resource_context *ctx = test->priv; - - kunit_init_test(&ctx->test, "testing_test_init_test"); - - KUNIT_EXPECT_TRUE(test, list_empty(&ctx->test.resources)); -} - -static void kunit_resource_test_alloc_resource(struct kunit *test) -{ - struct kunit_test_resource_context *ctx = test->priv; - struct kunit_resource *res; - kunit_resource_free_t free = fake_resource_free; - - res = kunit_alloc_and_get_resource(&ctx->test, - fake_resource_init, - fake_resource_free, - GFP_KERNEL, - ctx); - - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, res); - KUNIT_EXPECT_PTR_EQ(test, - &ctx->is_resource_initialized, - (bool *) res->allocation); - KUNIT_EXPECT_TRUE(test, list_is_last(&res->node, &ctx->test.resources)); - KUNIT_EXPECT_PTR_EQ(test, free, res->free); -} - -static void kunit_resource_test_destroy_resource(struct kunit *test) -{ - struct kunit_test_resource_context *ctx = test->priv; - struct kunit_resource *res = kunit_alloc_and_get_resource( - &ctx->test, - fake_resource_init, - fake_resource_free, - GFP_KERNEL, - ctx); - - KUNIT_ASSERT_FALSE(test, - kunit_resource_destroy(&ctx->test, - kunit_resource_instance_match, - res->free, - res->allocation)); - - KUNIT_EXPECT_FALSE(test, ctx->is_resource_initialized); - KUNIT_EXPECT_TRUE(test, list_empty(&ctx->test.resources)); -} - -static void kunit_resource_test_cleanup_resources(struct kunit *test) -{ - int i; - struct kunit_test_resource_context *ctx = test->priv; - struct kunit_resource *resources[5]; - - for (i = 0; i < ARRAY_SIZE(resources); i++) { - resources[i] = kunit_alloc_and_get_resource(&ctx->test, - fake_resource_init, - fake_resource_free, - GFP_KERNEL, - ctx); - } - - kunit_cleanup(&ctx->test); - - KUNIT_EXPECT_TRUE(test, list_empty(&ctx->test.resources)); -} - -static void kunit_resource_test_mark_order(int order_array[], - size_t order_size, - int key) -{ - int i; - - for (i = 0; i < order_size && order_array[i]; i++) - ; - - order_array[i] = key; -} - -#define KUNIT_RESOURCE_TEST_MARK_ORDER(ctx, order_field, key) \ - kunit_resource_test_mark_order(ctx->order_field, \ - ARRAY_SIZE(ctx->order_field), \ - key) - -static int fake_resource_2_init(struct kunit_resource *res, void *context) -{ - struct kunit_test_resource_context *ctx = context; - - KUNIT_RESOURCE_TEST_MARK_ORDER(ctx, allocate_order, 2); - - res->allocation = ctx; - - return 0; -} - -static void fake_resource_2_free(struct kunit_resource *res) -{ - struct kunit_test_resource_context *ctx = res->allocation; - - KUNIT_RESOURCE_TEST_MARK_ORDER(ctx, free_order, 2); -} - -static int fake_resource_1_init(struct kunit_resource *res, void *context) -{ - struct kunit_test_resource_context *ctx = context; - - kunit_alloc_and_get_resource(&ctx->test, - fake_resource_2_init, - fake_resource_2_free, - GFP_KERNEL, - ctx); - - KUNIT_RESOURCE_TEST_MARK_ORDER(ctx, allocate_order, 1); - - res->allocation = ctx; - - return 0; -} - -static void fake_resource_1_free(struct kunit_resource *res) -{ - struct kunit_test_resource_context *ctx = res->allocation; - - KUNIT_RESOURCE_TEST_MARK_ORDER(ctx, free_order, 1); -} - -/* - * TODO(brendanhiggins@google.com): replace the arrays that keep track of the - * order of allocation and freeing with strict mocks using the IN_SEQUENCE macro - * to assert allocation and freeing order when the feature becomes available. - */ -static void kunit_resource_test_proper_free_ordering(struct kunit *test) -{ - struct kunit_test_resource_context *ctx = test->priv; - - /* fake_resource_1 allocates a fake_resource_2 in its init. */ - kunit_alloc_and_get_resource(&ctx->test, - fake_resource_1_init, - fake_resource_1_free, - GFP_KERNEL, - ctx); - - /* - * Since fake_resource_2_init calls KUNIT_RESOURCE_TEST_MARK_ORDER - * before returning to fake_resource_1_init, it should be the first to - * put its key in the allocate_order array. - */ - KUNIT_EXPECT_EQ(test, ctx->allocate_order[0], 2); - KUNIT_EXPECT_EQ(test, ctx->allocate_order[1], 1); - - kunit_cleanup(&ctx->test); - - /* - * Because fake_resource_2 finishes allocation before fake_resource_1, - * fake_resource_1 should be freed first since it could depend on - * fake_resource_2. - */ - KUNIT_EXPECT_EQ(test, ctx->free_order[0], 1); - KUNIT_EXPECT_EQ(test, ctx->free_order[1], 2); -} - -static int kunit_resource_test_init(struct kunit *test) -{ - struct kunit_test_resource_context *ctx = - kzalloc(sizeof(*ctx), GFP_KERNEL); - - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx); - - test->priv = ctx; - - kunit_init_test(&ctx->test, "test_test_context"); - - return 0; -} - -static void kunit_resource_test_exit(struct kunit *test) -{ - struct kunit_test_resource_context *ctx = test->priv; - - kunit_cleanup(&ctx->test); - kfree(ctx); -} - -static struct kunit_case kunit_resource_test_cases[] = { - KUNIT_CASE(kunit_resource_test_init_resources), - KUNIT_CASE(kunit_resource_test_alloc_resource), - KUNIT_CASE(kunit_resource_test_destroy_resource), - KUNIT_CASE(kunit_resource_test_cleanup_resources), - KUNIT_CASE(kunit_resource_test_proper_free_ordering), - {} -}; - -static struct kunit_suite kunit_resource_test_suite = { - .name = "kunit-resource-test", - .init = kunit_resource_test_init, - .exit = kunit_resource_test_exit, - .test_cases = kunit_resource_test_cases, -}; -kunit_test_suite(kunit_resource_test_suite); diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 58a6227bb12c..87b5cf1f0fa8 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -173,6 +173,7 @@ void kunit_do_assertion(struct kunit *test, if (assert->type == KUNIT_ASSERTION) kunit_abort(test); } +EXPORT_SYMBOL_GPL(kunit_do_assertion); void kunit_init_test(struct kunit *test, const char *name) { @@ -181,6 +182,7 @@ void kunit_init_test(struct kunit *test, const char *name) test->name = name; test->success = true; } +EXPORT_SYMBOL_GPL(kunit_init_test); /* * Initializes and runs test case. Does not clean up or do post validations. @@ -319,6 +321,7 @@ int kunit_run_tests(struct kunit_suite *suite) return 0; } +EXPORT_SYMBOL_GPL(kunit_run_tests); struct kunit_resource *kunit_alloc_and_get_resource(struct kunit *test, kunit_resource_init_t init, @@ -344,6 +347,7 @@ struct kunit_resource *kunit_alloc_and_get_resource(struct kunit *test, return res; } +EXPORT_SYMBOL_GPL(kunit_alloc_and_get_resource); static void kunit_resource_free(struct kunit *test, struct kunit_resource *res) { @@ -402,6 +406,7 @@ int kunit_resource_destroy(struct kunit *test, kunit_resource_free(test, resource); return 0; } +EXPORT_SYMBOL_GPL(kunit_resource_destroy); struct kunit_kmalloc_params { size_t size; @@ -437,6 +442,7 @@ void *kunit_kmalloc(struct kunit *test, size_t size, gfp_t gfp) gfp, ¶ms); } +EXPORT_SYMBOL_GPL(kunit_kmalloc); void kunit_kfree(struct kunit *test, const void *ptr) { @@ -449,6 +455,7 @@ void kunit_kfree(struct kunit *test, const void *ptr) WARN_ON(rc); } +EXPORT_SYMBOL_GPL(kunit_kfree); void kunit_cleanup(struct kunit *test) { @@ -478,3 +485,4 @@ void kunit_cleanup(struct kunit *test) kunit_resource_free(test, resource); } } +EXPORT_SYMBOL_GPL(kunit_cleanup); diff --git a/lib/kunit/try-catch.c b/lib/kunit/try-catch.c index 4a66d1684b46..0247a281431a 100644 --- a/lib/kunit/try-catch.c +++ b/lib/kunit/try-catch.c @@ -20,6 +20,7 @@ void __noreturn kunit_try_catch_throw(struct kunit_try_catch *try_catch) try_catch->try_result = -EFAULT; complete_and_exit(try_catch->try_completion, -EFAULT); } +EXPORT_SYMBOL_GPL(kunit_try_catch_throw); static int kunit_generic_run_threadfn_adapter(void *data) { @@ -107,3 +108,4 @@ void kunit_try_catch_run(struct kunit_try_catch *try_catch, void *context) try_catch->catch(try_catch->context); } +EXPORT_SYMBOL_GPL(kunit_try_catch_run); diff --git a/lib/list-test.c b/lib/list-test.c index 363c600491c3..76babb1df889 100644 --- a/lib/list-test.c +++ b/lib/list-test.c @@ -743,4 +743,6 @@ static struct kunit_suite list_test_module = { .test_cases = list_test_cases, }; -kunit_test_suite(list_test_module); +kunit_test_suites(&list_test_module); + +MODULE_LICENSE("GPL v2"); -- cgit v1.2.3 From 1c024d45151b51c8f8d4749e65958b0bcf3e7c52 Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Mon, 6 Jan 2020 22:28:21 +0000 Subject: kunit: remove timeout dependence on sysctl_hung_task_timeout_seconds In discussion of how to handle timeouts, it was noted that if sysctl_hung_task_timeout_seconds is exceeded for a kunit test, the test task will be killed and an oops generated. This should suffice as a means of debugging such timeout issues for now. Hence remove use of sysctl_hung_task_timeout_secs, which has the added benefit of avoiding the need to export that symbol from the core kernel. Co-developed-by: Knut Omang Signed-off-by: Knut Omang Signed-off-by: Alan Maguire Reviewed-by: Stephen Boyd Acked-by: Brendan Higgins Signed-off-by: Shuah Khan --- lib/kunit/try-catch.c | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/kunit/try-catch.c b/lib/kunit/try-catch.c index 0247a281431a..0dd434e40487 100644 --- a/lib/kunit/try-catch.c +++ b/lib/kunit/try-catch.c @@ -11,7 +11,6 @@ #include #include #include -#include #include "try-catch-impl.h" @@ -33,8 +32,6 @@ static int kunit_generic_run_threadfn_adapter(void *data) static unsigned long kunit_test_timeout(void) { - unsigned long timeout_msecs; - /* * TODO(brendanhiggins@google.com): We should probably have some type of * variable timeout here. The only question is what that timeout value @@ -51,22 +48,11 @@ static unsigned long kunit_test_timeout(void) * * For more background on this topic, see: * https://mike-bland.com/2011/11/01/small-medium-large.html + * + * If tests timeout due to exceeding sysctl_hung_task_timeout_secs, + * the task will be killed and an oops generated. */ - if (sysctl_hung_task_timeout_secs) { - /* - * If sysctl_hung_task is active, just set the timeout to some - * value less than that. - * - * In regards to the above TODO, if we decide on variable - * timeouts, this logic will likely need to change. - */ - timeout_msecs = (sysctl_hung_task_timeout_secs - 1) * - MSEC_PER_SEC; - } else { - timeout_msecs = 300 * MSEC_PER_SEC; /* 5 min */ - } - - return timeout_msecs; + return 300 * MSEC_PER_SEC; /* 5 min */ } void kunit_try_catch_run(struct kunit_try_catch *try_catch, void *context) -- cgit v1.2.3 From 9fe124bf1b7788058ecfe5778fea1660b01e3e9c Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Mon, 6 Jan 2020 22:28:22 +0000 Subject: kunit: allow kunit to be loaded as a module Making kunit itself buildable as a module allows for "always-on" kunit configuration; specifying CONFIG_KUNIT=m means the module is built but only used when loaded. Kunit test modules will load kunit.ko as an implicit dependency, so simply running "modprobe my-kunit-tests" will load the tests along with the kunit module and run them. Co-developed-by: Knut Omang Signed-off-by: Knut Omang Signed-off-by: Alan Maguire Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- lib/kunit/Kconfig | 2 +- lib/kunit/Makefile | 4 +++- lib/kunit/test.c | 13 +++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/kunit/Kconfig b/lib/kunit/Kconfig index 9ebd5e6d15fd..065aa16f448b 100644 --- a/lib/kunit/Kconfig +++ b/lib/kunit/Kconfig @@ -3,7 +3,7 @@ # menuconfig KUNIT - bool "KUnit - Enable support for unit tests" + tristate "KUnit - Enable support for unit tests" help Enables support for kernel unit tests (KUnit), a lightweight unit testing and mocking framework for the Linux kernel. These tests are diff --git a/lib/kunit/Makefile b/lib/kunit/Makefile index bc6e5e549e84..fab55649b69a 100644 --- a/lib/kunit/Makefile +++ b/lib/kunit/Makefile @@ -1,4 +1,6 @@ -obj-$(CONFIG_KUNIT) += test.o \ +obj-$(CONFIG_KUNIT) += kunit.o + +kunit-objs += test.o \ string-stream.o \ assert.o \ try-catch.o diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 87b5cf1f0fa8..9242f932896c 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -486,3 +486,16 @@ void kunit_cleanup(struct kunit *test) } } EXPORT_SYMBOL_GPL(kunit_cleanup); + +static int __init kunit_init(void) +{ + return 0; +} +late_initcall(kunit_init); + +static void __exit kunit_exit(void) +{ +} +module_exit(kunit_exit); + +MODULE_LICENSE("GPL v2"); -- cgit v1.2.3 From ffd08731b2d632459428612431060cf902324a8d Mon Sep 17 00:00:00 2001 From: Vincenzo Frascino Date: Thu, 28 Nov 2019 11:17:19 +0000 Subject: lib/vdso: Make __cvdso_clock_getres() static Fix the following sparse warning in the generic vDSO library: linux/lib/vdso/gettimeofday.c:224:5: warning: symbol '__cvdso_clock_getres' was not declared. Should it be static? Make it static and also mark it __maybe_unsed. Fixes: 502a590a170b ("lib/vdso: Move fallback invocation to the callers") Reported-by: Marc Gonzalez Signed-off-by: Vincenzo Frascino Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20191128111719.8282-1-vincenzo.frascino@arm.com --- lib/vdso/gettimeofday.c | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index 9ecfd3b547ba..42bd8ab955fa 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -221,6 +221,7 @@ int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) return 0; } +static __maybe_unused int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) { int ret = __cvdso_clock_getres_common(clock, res); -- cgit v1.2.3 From 76db5a27a827c2c89e5120a3d486472da847863b Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Sat, 11 Jan 2020 01:03:32 +0900 Subject: bootconfig: Add Extra Boot Config support Extra Boot Config (XBC) allows admin to pass a tree-structured boot configuration file when boot up the kernel. This extends the kernel command line in an efficient way. Boot config will contain some key-value commands, e.g. key.word = value1 another.key.word = value2 It can fold same keys with braces, also you can write array data. For example, key { word1 { setting1 = data setting2 } word2.array = "val1", "val2" } User can access these key-value pair and tree structure via SKC APIs. Link: http://lkml.kernel.org/r/157867221257.17873.1775090991929862549.stgit@devnote2 Signed-off-by: Masami Hiramatsu Signed-off-by: Steven Rostedt (VMware) --- MAINTAINERS | 6 + include/linux/bootconfig.h | 224 +++++++++++++ init/Kconfig | 11 + lib/Kconfig | 3 + lib/Makefile | 2 + lib/bootconfig.c | 803 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 1049 insertions(+) create mode 100644 include/linux/bootconfig.h create mode 100644 lib/bootconfig.c (limited to 'lib') diff --git a/MAINTAINERS b/MAINTAINERS index 4017e6b760be..8597285eb7c8 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -15770,6 +15770,12 @@ W: http://www.stlinux.com S: Supported F: drivers/net/ethernet/stmicro/stmmac/ +EXTRA BOOT CONFIG +M: Masami Hiramatsu +S: Maintained +F: lib/bootconfig.c +F: include/linux/bootconfig.h + SUN3/3X M: Sam Creasey W: http://sammy.net/sun3/ diff --git a/include/linux/bootconfig.h b/include/linux/bootconfig.h new file mode 100644 index 000000000000..7e18c939663e --- /dev/null +++ b/include/linux/bootconfig.h @@ -0,0 +1,224 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_XBC_H +#define _LINUX_XBC_H +/* + * Extra Boot Config + * Copyright (C) 2019 Linaro Ltd. + * Author: Masami Hiramatsu + */ + +#include +#include + +/* XBC tree node */ +struct xbc_node { + u16 next; + u16 child; + u16 parent; + u16 data; +} __attribute__ ((__packed__)); + +#define XBC_KEY 0 +#define XBC_VALUE (1 << 15) +/* Maximum size of boot config is 32KB - 1 */ +#define XBC_DATA_MAX (XBC_VALUE - 1) + +#define XBC_NODE_MAX 1024 +#define XBC_KEYLEN_MAX 256 +#define XBC_DEPTH_MAX 16 + +/* Node tree access raw APIs */ +struct xbc_node * __init xbc_root_node(void); +int __init xbc_node_index(struct xbc_node *node); +struct xbc_node * __init xbc_node_get_parent(struct xbc_node *node); +struct xbc_node * __init xbc_node_get_child(struct xbc_node *node); +struct xbc_node * __init xbc_node_get_next(struct xbc_node *node); +const char * __init xbc_node_get_data(struct xbc_node *node); + +/** + * xbc_node_is_value() - Test the node is a value node + * @node: An XBC node. + * + * Test the @node is a value node and return true if a value node, false if not. + */ +static inline __init bool xbc_node_is_value(struct xbc_node *node) +{ + return node->data & XBC_VALUE; +} + +/** + * xbc_node_is_key() - Test the node is a key node + * @node: An XBC node. + * + * Test the @node is a key node and return true if a key node, false if not. + */ +static inline __init bool xbc_node_is_key(struct xbc_node *node) +{ + return !xbc_node_is_value(node); +} + +/** + * xbc_node_is_array() - Test the node is an arraied value node + * @node: An XBC node. + * + * Test the @node is an arraied value node. + */ +static inline __init bool xbc_node_is_array(struct xbc_node *node) +{ + return xbc_node_is_value(node) && node->next != 0; +} + +/** + * xbc_node_is_leaf() - Test the node is a leaf key node + * @node: An XBC node. + * + * Test the @node is a leaf key node which is a key node and has a value node + * or no child. Returns true if it is a leaf node, or false if not. + */ +static inline __init bool xbc_node_is_leaf(struct xbc_node *node) +{ + return xbc_node_is_key(node) && + (!node->child || xbc_node_is_value(xbc_node_get_child(node))); +} + +/* Tree-based key-value access APIs */ +struct xbc_node * __init xbc_node_find_child(struct xbc_node *parent, + const char *key); + +const char * __init xbc_node_find_value(struct xbc_node *parent, + const char *key, + struct xbc_node **vnode); + +struct xbc_node * __init xbc_node_find_next_leaf(struct xbc_node *root, + struct xbc_node *leaf); + +const char * __init xbc_node_find_next_key_value(struct xbc_node *root, + struct xbc_node **leaf); + +/** + * xbc_find_value() - Find a value which matches the key + * @key: Search key + * @vnode: A container pointer of XBC value node. + * + * Search a value whose key matches @key from whole of XBC tree and return + * the value if found. Found value node is stored in *@vnode. + * Note that this can return 0-length string and store NULL in *@vnode for + * key-only (non-value) entry. + */ +static inline const char * __init +xbc_find_value(const char *key, struct xbc_node **vnode) +{ + return xbc_node_find_value(NULL, key, vnode); +} + +/** + * xbc_find_node() - Find a node which matches the key + * @key: Search key + * + * Search a (key) node whose key matches @key from whole of XBC tree and + * return the node if found. If not found, returns NULL. + */ +static inline struct xbc_node * __init xbc_find_node(const char *key) +{ + return xbc_node_find_child(NULL, key); +} + +/** + * xbc_array_for_each_value() - Iterate value nodes on an array + * @anode: An XBC arraied value node + * @value: A value + * + * Iterate array value nodes and values starts from @anode. This is expected to + * be used with xbc_find_value() and xbc_node_find_value(), so that user can + * process each array entry node. + */ +#define xbc_array_for_each_value(anode, value) \ + for (value = xbc_node_get_data(anode); anode != NULL ; \ + anode = xbc_node_get_next(anode), \ + value = anode ? xbc_node_get_data(anode) : NULL) + +/** + * xbc_node_for_each_child() - Iterate child nodes + * @parent: An XBC node. + * @child: Iterated XBC node. + * + * Iterate child nodes of @parent. Each child nodes are stored to @child. + */ +#define xbc_node_for_each_child(parent, child) \ + for (child = xbc_node_get_child(parent); child != NULL ; \ + child = xbc_node_get_next(child)) + +/** + * xbc_node_for_each_array_value() - Iterate array entries of geven key + * @node: An XBC node. + * @key: A key string searched under @node + * @anode: Iterated XBC node of array entry. + * @value: Iterated value of array entry. + * + * Iterate array entries of given @key under @node. Each array entry node + * is stroed to @anode and @value. If the @node doesn't have @key node, + * it does nothing. + * Note that even if the found key node has only one value (not array) + * this executes block once. Hoever, if the found key node has no value + * (key-only node), this does nothing. So don't use this for testing the + * key-value pair existence. + */ +#define xbc_node_for_each_array_value(node, key, anode, value) \ + for (value = xbc_node_find_value(node, key, &anode); value != NULL; \ + anode = xbc_node_get_next(anode), \ + value = anode ? xbc_node_get_data(anode) : NULL) + +/** + * xbc_node_for_each_key_value() - Iterate key-value pairs under a node + * @node: An XBC node. + * @knode: Iterated key node + * @value: Iterated value string + * + * Iterate key-value pairs under @node. Each key node and value string are + * stored in @knode and @value respectively. + */ +#define xbc_node_for_each_key_value(node, knode, value) \ + for (knode = NULL, value = xbc_node_find_next_key_value(node, &knode);\ + knode != NULL; value = xbc_node_find_next_key_value(node, &knode)) + +/** + * xbc_for_each_key_value() - Iterate key-value pairs + * @knode: Iterated key node + * @value: Iterated value string + * + * Iterate key-value pairs in whole XBC tree. Each key node and value string + * are stored in @knode and @value respectively. + */ +#define xbc_for_each_key_value(knode, value) \ + xbc_node_for_each_key_value(NULL, knode, value) + +/* Compose partial key */ +int __init xbc_node_compose_key_after(struct xbc_node *root, + struct xbc_node *node, char *buf, size_t size); + +/** + * xbc_node_compose_key() - Compose full key string of the XBC node + * @node: An XBC node. + * @buf: A buffer to store the key. + * @size: The size of the @buf. + * + * Compose the full-length key of the @node into @buf. Returns the total + * length of the key stored in @buf. Or returns -EINVAL if @node is NULL, + * and -ERANGE if the key depth is deeper than max depth. + */ +static inline int __init xbc_node_compose_key(struct xbc_node *node, + char *buf, size_t size) +{ + return xbc_node_compose_key_after(NULL, node, buf, size); +} + +/* XBC node initializer */ +int __init xbc_init(char *buf); + +/* XBC cleanup data structures */ +void __init xbc_destroy_all(void); + +/* Debug dump functions */ +void __init xbc_debug_dump(void); + +#endif diff --git a/init/Kconfig b/init/Kconfig index a34064a031a5..63450d3bbf12 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1215,6 +1215,17 @@ source "usr/Kconfig" endif +config BOOT_CONFIG + bool "Boot config support" + select LIBXBC + default y + help + Extra boot config allows system admin to pass a config file as + complemental extension of kernel cmdline when booting. + The boot config file is usually attached at the end of initramfs. + + If unsure, say Y. + choice prompt "Compiler optimization level" default CC_OPTIMIZE_FOR_PERFORMANCE diff --git a/lib/Kconfig b/lib/Kconfig index 6e790dc55c5b..10012b646009 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -566,6 +566,9 @@ config DIMLIB config LIBFDT bool +config LIBXBC + bool + config OID_REGISTRY tristate help diff --git a/lib/Makefile b/lib/Makefile index 93217d44237f..75a64d2552a2 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -228,6 +228,8 @@ $(foreach file, $(libfdt_files), \ $(eval CFLAGS_$(file) = -I $(srctree)/scripts/dtc/libfdt)) lib-$(CONFIG_LIBFDT) += $(libfdt_files) +lib-$(CONFIG_LIBXBC) += bootconfig.o + obj-$(CONFIG_RBTREE_TEST) += rbtree_test.o obj-$(CONFIG_INTERVAL_TREE_TEST) += interval_tree_test.o diff --git a/lib/bootconfig.c b/lib/bootconfig.c new file mode 100644 index 000000000000..055014e233a5 --- /dev/null +++ b/lib/bootconfig.c @@ -0,0 +1,803 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Extra Boot Config + * Masami Hiramatsu + */ + +#define pr_fmt(fmt) "bootconfig: " fmt + +#include +#include +#include +#include +#include +#include +#include + +/* + * Extra Boot Config (XBC) is given as tree-structured ascii text of + * key-value pairs on memory. + * xbc_parse() parses the text to build a simple tree. Each tree node is + * simply a key word or a value. A key node may have a next key node or/and + * a child node (both key and value). A value node may have a next value + * node (for array). + */ + +static struct xbc_node xbc_nodes[XBC_NODE_MAX] __initdata; +static int xbc_node_num __initdata; +static char *xbc_data __initdata; +static size_t xbc_data_size __initdata; +static struct xbc_node *last_parent __initdata; + +static int __init xbc_parse_error(const char *msg, const char *p) +{ + int pos = p - xbc_data; + + pr_err("Parse error at pos %d: %s\n", pos, msg); + return -EINVAL; +} + +/** + * xbc_root_node() - Get the root node of extended boot config + * + * Return the address of root node of extended boot config. If the + * extended boot config is not initiized, return NULL. + */ +struct xbc_node * __init xbc_root_node(void) +{ + if (unlikely(!xbc_data)) + return NULL; + + return xbc_nodes; +} + +/** + * xbc_node_index() - Get the index of XBC node + * @node: A target node of getting index. + * + * Return the index number of @node in XBC node list. + */ +int __init xbc_node_index(struct xbc_node *node) +{ + return node - &xbc_nodes[0]; +} + +/** + * xbc_node_get_parent() - Get the parent XBC node + * @node: An XBC node. + * + * Return the parent node of @node. If the node is top node of the tree, + * return NULL. + */ +struct xbc_node * __init xbc_node_get_parent(struct xbc_node *node) +{ + return node->parent == XBC_NODE_MAX ? NULL : &xbc_nodes[node->parent]; +} + +/** + * xbc_node_get_child() - Get the child XBC node + * @node: An XBC node. + * + * Return the first child node of @node. If the node has no child, return + * NULL. + */ +struct xbc_node * __init xbc_node_get_child(struct xbc_node *node) +{ + return node->child ? &xbc_nodes[node->child] : NULL; +} + +/** + * xbc_node_get_next() - Get the next sibling XBC node + * @node: An XBC node. + * + * Return the NEXT sibling node of @node. If the node has no next sibling, + * return NULL. Note that even if this returns NULL, it doesn't mean @node + * has no siblings. (You also has to check whether the parent's child node + * is @node or not.) + */ +struct xbc_node * __init xbc_node_get_next(struct xbc_node *node) +{ + return node->next ? &xbc_nodes[node->next] : NULL; +} + +/** + * xbc_node_get_data() - Get the data of XBC node + * @node: An XBC node. + * + * Return the data (which is always a null terminated string) of @node. + * If the node has invalid data, warn and return NULL. + */ +const char * __init xbc_node_get_data(struct xbc_node *node) +{ + int offset = node->data & ~XBC_VALUE; + + if (WARN_ON(offset >= xbc_data_size)) + return NULL; + + return xbc_data + offset; +} + +static bool __init +xbc_node_match_prefix(struct xbc_node *node, const char **prefix) +{ + const char *p = xbc_node_get_data(node); + int len = strlen(p); + + if (strncmp(*prefix, p, len)) + return false; + + p = *prefix + len; + if (*p == '.') + p++; + else if (*p != '\0') + return false; + *prefix = p; + + return true; +} + +/** + * xbc_node_find_child() - Find a child node which matches given key + * @parent: An XBC node. + * @key: A key string. + * + * Search a node under @parent which matches @key. The @key can contain + * several words jointed with '.'. If @parent is NULL, this searches the + * node from whole tree. Return NULL if no node is matched. + */ +struct xbc_node * __init +xbc_node_find_child(struct xbc_node *parent, const char *key) +{ + struct xbc_node *node; + + if (parent) + node = xbc_node_get_child(parent); + else + node = xbc_root_node(); + + while (node && xbc_node_is_key(node)) { + if (!xbc_node_match_prefix(node, &key)) + node = xbc_node_get_next(node); + else if (*key != '\0') + node = xbc_node_get_child(node); + else + break; + } + + return node; +} + +/** + * xbc_node_find_value() - Find a value node which matches given key + * @parent: An XBC node. + * @key: A key string. + * @vnode: A container pointer of found XBC node. + * + * Search a value node under @parent whose (parent) key node matches @key, + * store it in *@vnode, and returns the value string. + * The @key can contain several words jointed with '.'. If @parent is NULL, + * this searches the node from whole tree. Return the value string if a + * matched key found, return NULL if no node is matched. + * Note that this returns 0-length string and stores NULL in *@vnode if the + * key has no value. And also it will return the value of the first entry if + * the value is an array. + */ +const char * __init +xbc_node_find_value(struct xbc_node *parent, const char *key, + struct xbc_node **vnode) +{ + struct xbc_node *node = xbc_node_find_child(parent, key); + + if (!node || !xbc_node_is_key(node)) + return NULL; + + node = xbc_node_get_child(node); + if (node && !xbc_node_is_value(node)) + return NULL; + + if (vnode) + *vnode = node; + + return node ? xbc_node_get_data(node) : ""; +} + +/** + * xbc_node_compose_key_after() - Compose partial key string of the XBC node + * @root: Root XBC node + * @node: Target XBC node. + * @buf: A buffer to store the key. + * @size: The size of the @buf. + * + * Compose the partial key of the @node into @buf, which is starting right + * after @root (@root is not included.) If @root is NULL, this returns full + * key words of @node. + * Returns the total length of the key stored in @buf. Returns -EINVAL + * if @node is NULL or @root is not the ancestor of @node or @root is @node, + * or returns -ERANGE if the key depth is deeper than max depth. + * This is expected to be used with xbc_find_node() to list up all (child) + * keys under given key. + */ +int __init xbc_node_compose_key_after(struct xbc_node *root, + struct xbc_node *node, + char *buf, size_t size) +{ + u16 keys[XBC_DEPTH_MAX]; + int depth = 0, ret = 0, total = 0; + + if (!node || node == root) + return -EINVAL; + + if (xbc_node_is_value(node)) + node = xbc_node_get_parent(node); + + while (node && node != root) { + keys[depth++] = xbc_node_index(node); + if (depth == XBC_DEPTH_MAX) + return -ERANGE; + node = xbc_node_get_parent(node); + } + if (!node && root) + return -EINVAL; + + while (--depth >= 0) { + node = xbc_nodes + keys[depth]; + ret = snprintf(buf, size, "%s%s", xbc_node_get_data(node), + depth ? "." : ""); + if (ret < 0) + return ret; + if (ret > size) { + size = 0; + } else { + size -= ret; + buf += ret; + } + total += ret; + } + + return total; +} + +/** + * xbc_node_find_next_leaf() - Find the next leaf node under given node + * @root: An XBC root node + * @node: An XBC node which starts from. + * + * Search the next leaf node (which means the terminal key node) of @node + * under @root node (including @root node itself). + * Return the next node or NULL if next leaf node is not found. + */ +struct xbc_node * __init xbc_node_find_next_leaf(struct xbc_node *root, + struct xbc_node *node) +{ + if (unlikely(!xbc_data)) + return NULL; + + if (!node) { /* First try */ + node = root; + if (!node) + node = xbc_nodes; + } else { + if (node == root) /* @root was a leaf, no child node. */ + return NULL; + + while (!node->next) { + node = xbc_node_get_parent(node); + if (node == root) + return NULL; + /* User passed a node which is not uder parent */ + if (WARN_ON(!node)) + return NULL; + } + node = xbc_node_get_next(node); + } + + while (node && !xbc_node_is_leaf(node)) + node = xbc_node_get_child(node); + + return node; +} + +/** + * xbc_node_find_next_key_value() - Find the next key-value pair nodes + * @root: An XBC root node + * @leaf: A container pointer of XBC node which starts from. + * + * Search the next leaf node (which means the terminal key node) of *@leaf + * under @root node. Returns the value and update *@leaf if next leaf node + * is found, or NULL if no next leaf node is found. + * Note that this returns 0-length string if the key has no value, or + * the value of the first entry if the value is an array. + */ +const char * __init xbc_node_find_next_key_value(struct xbc_node *root, + struct xbc_node **leaf) +{ + /* tip must be passed */ + if (WARN_ON(!leaf)) + return NULL; + + *leaf = xbc_node_find_next_leaf(root, *leaf); + if (!*leaf) + return NULL; + if ((*leaf)->child) + return xbc_node_get_data(xbc_node_get_child(*leaf)); + else + return ""; /* No value key */ +} + +/* XBC parse and tree build */ + +static struct xbc_node * __init xbc_add_node(char *data, u32 flag) +{ + struct xbc_node *node; + unsigned long offset; + + if (xbc_node_num == XBC_NODE_MAX) + return NULL; + + node = &xbc_nodes[xbc_node_num++]; + offset = data - xbc_data; + node->data = (u16)offset; + if (WARN_ON(offset >= XBC_DATA_MAX)) + return NULL; + node->data |= flag; + node->child = 0; + node->next = 0; + + return node; +} + +static inline __init struct xbc_node *xbc_last_sibling(struct xbc_node *node) +{ + while (node->next) + node = xbc_node_get_next(node); + + return node; +} + +static struct xbc_node * __init xbc_add_sibling(char *data, u32 flag) +{ + struct xbc_node *sib, *node = xbc_add_node(data, flag); + + if (node) { + if (!last_parent) { + node->parent = XBC_NODE_MAX; + sib = xbc_last_sibling(xbc_nodes); + sib->next = xbc_node_index(node); + } else { + node->parent = xbc_node_index(last_parent); + if (!last_parent->child) { + last_parent->child = xbc_node_index(node); + } else { + sib = xbc_node_get_child(last_parent); + sib = xbc_last_sibling(sib); + sib->next = xbc_node_index(node); + } + } + } + + return node; +} + +static inline __init struct xbc_node *xbc_add_child(char *data, u32 flag) +{ + struct xbc_node *node = xbc_add_sibling(data, flag); + + if (node) + last_parent = node; + + return node; +} + +static inline __init bool xbc_valid_keyword(char *key) +{ + if (key[0] == '\0') + return false; + + while (isalnum(*key) || *key == '-' || *key == '_') + key++; + + return *key == '\0'; +} + +static char *skip_comment(char *p) +{ + char *ret; + + ret = strchr(p, '\n'); + if (!ret) + ret = p + strlen(p); + else + ret++; + + return ret; +} + +static char *skip_spaces_until_newline(char *p) +{ + while (isspace(*p) && *p != '\n') + p++; + return p; +} + +static int __init __xbc_open_brace(void) +{ + /* Mark the last key as open brace */ + last_parent->next = XBC_NODE_MAX; + + return 0; +} + +static int __init __xbc_close_brace(char *p) +{ + struct xbc_node *node; + + if (!last_parent || last_parent->next != XBC_NODE_MAX) + return xbc_parse_error("Unexpected closing brace", p); + + node = last_parent; + node->next = 0; + do { + node = xbc_node_get_parent(node); + } while (node && node->next != XBC_NODE_MAX); + last_parent = node; + + return 0; +} + +/* + * Return delimiter or error, no node added. As same as lib/cmdline.c, + * you can use " around spaces, but can't escape " for value. + */ +static int __init __xbc_parse_value(char **__v, char **__n) +{ + char *p, *v = *__v; + int c, quotes = 0; + + v = skip_spaces(v); + while (*v == '#') { + v = skip_comment(v); + v = skip_spaces(v); + } + if (*v == '"' || *v == '\'') { + quotes = *v; + v++; + } + p = v - 1; + while ((c = *++p)) { + if (!isprint(c) && !isspace(c)) + return xbc_parse_error("Non printable value", p); + if (quotes) { + if (c != quotes) + continue; + quotes = 0; + *p++ = '\0'; + p = skip_spaces_until_newline(p); + c = *p; + if (c && !strchr(",;\n#}", c)) + return xbc_parse_error("No value delimiter", p); + if (*p) + p++; + break; + } + if (strchr(",;\n#}", c)) { + v = strim(v); + *p++ = '\0'; + break; + } + } + if (quotes) + return xbc_parse_error("No closing quotes", p); + if (c == '#') { + p = skip_comment(p); + c = '\n'; /* A comment must be treated as a newline */ + } + *__n = p; + *__v = v; + + return c; +} + +static int __init xbc_parse_array(char **__v) +{ + struct xbc_node *node; + char *next; + int c = 0; + + do { + c = __xbc_parse_value(__v, &next); + if (c < 0) + return c; + + node = xbc_add_sibling(*__v, XBC_VALUE); + if (!node) + return -ENOMEM; + *__v = next; + } while (c == ','); + node->next = 0; + + return c; +} + +static inline __init +struct xbc_node *find_match_node(struct xbc_node *node, char *k) +{ + while (node) { + if (!strcmp(xbc_node_get_data(node), k)) + break; + node = xbc_node_get_next(node); + } + return node; +} + +static int __init __xbc_add_key(char *k) +{ + struct xbc_node *node; + + if (!xbc_valid_keyword(k)) + return xbc_parse_error("Invalid keyword", k); + + if (unlikely(xbc_node_num == 0)) + goto add_node; + + if (!last_parent) /* the first level */ + node = find_match_node(xbc_nodes, k); + else + node = find_match_node(xbc_node_get_child(last_parent), k); + + if (node) + last_parent = node; + else { +add_node: + node = xbc_add_child(k, XBC_KEY); + if (!node) + return -ENOMEM; + } + return 0; +} + +static int __init __xbc_parse_keys(char *k) +{ + char *p; + int ret; + + k = strim(k); + while ((p = strchr(k, '.'))) { + *p++ = '\0'; + ret = __xbc_add_key(k); + if (ret) + return ret; + k = p; + } + + return __xbc_add_key(k); +} + +static int __init xbc_parse_kv(char **k, char *v) +{ + struct xbc_node *prev_parent = last_parent; + struct xbc_node *node; + char *next; + int c, ret; + + ret = __xbc_parse_keys(*k); + if (ret) + return ret; + + c = __xbc_parse_value(&v, &next); + if (c < 0) + return c; + + node = xbc_add_sibling(v, XBC_VALUE); + if (!node) + return -ENOMEM; + + if (c == ',') { /* Array */ + c = xbc_parse_array(&next); + if (c < 0) + return c; + } + + last_parent = prev_parent; + + if (c == '}') { + ret = __xbc_close_brace(next - 1); + if (ret < 0) + return ret; + } + + *k = next; + + return 0; +} + +static int __init xbc_parse_key(char **k, char *n) +{ + struct xbc_node *prev_parent = last_parent; + int ret; + + *k = strim(*k); + if (**k != '\0') { + ret = __xbc_parse_keys(*k); + if (ret) + return ret; + last_parent = prev_parent; + } + *k = n; + + return 0; +} + +static int __init xbc_open_brace(char **k, char *n) +{ + int ret; + + ret = __xbc_parse_keys(*k); + if (ret) + return ret; + *k = n; + + return __xbc_open_brace(); +} + +static int __init xbc_close_brace(char **k, char *n) +{ + int ret; + + ret = xbc_parse_key(k, n); + if (ret) + return ret; + /* k is updated in xbc_parse_key() */ + + return __xbc_close_brace(n - 1); +} + +static int __init xbc_verify_tree(void) +{ + int i, depth, len, wlen; + struct xbc_node *n, *m; + + /* Empty tree */ + if (xbc_node_num == 0) + return -ENOENT; + + for (i = 0; i < xbc_node_num; i++) { + if (xbc_nodes[i].next > xbc_node_num) { + return xbc_parse_error("No closing brace", + xbc_node_get_data(xbc_nodes + i)); + } + } + + /* Key tree limitation check */ + n = &xbc_nodes[0]; + depth = 1; + len = 0; + + while (n) { + wlen = strlen(xbc_node_get_data(n)) + 1; + len += wlen; + if (len > XBC_KEYLEN_MAX) + return xbc_parse_error("Too long key length", + xbc_node_get_data(n)); + + m = xbc_node_get_child(n); + if (m && xbc_node_is_key(m)) { + n = m; + depth++; + if (depth > XBC_DEPTH_MAX) + return xbc_parse_error("Too many key words", + xbc_node_get_data(n)); + continue; + } + len -= wlen; + m = xbc_node_get_next(n); + while (!m) { + n = xbc_node_get_parent(n); + if (!n) + break; + len -= strlen(xbc_node_get_data(n)) + 1; + depth--; + m = xbc_node_get_next(n); + } + n = m; + } + + return 0; +} + +/** + * xbc_destroy_all() - Clean up all parsed bootconfig + * + * This clears all data structures of parsed bootconfig on memory. + * If you need to reuse xbc_init() with new boot config, you can + * use this. + */ +void __init xbc_destroy_all(void) +{ + xbc_data = NULL; + xbc_data_size = 0; + xbc_node_num = 0; + memset(xbc_nodes, 0, sizeof(xbc_nodes)); +} + +/** + * xbc_init() - Parse given XBC file and build XBC internal tree + * @buf: boot config text + * + * This parses the boot config text in @buf. @buf must be a + * null terminated string and smaller than XBC_DATA_MAX. + * Return 0 if succeeded, or -errno if there is any error. + */ +int __init xbc_init(char *buf) +{ + char *p, *q; + int ret, c; + + if (xbc_data) + return -EBUSY; + + ret = strlen(buf); + if (ret > XBC_DATA_MAX - 1 || ret == 0) + return -ERANGE; + + xbc_data = buf; + xbc_data_size = ret + 1; + last_parent = NULL; + + p = buf; + do { + q = strpbrk(p, "{}=;\n#"); + if (!q) { + p = skip_spaces(p); + if (*p != '\0') + ret = xbc_parse_error("No delimiter", p); + break; + } + + c = *q; + *q++ = '\0'; + switch (c) { + case '=': + ret = xbc_parse_kv(&p, q); + break; + case '{': + ret = xbc_open_brace(&p, q); + break; + case '#': + q = skip_comment(q); + /* fall through */ + case ';': + case '\n': + ret = xbc_parse_key(&p, q); + break; + case '}': + ret = xbc_close_brace(&p, q); + break; + } + } while (!ret); + + if (!ret) + ret = xbc_verify_tree(); + + if (ret < 0) + xbc_destroy_all(); + + return ret; +} + +/** + * xbc_debug_dump() - Dump current XBC node list + * + * Dump the current XBC node list on printk buffer for debug. + */ +void __init xbc_debug_dump(void) +{ + int i; + + for (i = 0; i < xbc_node_num; i++) { + pr_debug("[%d] %s (%s) .next=%d, .child=%d .parent=%d\n", i, + xbc_node_get_data(xbc_nodes + i), + xbc_node_is_value(xbc_nodes + i) ? "value" : "key", + xbc_nodes[i].next, xbc_nodes[i].child, + xbc_nodes[i].parent); + } +} -- cgit v1.2.3 From 5e5ac01c2b8802921fee680518a986011cb59820 Mon Sep 17 00:00:00 2001 From: Zhengyuan Liu Date: Fri, 20 Dec 2019 10:21:27 +0800 Subject: raid6/test: fix a compilation warning The compilation warning is redefination showed as following: In file included from tables.c:2: ../../../include/linux/export.h:180: warning: "EXPORT_SYMBOL" redefined #define EXPORT_SYMBOL(sym) __EXPORT_SYMBOL(sym, "") In file included from tables.c:1: ../../../include/linux/raid/pq.h:61: note: this is the location of the previous definition #define EXPORT_SYMBOL(sym) Fixes: 69a94abb82ee ("export.h, genksyms: do not make genksyms calculate CRC of trimmed symbols") Signed-off-by: Zhengyuan Liu Signed-off-by: Song Liu --- include/linux/raid/pq.h | 2 ++ lib/raid6/mktables.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/include/linux/raid/pq.h b/include/linux/raid/pq.h index 0b6e7ad9cd2a..e0ddb47f4402 100644 --- a/include/linux/raid/pq.h +++ b/include/linux/raid/pq.h @@ -58,7 +58,9 @@ extern const char raid6_empty_zero_page[PAGE_SIZE]; #define enable_kernel_altivec() #define disable_kernel_altivec() +#undef EXPORT_SYMBOL #define EXPORT_SYMBOL(sym) +#undef EXPORT_SYMBOL_GPL #define EXPORT_SYMBOL_GPL(sym) #define MODULE_LICENSE(licence) #define MODULE_DESCRIPTION(desc) diff --git a/lib/raid6/mktables.c b/lib/raid6/mktables.c index 9c485df1308f..f02e10fa6238 100644 --- a/lib/raid6/mktables.c +++ b/lib/raid6/mktables.c @@ -56,8 +56,8 @@ int main(int argc, char *argv[]) uint8_t v; uint8_t exptbl[256], invtbl[256]; - printf("#include \n"); printf("#include \n"); + printf("#include \n"); /* Compute multiplication table */ printf("\nconst u8 __attribute__((aligned(256)))\n" -- cgit v1.2.3 From f591df3cc6d60cadf8ceff5d44af73ea6ba0a39a Mon Sep 17 00:00:00 2001 From: Zhengyuan Liu Date: Fri, 20 Dec 2019 10:21:28 +0800 Subject: md/raid6: fix algorithm choice under larger PAGE_SIZE There are several algorithms available for raid6 to generate xor and syndrome parity, including basic int1, int2 ... int32 and SIMD optimized implementation like sse and neon. To test and choose the best algorithms at the initial stage, we need provide enough disk data to feed the algorithms. However, the disk number we provided depends on page size and gfmul table, seeing bellow: const int disks = (65536/PAGE_SIZE) + 2; So when come to 64K PAGE_SIZE, there is only one data disk plus 2 parity disk, as a result the chosed algorithm is not reliable. For example, on my arm64 machine with 64K page enabled, it will choose intx32 as the best one, although the NEON implementation is better. This patch tries to fix the problem by defining a constant raid6 disk number to supporting arbitrary page size. Suggested-by: H. Peter Anvin Signed-off-by: Zhengyuan Liu Signed-off-by: Song Liu --- include/linux/raid/pq.h | 4 ++++ lib/raid6/algos.c | 63 +++++++++++++++++++++++++++++++------------------ 2 files changed, 44 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/include/linux/raid/pq.h b/include/linux/raid/pq.h index e0ddb47f4402..154e954b711d 100644 --- a/include/linux/raid/pq.h +++ b/include/linux/raid/pq.h @@ -28,6 +28,7 @@ extern const char raid6_empty_zero_page[PAGE_SIZE]; #include #include #include +#include #include #include #include @@ -43,6 +44,9 @@ typedef uint64_t u64; #ifndef PAGE_SIZE # define PAGE_SIZE 4096 #endif +#ifndef PAGE_SHIFT +# define PAGE_SHIFT 12 +#endif extern const char raid6_empty_zero_page[PAGE_SIZE]; #define __init diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c index 17417eee0866..bf1b4765c8f6 100644 --- a/lib/raid6/algos.c +++ b/lib/raid6/algos.c @@ -124,6 +124,9 @@ const struct raid6_recov_calls *const raid6_recov_algos[] = { #define time_before(x, y) ((x) < (y)) #endif +#define RAID6_TEST_DISKS 8 +#define RAID6_TEST_DISKS_ORDER 3 + static inline const struct raid6_recov_calls *raid6_choose_recov(void) { const struct raid6_recov_calls *const *algo; @@ -146,7 +149,7 @@ static inline const struct raid6_recov_calls *raid6_choose_recov(void) } static inline const struct raid6_calls *raid6_choose_gen( - void *(*const dptrs)[(65536/PAGE_SIZE)+2], const int disks) + void *(*const dptrs)[RAID6_TEST_DISKS], const int disks) { unsigned long perf, bestgenperf, bestxorperf, j0, j1; int start = (disks>>1)-1, stop = disks-3; /* work on the second half of the disks */ @@ -181,7 +184,8 @@ static inline const struct raid6_calls *raid6_choose_gen( best = *algo; } pr_info("raid6: %-8s gen() %5ld MB/s\n", (*algo)->name, - (perf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2)); + (perf * HZ * (disks-2)) >> + (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); if (!(*algo)->xor_syndrome) continue; @@ -204,17 +208,24 @@ static inline const struct raid6_calls *raid6_choose_gen( bestxorperf = perf; pr_info("raid6: %-8s xor() %5ld MB/s\n", (*algo)->name, - (perf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2+1)); + (perf * HZ * (disks-2)) >> + (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1)); } } if (best) { - pr_info("raid6: using algorithm %s gen() %ld MB/s\n", - best->name, - (bestgenperf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2)); - if (best->xor_syndrome) - pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n", - (bestxorperf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2+1)); + if (IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) { + pr_info("raid6: using algorithm %s gen() %ld MB/s\n", + best->name, + (bestgenperf * HZ * (disks-2)) >> + (20 - PAGE_SHIFT+RAID6_TIME_JIFFIES_LG2)); + if (best->xor_syndrome) + pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n", + (bestxorperf * HZ * (disks-2)) >> + (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1)); + } else + pr_info("raid6: skip pq benchmark and using algorithm %s\n", + best->name); raid6_call = *best; } else pr_err("raid6: Yikes! No algorithm found!\n"); @@ -228,27 +239,33 @@ static inline const struct raid6_calls *raid6_choose_gen( int __init raid6_select_algo(void) { - const int disks = (65536/PAGE_SIZE)+2; + const int disks = RAID6_TEST_DISKS; const struct raid6_calls *gen_best; const struct raid6_recov_calls *rec_best; - char *syndromes; - void *dptrs[(65536/PAGE_SIZE)+2]; - int i; - - for (i = 0; i < disks-2; i++) - dptrs[i] = ((char *)raid6_gfmul) + PAGE_SIZE*i; - - /* Normal code - use a 2-page allocation to avoid D$ conflict */ - syndromes = (void *) __get_free_pages(GFP_KERNEL, 1); + char *disk_ptr, *p; + void *dptrs[RAID6_TEST_DISKS]; + int i, cycle; - if (!syndromes) { + /* prepare the buffer and fill it circularly with gfmul table */ + disk_ptr = (char *)__get_free_pages(GFP_KERNEL, RAID6_TEST_DISKS_ORDER); + if (!disk_ptr) { pr_err("raid6: Yikes! No memory available.\n"); return -ENOMEM; } - dptrs[disks-2] = syndromes; - dptrs[disks-1] = syndromes + PAGE_SIZE; + p = disk_ptr; + for (i = 0; i < disks; i++) + dptrs[i] = p + PAGE_SIZE * i; + + cycle = ((disks - 2) * PAGE_SIZE) / 65536; + for (i = 0; i < cycle; i++) { + memcpy(p, raid6_gfmul, 65536); + p += 65536; + } + + if ((disks - 2) * PAGE_SIZE % 65536) + memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536); /* select raid gen_syndrome function */ gen_best = raid6_choose_gen(&dptrs, disks); @@ -256,7 +273,7 @@ int __init raid6_select_algo(void) /* select raid recover functions */ rec_best = raid6_choose_recov(); - free_pages((unsigned long)syndromes, 1); + free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER); return gen_best && rec_best ? 0 : -EINVAL; } -- cgit v1.2.3 From bf279849ad59538a1518c667c0795ec1fe9dbd66 Mon Sep 17 00:00:00 2001 From: Vincenzo Frascino Date: Fri, 30 Aug 2019 14:58:56 +0100 Subject: lib/vdso: Build 32 bit specific functions in the right context clock_gettime32 and clock_getres_time32 should be compiled only with a 32 bit vdso library. Exclude these symbols when BUILD_VDSO32 is not defined. Signed-off-by: Vincenzo Frascino Signed-off-by: Thomas Gleixner Reviewed-by: Andy Lutomirski Link: https://lore.kernel.org/r/20190830135902.20861-3-vincenzo.frascino@arm.com --- lib/vdso/gettimeofday.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index 42bd8ab955fa..8e77071a4a77 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -117,6 +117,7 @@ __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) return 0; } +#ifdef BUILD_VDSO32 static __maybe_unused int __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) { @@ -139,6 +140,7 @@ __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) } return ret; } +#endif /* BUILD_VDSO32 */ static __maybe_unused int __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) @@ -231,6 +233,7 @@ int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) return 0; } +#ifdef BUILD_VDSO32 static __maybe_unused int __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) { @@ -253,4 +256,5 @@ __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) } return ret; } +#endif /* BUILD_VDSO32 */ #endif /* VDSO_HAS_CLOCK_GETRES */ -- cgit v1.2.3 From b767081c07a400ff1c6f95b87639a9405886e7a6 Mon Sep 17 00:00:00 2001 From: Vincenzo Frascino Date: Fri, 30 Aug 2019 14:58:58 +0100 Subject: lib/vdso: Remove VDSO_HAS_32BIT_FALLBACK VDSO_HAS_32BIT_FALLBACK was introduced to address a regression which caused seccomp to deny access to the applications to clock_gettime64() and clock_getres64() because they are not enabled in the existing filters. The purpose of VDSO_HAS_32BIT_FALLBACK was to simplify the conditional implementation of __cvdso_clock_get*time32() variants. Now that all the architectures that support the generic vDSO library have been converted to support the 32 bit fallbacks the conditional can be removed. Signed-off-by: Vincenzo Frascino Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20190830135902.20861-5-vincenzo.frascino@arm.com References: c60a32ea4f45 ("lib/vdso/32: Provide legacy syscall fallbacks") --- lib/vdso/gettimeofday.c | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'lib') diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index 8e77071a4a77..cd3aacf1cf86 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -126,13 +126,8 @@ __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) ret = __cvdso_clock_gettime_common(clock, &ts); -#ifdef VDSO_HAS_32BIT_FALLBACK if (unlikely(ret)) return clock_gettime32_fallback(clock, res); -#else - if (unlikely(ret)) - ret = clock_gettime_fallback(clock, &ts); -#endif if (likely(!ret)) { res->tv_sec = ts.tv_sec; @@ -242,13 +237,8 @@ __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) ret = __cvdso_clock_getres_common(clock, &ts); -#ifdef VDSO_HAS_32BIT_FALLBACK if (unlikely(ret)) return clock_getres32_fallback(clock, res); -#else - if (unlikely(ret)) - ret = clock_getres_fallback(clock, &ts); -#endif if (likely(!ret && res)) { res->tv_sec = ts.tv_sec; -- cgit v1.2.3 From a279235ddbe975670afe2267162028ec0a312293 Mon Sep 17 00:00:00 2001 From: Vincenzo Frascino Date: Fri, 30 Aug 2019 14:58:59 +0100 Subject: lib/vdso: Remove checks on return value for 32 bit vDSO Since all the architectures that support the generic vDSO library have been converted to support the 32 bit fallbacks it is not required anymore to check the return value of __cvdso_clock_get*time32_common() before updating the old_timespec fields. Remove the related checks from the generic vdso library. References: c60a32ea4f45 ("lib/vdso/32: Provide legacy syscall fallbacks") Signed-off-by: Vincenzo Frascino Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20190830135902.20861-6-vincenzo.frascino@arm.com --- lib/vdso/gettimeofday.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index cd3aacf1cf86..b676a9845def 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -129,10 +129,10 @@ __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) if (unlikely(ret)) return clock_gettime32_fallback(clock, res); - if (likely(!ret)) { - res->tv_sec = ts.tv_sec; - res->tv_nsec = ts.tv_nsec; - } + /* For ret == 0 */ + res->tv_sec = ts.tv_sec; + res->tv_nsec = ts.tv_nsec; + return ret; } #endif /* BUILD_VDSO32 */ @@ -240,7 +240,7 @@ __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) if (unlikely(ret)) return clock_getres32_fallback(clock, res); - if (likely(!ret && res)) { + if (likely(res)) { res->tv_sec = ts.tv_sec; res->tv_nsec = ts.tv_nsec; } -- cgit v1.2.3 From 8463cf80529d0fd80b84cd5ab8b9b952b01c7eb9 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 23 Dec 2019 14:31:07 +0000 Subject: lib/vdso: Let do_coarse() return 0 to simplify the callsite do_coarse() is similar to do_hres() except that it never fails. Change its type to int instead of void and let it always return success (0) to simplify the call site. Signed-off-by: Christophe Leroy Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/21e8afa38c02ca8672c2690307383507fe63b454.1577111367.git.christophe.leroy@c-s.fr --- lib/vdso/gettimeofday.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index b676a9845def..5a5ec899a21a 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -68,7 +68,7 @@ static int do_hres(const struct vdso_data *vd, clockid_t clk, return 0; } -static void do_coarse(const struct vdso_data *vd, clockid_t clk, +static int do_coarse(const struct vdso_data *vd, clockid_t clk, struct __kernel_timespec *ts) { const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; @@ -79,6 +79,8 @@ static void do_coarse(const struct vdso_data *vd, clockid_t clk, ts->tv_sec = vdso_ts->sec; ts->tv_nsec = vdso_ts->nsec; } while (unlikely(vdso_read_retry(vd, seq))); + + return 0; } static __maybe_unused int @@ -96,14 +98,13 @@ __cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts) * clocks are handled in the VDSO directly. */ msk = 1U << clock; - if (likely(msk & VDSO_HRES)) { + if (likely(msk & VDSO_HRES)) return do_hres(&vd[CS_HRES_COARSE], clock, ts); - } else if (msk & VDSO_COARSE) { - do_coarse(&vd[CS_HRES_COARSE], clock, ts); - return 0; - } else if (msk & VDSO_RAW) { + else if (msk & VDSO_COARSE) + return do_coarse(&vd[CS_HRES_COARSE], clock, ts); + else if (msk & VDSO_RAW) return do_hres(&vd[CS_RAW], clock, ts); - } + return -1; } -- cgit v1.2.3 From cdb7c5a9c897ab2e5c56df647dd84c84e150e925 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 23 Dec 2019 14:31:09 +0000 Subject: lib/vdso: Avoid duplication in __cvdso_clock_getres() VDSO_HRES and VDSO_RAW clocks are handled the same way. Avoid the code duplication. Signed-off-by: Christophe Leroy Signed-off-by: Thomas Gleixner Reviewed-by: Andy Lutomirski Link: https://lore.kernel.org/r/fdf1a968a8f7edd61456f1689ac44082ebb19c15.1577111367.git.christophe.leroy@c-s.fr --- lib/vdso/gettimeofday.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index 5a5ec899a21a..fac9e86ef124 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -193,7 +193,7 @@ int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) * clocks are handled in the VDSO directly. */ msk = 1U << clock; - if (msk & VDSO_HRES) { + if (msk & (VDSO_HRES | VDSO_RAW)) { /* * Preserves the behaviour of posix_get_hrtimer_res(). */ @@ -203,11 +203,6 @@ int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) * Preserves the behaviour of posix_get_coarse_res(). */ ns = LOW_RES_NSEC; - } else if (msk & VDSO_RAW) { - /* - * Preserves the behaviour of posix_get_hrtimer_res(). - */ - ns = hrtimer_res; } else { return -1; } -- cgit v1.2.3 From c966533f8c6c45f93c52599f8460e7695f0b7eaa Mon Sep 17 00:00:00 2001 From: Andrei Vagin Date: Tue, 12 Nov 2019 01:26:51 +0000 Subject: lib/vdso: Mark do_hres() and do_coarse() as __always_inline Performance numbers for Intel(R) Core(TM) i5-6300U CPU @ 2.40GHz (more clock_gettime() cycles - the better): clock | before | after | diff ---------------------------------------------------------- monotonic | 153222105 | 166775025 | 8.8% monotonic-coarse | 671557054 | 691513017 | 3.0% monotonic-raw | 147116067 | 161057395 | 9.5% boottime | 153446224 | 166962668 | 9.1% The improvement for arm64 for monotonic and boottime is around 3.5%. clock | before | after | diff ================================================== monotonic 17326692 17951770 3.6% monotonic-coarse 43624027 44215292 1.3% monotonic-raw 17541809 17554932 0.1% boottime 17334982 17954361 3.5% [ tglx: Avoid the goto ] Signed-off-by: Andrei Vagin Signed-off-by: Dmitry Safonov Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20191112012724.250792-3-dima@arista.com --- lib/vdso/gettimeofday.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index fac9e86ef124..b453d2469b63 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -38,7 +38,7 @@ u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) } #endif -static int do_hres(const struct vdso_data *vd, clockid_t clk, +static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, struct __kernel_timespec *ts) { const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; @@ -68,8 +68,8 @@ static int do_hres(const struct vdso_data *vd, clockid_t clk, return 0; } -static int do_coarse(const struct vdso_data *vd, clockid_t clk, - struct __kernel_timespec *ts) +static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk, + struct __kernel_timespec *ts) { const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; u32 seq; @@ -99,13 +99,15 @@ __cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts) */ msk = 1U << clock; if (likely(msk & VDSO_HRES)) - return do_hres(&vd[CS_HRES_COARSE], clock, ts); + vd = &vd[CS_HRES_COARSE]; else if (msk & VDSO_COARSE) return do_coarse(&vd[CS_HRES_COARSE], clock, ts); else if (msk & VDSO_RAW) - return do_hres(&vd[CS_RAW], clock, ts); + vd = &vd[CS_RAW]; + else + return -1; - return -1; + return do_hres(vd, clock, ts); } static __maybe_unused int -- cgit v1.2.3 From 660fd04f9317172ae90f414c68b18a26ae88a829 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 12 Nov 2019 01:27:09 +0000 Subject: lib/vdso: Prepare for time namespace support To support time namespaces in the vdso with a minimal impact on regular non time namespace affected tasks, the namespace handling needs to be hidden in a slow path. The most obvious place is vdso_seq_begin(). If a task belongs to a time namespace then the VVAR page which contains the system wide vdso data is replaced with a namespace specific page which has the same layout as the VVAR page. That page has vdso_data->seq set to 1 to enforce the slow path and vdso_data->clock_mode set to VCLOCK_TIMENS to enforce the time namespace handling path. The extra check in the case that vdso_data->seq is odd, e.g. a concurrent update of the vdso data is in progress, is not really affecting regular tasks which are not part of a time namespace as the task is spin waiting for the update to finish and vdso_data->seq to become even again. If a time namespace task hits that code path, it invokes the corresponding time getter function which retrieves the real VVAR page, reads host time and then adds the offset for the requested clock which is stored in the special VVAR page. If VDSO time namespace support is disabled the whole magic is compiled out. Initial testing shows that the disabled case is almost identical to the host case which does not take the slow timens path. With the special timens page installed the performance hit is constant time and in the range of 5-7%. For the vdso functions which are not using the sequence count an unconditional check for vdso_data->clock_mode is added which switches to the real vdso when the clock_mode is VCLOCK_TIMENS. [avagin: Make do_hres_timens() work with raw clocks too: choose vdso_data pointer by CS_RAW offset.] Suggested-by: Andy Lutomirski Signed-off-by: Thomas Gleixner Signed-off-by: Andrei Vagin Signed-off-by: Dmitry Safonov Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20191112012724.250792-21-dima@arista.com --- include/linux/time.h | 6 ++ include/vdso/datapage.h | 19 ++++++- init/Kconfig | 1 + lib/vdso/Kconfig | 6 ++ lib/vdso/gettimeofday.c | 142 ++++++++++++++++++++++++++++++++++++++++++++++-- 5 files changed, 169 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/include/linux/time.h b/include/linux/time.h index 8e10b9dbd8c2..8ef5e5cc9f57 100644 --- a/include/linux/time.h +++ b/include/linux/time.h @@ -110,4 +110,10 @@ static inline bool itimerspec64_valid(const struct itimerspec64 *its) * Equivalent to !(time_before32(@t, @l) || time_after32(@t, @h)). */ #define time_between32(t, l, h) ((u32)(h) - (u32)(l) >= (u32)(t) - (u32)(l)) + +struct timens_offset { + s64 sec; + u64 nsec; +}; + #endif diff --git a/include/vdso/datapage.h b/include/vdso/datapage.h index 2e302c0f41f7..c5f347cc5e55 100644 --- a/include/vdso/datapage.h +++ b/include/vdso/datapage.h @@ -21,6 +21,8 @@ #define CS_RAW 1 #define CS_BASES (CS_RAW + 1) +#define VCLOCK_TIMENS UINT_MAX + /** * struct vdso_timestamp - basetime per clock_id * @sec: seconds @@ -48,6 +50,7 @@ struct vdso_timestamp { * @mult: clocksource multiplier * @shift: clocksource shift * @basetime[clock_id]: basetime per clock_id + * @offset[clock_id]: time namespace offset per clock_id * @tz_minuteswest: minutes west of Greenwich * @tz_dsttime: type of DST correction * @hrtimer_res: hrtimer resolution @@ -55,6 +58,17 @@ struct vdso_timestamp { * * vdso_data will be accessed by 64 bit and compat code at the same time * so we should be careful before modifying this structure. + * + * @basetime is used to store the base time for the system wide time getter + * VVAR page. + * + * @offset is used by the special time namespace VVAR pages which are + * installed instead of the real VVAR page. These namespace pages must set + * @seq to 1 and @clock_mode to VLOCK_TIMENS to force the code into the + * time namespace slow path. The namespace aware functions retrieve the + * real system wide VVAR page, read host time and add the per clock offset. + * For clocks which are not affected by time namespace adjustment the + * offset must be zero. */ struct vdso_data { u32 seq; @@ -65,7 +79,10 @@ struct vdso_data { u32 mult; u32 shift; - struct vdso_timestamp basetime[VDSO_BASES]; + union { + struct vdso_timestamp basetime[VDSO_BASES]; + struct timens_offset offset[VDSO_BASES]; + }; s32 tz_minuteswest; s32 tz_dsttime; diff --git a/init/Kconfig b/init/Kconfig index b34314fc75f7..9b7f144a6d35 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1082,6 +1082,7 @@ config UTS_NS config TIME_NS bool "TIME namespace" + depends on GENERIC_VDSO_TIME_NS default y help In this namespace boottime and monotonic clocks can be set. diff --git a/lib/vdso/Kconfig b/lib/vdso/Kconfig index 9fe698ff62ec..d883ac299508 100644 --- a/lib/vdso/Kconfig +++ b/lib/vdso/Kconfig @@ -24,4 +24,10 @@ config GENERIC_COMPAT_VDSO help This config option enables the compat VDSO layer. +config GENERIC_VDSO_TIME_NS + bool + help + Selected by architectures which support time namespaces in the + VDSO + endif diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index b453d2469b63..f342ac1fce77 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -38,15 +38,89 @@ u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) } #endif +#ifdef CONFIG_TIME_NS +static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, + struct __kernel_timespec *ts) +{ + const struct vdso_data *vd = __arch_get_timens_vdso_data(); + const struct timens_offset *offs = &vdns->offset[clk]; + const struct vdso_timestamp *vdso_ts; + u64 cycles, last, ns; + u32 seq; + s64 sec; + + if (clk != CLOCK_MONOTONIC_RAW) + vd = &vd[CS_HRES_COARSE]; + else + vd = &vd[CS_RAW]; + vdso_ts = &vd->basetime[clk]; + + do { + seq = vdso_read_begin(vd); + cycles = __arch_get_hw_counter(vd->clock_mode); + ns = vdso_ts->nsec; + last = vd->cycle_last; + if (unlikely((s64)cycles < 0)) + return -1; + + ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); + ns >>= vd->shift; + sec = vdso_ts->sec; + } while (unlikely(vdso_read_retry(vd, seq))); + + /* Add the namespace offset */ + sec += offs->sec; + ns += offs->nsec; + + /* + * Do this outside the loop: a race inside the loop could result + * in __iter_div_u64_rem() being extremely slow. + */ + ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); + ts->tv_nsec = ns; + + return 0; +} +#else +static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void) +{ + return NULL; +} + +static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, + struct __kernel_timespec *ts) +{ + return -EINVAL; +} +#endif + static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, - struct __kernel_timespec *ts) + struct __kernel_timespec *ts) { const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; u64 cycles, last, sec, ns; u32 seq; do { - seq = vdso_read_begin(vd); + /* + * Open coded to handle VCLOCK_TIMENS. Time namespace + * enabled tasks have a special VVAR page installed which + * has vd->seq set to 1 and vd->clock_mode set to + * VCLOCK_TIMENS. For non time namespace affected tasks + * this does not affect performance because if vd->seq is + * odd, i.e. a concurrent update is in progress the extra + * check for vd->clock_mode is just a few extra + * instructions while spin waiting for vd->seq to become + * even again. + */ + while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) { + if (IS_ENABLED(CONFIG_TIME_NS) && + vd->clock_mode == VCLOCK_TIMENS) + return do_hres_timens(vd, clk, ts); + cpu_relax(); + } + smp_rmb(); + cycles = __arch_get_hw_counter(vd->clock_mode); ns = vdso_ts->nsec; last = vd->cycle_last; @@ -68,6 +142,43 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, return 0; } +#ifdef CONFIG_TIME_NS +static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, + struct __kernel_timespec *ts) +{ + const struct vdso_data *vd = __arch_get_timens_vdso_data(); + const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; + const struct timens_offset *offs = &vdns->offset[clk]; + u64 nsec; + s64 sec; + s32 seq; + + do { + seq = vdso_read_begin(vd); + sec = vdso_ts->sec; + nsec = vdso_ts->nsec; + } while (unlikely(vdso_read_retry(vd, seq))); + + /* Add the namespace offset */ + sec += offs->sec; + nsec += offs->nsec; + + /* + * Do this outside the loop: a race inside the loop could result + * in __iter_div_u64_rem() being extremely slow. + */ + ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec); + ts->tv_nsec = nsec; + return 0; +} +#else +static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, + struct __kernel_timespec *ts) +{ + return -1; +} +#endif + static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk, struct __kernel_timespec *ts) { @@ -75,7 +186,18 @@ static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk, u32 seq; do { - seq = vdso_read_begin(vd); + /* + * Open coded to handle VCLOCK_TIMENS. See comment in + * do_hres(). + */ + while ((seq = READ_ONCE(vd->seq)) & 1) { + if (IS_ENABLED(CONFIG_TIME_NS) && + vd->clock_mode == VCLOCK_TIMENS) + return do_coarse_timens(vd, clk, ts); + cpu_relax(); + } + smp_rmb(); + ts->tv_sec = vdso_ts->sec; ts->tv_nsec = vdso_ts->nsec; } while (unlikely(vdso_read_retry(vd, seq))); @@ -156,6 +278,10 @@ __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) } if (unlikely(tz != NULL)) { + if (IS_ENABLED(CONFIG_TIME_NS) && + vd->clock_mode == VCLOCK_TIMENS) + vd = __arch_get_timens_vdso_data(); + tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; } @@ -167,7 +293,12 @@ __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) { const struct vdso_data *vd = __arch_get_vdso_data(); - __kernel_old_time_t t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); + __kernel_old_time_t t; + + if (IS_ENABLED(CONFIG_TIME_NS) && vd->clock_mode == VCLOCK_TIMENS) + vd = __arch_get_timens_vdso_data(); + + t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); if (time) *time = t; @@ -189,6 +320,9 @@ int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) if (unlikely((u32) clock >= MAX_CLOCKS)) return -1; + if (IS_ENABLED(CONFIG_TIME_NS) && vd->clock_mode == VCLOCK_TIMENS) + vd = __arch_get_timens_vdso_data(); + hrtimer_res = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res); /* * Convert the clockid to a bitmask and use it to check which -- cgit v1.2.3 From 1c08a104360f3e18f4ee6346c21cc3923efb952e Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Sun, 5 Jan 2020 22:40:46 -0500 Subject: crypto: poly1305 - add new 32 and 64-bit generic versions These two C implementations from Zinc -- a 32x32 one and a 64x64 one, depending on the platform -- come from Andrew Moon's public domain poly1305-donna portable code, modified for usage in the kernel. The precomputation in the 32-bit version and the use of 64x64 multiplies in the 64-bit version make these perform better than the code it replaces. Moon's code is also very widespread and has received many eyeballs of scrutiny. There's a bit of interference between the x86 implementation, which relies on internal details of the old scalar implementation. In the next commit, the x86 implementation will be replaced with a faster one that doesn't rely on this, so none of this matters much. But for now, to keep this passing the tests, we inline the bits of the old implementation that the x86 implementation relied on. Also, since we now support a slightly larger key space, via the union, some offsets had to be fixed up. Nonce calculation was folded in with the emit function, to take advantage of 64x64 arithmetic. However, Adiantum appeared to rely on no nonce handling in emit, so this path was conditionalized. We also introduced a new struct, poly1305_core_key, to represent the precise amount of space that particular implementation uses. Testing with kbench9000, depending on the CPU, the update function for the 32x32 version has been improved by 4%-7%, and for the 64x64 by 19%-30%. The 32x32 gains are small, but I think there's great value in having a parallel implementation to the 64x64 one so that the two can be compared side-by-side as nice stand-alone units. Signed-off-by: Jason A. Donenfeld Signed-off-by: Herbert Xu --- arch/x86/crypto/poly1305-avx2-x86_64.S | 20 +-- arch/x86/crypto/poly1305_glue.c | 215 +++++++++++++++++++++++++++++++-- crypto/adiantum.c | 4 +- crypto/nhpoly1305.c | 2 +- crypto/poly1305_generic.c | 25 +++- include/crypto/internal/poly1305.h | 45 ++----- include/crypto/nhpoly1305.h | 4 +- include/crypto/poly1305.h | 26 +++- lib/crypto/Makefile | 4 +- lib/crypto/poly1305-donna32.c | 204 +++++++++++++++++++++++++++++++ lib/crypto/poly1305-donna64.c | 185 ++++++++++++++++++++++++++++ lib/crypto/poly1305.c | 169 ++------------------------ 12 files changed, 675 insertions(+), 228 deletions(-) create mode 100644 lib/crypto/poly1305-donna32.c create mode 100644 lib/crypto/poly1305-donna64.c (limited to 'lib') diff --git a/arch/x86/crypto/poly1305-avx2-x86_64.S b/arch/x86/crypto/poly1305-avx2-x86_64.S index d6063feda9da..8f56989ea599 100644 --- a/arch/x86/crypto/poly1305-avx2-x86_64.S +++ b/arch/x86/crypto/poly1305-avx2-x86_64.S @@ -34,16 +34,16 @@ ORMASK: .octa 0x00000000010000000000000001000000 #define u2 0x08(%r8) #define u3 0x0c(%r8) #define u4 0x10(%r8) -#define w0 0x14(%r8) -#define w1 0x18(%r8) -#define w2 0x1c(%r8) -#define w3 0x20(%r8) -#define w4 0x24(%r8) -#define y0 0x28(%r8) -#define y1 0x2c(%r8) -#define y2 0x30(%r8) -#define y3 0x34(%r8) -#define y4 0x38(%r8) +#define w0 0x18(%r8) +#define w1 0x1c(%r8) +#define w2 0x20(%r8) +#define w3 0x24(%r8) +#define w4 0x28(%r8) +#define y0 0x30(%r8) +#define y1 0x34(%r8) +#define y2 0x38(%r8) +#define y3 0x3c(%r8) +#define y4 0x40(%r8) #define m %rsi #define hc0 %ymm0 #define hc1 %ymm1 diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c index 0cc4537e6617..edb7113e36f3 100644 --- a/arch/x86/crypto/poly1305_glue.c +++ b/arch/x86/crypto/poly1305_glue.c @@ -25,6 +25,21 @@ asmlinkage void poly1305_4block_avx2(u32 *h, const u8 *src, const u32 *r, static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_simd); static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx2); +static inline u64 mlt(u64 a, u64 b) +{ + return a * b; +} + +static inline u32 sr(u64 v, u_char n) +{ + return v >> n; +} + +static inline u32 and(u32 v, u32 mask) +{ + return v & mask; +} + static void poly1305_simd_mult(u32 *a, const u32 *b) { u8 m[POLY1305_BLOCK_SIZE]; @@ -36,6 +51,168 @@ static void poly1305_simd_mult(u32 *a, const u32 *b) poly1305_block_sse2(a, m, b, 1); } +static void poly1305_integer_setkey(struct poly1305_key *key, const u8 *raw_key) +{ + /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ + key->r[0] = (get_unaligned_le32(raw_key + 0) >> 0) & 0x3ffffff; + key->r[1] = (get_unaligned_le32(raw_key + 3) >> 2) & 0x3ffff03; + key->r[2] = (get_unaligned_le32(raw_key + 6) >> 4) & 0x3ffc0ff; + key->r[3] = (get_unaligned_le32(raw_key + 9) >> 6) & 0x3f03fff; + key->r[4] = (get_unaligned_le32(raw_key + 12) >> 8) & 0x00fffff; +} + +static void poly1305_integer_blocks(struct poly1305_state *state, + const struct poly1305_key *key, + const void *src, + unsigned int nblocks, u32 hibit) +{ + u32 r0, r1, r2, r3, r4; + u32 s1, s2, s3, s4; + u32 h0, h1, h2, h3, h4; + u64 d0, d1, d2, d3, d4; + + if (!nblocks) + return; + + r0 = key->r[0]; + r1 = key->r[1]; + r2 = key->r[2]; + r3 = key->r[3]; + r4 = key->r[4]; + + s1 = r1 * 5; + s2 = r2 * 5; + s3 = r3 * 5; + s4 = r4 * 5; + + h0 = state->h[0]; + h1 = state->h[1]; + h2 = state->h[2]; + h3 = state->h[3]; + h4 = state->h[4]; + + do { + /* h += m[i] */ + h0 += (get_unaligned_le32(src + 0) >> 0) & 0x3ffffff; + h1 += (get_unaligned_le32(src + 3) >> 2) & 0x3ffffff; + h2 += (get_unaligned_le32(src + 6) >> 4) & 0x3ffffff; + h3 += (get_unaligned_le32(src + 9) >> 6) & 0x3ffffff; + h4 += (get_unaligned_le32(src + 12) >> 8) | (hibit << 24); + + /* h *= r */ + d0 = mlt(h0, r0) + mlt(h1, s4) + mlt(h2, s3) + + mlt(h3, s2) + mlt(h4, s1); + d1 = mlt(h0, r1) + mlt(h1, r0) + mlt(h2, s4) + + mlt(h3, s3) + mlt(h4, s2); + d2 = mlt(h0, r2) + mlt(h1, r1) + mlt(h2, r0) + + mlt(h3, s4) + mlt(h4, s3); + d3 = mlt(h0, r3) + mlt(h1, r2) + mlt(h2, r1) + + mlt(h3, r0) + mlt(h4, s4); + d4 = mlt(h0, r4) + mlt(h1, r3) + mlt(h2, r2) + + mlt(h3, r1) + mlt(h4, r0); + + /* (partial) h %= p */ + d1 += sr(d0, 26); h0 = and(d0, 0x3ffffff); + d2 += sr(d1, 26); h1 = and(d1, 0x3ffffff); + d3 += sr(d2, 26); h2 = and(d2, 0x3ffffff); + d4 += sr(d3, 26); h3 = and(d3, 0x3ffffff); + h0 += sr(d4, 26) * 5; h4 = and(d4, 0x3ffffff); + h1 += h0 >> 26; h0 = h0 & 0x3ffffff; + + src += POLY1305_BLOCK_SIZE; + } while (--nblocks); + + state->h[0] = h0; + state->h[1] = h1; + state->h[2] = h2; + state->h[3] = h3; + state->h[4] = h4; +} + +static void poly1305_integer_emit(const struct poly1305_state *state, void *dst) +{ + u32 h0, h1, h2, h3, h4; + u32 g0, g1, g2, g3, g4; + u32 mask; + + /* fully carry h */ + h0 = state->h[0]; + h1 = state->h[1]; + h2 = state->h[2]; + h3 = state->h[3]; + h4 = state->h[4]; + + h2 += (h1 >> 26); h1 = h1 & 0x3ffffff; + h3 += (h2 >> 26); h2 = h2 & 0x3ffffff; + h4 += (h3 >> 26); h3 = h3 & 0x3ffffff; + h0 += (h4 >> 26) * 5; h4 = h4 & 0x3ffffff; + h1 += (h0 >> 26); h0 = h0 & 0x3ffffff; + + /* compute h + -p */ + g0 = h0 + 5; + g1 = h1 + (g0 >> 26); g0 &= 0x3ffffff; + g2 = h2 + (g1 >> 26); g1 &= 0x3ffffff; + g3 = h3 + (g2 >> 26); g2 &= 0x3ffffff; + g4 = h4 + (g3 >> 26) - (1 << 26); g3 &= 0x3ffffff; + + /* select h if h < p, or h + -p if h >= p */ + mask = (g4 >> ((sizeof(u32) * 8) - 1)) - 1; + g0 &= mask; + g1 &= mask; + g2 &= mask; + g3 &= mask; + g4 &= mask; + mask = ~mask; + h0 = (h0 & mask) | g0; + h1 = (h1 & mask) | g1; + h2 = (h2 & mask) | g2; + h3 = (h3 & mask) | g3; + h4 = (h4 & mask) | g4; + + /* h = h % (2^128) */ + put_unaligned_le32((h0 >> 0) | (h1 << 26), dst + 0); + put_unaligned_le32((h1 >> 6) | (h2 << 20), dst + 4); + put_unaligned_le32((h2 >> 12) | (h3 << 14), dst + 8); + put_unaligned_le32((h3 >> 18) | (h4 << 8), dst + 12); +} + +void poly1305_init_arch(struct poly1305_desc_ctx *desc, const u8 *key) +{ + poly1305_integer_setkey(desc->opaque_r, key); + desc->s[0] = get_unaligned_le32(key + 16); + desc->s[1] = get_unaligned_le32(key + 20); + desc->s[2] = get_unaligned_le32(key + 24); + desc->s[3] = get_unaligned_le32(key + 28); + poly1305_core_init(&desc->h); + desc->buflen = 0; + desc->sset = true; + desc->rset = 1; +} +EXPORT_SYMBOL_GPL(poly1305_init_arch); + +static unsigned int crypto_poly1305_setdesckey(struct poly1305_desc_ctx *dctx, + const u8 *src, unsigned int srclen) +{ + if (!dctx->sset) { + if (!dctx->rset && srclen >= POLY1305_BLOCK_SIZE) { + poly1305_integer_setkey(dctx->r, src); + src += POLY1305_BLOCK_SIZE; + srclen -= POLY1305_BLOCK_SIZE; + dctx->rset = 1; + } + if (srclen >= POLY1305_BLOCK_SIZE) { + dctx->s[0] = get_unaligned_le32(src + 0); + dctx->s[1] = get_unaligned_le32(src + 4); + dctx->s[2] = get_unaligned_le32(src + 8); + dctx->s[3] = get_unaligned_le32(src + 12); + src += POLY1305_BLOCK_SIZE; + srclen -= POLY1305_BLOCK_SIZE; + dctx->sset = true; + } + } + return srclen; +} + static unsigned int poly1305_scalar_blocks(struct poly1305_desc_ctx *dctx, const u8 *src, unsigned int srclen) { @@ -47,8 +224,8 @@ static unsigned int poly1305_scalar_blocks(struct poly1305_desc_ctx *dctx, srclen = datalen; } if (srclen >= POLY1305_BLOCK_SIZE) { - poly1305_core_blocks(&dctx->h, dctx->r, src, - srclen / POLY1305_BLOCK_SIZE, 1); + poly1305_integer_blocks(&dctx->h, dctx->opaque_r, src, + srclen / POLY1305_BLOCK_SIZE, 1); srclen %= POLY1305_BLOCK_SIZE; } return srclen; @@ -105,12 +282,6 @@ static unsigned int poly1305_simd_blocks(struct poly1305_desc_ctx *dctx, return srclen; } -void poly1305_init_arch(struct poly1305_desc_ctx *desc, const u8 *key) -{ - poly1305_init_generic(desc, key); -} -EXPORT_SYMBOL(poly1305_init_arch); - void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, unsigned int srclen) { @@ -158,9 +329,31 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, } EXPORT_SYMBOL(poly1305_update_arch); -void poly1305_final_arch(struct poly1305_desc_ctx *desc, u8 *digest) +void poly1305_final_arch(struct poly1305_desc_ctx *desc, u8 *dst) { - poly1305_final_generic(desc, digest); + __le32 digest[4]; + u64 f = 0; + + if (unlikely(desc->buflen)) { + desc->buf[desc->buflen++] = 1; + memset(desc->buf + desc->buflen, 0, + POLY1305_BLOCK_SIZE - desc->buflen); + poly1305_integer_blocks(&desc->h, desc->opaque_r, desc->buf, 1, 0); + } + + poly1305_integer_emit(&desc->h, digest); + + /* mac = (h + s) % (2^128) */ + f = (f >> 32) + le32_to_cpu(digest[0]) + desc->s[0]; + put_unaligned_le32(f, dst + 0); + f = (f >> 32) + le32_to_cpu(digest[1]) + desc->s[1]; + put_unaligned_le32(f, dst + 4); + f = (f >> 32) + le32_to_cpu(digest[2]) + desc->s[2]; + put_unaligned_le32(f, dst + 8); + f = (f >> 32) + le32_to_cpu(digest[3]) + desc->s[3]; + put_unaligned_le32(f, dst + 12); + + *desc = (struct poly1305_desc_ctx){}; } EXPORT_SYMBOL(poly1305_final_arch); @@ -183,7 +376,7 @@ static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst) if (unlikely(!dctx->sset)) return -ENOKEY; - poly1305_final_generic(dctx, dst); + poly1305_final_arch(dctx, dst); return 0; } diff --git a/crypto/adiantum.c b/crypto/adiantum.c index 4d7a6cac82ed..53d5e705a425 100644 --- a/crypto/adiantum.c +++ b/crypto/adiantum.c @@ -70,7 +70,7 @@ struct adiantum_tfm_ctx { struct crypto_skcipher *streamcipher; struct crypto_cipher *blockcipher; struct crypto_shash *hash; - struct poly1305_key header_hash_key; + struct poly1305_core_key header_hash_key; }; struct adiantum_request_ctx { @@ -239,7 +239,7 @@ static void adiantum_hash_header(struct skcipher_request *req) poly1305_core_blocks(&state, &tctx->header_hash_key, req->iv, TWEAK_SIZE / POLY1305_BLOCK_SIZE, 1); - poly1305_core_emit(&state, &rctx->header_hash); + poly1305_core_emit(&state, NULL, &rctx->header_hash); } /* Hash the left-hand part (the "bulk") of the message using NHPoly1305 */ diff --git a/crypto/nhpoly1305.c b/crypto/nhpoly1305.c index f6b6a52092b4..8a3006c3b51b 100644 --- a/crypto/nhpoly1305.c +++ b/crypto/nhpoly1305.c @@ -210,7 +210,7 @@ int crypto_nhpoly1305_final_helper(struct shash_desc *desc, u8 *dst, nh_t nh_fn) if (state->nh_remaining) process_nh_hash_value(state, key); - poly1305_core_emit(&state->poly_state, dst); + poly1305_core_emit(&state->poly_state, NULL, dst); return 0; } EXPORT_SYMBOL(crypto_nhpoly1305_final_helper); diff --git a/crypto/poly1305_generic.c b/crypto/poly1305_generic.c index 21edbd8c99fb..94af47eb6fa6 100644 --- a/crypto/poly1305_generic.c +++ b/crypto/poly1305_generic.c @@ -31,6 +31,29 @@ static int crypto_poly1305_init(struct shash_desc *desc) return 0; } +static unsigned int crypto_poly1305_setdesckey(struct poly1305_desc_ctx *dctx, + const u8 *src, unsigned int srclen) +{ + if (!dctx->sset) { + if (!dctx->rset && srclen >= POLY1305_BLOCK_SIZE) { + poly1305_core_setkey(&dctx->core_r, src); + src += POLY1305_BLOCK_SIZE; + srclen -= POLY1305_BLOCK_SIZE; + dctx->rset = 2; + } + if (srclen >= POLY1305_BLOCK_SIZE) { + dctx->s[0] = get_unaligned_le32(src + 0); + dctx->s[1] = get_unaligned_le32(src + 4); + dctx->s[2] = get_unaligned_le32(src + 8); + dctx->s[3] = get_unaligned_le32(src + 12); + src += POLY1305_BLOCK_SIZE; + srclen -= POLY1305_BLOCK_SIZE; + dctx->sset = true; + } + } + return srclen; +} + static void poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src, unsigned int srclen) { @@ -42,7 +65,7 @@ static void poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src, srclen = datalen; } - poly1305_core_blocks(&dctx->h, dctx->r, src, + poly1305_core_blocks(&dctx->h, &dctx->core_r, src, srclen / POLY1305_BLOCK_SIZE, 1); } diff --git a/include/crypto/internal/poly1305.h b/include/crypto/internal/poly1305.h index 479b0cab2a1a..064e52ca5248 100644 --- a/include/crypto/internal/poly1305.h +++ b/include/crypto/internal/poly1305.h @@ -11,48 +11,23 @@ #include /* - * Poly1305 core functions. These implement the ε-almost-∆-universal hash - * function underlying the Poly1305 MAC, i.e. they don't add an encrypted nonce - * ("s key") at the end. They also only support block-aligned inputs. + * Poly1305 core functions. These only accept whole blocks; the caller must + * handle any needed block buffering and padding. 'hibit' must be 1 for any + * full blocks, or 0 for the final block if it had to be padded. If 'nonce' is + * non-NULL, then it's added at the end to compute the Poly1305 MAC. Otherwise, + * only the ε-almost-∆-universal hash function (not the full MAC) is computed. */ -void poly1305_core_setkey(struct poly1305_key *key, const u8 *raw_key); + +void poly1305_core_setkey(struct poly1305_core_key *key, const u8 *raw_key); static inline void poly1305_core_init(struct poly1305_state *state) { *state = (struct poly1305_state){}; } void poly1305_core_blocks(struct poly1305_state *state, - const struct poly1305_key *key, const void *src, + const struct poly1305_core_key *key, const void *src, unsigned int nblocks, u32 hibit); -void poly1305_core_emit(const struct poly1305_state *state, void *dst); - -/* - * Poly1305 requires a unique key for each tag, which implies that we can't set - * it on the tfm that gets accessed by multiple users simultaneously. Instead we - * expect the key as the first 32 bytes in the update() call. - */ -static inline -unsigned int crypto_poly1305_setdesckey(struct poly1305_desc_ctx *dctx, - const u8 *src, unsigned int srclen) -{ - if (!dctx->sset) { - if (!dctx->rset && srclen >= POLY1305_BLOCK_SIZE) { - poly1305_core_setkey(dctx->r, src); - src += POLY1305_BLOCK_SIZE; - srclen -= POLY1305_BLOCK_SIZE; - dctx->rset = 1; - } - if (srclen >= POLY1305_BLOCK_SIZE) { - dctx->s[0] = get_unaligned_le32(src + 0); - dctx->s[1] = get_unaligned_le32(src + 4); - dctx->s[2] = get_unaligned_le32(src + 8); - dctx->s[3] = get_unaligned_le32(src + 12); - src += POLY1305_BLOCK_SIZE; - srclen -= POLY1305_BLOCK_SIZE; - dctx->sset = true; - } - } - return srclen; -} +void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4], + void *dst); #endif diff --git a/include/crypto/nhpoly1305.h b/include/crypto/nhpoly1305.h index 53c04423c582..306925fea190 100644 --- a/include/crypto/nhpoly1305.h +++ b/include/crypto/nhpoly1305.h @@ -7,7 +7,7 @@ #define _NHPOLY1305_H #include -#include +#include /* NH parameterization: */ @@ -33,7 +33,7 @@ #define NHPOLY1305_KEY_SIZE (POLY1305_BLOCK_SIZE + NH_KEY_BYTES) struct nhpoly1305_key { - struct poly1305_key poly_key; + struct poly1305_core_key poly_key; u32 nh_key[NH_KEY_WORDS]; }; diff --git a/include/crypto/poly1305.h b/include/crypto/poly1305.h index 74c6e1cd73ee..f1f67fc749cf 100644 --- a/include/crypto/poly1305.h +++ b/include/crypto/poly1305.h @@ -13,12 +13,29 @@ #define POLY1305_KEY_SIZE 32 #define POLY1305_DIGEST_SIZE 16 +/* The poly1305_key and poly1305_state types are mostly opaque and + * implementation-defined. Limbs might be in base 2^64 or base 2^26, or + * different yet. The union type provided keeps these 64-bit aligned for the + * case in which this is implemented using 64x64 multiplies. + */ + struct poly1305_key { - u32 r[5]; /* key, base 2^26 */ + union { + u32 r[5]; + u64 r64[3]; + }; +}; + +struct poly1305_core_key { + struct poly1305_key key; + struct poly1305_key precomputed_s; }; struct poly1305_state { - u32 h[5]; /* accumulator, base 2^26 */ + union { + u32 h[5]; + u64 h64[3]; + }; }; struct poly1305_desc_ctx { @@ -35,7 +52,10 @@ struct poly1305_desc_ctx { /* accumulator */ struct poly1305_state h; /* key */ - struct poly1305_key r[CONFIG_CRYPTO_LIB_POLY1305_RSIZE]; + union { + struct poly1305_key opaque_r[CONFIG_CRYPTO_LIB_POLY1305_RSIZE]; + struct poly1305_core_key core_r; + }; }; void poly1305_init_arch(struct poly1305_desc_ctx *desc, const u8 *key); diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index f97f9b941110..6ecaf83a5a9a 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -28,7 +28,9 @@ obj-$(CONFIG_CRYPTO_LIB_DES) += libdes.o libdes-y := des.o obj-$(CONFIG_CRYPTO_LIB_POLY1305_GENERIC) += libpoly1305.o -libpoly1305-y := poly1305.o +libpoly1305-y := poly1305-donna32.o +libpoly1305-$(CONFIG_ARCH_SUPPORTS_INT128) := poly1305-donna64.o +libpoly1305-y += poly1305.o obj-$(CONFIG_CRYPTO_LIB_SHA256) += libsha256.o libsha256-y := sha256.o diff --git a/lib/crypto/poly1305-donna32.c b/lib/crypto/poly1305-donna32.c new file mode 100644 index 000000000000..3cc77d94390b --- /dev/null +++ b/lib/crypto/poly1305-donna32.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is based in part on Andrew Moon's poly1305-donna, which is in the + * public domain. + */ + +#include +#include +#include + +void poly1305_core_setkey(struct poly1305_core_key *key, const u8 raw_key[16]) +{ + /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ + key->key.r[0] = (get_unaligned_le32(&raw_key[0])) & 0x3ffffff; + key->key.r[1] = (get_unaligned_le32(&raw_key[3]) >> 2) & 0x3ffff03; + key->key.r[2] = (get_unaligned_le32(&raw_key[6]) >> 4) & 0x3ffc0ff; + key->key.r[3] = (get_unaligned_le32(&raw_key[9]) >> 6) & 0x3f03fff; + key->key.r[4] = (get_unaligned_le32(&raw_key[12]) >> 8) & 0x00fffff; + + /* s = 5*r */ + key->precomputed_s.r[0] = key->key.r[1] * 5; + key->precomputed_s.r[1] = key->key.r[2] * 5; + key->precomputed_s.r[2] = key->key.r[3] * 5; + key->precomputed_s.r[3] = key->key.r[4] * 5; +} +EXPORT_SYMBOL(poly1305_core_setkey); + +void poly1305_core_blocks(struct poly1305_state *state, + const struct poly1305_core_key *key, const void *src, + unsigned int nblocks, u32 hibit) +{ + const u8 *input = src; + u32 r0, r1, r2, r3, r4; + u32 s1, s2, s3, s4; + u32 h0, h1, h2, h3, h4; + u64 d0, d1, d2, d3, d4; + u32 c; + + if (!nblocks) + return; + + hibit <<= 24; + + r0 = key->key.r[0]; + r1 = key->key.r[1]; + r2 = key->key.r[2]; + r3 = key->key.r[3]; + r4 = key->key.r[4]; + + s1 = key->precomputed_s.r[0]; + s2 = key->precomputed_s.r[1]; + s3 = key->precomputed_s.r[2]; + s4 = key->precomputed_s.r[3]; + + h0 = state->h[0]; + h1 = state->h[1]; + h2 = state->h[2]; + h3 = state->h[3]; + h4 = state->h[4]; + + do { + /* h += m[i] */ + h0 += (get_unaligned_le32(&input[0])) & 0x3ffffff; + h1 += (get_unaligned_le32(&input[3]) >> 2) & 0x3ffffff; + h2 += (get_unaligned_le32(&input[6]) >> 4) & 0x3ffffff; + h3 += (get_unaligned_le32(&input[9]) >> 6) & 0x3ffffff; + h4 += (get_unaligned_le32(&input[12]) >> 8) | hibit; + + /* h *= r */ + d0 = ((u64)h0 * r0) + ((u64)h1 * s4) + + ((u64)h2 * s3) + ((u64)h3 * s2) + + ((u64)h4 * s1); + d1 = ((u64)h0 * r1) + ((u64)h1 * r0) + + ((u64)h2 * s4) + ((u64)h3 * s3) + + ((u64)h4 * s2); + d2 = ((u64)h0 * r2) + ((u64)h1 * r1) + + ((u64)h2 * r0) + ((u64)h3 * s4) + + ((u64)h4 * s3); + d3 = ((u64)h0 * r3) + ((u64)h1 * r2) + + ((u64)h2 * r1) + ((u64)h3 * r0) + + ((u64)h4 * s4); + d4 = ((u64)h0 * r4) + ((u64)h1 * r3) + + ((u64)h2 * r2) + ((u64)h3 * r1) + + ((u64)h4 * r0); + + /* (partial) h %= p */ + c = (u32)(d0 >> 26); + h0 = (u32)d0 & 0x3ffffff; + d1 += c; + c = (u32)(d1 >> 26); + h1 = (u32)d1 & 0x3ffffff; + d2 += c; + c = (u32)(d2 >> 26); + h2 = (u32)d2 & 0x3ffffff; + d3 += c; + c = (u32)(d3 >> 26); + h3 = (u32)d3 & 0x3ffffff; + d4 += c; + c = (u32)(d4 >> 26); + h4 = (u32)d4 & 0x3ffffff; + h0 += c * 5; + c = (h0 >> 26); + h0 = h0 & 0x3ffffff; + h1 += c; + + input += POLY1305_BLOCK_SIZE; + } while (--nblocks); + + state->h[0] = h0; + state->h[1] = h1; + state->h[2] = h2; + state->h[3] = h3; + state->h[4] = h4; +} +EXPORT_SYMBOL(poly1305_core_blocks); + +void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4], + void *dst) +{ + u8 *mac = dst; + u32 h0, h1, h2, h3, h4, c; + u32 g0, g1, g2, g3, g4; + u64 f; + u32 mask; + + /* fully carry h */ + h0 = state->h[0]; + h1 = state->h[1]; + h2 = state->h[2]; + h3 = state->h[3]; + h4 = state->h[4]; + + c = h1 >> 26; + h1 = h1 & 0x3ffffff; + h2 += c; + c = h2 >> 26; + h2 = h2 & 0x3ffffff; + h3 += c; + c = h3 >> 26; + h3 = h3 & 0x3ffffff; + h4 += c; + c = h4 >> 26; + h4 = h4 & 0x3ffffff; + h0 += c * 5; + c = h0 >> 26; + h0 = h0 & 0x3ffffff; + h1 += c; + + /* compute h + -p */ + g0 = h0 + 5; + c = g0 >> 26; + g0 &= 0x3ffffff; + g1 = h1 + c; + c = g1 >> 26; + g1 &= 0x3ffffff; + g2 = h2 + c; + c = g2 >> 26; + g2 &= 0x3ffffff; + g3 = h3 + c; + c = g3 >> 26; + g3 &= 0x3ffffff; + g4 = h4 + c - (1UL << 26); + + /* select h if h < p, or h + -p if h >= p */ + mask = (g4 >> ((sizeof(u32) * 8) - 1)) - 1; + g0 &= mask; + g1 &= mask; + g2 &= mask; + g3 &= mask; + g4 &= mask; + mask = ~mask; + + h0 = (h0 & mask) | g0; + h1 = (h1 & mask) | g1; + h2 = (h2 & mask) | g2; + h3 = (h3 & mask) | g3; + h4 = (h4 & mask) | g4; + + /* h = h % (2^128) */ + h0 = ((h0) | (h1 << 26)) & 0xffffffff; + h1 = ((h1 >> 6) | (h2 << 20)) & 0xffffffff; + h2 = ((h2 >> 12) | (h3 << 14)) & 0xffffffff; + h3 = ((h3 >> 18) | (h4 << 8)) & 0xffffffff; + + if (likely(nonce)) { + /* mac = (h + nonce) % (2^128) */ + f = (u64)h0 + nonce[0]; + h0 = (u32)f; + f = (u64)h1 + nonce[1] + (f >> 32); + h1 = (u32)f; + f = (u64)h2 + nonce[2] + (f >> 32); + h2 = (u32)f; + f = (u64)h3 + nonce[3] + (f >> 32); + h3 = (u32)f; + } + + put_unaligned_le32(h0, &mac[0]); + put_unaligned_le32(h1, &mac[4]); + put_unaligned_le32(h2, &mac[8]); + put_unaligned_le32(h3, &mac[12]); +} +EXPORT_SYMBOL(poly1305_core_emit); diff --git a/lib/crypto/poly1305-donna64.c b/lib/crypto/poly1305-donna64.c new file mode 100644 index 000000000000..6ae181bb4345 --- /dev/null +++ b/lib/crypto/poly1305-donna64.c @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is based in part on Andrew Moon's poly1305-donna, which is in the + * public domain. + */ + +#include +#include +#include + +typedef __uint128_t u128; + +void poly1305_core_setkey(struct poly1305_core_key *key, const u8 raw_key[16]) +{ + u64 t0, t1; + + /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ + t0 = get_unaligned_le64(&raw_key[0]); + t1 = get_unaligned_le64(&raw_key[8]); + + key->key.r64[0] = t0 & 0xffc0fffffffULL; + key->key.r64[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffffULL; + key->key.r64[2] = ((t1 >> 24)) & 0x00ffffffc0fULL; + + /* s = 20*r */ + key->precomputed_s.r64[0] = key->key.r64[1] * 20; + key->precomputed_s.r64[1] = key->key.r64[2] * 20; +} +EXPORT_SYMBOL(poly1305_core_setkey); + +void poly1305_core_blocks(struct poly1305_state *state, + const struct poly1305_core_key *key, const void *src, + unsigned int nblocks, u32 hibit) +{ + const u8 *input = src; + u64 hibit64; + u64 r0, r1, r2; + u64 s1, s2; + u64 h0, h1, h2; + u64 c; + u128 d0, d1, d2, d; + + if (!nblocks) + return; + + hibit64 = ((u64)hibit) << 40; + + r0 = key->key.r64[0]; + r1 = key->key.r64[1]; + r2 = key->key.r64[2]; + + h0 = state->h64[0]; + h1 = state->h64[1]; + h2 = state->h64[2]; + + s1 = key->precomputed_s.r64[0]; + s2 = key->precomputed_s.r64[1]; + + do { + u64 t0, t1; + + /* h += m[i] */ + t0 = get_unaligned_le64(&input[0]); + t1 = get_unaligned_le64(&input[8]); + + h0 += t0 & 0xfffffffffffULL; + h1 += ((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL; + h2 += (((t1 >> 24)) & 0x3ffffffffffULL) | hibit64; + + /* h *= r */ + d0 = (u128)h0 * r0; + d = (u128)h1 * s2; + d0 += d; + d = (u128)h2 * s1; + d0 += d; + d1 = (u128)h0 * r1; + d = (u128)h1 * r0; + d1 += d; + d = (u128)h2 * s2; + d1 += d; + d2 = (u128)h0 * r2; + d = (u128)h1 * r1; + d2 += d; + d = (u128)h2 * r0; + d2 += d; + + /* (partial) h %= p */ + c = (u64)(d0 >> 44); + h0 = (u64)d0 & 0xfffffffffffULL; + d1 += c; + c = (u64)(d1 >> 44); + h1 = (u64)d1 & 0xfffffffffffULL; + d2 += c; + c = (u64)(d2 >> 42); + h2 = (u64)d2 & 0x3ffffffffffULL; + h0 += c * 5; + c = h0 >> 44; + h0 = h0 & 0xfffffffffffULL; + h1 += c; + + input += POLY1305_BLOCK_SIZE; + } while (--nblocks); + + state->h64[0] = h0; + state->h64[1] = h1; + state->h64[2] = h2; +} +EXPORT_SYMBOL(poly1305_core_blocks); + +void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4], + void *dst) +{ + u8 *mac = dst; + u64 h0, h1, h2, c; + u64 g0, g1, g2; + u64 t0, t1; + + /* fully carry h */ + h0 = state->h64[0]; + h1 = state->h64[1]; + h2 = state->h64[2]; + + c = h1 >> 44; + h1 &= 0xfffffffffffULL; + h2 += c; + c = h2 >> 42; + h2 &= 0x3ffffffffffULL; + h0 += c * 5; + c = h0 >> 44; + h0 &= 0xfffffffffffULL; + h1 += c; + c = h1 >> 44; + h1 &= 0xfffffffffffULL; + h2 += c; + c = h2 >> 42; + h2 &= 0x3ffffffffffULL; + h0 += c * 5; + c = h0 >> 44; + h0 &= 0xfffffffffffULL; + h1 += c; + + /* compute h + -p */ + g0 = h0 + 5; + c = g0 >> 44; + g0 &= 0xfffffffffffULL; + g1 = h1 + c; + c = g1 >> 44; + g1 &= 0xfffffffffffULL; + g2 = h2 + c - (1ULL << 42); + + /* select h if h < p, or h + -p if h >= p */ + c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1; + g0 &= c; + g1 &= c; + g2 &= c; + c = ~c; + h0 = (h0 & c) | g0; + h1 = (h1 & c) | g1; + h2 = (h2 & c) | g2; + + if (likely(nonce)) { + /* h = (h + nonce) */ + t0 = ((u64)nonce[1] << 32) | nonce[0]; + t1 = ((u64)nonce[3] << 32) | nonce[2]; + + h0 += t0 & 0xfffffffffffULL; + c = h0 >> 44; + h0 &= 0xfffffffffffULL; + h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL) + c; + c = h1 >> 44; + h1 &= 0xfffffffffffULL; + h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c; + h2 &= 0x3ffffffffffULL; + } + + /* mac = h % (2^128) */ + h0 = h0 | (h1 << 44); + h1 = (h1 >> 20) | (h2 << 24); + + put_unaligned_le64(h0, &mac[0]); + put_unaligned_le64(h1, &mac[8]); +} +EXPORT_SYMBOL(poly1305_core_emit); diff --git a/lib/crypto/poly1305.c b/lib/crypto/poly1305.c index 32ec293c65ae..9d2d14df0fee 100644 --- a/lib/crypto/poly1305.c +++ b/lib/crypto/poly1305.c @@ -12,151 +12,9 @@ #include #include -static inline u64 mlt(u64 a, u64 b) -{ - return a * b; -} - -static inline u32 sr(u64 v, u_char n) -{ - return v >> n; -} - -static inline u32 and(u32 v, u32 mask) -{ - return v & mask; -} - -void poly1305_core_setkey(struct poly1305_key *key, const u8 *raw_key) -{ - /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ - key->r[0] = (get_unaligned_le32(raw_key + 0) >> 0) & 0x3ffffff; - key->r[1] = (get_unaligned_le32(raw_key + 3) >> 2) & 0x3ffff03; - key->r[2] = (get_unaligned_le32(raw_key + 6) >> 4) & 0x3ffc0ff; - key->r[3] = (get_unaligned_le32(raw_key + 9) >> 6) & 0x3f03fff; - key->r[4] = (get_unaligned_le32(raw_key + 12) >> 8) & 0x00fffff; -} -EXPORT_SYMBOL_GPL(poly1305_core_setkey); - -void poly1305_core_blocks(struct poly1305_state *state, - const struct poly1305_key *key, const void *src, - unsigned int nblocks, u32 hibit) -{ - u32 r0, r1, r2, r3, r4; - u32 s1, s2, s3, s4; - u32 h0, h1, h2, h3, h4; - u64 d0, d1, d2, d3, d4; - - if (!nblocks) - return; - - r0 = key->r[0]; - r1 = key->r[1]; - r2 = key->r[2]; - r3 = key->r[3]; - r4 = key->r[4]; - - s1 = r1 * 5; - s2 = r2 * 5; - s3 = r3 * 5; - s4 = r4 * 5; - - h0 = state->h[0]; - h1 = state->h[1]; - h2 = state->h[2]; - h3 = state->h[3]; - h4 = state->h[4]; - - do { - /* h += m[i] */ - h0 += (get_unaligned_le32(src + 0) >> 0) & 0x3ffffff; - h1 += (get_unaligned_le32(src + 3) >> 2) & 0x3ffffff; - h2 += (get_unaligned_le32(src + 6) >> 4) & 0x3ffffff; - h3 += (get_unaligned_le32(src + 9) >> 6) & 0x3ffffff; - h4 += (get_unaligned_le32(src + 12) >> 8) | (hibit << 24); - - /* h *= r */ - d0 = mlt(h0, r0) + mlt(h1, s4) + mlt(h2, s3) + - mlt(h3, s2) + mlt(h4, s1); - d1 = mlt(h0, r1) + mlt(h1, r0) + mlt(h2, s4) + - mlt(h3, s3) + mlt(h4, s2); - d2 = mlt(h0, r2) + mlt(h1, r1) + mlt(h2, r0) + - mlt(h3, s4) + mlt(h4, s3); - d3 = mlt(h0, r3) + mlt(h1, r2) + mlt(h2, r1) + - mlt(h3, r0) + mlt(h4, s4); - d4 = mlt(h0, r4) + mlt(h1, r3) + mlt(h2, r2) + - mlt(h3, r1) + mlt(h4, r0); - - /* (partial) h %= p */ - d1 += sr(d0, 26); h0 = and(d0, 0x3ffffff); - d2 += sr(d1, 26); h1 = and(d1, 0x3ffffff); - d3 += sr(d2, 26); h2 = and(d2, 0x3ffffff); - d4 += sr(d3, 26); h3 = and(d3, 0x3ffffff); - h0 += sr(d4, 26) * 5; h4 = and(d4, 0x3ffffff); - h1 += h0 >> 26; h0 = h0 & 0x3ffffff; - - src += POLY1305_BLOCK_SIZE; - } while (--nblocks); - - state->h[0] = h0; - state->h[1] = h1; - state->h[2] = h2; - state->h[3] = h3; - state->h[4] = h4; -} -EXPORT_SYMBOL_GPL(poly1305_core_blocks); - -void poly1305_core_emit(const struct poly1305_state *state, void *dst) -{ - u32 h0, h1, h2, h3, h4; - u32 g0, g1, g2, g3, g4; - u32 mask; - - /* fully carry h */ - h0 = state->h[0]; - h1 = state->h[1]; - h2 = state->h[2]; - h3 = state->h[3]; - h4 = state->h[4]; - - h2 += (h1 >> 26); h1 = h1 & 0x3ffffff; - h3 += (h2 >> 26); h2 = h2 & 0x3ffffff; - h4 += (h3 >> 26); h3 = h3 & 0x3ffffff; - h0 += (h4 >> 26) * 5; h4 = h4 & 0x3ffffff; - h1 += (h0 >> 26); h0 = h0 & 0x3ffffff; - - /* compute h + -p */ - g0 = h0 + 5; - g1 = h1 + (g0 >> 26); g0 &= 0x3ffffff; - g2 = h2 + (g1 >> 26); g1 &= 0x3ffffff; - g3 = h3 + (g2 >> 26); g2 &= 0x3ffffff; - g4 = h4 + (g3 >> 26) - (1 << 26); g3 &= 0x3ffffff; - - /* select h if h < p, or h + -p if h >= p */ - mask = (g4 >> ((sizeof(u32) * 8) - 1)) - 1; - g0 &= mask; - g1 &= mask; - g2 &= mask; - g3 &= mask; - g4 &= mask; - mask = ~mask; - h0 = (h0 & mask) | g0; - h1 = (h1 & mask) | g1; - h2 = (h2 & mask) | g2; - h3 = (h3 & mask) | g3; - h4 = (h4 & mask) | g4; - - /* h = h % (2^128) */ - put_unaligned_le32((h0 >> 0) | (h1 << 26), dst + 0); - put_unaligned_le32((h1 >> 6) | (h2 << 20), dst + 4); - put_unaligned_le32((h2 >> 12) | (h3 << 14), dst + 8); - put_unaligned_le32((h3 >> 18) | (h4 << 8), dst + 12); -} -EXPORT_SYMBOL_GPL(poly1305_core_emit); - void poly1305_init_generic(struct poly1305_desc_ctx *desc, const u8 *key) { - poly1305_core_setkey(desc->r, key); + poly1305_core_setkey(&desc->core_r, key); desc->s[0] = get_unaligned_le32(key + 16); desc->s[1] = get_unaligned_le32(key + 20); desc->s[2] = get_unaligned_le32(key + 24); @@ -164,7 +22,7 @@ void poly1305_init_generic(struct poly1305_desc_ctx *desc, const u8 *key) poly1305_core_init(&desc->h); desc->buflen = 0; desc->sset = true; - desc->rset = 1; + desc->rset = 2; } EXPORT_SYMBOL_GPL(poly1305_init_generic); @@ -181,13 +39,14 @@ void poly1305_update_generic(struct poly1305_desc_ctx *desc, const u8 *src, desc->buflen += bytes; if (desc->buflen == POLY1305_BLOCK_SIZE) { - poly1305_core_blocks(&desc->h, desc->r, desc->buf, 1, 1); + poly1305_core_blocks(&desc->h, &desc->core_r, desc->buf, + 1, 1); desc->buflen = 0; } } if (likely(nbytes >= POLY1305_BLOCK_SIZE)) { - poly1305_core_blocks(&desc->h, desc->r, src, + poly1305_core_blocks(&desc->h, &desc->core_r, src, nbytes / POLY1305_BLOCK_SIZE, 1); src += nbytes - (nbytes % POLY1305_BLOCK_SIZE); nbytes %= POLY1305_BLOCK_SIZE; @@ -202,28 +61,14 @@ EXPORT_SYMBOL_GPL(poly1305_update_generic); void poly1305_final_generic(struct poly1305_desc_ctx *desc, u8 *dst) { - __le32 digest[4]; - u64 f = 0; - if (unlikely(desc->buflen)) { desc->buf[desc->buflen++] = 1; memset(desc->buf + desc->buflen, 0, POLY1305_BLOCK_SIZE - desc->buflen); - poly1305_core_blocks(&desc->h, desc->r, desc->buf, 1, 0); + poly1305_core_blocks(&desc->h, &desc->core_r, desc->buf, 1, 0); } - poly1305_core_emit(&desc->h, digest); - - /* mac = (h + s) % (2^128) */ - f = (f >> 32) + le32_to_cpu(digest[0]) + desc->s[0]; - put_unaligned_le32(f, dst + 0); - f = (f >> 32) + le32_to_cpu(digest[1]) + desc->s[1]; - put_unaligned_le32(f, dst + 4); - f = (f >> 32) + le32_to_cpu(digest[2]) + desc->s[2]; - put_unaligned_le32(f, dst + 8); - f = (f >> 32) + le32_to_cpu(digest[3]) + desc->s[3]; - put_unaligned_le32(f, dst + 12); - + poly1305_core_emit(&desc->h, desc->s, dst); *desc = (struct poly1305_desc_ctx){}; } EXPORT_SYMBOL_GPL(poly1305_final_generic); -- cgit v1.2.3 From d7d7b853566254648df59f7ea27ea05952a6cfa8 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Sun, 5 Jan 2020 22:40:48 -0500 Subject: crypto: x86/poly1305 - wire up faster implementations for kernel These x86_64 vectorized implementations support AVX, AVX-2, and AVX512F. The AVX-512F implementation is disabled on Skylake, due to throttling, but it is quite fast on >= Cannonlake. On the left is cycle counts on a Core i7 6700HQ using the AVX-2 codepath, comparing this implementation ("new") to the implementation in the current crypto api ("old"). On the right are benchmarks on a Xeon Gold 5120 using the AVX-512 codepath. The new implementation is faster on all benchmarks. AVX-2 AVX-512 --------- ----------- size old new size old new ---- ---- ---- ---- ---- ---- 0 70 68 0 74 70 16 92 90 16 96 92 32 134 104 32 136 106 48 172 120 48 184 124 64 218 136 64 218 138 80 254 158 80 260 160 96 298 174 96 300 176 112 342 192 112 342 194 128 388 212 128 384 212 144 428 228 144 420 226 160 466 246 160 464 248 176 510 264 176 504 264 192 550 282 192 544 282 208 594 302 208 582 300 224 628 316 224 624 318 240 676 334 240 662 338 256 716 354 256 708 358 272 764 374 272 748 372 288 802 352 288 788 358 304 420 366 304 422 370 320 428 360 320 432 364 336 484 378 336 486 380 352 426 384 352 434 390 368 478 400 368 480 408 384 488 394 384 490 398 400 542 408 400 542 412 416 486 416 416 492 426 432 534 430 432 538 436 448 544 422 448 546 432 464 600 438 464 600 448 480 540 448 480 548 456 496 594 464 496 594 476 512 602 456 512 606 470 528 656 476 528 656 480 544 600 480 544 606 498 560 650 494 560 652 512 576 664 490 576 662 508 592 714 508 592 716 522 608 656 514 608 664 538 624 708 532 624 710 552 640 716 524 640 720 516 656 770 536 656 772 526 672 716 548 672 722 544 688 770 562 688 768 556 704 774 552 704 778 556 720 826 568 720 832 568 736 768 574 736 780 584 752 822 592 752 826 600 768 830 584 768 836 560 784 884 602 784 888 572 800 828 610 800 838 588 816 884 628 816 884 604 832 888 618 832 894 598 848 942 632 848 946 612 864 884 644 864 896 628 880 936 660 880 942 644 896 948 652 896 952 608 912 1000 664 912 1004 616 928 942 676 928 954 634 944 994 690 944 1000 646 960 1002 680 960 1008 646 976 1054 694 976 1062 658 992 1002 706 992 1012 674 1008 1052 720 1008 1058 690 This commit wires in the prior implementation from Andy, and makes the following changes to be suitable for kernel land. - Some cosmetic and structural changes, like renaming labels to .Lname, constants, and other Linux conventions, as well as making the code easy for us to maintain moving forward. - CPU feature checking is done in C by the glue code. - We avoid jumping into the middle of functions, to appease objtool, and instead parameterize shared code. - We maintain frame pointers so that stack traces make sense. - We remove the dependency on the perl xlate code, which transforms the output into things that assemblers we don't care about use. Importantly, none of our changes affect the arithmetic or core code, but just involve the differing environment of kernel space. Signed-off-by: Jason A. Donenfeld Signed-off-by: Samuel Neves Co-developed-by: Samuel Neves Signed-off-by: Herbert Xu --- arch/x86/crypto/.gitignore | 1 + arch/x86/crypto/Makefile | 11 +- arch/x86/crypto/poly1305-avx2-x86_64.S | 390 --------------- arch/x86/crypto/poly1305-sse2-x86_64.S | 590 ---------------------- arch/x86/crypto/poly1305-x86_64-cryptogams.pl | 682 +++++++++++++++----------- arch/x86/crypto/poly1305_glue.c | 473 +++++++----------- lib/crypto/Kconfig | 2 +- 7 files changed, 572 insertions(+), 1577 deletions(-) create mode 100644 arch/x86/crypto/.gitignore delete mode 100644 arch/x86/crypto/poly1305-avx2-x86_64.S delete mode 100644 arch/x86/crypto/poly1305-sse2-x86_64.S (limited to 'lib') diff --git a/arch/x86/crypto/.gitignore b/arch/x86/crypto/.gitignore new file mode 100644 index 000000000000..c406ea6571fa --- /dev/null +++ b/arch/x86/crypto/.gitignore @@ -0,0 +1 @@ +poly1305-x86_64.S diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 958440eae27e..b69e00bf20b8 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -73,6 +73,10 @@ aegis128-aesni-y := aegis128-aesni-asm.o aegis128-aesni-glue.o nhpoly1305-sse2-y := nh-sse2-x86_64.o nhpoly1305-sse2-glue.o blake2s-x86_64-y := blake2s-core.o blake2s-glue.o +poly1305-x86_64-y := poly1305-x86_64-cryptogams.o poly1305_glue.o +ifneq ($(CONFIG_CRYPTO_POLY1305_X86_64),) +targets += poly1305-x86_64-cryptogams.S +endif ifeq ($(avx_supported),yes) camellia-aesni-avx-x86_64-y := camellia-aesni-avx-asm_64.o \ @@ -101,10 +105,8 @@ aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o aesni-intel-$(CONFIG_64BIT) += aesni-intel_avx-x86_64.o aes_ctrby8_avx-x86_64.o ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o sha1-ssse3-y := sha1_ssse3_asm.o sha1_ssse3_glue.o -poly1305-x86_64-y := poly1305-sse2-x86_64.o poly1305_glue.o ifeq ($(avx2_supported),yes) sha1-ssse3-y += sha1_avx2_x86_64_asm.o -poly1305-x86_64-y += poly1305-avx2-x86_64.o endif ifeq ($(sha1_ni_supported),yes) sha1-ssse3-y += sha1_ni_asm.o @@ -118,3 +120,8 @@ sha256-ssse3-y += sha256_ni_asm.o endif sha512-ssse3-y := sha512-ssse3-asm.o sha512-avx-asm.o sha512-avx2-asm.o sha512_ssse3_glue.o crct10dif-pclmul-y := crct10dif-pcl-asm_64.o crct10dif-pclmul_glue.o + +quiet_cmd_perlasm = PERLASM $@ + cmd_perlasm = $(PERL) $< > $@ +$(obj)/%.S: $(src)/%.pl FORCE + $(call if_changed,perlasm) diff --git a/arch/x86/crypto/poly1305-avx2-x86_64.S b/arch/x86/crypto/poly1305-avx2-x86_64.S deleted file mode 100644 index 8f56989ea599..000000000000 --- a/arch/x86/crypto/poly1305-avx2-x86_64.S +++ /dev/null @@ -1,390 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Poly1305 authenticator algorithm, RFC7539, x64 AVX2 functions - * - * Copyright (C) 2015 Martin Willi - */ - -#include - -.section .rodata.cst32.ANMASK, "aM", @progbits, 32 -.align 32 -ANMASK: .octa 0x0000000003ffffff0000000003ffffff - .octa 0x0000000003ffffff0000000003ffffff - -.section .rodata.cst32.ORMASK, "aM", @progbits, 32 -.align 32 -ORMASK: .octa 0x00000000010000000000000001000000 - .octa 0x00000000010000000000000001000000 - -.text - -#define h0 0x00(%rdi) -#define h1 0x04(%rdi) -#define h2 0x08(%rdi) -#define h3 0x0c(%rdi) -#define h4 0x10(%rdi) -#define r0 0x00(%rdx) -#define r1 0x04(%rdx) -#define r2 0x08(%rdx) -#define r3 0x0c(%rdx) -#define r4 0x10(%rdx) -#define u0 0x00(%r8) -#define u1 0x04(%r8) -#define u2 0x08(%r8) -#define u3 0x0c(%r8) -#define u4 0x10(%r8) -#define w0 0x18(%r8) -#define w1 0x1c(%r8) -#define w2 0x20(%r8) -#define w3 0x24(%r8) -#define w4 0x28(%r8) -#define y0 0x30(%r8) -#define y1 0x34(%r8) -#define y2 0x38(%r8) -#define y3 0x3c(%r8) -#define y4 0x40(%r8) -#define m %rsi -#define hc0 %ymm0 -#define hc1 %ymm1 -#define hc2 %ymm2 -#define hc3 %ymm3 -#define hc4 %ymm4 -#define hc0x %xmm0 -#define hc1x %xmm1 -#define hc2x %xmm2 -#define hc3x %xmm3 -#define hc4x %xmm4 -#define t1 %ymm5 -#define t2 %ymm6 -#define t1x %xmm5 -#define t2x %xmm6 -#define ruwy0 %ymm7 -#define ruwy1 %ymm8 -#define ruwy2 %ymm9 -#define ruwy3 %ymm10 -#define ruwy4 %ymm11 -#define ruwy0x %xmm7 -#define ruwy1x %xmm8 -#define ruwy2x %xmm9 -#define ruwy3x %xmm10 -#define ruwy4x %xmm11 -#define svxz1 %ymm12 -#define svxz2 %ymm13 -#define svxz3 %ymm14 -#define svxz4 %ymm15 -#define d0 %r9 -#define d1 %r10 -#define d2 %r11 -#define d3 %r12 -#define d4 %r13 - -SYM_FUNC_START(poly1305_4block_avx2) - # %rdi: Accumulator h[5] - # %rsi: 64 byte input block m - # %rdx: Poly1305 key r[5] - # %rcx: Quadblock count - # %r8: Poly1305 derived key r^2 u[5], r^3 w[5], r^4 y[5], - - # This four-block variant uses loop unrolled block processing. It - # requires 4 Poly1305 keys: r, r^2, r^3 and r^4: - # h = (h + m) * r => h = (h + m1) * r^4 + m2 * r^3 + m3 * r^2 + m4 * r - - vzeroupper - push %rbx - push %r12 - push %r13 - - # combine r0,u0,w0,y0 - vmovd y0,ruwy0x - vmovd w0,t1x - vpunpcklqdq t1,ruwy0,ruwy0 - vmovd u0,t1x - vmovd r0,t2x - vpunpcklqdq t2,t1,t1 - vperm2i128 $0x20,t1,ruwy0,ruwy0 - - # combine r1,u1,w1,y1 and s1=r1*5,v1=u1*5,x1=w1*5,z1=y1*5 - vmovd y1,ruwy1x - vmovd w1,t1x - vpunpcklqdq t1,ruwy1,ruwy1 - vmovd u1,t1x - vmovd r1,t2x - vpunpcklqdq t2,t1,t1 - vperm2i128 $0x20,t1,ruwy1,ruwy1 - vpslld $2,ruwy1,svxz1 - vpaddd ruwy1,svxz1,svxz1 - - # combine r2,u2,w2,y2 and s2=r2*5,v2=u2*5,x2=w2*5,z2=y2*5 - vmovd y2,ruwy2x - vmovd w2,t1x - vpunpcklqdq t1,ruwy2,ruwy2 - vmovd u2,t1x - vmovd r2,t2x - vpunpcklqdq t2,t1,t1 - vperm2i128 $0x20,t1,ruwy2,ruwy2 - vpslld $2,ruwy2,svxz2 - vpaddd ruwy2,svxz2,svxz2 - - # combine r3,u3,w3,y3 and s3=r3*5,v3=u3*5,x3=w3*5,z3=y3*5 - vmovd y3,ruwy3x - vmovd w3,t1x - vpunpcklqdq t1,ruwy3,ruwy3 - vmovd u3,t1x - vmovd r3,t2x - vpunpcklqdq t2,t1,t1 - vperm2i128 $0x20,t1,ruwy3,ruwy3 - vpslld $2,ruwy3,svxz3 - vpaddd ruwy3,svxz3,svxz3 - - # combine r4,u4,w4,y4 and s4=r4*5,v4=u4*5,x4=w4*5,z4=y4*5 - vmovd y4,ruwy4x - vmovd w4,t1x - vpunpcklqdq t1,ruwy4,ruwy4 - vmovd u4,t1x - vmovd r4,t2x - vpunpcklqdq t2,t1,t1 - vperm2i128 $0x20,t1,ruwy4,ruwy4 - vpslld $2,ruwy4,svxz4 - vpaddd ruwy4,svxz4,svxz4 - -.Ldoblock4: - # hc0 = [m[48-51] & 0x3ffffff, m[32-35] & 0x3ffffff, - # m[16-19] & 0x3ffffff, m[ 0- 3] & 0x3ffffff + h0] - vmovd 0x00(m),hc0x - vmovd 0x10(m),t1x - vpunpcklqdq t1,hc0,hc0 - vmovd 0x20(m),t1x - vmovd 0x30(m),t2x - vpunpcklqdq t2,t1,t1 - vperm2i128 $0x20,t1,hc0,hc0 - vpand ANMASK(%rip),hc0,hc0 - vmovd h0,t1x - vpaddd t1,hc0,hc0 - # hc1 = [(m[51-54] >> 2) & 0x3ffffff, (m[35-38] >> 2) & 0x3ffffff, - # (m[19-22] >> 2) & 0x3ffffff, (m[ 3- 6] >> 2) & 0x3ffffff + h1] - vmovd 0x03(m),hc1x - vmovd 0x13(m),t1x - vpunpcklqdq t1,hc1,hc1 - vmovd 0x23(m),t1x - vmovd 0x33(m),t2x - vpunpcklqdq t2,t1,t1 - vperm2i128 $0x20,t1,hc1,hc1 - vpsrld $2,hc1,hc1 - vpand ANMASK(%rip),hc1,hc1 - vmovd h1,t1x - vpaddd t1,hc1,hc1 - # hc2 = [(m[54-57] >> 4) & 0x3ffffff, (m[38-41] >> 4) & 0x3ffffff, - # (m[22-25] >> 4) & 0x3ffffff, (m[ 6- 9] >> 4) & 0x3ffffff + h2] - vmovd 0x06(m),hc2x - vmovd 0x16(m),t1x - vpunpcklqdq t1,hc2,hc2 - vmovd 0x26(m),t1x - vmovd 0x36(m),t2x - vpunpcklqdq t2,t1,t1 - vperm2i128 $0x20,t1,hc2,hc2 - vpsrld $4,hc2,hc2 - vpand ANMASK(%rip),hc2,hc2 - vmovd h2,t1x - vpaddd t1,hc2,hc2 - # hc3 = [(m[57-60] >> 6) & 0x3ffffff, (m[41-44] >> 6) & 0x3ffffff, - # (m[25-28] >> 6) & 0x3ffffff, (m[ 9-12] >> 6) & 0x3ffffff + h3] - vmovd 0x09(m),hc3x - vmovd 0x19(m),t1x - vpunpcklqdq t1,hc3,hc3 - vmovd 0x29(m),t1x - vmovd 0x39(m),t2x - vpunpcklqdq t2,t1,t1 - vperm2i128 $0x20,t1,hc3,hc3 - vpsrld $6,hc3,hc3 - vpand ANMASK(%rip),hc3,hc3 - vmovd h3,t1x - vpaddd t1,hc3,hc3 - # hc4 = [(m[60-63] >> 8) | (1<<24), (m[44-47] >> 8) | (1<<24), - # (m[28-31] >> 8) | (1<<24), (m[12-15] >> 8) | (1<<24) + h4] - vmovd 0x0c(m),hc4x - vmovd 0x1c(m),t1x - vpunpcklqdq t1,hc4,hc4 - vmovd 0x2c(m),t1x - vmovd 0x3c(m),t2x - vpunpcklqdq t2,t1,t1 - vperm2i128 $0x20,t1,hc4,hc4 - vpsrld $8,hc4,hc4 - vpor ORMASK(%rip),hc4,hc4 - vmovd h4,t1x - vpaddd t1,hc4,hc4 - - # t1 = [ hc0[3] * r0, hc0[2] * u0, hc0[1] * w0, hc0[0] * y0 ] - vpmuludq hc0,ruwy0,t1 - # t1 += [ hc1[3] * s4, hc1[2] * v4, hc1[1] * x4, hc1[0] * z4 ] - vpmuludq hc1,svxz4,t2 - vpaddq t2,t1,t1 - # t1 += [ hc2[3] * s3, hc2[2] * v3, hc2[1] * x3, hc2[0] * z3 ] - vpmuludq hc2,svxz3,t2 - vpaddq t2,t1,t1 - # t1 += [ hc3[3] * s2, hc3[2] * v2, hc3[1] * x2, hc3[0] * z2 ] - vpmuludq hc3,svxz2,t2 - vpaddq t2,t1,t1 - # t1 += [ hc4[3] * s1, hc4[2] * v1, hc4[1] * x1, hc4[0] * z1 ] - vpmuludq hc4,svxz1,t2 - vpaddq t2,t1,t1 - # d0 = t1[0] + t1[1] + t[2] + t[3] - vpermq $0xee,t1,t2 - vpaddq t2,t1,t1 - vpsrldq $8,t1,t2 - vpaddq t2,t1,t1 - vmovq t1x,d0 - - # t1 = [ hc0[3] * r1, hc0[2] * u1,hc0[1] * w1, hc0[0] * y1 ] - vpmuludq hc0,ruwy1,t1 - # t1 += [ hc1[3] * r0, hc1[2] * u0, hc1[1] * w0, hc1[0] * y0 ] - vpmuludq hc1,ruwy0,t2 - vpaddq t2,t1,t1 - # t1 += [ hc2[3] * s4, hc2[2] * v4, hc2[1] * x4, hc2[0] * z4 ] - vpmuludq hc2,svxz4,t2 - vpaddq t2,t1,t1 - # t1 += [ hc3[3] * s3, hc3[2] * v3, hc3[1] * x3, hc3[0] * z3 ] - vpmuludq hc3,svxz3,t2 - vpaddq t2,t1,t1 - # t1 += [ hc4[3] * s2, hc4[2] * v2, hc4[1] * x2, hc4[0] * z2 ] - vpmuludq hc4,svxz2,t2 - vpaddq t2,t1,t1 - # d1 = t1[0] + t1[1] + t1[3] + t1[4] - vpermq $0xee,t1,t2 - vpaddq t2,t1,t1 - vpsrldq $8,t1,t2 - vpaddq t2,t1,t1 - vmovq t1x,d1 - - # t1 = [ hc0[3] * r2, hc0[2] * u2, hc0[1] * w2, hc0[0] * y2 ] - vpmuludq hc0,ruwy2,t1 - # t1 += [ hc1[3] * r1, hc1[2] * u1, hc1[1] * w1, hc1[0] * y1 ] - vpmuludq hc1,ruwy1,t2 - vpaddq t2,t1,t1 - # t1 += [ hc2[3] * r0, hc2[2] * u0, hc2[1] * w0, hc2[0] * y0 ] - vpmuludq hc2,ruwy0,t2 - vpaddq t2,t1,t1 - # t1 += [ hc3[3] * s4, hc3[2] * v4, hc3[1] * x4, hc3[0] * z4 ] - vpmuludq hc3,svxz4,t2 - vpaddq t2,t1,t1 - # t1 += [ hc4[3] * s3, hc4[2] * v3, hc4[1] * x3, hc4[0] * z3 ] - vpmuludq hc4,svxz3,t2 - vpaddq t2,t1,t1 - # d2 = t1[0] + t1[1] + t1[2] + t1[3] - vpermq $0xee,t1,t2 - vpaddq t2,t1,t1 - vpsrldq $8,t1,t2 - vpaddq t2,t1,t1 - vmovq t1x,d2 - - # t1 = [ hc0[3] * r3, hc0[2] * u3, hc0[1] * w3, hc0[0] * y3 ] - vpmuludq hc0,ruwy3,t1 - # t1 += [ hc1[3] * r2, hc1[2] * u2, hc1[1] * w2, hc1[0] * y2 ] - vpmuludq hc1,ruwy2,t2 - vpaddq t2,t1,t1 - # t1 += [ hc2[3] * r1, hc2[2] * u1, hc2[1] * w1, hc2[0] * y1 ] - vpmuludq hc2,ruwy1,t2 - vpaddq t2,t1,t1 - # t1 += [ hc3[3] * r0, hc3[2] * u0, hc3[1] * w0, hc3[0] * y0 ] - vpmuludq hc3,ruwy0,t2 - vpaddq t2,t1,t1 - # t1 += [ hc4[3] * s4, hc4[2] * v4, hc4[1] * x4, hc4[0] * z4 ] - vpmuludq hc4,svxz4,t2 - vpaddq t2,t1,t1 - # d3 = t1[0] + t1[1] + t1[2] + t1[3] - vpermq $0xee,t1,t2 - vpaddq t2,t1,t1 - vpsrldq $8,t1,t2 - vpaddq t2,t1,t1 - vmovq t1x,d3 - - # t1 = [ hc0[3] * r4, hc0[2] * u4, hc0[1] * w4, hc0[0] * y4 ] - vpmuludq hc0,ruwy4,t1 - # t1 += [ hc1[3] * r3, hc1[2] * u3, hc1[1] * w3, hc1[0] * y3 ] - vpmuludq hc1,ruwy3,t2 - vpaddq t2,t1,t1 - # t1 += [ hc2[3] * r2, hc2[2] * u2, hc2[1] * w2, hc2[0] * y2 ] - vpmuludq hc2,ruwy2,t2 - vpaddq t2,t1,t1 - # t1 += [ hc3[3] * r1, hc3[2] * u1, hc3[1] * w1, hc3[0] * y1 ] - vpmuludq hc3,ruwy1,t2 - vpaddq t2,t1,t1 - # t1 += [ hc4[3] * r0, hc4[2] * u0, hc4[1] * w0, hc4[0] * y0 ] - vpmuludq hc4,ruwy0,t2 - vpaddq t2,t1,t1 - # d4 = t1[0] + t1[1] + t1[2] + t1[3] - vpermq $0xee,t1,t2 - vpaddq t2,t1,t1 - vpsrldq $8,t1,t2 - vpaddq t2,t1,t1 - vmovq t1x,d4 - - # Now do a partial reduction mod (2^130)-5, carrying h0 -> h1 -> h2 -> - # h3 -> h4 -> h0 -> h1 to get h0,h2,h3,h4 < 2^26 and h1 < 2^26 + a small - # amount. Careful: we must not assume the carry bits 'd0 >> 26', - # 'd1 >> 26', 'd2 >> 26', 'd3 >> 26', and '(d4 >> 26) * 5' fit in 32-bit - # integers. It's true in a single-block implementation, but not here. - - # d1 += d0 >> 26 - mov d0,%rax - shr $26,%rax - add %rax,d1 - # h0 = d0 & 0x3ffffff - mov d0,%rbx - and $0x3ffffff,%ebx - - # d2 += d1 >> 26 - mov d1,%rax - shr $26,%rax - add %rax,d2 - # h1 = d1 & 0x3ffffff - mov d1,%rax - and $0x3ffffff,%eax - mov %eax,h1 - - # d3 += d2 >> 26 - mov d2,%rax - shr $26,%rax - add %rax,d3 - # h2 = d2 & 0x3ffffff - mov d2,%rax - and $0x3ffffff,%eax - mov %eax,h2 - - # d4 += d3 >> 26 - mov d3,%rax - shr $26,%rax - add %rax,d4 - # h3 = d3 & 0x3ffffff - mov d3,%rax - and $0x3ffffff,%eax - mov %eax,h3 - - # h0 += (d4 >> 26) * 5 - mov d4,%rax - shr $26,%rax - lea (%rax,%rax,4),%rax - add %rax,%rbx - # h4 = d4 & 0x3ffffff - mov d4,%rax - and $0x3ffffff,%eax - mov %eax,h4 - - # h1 += h0 >> 26 - mov %rbx,%rax - shr $26,%rax - add %eax,h1 - # h0 = h0 & 0x3ffffff - andl $0x3ffffff,%ebx - mov %ebx,h0 - - add $0x40,m - dec %rcx - jnz .Ldoblock4 - - vzeroupper - pop %r13 - pop %r12 - pop %rbx - ret -SYM_FUNC_END(poly1305_4block_avx2) diff --git a/arch/x86/crypto/poly1305-sse2-x86_64.S b/arch/x86/crypto/poly1305-sse2-x86_64.S deleted file mode 100644 index d8ea29b96640..000000000000 --- a/arch/x86/crypto/poly1305-sse2-x86_64.S +++ /dev/null @@ -1,590 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Poly1305 authenticator algorithm, RFC7539, x64 SSE2 functions - * - * Copyright (C) 2015 Martin Willi - */ - -#include - -.section .rodata.cst16.ANMASK, "aM", @progbits, 16 -.align 16 -ANMASK: .octa 0x0000000003ffffff0000000003ffffff - -.section .rodata.cst16.ORMASK, "aM", @progbits, 16 -.align 16 -ORMASK: .octa 0x00000000010000000000000001000000 - -.text - -#define h0 0x00(%rdi) -#define h1 0x04(%rdi) -#define h2 0x08(%rdi) -#define h3 0x0c(%rdi) -#define h4 0x10(%rdi) -#define r0 0x00(%rdx) -#define r1 0x04(%rdx) -#define r2 0x08(%rdx) -#define r3 0x0c(%rdx) -#define r4 0x10(%rdx) -#define s1 0x00(%rsp) -#define s2 0x04(%rsp) -#define s3 0x08(%rsp) -#define s4 0x0c(%rsp) -#define m %rsi -#define h01 %xmm0 -#define h23 %xmm1 -#define h44 %xmm2 -#define t1 %xmm3 -#define t2 %xmm4 -#define t3 %xmm5 -#define t4 %xmm6 -#define mask %xmm7 -#define d0 %r8 -#define d1 %r9 -#define d2 %r10 -#define d3 %r11 -#define d4 %r12 - -SYM_FUNC_START(poly1305_block_sse2) - # %rdi: Accumulator h[5] - # %rsi: 16 byte input block m - # %rdx: Poly1305 key r[5] - # %rcx: Block count - - # This single block variant tries to improve performance by doing two - # multiplications in parallel using SSE instructions. There is quite - # some quardword packing involved, hence the speedup is marginal. - - push %rbx - push %r12 - sub $0x10,%rsp - - # s1..s4 = r1..r4 * 5 - mov r1,%eax - lea (%eax,%eax,4),%eax - mov %eax,s1 - mov r2,%eax - lea (%eax,%eax,4),%eax - mov %eax,s2 - mov r3,%eax - lea (%eax,%eax,4),%eax - mov %eax,s3 - mov r4,%eax - lea (%eax,%eax,4),%eax - mov %eax,s4 - - movdqa ANMASK(%rip),mask - -.Ldoblock: - # h01 = [0, h1, 0, h0] - # h23 = [0, h3, 0, h2] - # h44 = [0, h4, 0, h4] - movd h0,h01 - movd h1,t1 - movd h2,h23 - movd h3,t2 - movd h4,h44 - punpcklqdq t1,h01 - punpcklqdq t2,h23 - punpcklqdq h44,h44 - - # h01 += [ (m[3-6] >> 2) & 0x3ffffff, m[0-3] & 0x3ffffff ] - movd 0x00(m),t1 - movd 0x03(m),t2 - psrld $2,t2 - punpcklqdq t2,t1 - pand mask,t1 - paddd t1,h01 - # h23 += [ (m[9-12] >> 6) & 0x3ffffff, (m[6-9] >> 4) & 0x3ffffff ] - movd 0x06(m),t1 - movd 0x09(m),t2 - psrld $4,t1 - psrld $6,t2 - punpcklqdq t2,t1 - pand mask,t1 - paddd t1,h23 - # h44 += [ (m[12-15] >> 8) | (1 << 24), (m[12-15] >> 8) | (1 << 24) ] - mov 0x0c(m),%eax - shr $8,%eax - or $0x01000000,%eax - movd %eax,t1 - pshufd $0xc4,t1,t1 - paddd t1,h44 - - # t1[0] = h0 * r0 + h2 * s3 - # t1[1] = h1 * s4 + h3 * s2 - movd r0,t1 - movd s4,t2 - punpcklqdq t2,t1 - pmuludq h01,t1 - movd s3,t2 - movd s2,t3 - punpcklqdq t3,t2 - pmuludq h23,t2 - paddq t2,t1 - # t2[0] = h0 * r1 + h2 * s4 - # t2[1] = h1 * r0 + h3 * s3 - movd r1,t2 - movd r0,t3 - punpcklqdq t3,t2 - pmuludq h01,t2 - movd s4,t3 - movd s3,t4 - punpcklqdq t4,t3 - pmuludq h23,t3 - paddq t3,t2 - # t3[0] = h4 * s1 - # t3[1] = h4 * s2 - movd s1,t3 - movd s2,t4 - punpcklqdq t4,t3 - pmuludq h44,t3 - # d0 = t1[0] + t1[1] + t3[0] - # d1 = t2[0] + t2[1] + t3[1] - movdqa t1,t4 - punpcklqdq t2,t4 - punpckhqdq t2,t1 - paddq t4,t1 - paddq t3,t1 - movq t1,d0 - psrldq $8,t1 - movq t1,d1 - - # t1[0] = h0 * r2 + h2 * r0 - # t1[1] = h1 * r1 + h3 * s4 - movd r2,t1 - movd r1,t2 - punpcklqdq t2,t1 - pmuludq h01,t1 - movd r0,t2 - movd s4,t3 - punpcklqdq t3,t2 - pmuludq h23,t2 - paddq t2,t1 - # t2[0] = h0 * r3 + h2 * r1 - # t2[1] = h1 * r2 + h3 * r0 - movd r3,t2 - movd r2,t3 - punpcklqdq t3,t2 - pmuludq h01,t2 - movd r1,t3 - movd r0,t4 - punpcklqdq t4,t3 - pmuludq h23,t3 - paddq t3,t2 - # t3[0] = h4 * s3 - # t3[1] = h4 * s4 - movd s3,t3 - movd s4,t4 - punpcklqdq t4,t3 - pmuludq h44,t3 - # d2 = t1[0] + t1[1] + t3[0] - # d3 = t2[0] + t2[1] + t3[1] - movdqa t1,t4 - punpcklqdq t2,t4 - punpckhqdq t2,t1 - paddq t4,t1 - paddq t3,t1 - movq t1,d2 - psrldq $8,t1 - movq t1,d3 - - # t1[0] = h0 * r4 + h2 * r2 - # t1[1] = h1 * r3 + h3 * r1 - movd r4,t1 - movd r3,t2 - punpcklqdq t2,t1 - pmuludq h01,t1 - movd r2,t2 - movd r1,t3 - punpcklqdq t3,t2 - pmuludq h23,t2 - paddq t2,t1 - # t3[0] = h4 * r0 - movd r0,t3 - pmuludq h44,t3 - # d4 = t1[0] + t1[1] + t3[0] - movdqa t1,t4 - psrldq $8,t4 - paddq t4,t1 - paddq t3,t1 - movq t1,d4 - - # d1 += d0 >> 26 - mov d0,%rax - shr $26,%rax - add %rax,d1 - # h0 = d0 & 0x3ffffff - mov d0,%rbx - and $0x3ffffff,%ebx - - # d2 += d1 >> 26 - mov d1,%rax - shr $26,%rax - add %rax,d2 - # h1 = d1 & 0x3ffffff - mov d1,%rax - and $0x3ffffff,%eax - mov %eax,h1 - - # d3 += d2 >> 26 - mov d2,%rax - shr $26,%rax - add %rax,d3 - # h2 = d2 & 0x3ffffff - mov d2,%rax - and $0x3ffffff,%eax - mov %eax,h2 - - # d4 += d3 >> 26 - mov d3,%rax - shr $26,%rax - add %rax,d4 - # h3 = d3 & 0x3ffffff - mov d3,%rax - and $0x3ffffff,%eax - mov %eax,h3 - - # h0 += (d4 >> 26) * 5 - mov d4,%rax - shr $26,%rax - lea (%rax,%rax,4),%rax - add %rax,%rbx - # h4 = d4 & 0x3ffffff - mov d4,%rax - and $0x3ffffff,%eax - mov %eax,h4 - - # h1 += h0 >> 26 - mov %rbx,%rax - shr $26,%rax - add %eax,h1 - # h0 = h0 & 0x3ffffff - andl $0x3ffffff,%ebx - mov %ebx,h0 - - add $0x10,m - dec %rcx - jnz .Ldoblock - - # Zeroing of key material - mov %rcx,0x00(%rsp) - mov %rcx,0x08(%rsp) - - add $0x10,%rsp - pop %r12 - pop %rbx - ret -SYM_FUNC_END(poly1305_block_sse2) - - -#define u0 0x00(%r8) -#define u1 0x04(%r8) -#define u2 0x08(%r8) -#define u3 0x0c(%r8) -#define u4 0x10(%r8) -#define hc0 %xmm0 -#define hc1 %xmm1 -#define hc2 %xmm2 -#define hc3 %xmm5 -#define hc4 %xmm6 -#define ru0 %xmm7 -#define ru1 %xmm8 -#define ru2 %xmm9 -#define ru3 %xmm10 -#define ru4 %xmm11 -#define sv1 %xmm12 -#define sv2 %xmm13 -#define sv3 %xmm14 -#define sv4 %xmm15 -#undef d0 -#define d0 %r13 - -SYM_FUNC_START(poly1305_2block_sse2) - # %rdi: Accumulator h[5] - # %rsi: 16 byte input block m - # %rdx: Poly1305 key r[5] - # %rcx: Doubleblock count - # %r8: Poly1305 derived key r^2 u[5] - - # This two-block variant further improves performance by using loop - # unrolled block processing. This is more straight forward and does - # less byte shuffling, but requires a second Poly1305 key r^2: - # h = (h + m) * r => h = (h + m1) * r^2 + m2 * r - - push %rbx - push %r12 - push %r13 - - # combine r0,u0 - movd u0,ru0 - movd r0,t1 - punpcklqdq t1,ru0 - - # combine r1,u1 and s1=r1*5,v1=u1*5 - movd u1,ru1 - movd r1,t1 - punpcklqdq t1,ru1 - movdqa ru1,sv1 - pslld $2,sv1 - paddd ru1,sv1 - - # combine r2,u2 and s2=r2*5,v2=u2*5 - movd u2,ru2 - movd r2,t1 - punpcklqdq t1,ru2 - movdqa ru2,sv2 - pslld $2,sv2 - paddd ru2,sv2 - - # combine r3,u3 and s3=r3*5,v3=u3*5 - movd u3,ru3 - movd r3,t1 - punpcklqdq t1,ru3 - movdqa ru3,sv3 - pslld $2,sv3 - paddd ru3,sv3 - - # combine r4,u4 and s4=r4*5,v4=u4*5 - movd u4,ru4 - movd r4,t1 - punpcklqdq t1,ru4 - movdqa ru4,sv4 - pslld $2,sv4 - paddd ru4,sv4 - -.Ldoblock2: - # hc0 = [ m[16-19] & 0x3ffffff, h0 + m[0-3] & 0x3ffffff ] - movd 0x00(m),hc0 - movd 0x10(m),t1 - punpcklqdq t1,hc0 - pand ANMASK(%rip),hc0 - movd h0,t1 - paddd t1,hc0 - # hc1 = [ (m[19-22] >> 2) & 0x3ffffff, h1 + (m[3-6] >> 2) & 0x3ffffff ] - movd 0x03(m),hc1 - movd 0x13(m),t1 - punpcklqdq t1,hc1 - psrld $2,hc1 - pand ANMASK(%rip),hc1 - movd h1,t1 - paddd t1,hc1 - # hc2 = [ (m[22-25] >> 4) & 0x3ffffff, h2 + (m[6-9] >> 4) & 0x3ffffff ] - movd 0x06(m),hc2 - movd 0x16(m),t1 - punpcklqdq t1,hc2 - psrld $4,hc2 - pand ANMASK(%rip),hc2 - movd h2,t1 - paddd t1,hc2 - # hc3 = [ (m[25-28] >> 6) & 0x3ffffff, h3 + (m[9-12] >> 6) & 0x3ffffff ] - movd 0x09(m),hc3 - movd 0x19(m),t1 - punpcklqdq t1,hc3 - psrld $6,hc3 - pand ANMASK(%rip),hc3 - movd h3,t1 - paddd t1,hc3 - # hc4 = [ (m[28-31] >> 8) | (1<<24), h4 + (m[12-15] >> 8) | (1<<24) ] - movd 0x0c(m),hc4 - movd 0x1c(m),t1 - punpcklqdq t1,hc4 - psrld $8,hc4 - por ORMASK(%rip),hc4 - movd h4,t1 - paddd t1,hc4 - - # t1 = [ hc0[1] * r0, hc0[0] * u0 ] - movdqa ru0,t1 - pmuludq hc0,t1 - # t1 += [ hc1[1] * s4, hc1[0] * v4 ] - movdqa sv4,t2 - pmuludq hc1,t2 - paddq t2,t1 - # t1 += [ hc2[1] * s3, hc2[0] * v3 ] - movdqa sv3,t2 - pmuludq hc2,t2 - paddq t2,t1 - # t1 += [ hc3[1] * s2, hc3[0] * v2 ] - movdqa sv2,t2 - pmuludq hc3,t2 - paddq t2,t1 - # t1 += [ hc4[1] * s1, hc4[0] * v1 ] - movdqa sv1,t2 - pmuludq hc4,t2 - paddq t2,t1 - # d0 = t1[0] + t1[1] - movdqa t1,t2 - psrldq $8,t2 - paddq t2,t1 - movq t1,d0 - - # t1 = [ hc0[1] * r1, hc0[0] * u1 ] - movdqa ru1,t1 - pmuludq hc0,t1 - # t1 += [ hc1[1] * r0, hc1[0] * u0 ] - movdqa ru0,t2 - pmuludq hc1,t2 - paddq t2,t1 - # t1 += [ hc2[1] * s4, hc2[0] * v4 ] - movdqa sv4,t2 - pmuludq hc2,t2 - paddq t2,t1 - # t1 += [ hc3[1] * s3, hc3[0] * v3 ] - movdqa sv3,t2 - pmuludq hc3,t2 - paddq t2,t1 - # t1 += [ hc4[1] * s2, hc4[0] * v2 ] - movdqa sv2,t2 - pmuludq hc4,t2 - paddq t2,t1 - # d1 = t1[0] + t1[1] - movdqa t1,t2 - psrldq $8,t2 - paddq t2,t1 - movq t1,d1 - - # t1 = [ hc0[1] * r2, hc0[0] * u2 ] - movdqa ru2,t1 - pmuludq hc0,t1 - # t1 += [ hc1[1] * r1, hc1[0] * u1 ] - movdqa ru1,t2 - pmuludq hc1,t2 - paddq t2,t1 - # t1 += [ hc2[1] * r0, hc2[0] * u0 ] - movdqa ru0,t2 - pmuludq hc2,t2 - paddq t2,t1 - # t1 += [ hc3[1] * s4, hc3[0] * v4 ] - movdqa sv4,t2 - pmuludq hc3,t2 - paddq t2,t1 - # t1 += [ hc4[1] * s3, hc4[0] * v3 ] - movdqa sv3,t2 - pmuludq hc4,t2 - paddq t2,t1 - # d2 = t1[0] + t1[1] - movdqa t1,t2 - psrldq $8,t2 - paddq t2,t1 - movq t1,d2 - - # t1 = [ hc0[1] * r3, hc0[0] * u3 ] - movdqa ru3,t1 - pmuludq hc0,t1 - # t1 += [ hc1[1] * r2, hc1[0] * u2 ] - movdqa ru2,t2 - pmuludq hc1,t2 - paddq t2,t1 - # t1 += [ hc2[1] * r1, hc2[0] * u1 ] - movdqa ru1,t2 - pmuludq hc2,t2 - paddq t2,t1 - # t1 += [ hc3[1] * r0, hc3[0] * u0 ] - movdqa ru0,t2 - pmuludq hc3,t2 - paddq t2,t1 - # t1 += [ hc4[1] * s4, hc4[0] * v4 ] - movdqa sv4,t2 - pmuludq hc4,t2 - paddq t2,t1 - # d3 = t1[0] + t1[1] - movdqa t1,t2 - psrldq $8,t2 - paddq t2,t1 - movq t1,d3 - - # t1 = [ hc0[1] * r4, hc0[0] * u4 ] - movdqa ru4,t1 - pmuludq hc0,t1 - # t1 += [ hc1[1] * r3, hc1[0] * u3 ] - movdqa ru3,t2 - pmuludq hc1,t2 - paddq t2,t1 - # t1 += [ hc2[1] * r2, hc2[0] * u2 ] - movdqa ru2,t2 - pmuludq hc2,t2 - paddq t2,t1 - # t1 += [ hc3[1] * r1, hc3[0] * u1 ] - movdqa ru1,t2 - pmuludq hc3,t2 - paddq t2,t1 - # t1 += [ hc4[1] * r0, hc4[0] * u0 ] - movdqa ru0,t2 - pmuludq hc4,t2 - paddq t2,t1 - # d4 = t1[0] + t1[1] - movdqa t1,t2 - psrldq $8,t2 - paddq t2,t1 - movq t1,d4 - - # Now do a partial reduction mod (2^130)-5, carrying h0 -> h1 -> h2 -> - # h3 -> h4 -> h0 -> h1 to get h0,h2,h3,h4 < 2^26 and h1 < 2^26 + a small - # amount. Careful: we must not assume the carry bits 'd0 >> 26', - # 'd1 >> 26', 'd2 >> 26', 'd3 >> 26', and '(d4 >> 26) * 5' fit in 32-bit - # integers. It's true in a single-block implementation, but not here. - - # d1 += d0 >> 26 - mov d0,%rax - shr $26,%rax - add %rax,d1 - # h0 = d0 & 0x3ffffff - mov d0,%rbx - and $0x3ffffff,%ebx - - # d2 += d1 >> 26 - mov d1,%rax - shr $26,%rax - add %rax,d2 - # h1 = d1 & 0x3ffffff - mov d1,%rax - and $0x3ffffff,%eax - mov %eax,h1 - - # d3 += d2 >> 26 - mov d2,%rax - shr $26,%rax - add %rax,d3 - # h2 = d2 & 0x3ffffff - mov d2,%rax - and $0x3ffffff,%eax - mov %eax,h2 - - # d4 += d3 >> 26 - mov d3,%rax - shr $26,%rax - add %rax,d4 - # h3 = d3 & 0x3ffffff - mov d3,%rax - and $0x3ffffff,%eax - mov %eax,h3 - - # h0 += (d4 >> 26) * 5 - mov d4,%rax - shr $26,%rax - lea (%rax,%rax,4),%rax - add %rax,%rbx - # h4 = d4 & 0x3ffffff - mov d4,%rax - and $0x3ffffff,%eax - mov %eax,h4 - - # h1 += h0 >> 26 - mov %rbx,%rax - shr $26,%rax - add %eax,h1 - # h0 = h0 & 0x3ffffff - andl $0x3ffffff,%ebx - mov %ebx,h0 - - add $0x20,m - dec %rcx - jnz .Ldoblock2 - - pop %r13 - pop %r12 - pop %rbx - ret -SYM_FUNC_END(poly1305_2block_sse2) diff --git a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl b/arch/x86/crypto/poly1305-x86_64-cryptogams.pl index 342ad7f18aa7..7a6b5380a46f 100644 --- a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl +++ b/arch/x86/crypto/poly1305-x86_64-cryptogams.pl @@ -1,11 +1,14 @@ -#! /usr/bin/env perl -# Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved. +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause # -# Licensed under the OpenSSL license (the "License"). You may not use -# this file except in compliance with the License. You can obtain a copy -# in the file LICENSE in the source distribution or at -# https://www.openssl.org/source/license.html - +# Copyright (C) 2017-2018 Samuel Neves . All Rights Reserved. +# Copyright (C) 2017-2019 Jason A. Donenfeld . All Rights Reserved. +# Copyright (C) 2006-2017 CRYPTOGAMS by . All Rights Reserved. +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. # # ==================================================================== # Written by Andy Polyakov for the OpenSSL @@ -32,7 +35,7 @@ # Skylake-X system performance. Since we are likely to suppress # AVX512F capability flag [at least on Skylake-X], conversion serves # as kind of "investment protection". Note that next *lake processor, -# Cannolake, has AVX512IFMA code path to execute... +# Cannonlake, has AVX512IFMA code path to execute... # # Numbers are cycles per processed byte with poly1305_blocks alone, # measured with rdtsc at fixed clock frequency. @@ -68,39 +71,114 @@ $output = shift; if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); - -$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; -( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or -( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or -die "can't locate x86_64-xlate.pl"; - -if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` - =~ /GNU assembler version ([2-9]\.[0-9]+)/) { - $avx = ($1>=2.19) + ($1>=2.22) + ($1>=2.25) + ($1>=2.26); +$kernel=0; $kernel=1 if (!$flavour && !$output); + +if (!$kernel) { + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; + ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or + ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or + die "can't locate x86_64-xlate.pl"; + + open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""; + *STDOUT=*OUT; + + if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.19) + ($1>=2.22) + ($1>=2.25); + } + + if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) { + $avx = ($1>=2.09) + ($1>=2.10) + ($1>=2.12); + $avx += 1 if ($1==2.11 && $2>=8); + } + + if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $avx = ($1>=10) + ($1>=11); + } + + if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) { + $avx = ($2>=3.0) + ($2>3.0); + } +} else { + $avx = 4; # The kernel uses ifdefs for this. } -if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && - `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) { - $avx = ($1>=2.09) + ($1>=2.10) + 2 * ($1>=2.12); - $avx += 2 if ($1==2.11 && $2>=8); +sub declare_function() { + my ($name, $align, $nargs) = @_; + if($kernel) { + $code .= ".align $align\n"; + $code .= "SYM_FUNC_START($name)\n"; + $code .= ".L$name:\n"; + } else { + $code .= ".globl $name\n"; + $code .= ".type $name,\@function,$nargs\n"; + $code .= ".align $align\n"; + $code .= "$name:\n"; + } } -if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && - `ml64 2>&1` =~ /Version ([0-9]+)\./) { - $avx = ($1>=10) + ($1>=12); +sub end_function() { + my ($name) = @_; + if($kernel) { + $code .= "SYM_FUNC_END($name)\n"; + } else { + $code .= ".size $name,.-$name\n"; + } } -if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) { - $avx = ($2>=3.0) + ($2>3.0); -} +$code.=<<___ if $kernel; +#include +___ + +if ($avx) { +$code.=<<___ if $kernel; +.section .rodata +___ +$code.=<<___; +.align 64 +.Lconst: +.Lmask24: +.long 0x0ffffff,0,0x0ffffff,0,0x0ffffff,0,0x0ffffff,0 +.L129: +.long `1<<24`,0,`1<<24`,0,`1<<24`,0,`1<<24`,0 +.Lmask26: +.long 0x3ffffff,0,0x3ffffff,0,0x3ffffff,0,0x3ffffff,0 +.Lpermd_avx2: +.long 2,2,2,3,2,0,2,1 +.Lpermd_avx512: +.long 0,0,0,1, 0,2,0,3, 0,4,0,5, 0,6,0,7 + +.L2_44_inp_permd: +.long 0,1,1,2,2,3,7,7 +.L2_44_inp_shift: +.quad 0,12,24,64 +.L2_44_mask: +.quad 0xfffffffffff,0xfffffffffff,0x3ffffffffff,0xffffffffffffffff +.L2_44_shift_rgt: +.quad 44,44,42,64 +.L2_44_shift_lft: +.quad 8,8,10,64 -open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""; -*STDOUT=*OUT; +.align 64 +.Lx_mask44: +.quad 0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff +.quad 0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff +.Lx_mask42: +.quad 0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff +.quad 0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff +___ +} +$code.=<<___ if (!$kernel); +.asciz "Poly1305 for x86_64, CRYPTOGAMS by " +.align 16 +___ my ($ctx,$inp,$len,$padbit)=("%rdi","%rsi","%rdx","%rcx"); my ($mac,$nonce)=($inp,$len); # *_emit arguments -my ($d1,$d2,$d3, $r0,$r1,$s1)=map("%r$_",(8..13)); -my ($h0,$h1,$h2)=("%r14","%rbx","%rbp"); +my ($d1,$d2,$d3, $r0,$r1,$s1)=("%r8","%r9","%rdi","%r11","%r12","%r13"); +my ($h0,$h1,$h2)=("%r14","%rbx","%r10"); sub poly1305_iteration { # input: copy of $r1 in %rax, $h0-$h2, $r0-$r1 @@ -155,19 +233,19 @@ ___ $code.=<<___; .text - +___ +$code.=<<___ if (!$kernel); .extern OPENSSL_ia32cap_P -.globl poly1305_init -.hidden poly1305_init -.globl poly1305_blocks -.hidden poly1305_blocks -.globl poly1305_emit -.hidden poly1305_emit - -.type poly1305_init,\@function,3 -.align 32 -poly1305_init: +.globl poly1305_init_x86_64 +.hidden poly1305_init_x86_64 +.globl poly1305_blocks_x86_64 +.hidden poly1305_blocks_x86_64 +.globl poly1305_emit_x86_64 +.hidden poly1305_emit_x86_64 +___ +&declare_function("poly1305_init_x86_64", 32, 3); +$code.=<<___; xor %rax,%rax mov %rax,0($ctx) # initialize hash value mov %rax,8($ctx) @@ -175,11 +253,12 @@ poly1305_init: cmp \$0,$inp je .Lno_key - - lea poly1305_blocks(%rip),%r10 - lea poly1305_emit(%rip),%r11 ___ -$code.=<<___ if ($avx); +$code.=<<___ if (!$kernel); + lea poly1305_blocks_x86_64(%rip),%r10 + lea poly1305_emit_x86_64(%rip),%r11 +___ +$code.=<<___ if (!$kernel && $avx); mov OPENSSL_ia32cap_P+4(%rip),%r9 lea poly1305_blocks_avx(%rip),%rax lea poly1305_emit_avx(%rip),%rcx @@ -187,12 +266,12 @@ $code.=<<___ if ($avx); cmovc %rax,%r10 cmovc %rcx,%r11 ___ -$code.=<<___ if ($avx>1); +$code.=<<___ if (!$kernel && $avx>1); lea poly1305_blocks_avx2(%rip),%rax bt \$`5+32`,%r9 # AVX2? cmovc %rax,%r10 ___ -$code.=<<___ if ($avx>3); +$code.=<<___ if (!$kernel && $avx>3); mov \$`(1<<31|1<<21|1<<16)`,%rax shr \$32,%r9 and %rax,%r9 @@ -207,11 +286,11 @@ $code.=<<___; mov %rax,24($ctx) mov %rcx,32($ctx) ___ -$code.=<<___ if ($flavour !~ /elf32/); +$code.=<<___ if (!$kernel && $flavour !~ /elf32/); mov %r10,0(%rdx) mov %r11,8(%rdx) ___ -$code.=<<___ if ($flavour =~ /elf32/); +$code.=<<___ if (!$kernel && $flavour =~ /elf32/); mov %r10d,0(%rdx) mov %r11d,4(%rdx) ___ @@ -219,11 +298,11 @@ $code.=<<___; mov \$1,%eax .Lno_key: ret -.size poly1305_init,.-poly1305_init +___ +&end_function("poly1305_init_x86_64"); -.type poly1305_blocks,\@function,4 -.align 32 -poly1305_blocks: +&declare_function("poly1305_blocks_x86_64", 32, 4); +$code.=<<___; .cfi_startproc .Lblocks: shr \$4,$len @@ -231,8 +310,6 @@ poly1305_blocks: push %rbx .cfi_push %rbx - push %rbp -.cfi_push %rbp push %r12 .cfi_push %r12 push %r13 @@ -241,6 +318,8 @@ poly1305_blocks: .cfi_push %r14 push %r15 .cfi_push %r15 + push $ctx +.cfi_push $ctx .Lblocks_body: mov $len,%r15 # reassign $len @@ -265,26 +344,29 @@ poly1305_blocks: lea 16($inp),$inp adc $padbit,$h2 ___ + &poly1305_iteration(); + $code.=<<___; mov $r1,%rax dec %r15 # len-=16 jnz .Loop + mov 0(%rsp),$ctx +.cfi_restore $ctx + mov $h0,0($ctx) # store hash value mov $h1,8($ctx) mov $h2,16($ctx) - mov 0(%rsp),%r15 + mov 8(%rsp),%r15 .cfi_restore %r15 - mov 8(%rsp),%r14 + mov 16(%rsp),%r14 .cfi_restore %r14 - mov 16(%rsp),%r13 + mov 24(%rsp),%r13 .cfi_restore %r13 - mov 24(%rsp),%r12 + mov 32(%rsp),%r12 .cfi_restore %r12 - mov 32(%rsp),%rbp -.cfi_restore %rbp mov 40(%rsp),%rbx .cfi_restore %rbx lea 48(%rsp),%rsp @@ -293,11 +375,11 @@ $code.=<<___; .Lblocks_epilogue: ret .cfi_endproc -.size poly1305_blocks,.-poly1305_blocks +___ +&end_function("poly1305_blocks_x86_64"); -.type poly1305_emit,\@function,3 -.align 32 -poly1305_emit: +&declare_function("poly1305_emit_x86_64", 32, 3); +$code.=<<___; .Lemit: mov 0($ctx),%r8 # load hash value mov 8($ctx),%r9 @@ -318,10 +400,14 @@ poly1305_emit: mov %rcx,8($mac) ret -.size poly1305_emit,.-poly1305_emit ___ +&end_function("poly1305_emit_x86_64"); if ($avx) { +if($kernel) { + $code .= "#ifdef CONFIG_AS_AVX\n"; +} + ######################################################################## # Layout of opaque area is following. # @@ -342,15 +428,19 @@ $code.=<<___; .type __poly1305_block,\@abi-omnipotent .align 32 __poly1305_block: + push $ctx ___ &poly1305_iteration(); $code.=<<___; + pop $ctx ret .size __poly1305_block,.-__poly1305_block .type __poly1305_init_avx,\@abi-omnipotent .align 32 __poly1305_init_avx: + push %rbp + mov %rsp,%rbp mov $r0,$h0 mov $r1,$h1 xor $h2,$h2 @@ -507,12 +597,13 @@ __poly1305_init_avx: mov $d1#d,`16*8+8-64`($ctx) lea -48-64($ctx),$ctx # size [de-]optimization + pop %rbp ret .size __poly1305_init_avx,.-__poly1305_init_avx +___ -.type poly1305_blocks_avx,\@function,4 -.align 32 -poly1305_blocks_avx: +&declare_function("poly1305_blocks_avx", 32, 4); +$code.=<<___; .cfi_startproc mov 20($ctx),%r8d # is_base2_26 cmp \$128,$len @@ -532,10 +623,11 @@ poly1305_blocks_avx: test \$31,$len jz .Leven_avx - push %rbx -.cfi_push %rbx push %rbp .cfi_push %rbp + mov %rsp,%rbp + push %rbx +.cfi_push %rbx push %r12 .cfi_push %r12 push %r13 @@ -645,20 +737,18 @@ poly1305_blocks_avx: mov $h2#d,16($ctx) .align 16 .Ldone_avx: - mov 0(%rsp),%r15 + pop %r15 .cfi_restore %r15 - mov 8(%rsp),%r14 + pop %r14 .cfi_restore %r14 - mov 16(%rsp),%r13 + pop %r13 .cfi_restore %r13 - mov 24(%rsp),%r12 + pop %r12 .cfi_restore %r12 - mov 32(%rsp),%rbp -.cfi_restore %rbp - mov 40(%rsp),%rbx + pop %rbx .cfi_restore %rbx - lea 48(%rsp),%rsp -.cfi_adjust_cfa_offset -48 + pop %rbp +.cfi_restore %rbp .Lno_data_avx: .Lblocks_avx_epilogue: ret @@ -667,10 +757,11 @@ poly1305_blocks_avx: .align 32 .Lbase2_64_avx: .cfi_startproc - push %rbx -.cfi_push %rbx push %rbp .cfi_push %rbp + mov %rsp,%rbp + push %rbx +.cfi_push %rbx push %r12 .cfi_push %r12 push %r13 @@ -736,22 +827,18 @@ poly1305_blocks_avx: .Lproceed_avx: mov %r15,$len - - mov 0(%rsp),%r15 + pop %r15 .cfi_restore %r15 - mov 8(%rsp),%r14 + pop %r14 .cfi_restore %r14 - mov 16(%rsp),%r13 + pop %r13 .cfi_restore %r13 - mov 24(%rsp),%r12 + pop %r12 .cfi_restore %r12 - mov 32(%rsp),%rbp -.cfi_restore %rbp - mov 40(%rsp),%rbx + pop %rbx .cfi_restore %rbx - lea 48(%rsp),%rax - lea 48(%rsp),%rsp -.cfi_adjust_cfa_offset -48 + pop %rbp +.cfi_restore %rbp .Lbase2_64_avx_epilogue: jmp .Ldo_avx .cfi_endproc @@ -768,8 +855,11 @@ poly1305_blocks_avx: .Ldo_avx: ___ $code.=<<___ if (!$win64); + lea 8(%rsp),%r10 +.cfi_def_cfa_register %r10 + and \$-32,%rsp + sub \$-8,%rsp lea -0x58(%rsp),%r11 -.cfi_def_cfa %r11,0x60 sub \$0x178,%rsp ___ $code.=<<___ if ($win64); @@ -1361,18 +1451,18 @@ $code.=<<___ if ($win64); .Ldo_avx_epilogue: ___ $code.=<<___ if (!$win64); - lea 0x58(%r11),%rsp -.cfi_def_cfa %rsp,8 + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp ___ $code.=<<___; vzeroupper ret .cfi_endproc -.size poly1305_blocks_avx,.-poly1305_blocks_avx +___ +&end_function("poly1305_blocks_avx"); -.type poly1305_emit_avx,\@function,3 -.align 32 -poly1305_emit_avx: +&declare_function("poly1305_emit_avx", 32, 3); +$code.=<<___; cmpl \$0,20($ctx) # is_base2_26? je .Lemit @@ -1423,41 +1513,51 @@ poly1305_emit_avx: mov %rcx,8($mac) ret -.size poly1305_emit_avx,.-poly1305_emit_avx ___ +&end_function("poly1305_emit_avx"); + +if ($kernel) { + $code .= "#endif\n"; +} if ($avx>1) { + +if ($kernel) { + $code .= "#ifdef CONFIG_AS_AVX2\n"; +} + my ($H0,$H1,$H2,$H3,$H4, $MASK, $T4,$T0,$T1,$T2,$T3, $D0,$D1,$D2,$D3,$D4) = map("%ymm$_",(0..15)); my $S4=$MASK; +sub poly1305_blocks_avxN { + my ($avx512) = @_; + my $suffix = $avx512 ? "_avx512" : ""; $code.=<<___; -.type poly1305_blocks_avx2,\@function,4 -.align 32 -poly1305_blocks_avx2: .cfi_startproc mov 20($ctx),%r8d # is_base2_26 cmp \$128,$len - jae .Lblocks_avx2 + jae .Lblocks_avx2$suffix test %r8d,%r8d jz .Lblocks -.Lblocks_avx2: +.Lblocks_avx2$suffix: and \$-16,$len - jz .Lno_data_avx2 + jz .Lno_data_avx2$suffix vzeroupper test %r8d,%r8d - jz .Lbase2_64_avx2 + jz .Lbase2_64_avx2$suffix test \$63,$len - jz .Leven_avx2 + jz .Leven_avx2$suffix - push %rbx -.cfi_push %rbx push %rbp .cfi_push %rbp + mov %rsp,%rbp + push %rbx +.cfi_push %rbx push %r12 .cfi_push %r12 push %r13 @@ -1466,7 +1566,7 @@ poly1305_blocks_avx2: .cfi_push %r14 push %r15 .cfi_push %r15 -.Lblocks_avx2_body: +.Lblocks_avx2_body$suffix: mov $len,%r15 # reassign $len @@ -1513,7 +1613,7 @@ poly1305_blocks_avx2: shr \$2,$s1 add $r1,$s1 # s1 = r1 + (r1 >> 2) -.Lbase2_26_pre_avx2: +.Lbase2_26_pre_avx2$suffix: add 0($inp),$h0 # accumulate input adc 8($inp),$h1 lea 16($inp),$inp @@ -1524,10 +1624,10 @@ poly1305_blocks_avx2: mov $r1,%rax test \$63,%r15 - jnz .Lbase2_26_pre_avx2 + jnz .Lbase2_26_pre_avx2$suffix test $padbit,$padbit # if $padbit is zero, - jz .Lstore_base2_64_avx2 # store hash in base 2^64 format + jz .Lstore_base2_64_avx2$suffix # store hash in base 2^64 format ################################# base 2^64 -> base 2^26 mov $h0,%rax @@ -1548,57 +1648,56 @@ poly1305_blocks_avx2: or $r1,$h2 # h[4] test %r15,%r15 - jz .Lstore_base2_26_avx2 + jz .Lstore_base2_26_avx2$suffix vmovd %rax#d,%x#$H0 vmovd %rdx#d,%x#$H1 vmovd $h0#d,%x#$H2 vmovd $h1#d,%x#$H3 vmovd $h2#d,%x#$H4 - jmp .Lproceed_avx2 + jmp .Lproceed_avx2$suffix .align 32 -.Lstore_base2_64_avx2: +.Lstore_base2_64_avx2$suffix: mov $h0,0($ctx) mov $h1,8($ctx) mov $h2,16($ctx) # note that is_base2_26 is zeroed - jmp .Ldone_avx2 + jmp .Ldone_avx2$suffix .align 16 -.Lstore_base2_26_avx2: +.Lstore_base2_26_avx2$suffix: mov %rax#d,0($ctx) # store hash value base 2^26 mov %rdx#d,4($ctx) mov $h0#d,8($ctx) mov $h1#d,12($ctx) mov $h2#d,16($ctx) .align 16 -.Ldone_avx2: - mov 0(%rsp),%r15 +.Ldone_avx2$suffix: + pop %r15 .cfi_restore %r15 - mov 8(%rsp),%r14 + pop %r14 .cfi_restore %r14 - mov 16(%rsp),%r13 + pop %r13 .cfi_restore %r13 - mov 24(%rsp),%r12 + pop %r12 .cfi_restore %r12 - mov 32(%rsp),%rbp -.cfi_restore %rbp - mov 40(%rsp),%rbx + pop %rbx .cfi_restore %rbx - lea 48(%rsp),%rsp -.cfi_adjust_cfa_offset -48 -.Lno_data_avx2: -.Lblocks_avx2_epilogue: + pop %rbp +.cfi_restore %rbp +.Lno_data_avx2$suffix: +.Lblocks_avx2_epilogue$suffix: ret .cfi_endproc .align 32 -.Lbase2_64_avx2: +.Lbase2_64_avx2$suffix: .cfi_startproc - push %rbx -.cfi_push %rbx push %rbp .cfi_push %rbp + mov %rsp,%rbp + push %rbx +.cfi_push %rbx push %r12 .cfi_push %r12 push %r13 @@ -1607,7 +1706,7 @@ poly1305_blocks_avx2: .cfi_push %r14 push %r15 .cfi_push %r15 -.Lbase2_64_avx2_body: +.Lbase2_64_avx2_body$suffix: mov $len,%r15 # reassign $len @@ -1624,9 +1723,9 @@ poly1305_blocks_avx2: add $r1,$s1 # s1 = r1 + (r1 >> 2) test \$63,$len - jz .Linit_avx2 + jz .Linit_avx2$suffix -.Lbase2_64_pre_avx2: +.Lbase2_64_pre_avx2$suffix: add 0($inp),$h0 # accumulate input adc 8($inp),$h1 lea 16($inp),$inp @@ -1637,9 +1736,9 @@ poly1305_blocks_avx2: mov $r1,%rax test \$63,%r15 - jnz .Lbase2_64_pre_avx2 + jnz .Lbase2_64_pre_avx2$suffix -.Linit_avx2: +.Linit_avx2$suffix: ################################# base 2^64 -> base 2^26 mov $h0,%rax mov $h0,%rdx @@ -1667,69 +1766,77 @@ poly1305_blocks_avx2: call __poly1305_init_avx -.Lproceed_avx2: +.Lproceed_avx2$suffix: mov %r15,$len # restore $len - mov OPENSSL_ia32cap_P+8(%rip),%r10d +___ +$code.=<<___ if (!$kernel); + mov OPENSSL_ia32cap_P+8(%rip),%r9d mov \$`(1<<31|1<<30|1<<16)`,%r11d - - mov 0(%rsp),%r15 +___ +$code.=<<___; + pop %r15 .cfi_restore %r15 - mov 8(%rsp),%r14 + pop %r14 .cfi_restore %r14 - mov 16(%rsp),%r13 + pop %r13 .cfi_restore %r13 - mov 24(%rsp),%r12 + pop %r12 .cfi_restore %r12 - mov 32(%rsp),%rbp -.cfi_restore %rbp - mov 40(%rsp),%rbx + pop %rbx .cfi_restore %rbx - lea 48(%rsp),%rax - lea 48(%rsp),%rsp -.cfi_adjust_cfa_offset -48 -.Lbase2_64_avx2_epilogue: - jmp .Ldo_avx2 + pop %rbp +.cfi_restore %rbp +.Lbase2_64_avx2_epilogue$suffix: + jmp .Ldo_avx2$suffix .cfi_endproc .align 32 -.Leven_avx2: +.Leven_avx2$suffix: .cfi_startproc - mov OPENSSL_ia32cap_P+8(%rip),%r10d +___ +$code.=<<___ if (!$kernel); + mov OPENSSL_ia32cap_P+8(%rip),%r9d +___ +$code.=<<___; vmovd 4*0($ctx),%x#$H0 # load hash value base 2^26 vmovd 4*1($ctx),%x#$H1 vmovd 4*2($ctx),%x#$H2 vmovd 4*3($ctx),%x#$H3 vmovd 4*4($ctx),%x#$H4 -.Ldo_avx2: +.Ldo_avx2$suffix: ___ -$code.=<<___ if ($avx>2); +$code.=<<___ if (!$kernel && $avx>2); cmp \$512,$len jb .Lskip_avx512 - and %r11d,%r10d - test \$`1<<16`,%r10d # check for AVX512F + and %r11d,%r9d + test \$`1<<16`,%r9d # check for AVX512F jnz .Lblocks_avx512 -.Lskip_avx512: +.Lskip_avx512$suffix: +___ +$code.=<<___ if ($avx > 2 && $avx512 && $kernel); + cmp \$512,$len + jae .Lblocks_avx512 ___ $code.=<<___ if (!$win64); - lea -8(%rsp),%r11 -.cfi_def_cfa %r11,16 + lea 8(%rsp),%r10 +.cfi_def_cfa_register %r10 sub \$0x128,%rsp ___ $code.=<<___ if ($win64); - lea -0xf8(%rsp),%r11 + lea 8(%rsp),%r10 sub \$0x1c8,%rsp - vmovdqa %xmm6,0x50(%r11) - vmovdqa %xmm7,0x60(%r11) - vmovdqa %xmm8,0x70(%r11) - vmovdqa %xmm9,0x80(%r11) - vmovdqa %xmm10,0x90(%r11) - vmovdqa %xmm11,0xa0(%r11) - vmovdqa %xmm12,0xb0(%r11) - vmovdqa %xmm13,0xc0(%r11) - vmovdqa %xmm14,0xd0(%r11) - vmovdqa %xmm15,0xe0(%r11) -.Ldo_avx2_body: + vmovdqa %xmm6,-0xb0(%r10) + vmovdqa %xmm7,-0xa0(%r10) + vmovdqa %xmm8,-0x90(%r10) + vmovdqa %xmm9,-0x80(%r10) + vmovdqa %xmm10,-0x70(%r10) + vmovdqa %xmm11,-0x60(%r10) + vmovdqa %xmm12,-0x50(%r10) + vmovdqa %xmm13,-0x40(%r10) + vmovdqa %xmm14,-0x30(%r10) + vmovdqa %xmm15,-0x20(%r10) +.Ldo_avx2_body$suffix: ___ $code.=<<___; lea .Lconst(%rip),%rcx @@ -1794,11 +1901,11 @@ $code.=<<___; vpaddq $H2,$T2,$H2 # accumulate input sub \$64,$len - jz .Ltail_avx2 - jmp .Loop_avx2 + jz .Ltail_avx2$suffix + jmp .Loop_avx2$suffix .align 32 -.Loop_avx2: +.Loop_avx2$suffix: ################################################################ # ((inp[0]*r^4+inp[4])*r^4+inp[ 8])*r^4 # ((inp[1]*r^4+inp[5])*r^4+inp[ 9])*r^3 @@ -1946,10 +2053,10 @@ $code.=<<___; vpor 32(%rcx),$T4,$T4 # padbit, yes, always sub \$64,$len - jnz .Loop_avx2 + jnz .Loop_avx2$suffix .byte 0x66,0x90 -.Ltail_avx2: +.Ltail_avx2$suffix: ################################################################ # while above multiplications were by r^4 in all lanes, in last # iteration we multiply least significant lane by r^4 and most @@ -2087,37 +2194,29 @@ $code.=<<___; vmovd %x#$H4,`4*4-48-64`($ctx) ___ $code.=<<___ if ($win64); - vmovdqa 0x50(%r11),%xmm6 - vmovdqa 0x60(%r11),%xmm7 - vmovdqa 0x70(%r11),%xmm8 - vmovdqa 0x80(%r11),%xmm9 - vmovdqa 0x90(%r11),%xmm10 - vmovdqa 0xa0(%r11),%xmm11 - vmovdqa 0xb0(%r11),%xmm12 - vmovdqa 0xc0(%r11),%xmm13 - vmovdqa 0xd0(%r11),%xmm14 - vmovdqa 0xe0(%r11),%xmm15 - lea 0xf8(%r11),%rsp -.Ldo_avx2_epilogue: + vmovdqa -0xb0(%r10),%xmm6 + vmovdqa -0xa0(%r10),%xmm7 + vmovdqa -0x90(%r10),%xmm8 + vmovdqa -0x80(%r10),%xmm9 + vmovdqa -0x70(%r10),%xmm10 + vmovdqa -0x60(%r10),%xmm11 + vmovdqa -0x50(%r10),%xmm12 + vmovdqa -0x40(%r10),%xmm13 + vmovdqa -0x30(%r10),%xmm14 + vmovdqa -0x20(%r10),%xmm15 + lea -8(%r10),%rsp +.Ldo_avx2_epilogue$suffix: ___ $code.=<<___ if (!$win64); - lea 8(%r11),%rsp -.cfi_def_cfa %rsp,8 + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp ___ $code.=<<___; vzeroupper ret .cfi_endproc -.size poly1305_blocks_avx2,.-poly1305_blocks_avx2 ___ -####################################################################### -if ($avx>2) { -# On entry we have input length divisible by 64. But since inner loop -# processes 128 bytes per iteration, cases when length is not divisible -# by 128 are handled by passing tail 64 bytes to .Ltail_avx2. For this -# reason stack layout is kept identical to poly1305_blocks_avx2. If not -# for this tail, we wouldn't have to even allocate stack frame... - +if($avx > 2 && $avx512) { my ($R0,$R1,$R2,$R3,$R4, $S1,$S2,$S3,$S4) = map("%zmm$_",(16..24)); my ($M0,$M1,$M2,$M3,$M4) = map("%zmm$_",(25..29)); my $PADBIT="%zmm30"; @@ -2128,32 +2227,29 @@ map(s/%y/%z/,($H0,$H1,$H2,$H3,$H4)); map(s/%y/%z/,($MASK)); $code.=<<___; -.type poly1305_blocks_avx512,\@function,4 -.align 32 -poly1305_blocks_avx512: .cfi_startproc .Lblocks_avx512: mov \$15,%eax kmovw %eax,%k2 ___ $code.=<<___ if (!$win64); - lea -8(%rsp),%r11 -.cfi_def_cfa %r11,16 + lea 8(%rsp),%r10 +.cfi_def_cfa_register %r10 sub \$0x128,%rsp ___ $code.=<<___ if ($win64); - lea -0xf8(%rsp),%r11 + lea 8(%rsp),%r10 sub \$0x1c8,%rsp - vmovdqa %xmm6,0x50(%r11) - vmovdqa %xmm7,0x60(%r11) - vmovdqa %xmm8,0x70(%r11) - vmovdqa %xmm9,0x80(%r11) - vmovdqa %xmm10,0x90(%r11) - vmovdqa %xmm11,0xa0(%r11) - vmovdqa %xmm12,0xb0(%r11) - vmovdqa %xmm13,0xc0(%r11) - vmovdqa %xmm14,0xd0(%r11) - vmovdqa %xmm15,0xe0(%r11) + vmovdqa %xmm6,-0xb0(%r10) + vmovdqa %xmm7,-0xa0(%r10) + vmovdqa %xmm8,-0x90(%r10) + vmovdqa %xmm9,-0x80(%r10) + vmovdqa %xmm10,-0x70(%r10) + vmovdqa %xmm11,-0x60(%r10) + vmovdqa %xmm12,-0x50(%r10) + vmovdqa %xmm13,-0x40(%r10) + vmovdqa %xmm14,-0x30(%r10) + vmovdqa %xmm15,-0x20(%r10) .Ldo_avx512_body: ___ $code.=<<___; @@ -2679,7 +2775,7 @@ $code.=<<___; lea 0x90(%rsp),%rax # size optimization for .Ltail_avx2 add \$64,$len - jnz .Ltail_avx2 + jnz .Ltail_avx2$suffix vpsubq $T2,$H2,$H2 # undo input accumulation vmovd %x#$H0,`4*0-48-64`($ctx)# save partially reduced @@ -2690,29 +2786,61 @@ $code.=<<___; vzeroall ___ $code.=<<___ if ($win64); - movdqa 0x50(%r11),%xmm6 - movdqa 0x60(%r11),%xmm7 - movdqa 0x70(%r11),%xmm8 - movdqa 0x80(%r11),%xmm9 - movdqa 0x90(%r11),%xmm10 - movdqa 0xa0(%r11),%xmm11 - movdqa 0xb0(%r11),%xmm12 - movdqa 0xc0(%r11),%xmm13 - movdqa 0xd0(%r11),%xmm14 - movdqa 0xe0(%r11),%xmm15 - lea 0xf8(%r11),%rsp + movdqa -0xb0(%r10),%xmm6 + movdqa -0xa0(%r10),%xmm7 + movdqa -0x90(%r10),%xmm8 + movdqa -0x80(%r10),%xmm9 + movdqa -0x70(%r10),%xmm10 + movdqa -0x60(%r10),%xmm11 + movdqa -0x50(%r10),%xmm12 + movdqa -0x40(%r10),%xmm13 + movdqa -0x30(%r10),%xmm14 + movdqa -0x20(%r10),%xmm15 + lea -8(%r10),%rsp .Ldo_avx512_epilogue: ___ $code.=<<___ if (!$win64); - lea 8(%r11),%rsp -.cfi_def_cfa %rsp,8 + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp ___ $code.=<<___; ret .cfi_endproc -.size poly1305_blocks_avx512,.-poly1305_blocks_avx512 ___ -if ($avx>3) { + +} + +} + +&declare_function("poly1305_blocks_avx2", 32, 4); +poly1305_blocks_avxN(0); +&end_function("poly1305_blocks_avx2"); + +if($kernel) { + $code .= "#endif\n"; +} + +####################################################################### +if ($avx>2) { +# On entry we have input length divisible by 64. But since inner loop +# processes 128 bytes per iteration, cases when length is not divisible +# by 128 are handled by passing tail 64 bytes to .Ltail_avx2. For this +# reason stack layout is kept identical to poly1305_blocks_avx2. If not +# for this tail, we wouldn't have to even allocate stack frame... + +if($kernel) { + $code .= "#ifdef CONFIG_AS_AVX512\n"; +} + +&declare_function("poly1305_blocks_avx512", 32, 4); +poly1305_blocks_avxN(1); +&end_function("poly1305_blocks_avx512"); + +if ($kernel) { + $code .= "#endif\n"; +} + +if (!$kernel && $avx>3) { ######################################################################## # VPMADD52 version using 2^44 radix. # @@ -3753,45 +3881,9 @@ poly1305_emit_base2_44: .size poly1305_emit_base2_44,.-poly1305_emit_base2_44 ___ } } } -$code.=<<___; -.align 64 -.Lconst: -.Lmask24: -.long 0x0ffffff,0,0x0ffffff,0,0x0ffffff,0,0x0ffffff,0 -.L129: -.long `1<<24`,0,`1<<24`,0,`1<<24`,0,`1<<24`,0 -.Lmask26: -.long 0x3ffffff,0,0x3ffffff,0,0x3ffffff,0,0x3ffffff,0 -.Lpermd_avx2: -.long 2,2,2,3,2,0,2,1 -.Lpermd_avx512: -.long 0,0,0,1, 0,2,0,3, 0,4,0,5, 0,6,0,7 - -.L2_44_inp_permd: -.long 0,1,1,2,2,3,7,7 -.L2_44_inp_shift: -.quad 0,12,24,64 -.L2_44_mask: -.quad 0xfffffffffff,0xfffffffffff,0x3ffffffffff,0xffffffffffffffff -.L2_44_shift_rgt: -.quad 44,44,42,64 -.L2_44_shift_lft: -.quad 8,8,10,64 - -.align 64 -.Lx_mask44: -.quad 0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff -.quad 0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff -.Lx_mask42: -.quad 0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff -.quad 0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff -___ } -$code.=<<___; -.asciz "Poly1305 for x86_64, CRYPTOGAMS by " -.align 16 -___ +if (!$kernel) { # chacha20-poly1305 helpers my ($out,$inp,$otp,$len)=$win64 ? ("%rcx","%rdx","%r8", "%r9") : # Win64 order ("%rdi","%rsi","%rdx","%rcx"); # Unix order @@ -4038,17 +4130,17 @@ avx_handler: .section .pdata .align 4 - .rva .LSEH_begin_poly1305_init - .rva .LSEH_end_poly1305_init - .rva .LSEH_info_poly1305_init + .rva .LSEH_begin_poly1305_init_x86_64 + .rva .LSEH_end_poly1305_init_x86_64 + .rva .LSEH_info_poly1305_init_x86_64 - .rva .LSEH_begin_poly1305_blocks - .rva .LSEH_end_poly1305_blocks - .rva .LSEH_info_poly1305_blocks + .rva .LSEH_begin_poly1305_blocks_x86_64 + .rva .LSEH_end_poly1305_blocks_x86_64 + .rva .LSEH_info_poly1305_blocks_x86_64 - .rva .LSEH_begin_poly1305_emit - .rva .LSEH_end_poly1305_emit - .rva .LSEH_info_poly1305_emit + .rva .LSEH_begin_poly1305_emit_x86_64 + .rva .LSEH_end_poly1305_emit_x86_64 + .rva .LSEH_info_poly1305_emit_x86_64 ___ $code.=<<___ if ($avx); .rva .LSEH_begin_poly1305_blocks_avx @@ -4088,20 +4180,20 @@ ___ $code.=<<___; .section .xdata .align 8 -.LSEH_info_poly1305_init: +.LSEH_info_poly1305_init_x86_64: .byte 9,0,0,0 .rva se_handler - .rva .LSEH_begin_poly1305_init,.LSEH_begin_poly1305_init + .rva .LSEH_begin_poly1305_init_x86_64,.LSEH_begin_poly1305_init_x86_64 -.LSEH_info_poly1305_blocks: +.LSEH_info_poly1305_blocks_x86_64: .byte 9,0,0,0 .rva se_handler .rva .Lblocks_body,.Lblocks_epilogue -.LSEH_info_poly1305_emit: +.LSEH_info_poly1305_emit_x86_64: .byte 9,0,0,0 .rva se_handler - .rva .LSEH_begin_poly1305_emit,.LSEH_begin_poly1305_emit + .rva .LSEH_begin_poly1305_emit_x86_64,.LSEH_begin_poly1305_emit_x86_64 ___ $code.=<<___ if ($avx); .LSEH_info_poly1305_blocks_avx_1: @@ -4148,12 +4240,26 @@ $code.=<<___ if ($avx>2); ___ } +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + foreach (split('\n',$code)) { s/\`([^\`]*)\`/eval($1)/ge; s/%r([a-z]+)#d/%e$1/g; s/%r([0-9]+)#d/%r$1d/g; s/%x#%[yz]/%x/g or s/%y#%z/%y/g or s/%z#%[yz]/%z/g; + if ($kernel) { + s/(^\.type.*),[0-9]+$/\1/; + s/(^\.type.*),\@abi-omnipotent+$/\1,\@function/; + next if /^\.cfi.*/; + } + print $_,"\n"; } close STDOUT; diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c index edb7113e36f3..657363588e0c 100644 --- a/arch/x86/crypto/poly1305_glue.c +++ b/arch/x86/crypto/poly1305_glue.c @@ -1,8 +1,6 @@ -// SPDX-License-Identifier: GPL-2.0-or-later +// SPDX-License-Identifier: GPL-2.0 OR MIT /* - * Poly1305 authenticator algorithm, RFC7539, SIMD glue code - * - * Copyright (C) 2015 Martin Willi + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. */ #include @@ -13,279 +11,170 @@ #include #include #include +#include #include -asmlinkage void poly1305_block_sse2(u32 *h, const u8 *src, - const u32 *r, unsigned int blocks); -asmlinkage void poly1305_2block_sse2(u32 *h, const u8 *src, const u32 *r, - unsigned int blocks, const u32 *u); -asmlinkage void poly1305_4block_avx2(u32 *h, const u8 *src, const u32 *r, - unsigned int blocks, const u32 *u); - -static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_simd); +asmlinkage void poly1305_init_x86_64(void *ctx, + const u8 key[POLY1305_KEY_SIZE]); +asmlinkage void poly1305_blocks_x86_64(void *ctx, const u8 *inp, + const size_t len, const u32 padbit); +asmlinkage void poly1305_emit_x86_64(void *ctx, u8 mac[POLY1305_DIGEST_SIZE], + const u32 nonce[4]); +asmlinkage void poly1305_emit_avx(void *ctx, u8 mac[POLY1305_DIGEST_SIZE], + const u32 nonce[4]); +asmlinkage void poly1305_blocks_avx(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_blocks_avx2(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_blocks_avx512(void *ctx, const u8 *inp, + const size_t len, const u32 padbit); + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx); static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx2); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx512); + +struct poly1305_arch_internal { + union { + struct { + u32 h[5]; + u32 is_base2_26; + }; + u64 hs[3]; + }; + u64 r[2]; + u64 pad; + struct { u32 r2, r1, r4, r3; } rn[9]; +}; -static inline u64 mlt(u64 a, u64 b) +/* The AVX code uses base 2^26, while the scalar code uses base 2^64. If we hit + * the unfortunate situation of using AVX and then having to go back to scalar + * -- because the user is silly and has called the update function from two + * separate contexts -- then we need to convert back to the original base before + * proceeding. It is possible to reason that the initial reduction below is + * sufficient given the implementation invariants. However, for an avoidance of + * doubt and because this is not performance critical, we do the full reduction + * anyway. Z3 proof of below function: https://xn--4db.cc/ltPtHCKN/py + */ +static void convert_to_base2_64(void *ctx) { - return a * b; -} + struct poly1305_arch_internal *state = ctx; + u32 cy; -static inline u32 sr(u64 v, u_char n) -{ - return v >> n; -} + if (!state->is_base2_26) + return; -static inline u32 and(u32 v, u32 mask) -{ - return v & mask; + cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy; + cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy; + cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy; + cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy; + state->hs[0] = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0]; + state->hs[1] = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12); + state->hs[2] = state->h[4] >> 24; +#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1)) + cy = (state->hs[2] >> 2) + (state->hs[2] & ~3ULL); + state->hs[2] &= 3; + state->hs[0] += cy; + state->hs[1] += (cy = ULT(state->hs[0], cy)); + state->hs[2] += ULT(state->hs[1], cy); +#undef ULT + state->is_base2_26 = 0; } -static void poly1305_simd_mult(u32 *a, const u32 *b) +static void poly1305_simd_init(void *ctx, const u8 key[POLY1305_KEY_SIZE]) { - u8 m[POLY1305_BLOCK_SIZE]; - - memset(m, 0, sizeof(m)); - /* The poly1305 block function adds a hi-bit to the accumulator which - * we don't need for key multiplication; compensate for it. */ - a[4] -= 1 << 24; - poly1305_block_sse2(a, m, b, 1); + poly1305_init_x86_64(ctx, key); } -static void poly1305_integer_setkey(struct poly1305_key *key, const u8 *raw_key) +static void poly1305_simd_blocks(void *ctx, const u8 *inp, size_t len, + const u32 padbit) { - /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ - key->r[0] = (get_unaligned_le32(raw_key + 0) >> 0) & 0x3ffffff; - key->r[1] = (get_unaligned_le32(raw_key + 3) >> 2) & 0x3ffff03; - key->r[2] = (get_unaligned_le32(raw_key + 6) >> 4) & 0x3ffc0ff; - key->r[3] = (get_unaligned_le32(raw_key + 9) >> 6) & 0x3f03fff; - key->r[4] = (get_unaligned_le32(raw_key + 12) >> 8) & 0x00fffff; -} + struct poly1305_arch_internal *state = ctx; -static void poly1305_integer_blocks(struct poly1305_state *state, - const struct poly1305_key *key, - const void *src, - unsigned int nblocks, u32 hibit) -{ - u32 r0, r1, r2, r3, r4; - u32 s1, s2, s3, s4; - u32 h0, h1, h2, h3, h4; - u64 d0, d1, d2, d3, d4; + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(PAGE_SIZE < POLY1305_BLOCK_SIZE || + PAGE_SIZE % POLY1305_BLOCK_SIZE); - if (!nblocks) + if (!IS_ENABLED(CONFIG_AS_AVX) || !static_branch_likely(&poly1305_use_avx) || + (len < (POLY1305_BLOCK_SIZE * 18) && !state->is_base2_26) || + !crypto_simd_usable()) { + convert_to_base2_64(ctx); + poly1305_blocks_x86_64(ctx, inp, len, padbit); return; + } - r0 = key->r[0]; - r1 = key->r[1]; - r2 = key->r[2]; - r3 = key->r[3]; - r4 = key->r[4]; - - s1 = r1 * 5; - s2 = r2 * 5; - s3 = r3 * 5; - s4 = r4 * 5; - - h0 = state->h[0]; - h1 = state->h[1]; - h2 = state->h[2]; - h3 = state->h[3]; - h4 = state->h[4]; - - do { - /* h += m[i] */ - h0 += (get_unaligned_le32(src + 0) >> 0) & 0x3ffffff; - h1 += (get_unaligned_le32(src + 3) >> 2) & 0x3ffffff; - h2 += (get_unaligned_le32(src + 6) >> 4) & 0x3ffffff; - h3 += (get_unaligned_le32(src + 9) >> 6) & 0x3ffffff; - h4 += (get_unaligned_le32(src + 12) >> 8) | (hibit << 24); - - /* h *= r */ - d0 = mlt(h0, r0) + mlt(h1, s4) + mlt(h2, s3) + - mlt(h3, s2) + mlt(h4, s1); - d1 = mlt(h0, r1) + mlt(h1, r0) + mlt(h2, s4) + - mlt(h3, s3) + mlt(h4, s2); - d2 = mlt(h0, r2) + mlt(h1, r1) + mlt(h2, r0) + - mlt(h3, s4) + mlt(h4, s3); - d3 = mlt(h0, r3) + mlt(h1, r2) + mlt(h2, r1) + - mlt(h3, r0) + mlt(h4, s4); - d4 = mlt(h0, r4) + mlt(h1, r3) + mlt(h2, r2) + - mlt(h3, r1) + mlt(h4, r0); - - /* (partial) h %= p */ - d1 += sr(d0, 26); h0 = and(d0, 0x3ffffff); - d2 += sr(d1, 26); h1 = and(d1, 0x3ffffff); - d3 += sr(d2, 26); h2 = and(d2, 0x3ffffff); - d4 += sr(d3, 26); h3 = and(d3, 0x3ffffff); - h0 += sr(d4, 26) * 5; h4 = and(d4, 0x3ffffff); - h1 += h0 >> 26; h0 = h0 & 0x3ffffff; - - src += POLY1305_BLOCK_SIZE; - } while (--nblocks); - - state->h[0] = h0; - state->h[1] = h1; - state->h[2] = h2; - state->h[3] = h3; - state->h[4] = h4; + for (;;) { + const size_t bytes = min_t(size_t, len, PAGE_SIZE); + + kernel_fpu_begin(); + if (IS_ENABLED(CONFIG_AS_AVX512) && static_branch_likely(&poly1305_use_avx512)) + poly1305_blocks_avx512(ctx, inp, bytes, padbit); + else if (IS_ENABLED(CONFIG_AS_AVX2) && static_branch_likely(&poly1305_use_avx2)) + poly1305_blocks_avx2(ctx, inp, bytes, padbit); + else + poly1305_blocks_avx(ctx, inp, bytes, padbit); + kernel_fpu_end(); + len -= bytes; + if (!len) + break; + inp += bytes; + } } -static void poly1305_integer_emit(const struct poly1305_state *state, void *dst) +static void poly1305_simd_emit(void *ctx, u8 mac[POLY1305_DIGEST_SIZE], + const u32 nonce[4]) { - u32 h0, h1, h2, h3, h4; - u32 g0, g1, g2, g3, g4; - u32 mask; - - /* fully carry h */ - h0 = state->h[0]; - h1 = state->h[1]; - h2 = state->h[2]; - h3 = state->h[3]; - h4 = state->h[4]; - - h2 += (h1 >> 26); h1 = h1 & 0x3ffffff; - h3 += (h2 >> 26); h2 = h2 & 0x3ffffff; - h4 += (h3 >> 26); h3 = h3 & 0x3ffffff; - h0 += (h4 >> 26) * 5; h4 = h4 & 0x3ffffff; - h1 += (h0 >> 26); h0 = h0 & 0x3ffffff; - - /* compute h + -p */ - g0 = h0 + 5; - g1 = h1 + (g0 >> 26); g0 &= 0x3ffffff; - g2 = h2 + (g1 >> 26); g1 &= 0x3ffffff; - g3 = h3 + (g2 >> 26); g2 &= 0x3ffffff; - g4 = h4 + (g3 >> 26) - (1 << 26); g3 &= 0x3ffffff; - - /* select h if h < p, or h + -p if h >= p */ - mask = (g4 >> ((sizeof(u32) * 8) - 1)) - 1; - g0 &= mask; - g1 &= mask; - g2 &= mask; - g3 &= mask; - g4 &= mask; - mask = ~mask; - h0 = (h0 & mask) | g0; - h1 = (h1 & mask) | g1; - h2 = (h2 & mask) | g2; - h3 = (h3 & mask) | g3; - h4 = (h4 & mask) | g4; - - /* h = h % (2^128) */ - put_unaligned_le32((h0 >> 0) | (h1 << 26), dst + 0); - put_unaligned_le32((h1 >> 6) | (h2 << 20), dst + 4); - put_unaligned_le32((h2 >> 12) | (h3 << 14), dst + 8); - put_unaligned_le32((h3 >> 18) | (h4 << 8), dst + 12); + struct poly1305_arch_internal *state = ctx; + + if (!IS_ENABLED(CONFIG_AS_AVX) || !static_branch_likely(&poly1305_use_avx) || + !state->is_base2_26 || !crypto_simd_usable()) { + convert_to_base2_64(ctx); + poly1305_emit_x86_64(ctx, mac, nonce); + } else + poly1305_emit_avx(ctx, mac, nonce); } -void poly1305_init_arch(struct poly1305_desc_ctx *desc, const u8 *key) +void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 *key) { - poly1305_integer_setkey(desc->opaque_r, key); - desc->s[0] = get_unaligned_le32(key + 16); - desc->s[1] = get_unaligned_le32(key + 20); - desc->s[2] = get_unaligned_le32(key + 24); - desc->s[3] = get_unaligned_le32(key + 28); - poly1305_core_init(&desc->h); - desc->buflen = 0; - desc->sset = true; - desc->rset = 1; + poly1305_simd_init(&dctx->h, key); + dctx->s[0] = get_unaligned_le32(&key[16]); + dctx->s[1] = get_unaligned_le32(&key[20]); + dctx->s[2] = get_unaligned_le32(&key[24]); + dctx->s[3] = get_unaligned_le32(&key[28]); + dctx->buflen = 0; + dctx->sset = true; } -EXPORT_SYMBOL_GPL(poly1305_init_arch); +EXPORT_SYMBOL(poly1305_init_arch); -static unsigned int crypto_poly1305_setdesckey(struct poly1305_desc_ctx *dctx, - const u8 *src, unsigned int srclen) +static unsigned int crypto_poly1305_setdctxkey(struct poly1305_desc_ctx *dctx, + const u8 *inp, unsigned int len) { - if (!dctx->sset) { - if (!dctx->rset && srclen >= POLY1305_BLOCK_SIZE) { - poly1305_integer_setkey(dctx->r, src); - src += POLY1305_BLOCK_SIZE; - srclen -= POLY1305_BLOCK_SIZE; + unsigned int acc = 0; + if (unlikely(!dctx->sset)) { + if (!dctx->rset && len >= POLY1305_BLOCK_SIZE) { + poly1305_simd_init(&dctx->h, inp); + inp += POLY1305_BLOCK_SIZE; + len -= POLY1305_BLOCK_SIZE; + acc += POLY1305_BLOCK_SIZE; dctx->rset = 1; } - if (srclen >= POLY1305_BLOCK_SIZE) { - dctx->s[0] = get_unaligned_le32(src + 0); - dctx->s[1] = get_unaligned_le32(src + 4); - dctx->s[2] = get_unaligned_le32(src + 8); - dctx->s[3] = get_unaligned_le32(src + 12); - src += POLY1305_BLOCK_SIZE; - srclen -= POLY1305_BLOCK_SIZE; + if (len >= POLY1305_BLOCK_SIZE) { + dctx->s[0] = get_unaligned_le32(&inp[0]); + dctx->s[1] = get_unaligned_le32(&inp[4]); + dctx->s[2] = get_unaligned_le32(&inp[8]); + dctx->s[3] = get_unaligned_le32(&inp[12]); + inp += POLY1305_BLOCK_SIZE; + len -= POLY1305_BLOCK_SIZE; + acc += POLY1305_BLOCK_SIZE; dctx->sset = true; } } - return srclen; -} - -static unsigned int poly1305_scalar_blocks(struct poly1305_desc_ctx *dctx, - const u8 *src, unsigned int srclen) -{ - unsigned int datalen; - - if (unlikely(!dctx->sset)) { - datalen = crypto_poly1305_setdesckey(dctx, src, srclen); - src += srclen - datalen; - srclen = datalen; - } - if (srclen >= POLY1305_BLOCK_SIZE) { - poly1305_integer_blocks(&dctx->h, dctx->opaque_r, src, - srclen / POLY1305_BLOCK_SIZE, 1); - srclen %= POLY1305_BLOCK_SIZE; - } - return srclen; -} - -static unsigned int poly1305_simd_blocks(struct poly1305_desc_ctx *dctx, - const u8 *src, unsigned int srclen) -{ - unsigned int blocks, datalen; - - if (unlikely(!dctx->sset)) { - datalen = crypto_poly1305_setdesckey(dctx, src, srclen); - src += srclen - datalen; - srclen = datalen; - } - - if (IS_ENABLED(CONFIG_AS_AVX2) && - static_branch_likely(&poly1305_use_avx2) && - srclen >= POLY1305_BLOCK_SIZE * 4) { - if (unlikely(dctx->rset < 4)) { - if (dctx->rset < 2) { - dctx->r[1] = dctx->r[0]; - poly1305_simd_mult(dctx->r[1].r, dctx->r[0].r); - } - dctx->r[2] = dctx->r[1]; - poly1305_simd_mult(dctx->r[2].r, dctx->r[0].r); - dctx->r[3] = dctx->r[2]; - poly1305_simd_mult(dctx->r[3].r, dctx->r[0].r); - dctx->rset = 4; - } - blocks = srclen / (POLY1305_BLOCK_SIZE * 4); - poly1305_4block_avx2(dctx->h.h, src, dctx->r[0].r, blocks, - dctx->r[1].r); - src += POLY1305_BLOCK_SIZE * 4 * blocks; - srclen -= POLY1305_BLOCK_SIZE * 4 * blocks; - } - - if (likely(srclen >= POLY1305_BLOCK_SIZE * 2)) { - if (unlikely(dctx->rset < 2)) { - dctx->r[1] = dctx->r[0]; - poly1305_simd_mult(dctx->r[1].r, dctx->r[0].r); - dctx->rset = 2; - } - blocks = srclen / (POLY1305_BLOCK_SIZE * 2); - poly1305_2block_sse2(dctx->h.h, src, dctx->r[0].r, - blocks, dctx->r[1].r); - src += POLY1305_BLOCK_SIZE * 2 * blocks; - srclen -= POLY1305_BLOCK_SIZE * 2 * blocks; - } - if (srclen >= POLY1305_BLOCK_SIZE) { - poly1305_block_sse2(dctx->h.h, src, dctx->r[0].r, 1); - srclen -= POLY1305_BLOCK_SIZE; - } - return srclen; + return acc; } void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, unsigned int srclen) { - unsigned int bytes; + unsigned int bytes, used; if (unlikely(dctx->buflen)) { bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); @@ -295,31 +184,19 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, dctx->buflen += bytes; if (dctx->buflen == POLY1305_BLOCK_SIZE) { - if (static_branch_likely(&poly1305_use_simd) && - likely(crypto_simd_usable())) { - kernel_fpu_begin(); - poly1305_simd_blocks(dctx, dctx->buf, - POLY1305_BLOCK_SIZE); - kernel_fpu_end(); - } else { - poly1305_scalar_blocks(dctx, dctx->buf, - POLY1305_BLOCK_SIZE); - } + if (likely(!crypto_poly1305_setdctxkey(dctx, dctx->buf, POLY1305_BLOCK_SIZE))) + poly1305_simd_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1); dctx->buflen = 0; } } if (likely(srclen >= POLY1305_BLOCK_SIZE)) { - if (static_branch_likely(&poly1305_use_simd) && - likely(crypto_simd_usable())) { - kernel_fpu_begin(); - bytes = poly1305_simd_blocks(dctx, src, srclen); - kernel_fpu_end(); - } else { - bytes = poly1305_scalar_blocks(dctx, src, srclen); - } - src += srclen - bytes; - srclen = bytes; + bytes = round_down(srclen, POLY1305_BLOCK_SIZE); + srclen -= bytes; + used = crypto_poly1305_setdctxkey(dctx, src, bytes); + if (likely(bytes - used)) + poly1305_simd_blocks(&dctx->h, src + used, bytes - used, 1); + src += bytes; } if (unlikely(srclen)) { @@ -329,31 +206,17 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, } EXPORT_SYMBOL(poly1305_update_arch); -void poly1305_final_arch(struct poly1305_desc_ctx *desc, u8 *dst) +void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) { - __le32 digest[4]; - u64 f = 0; - - if (unlikely(desc->buflen)) { - desc->buf[desc->buflen++] = 1; - memset(desc->buf + desc->buflen, 0, - POLY1305_BLOCK_SIZE - desc->buflen); - poly1305_integer_blocks(&desc->h, desc->opaque_r, desc->buf, 1, 0); + if (unlikely(dctx->buflen)) { + dctx->buf[dctx->buflen++] = 1; + memset(dctx->buf + dctx->buflen, 0, + POLY1305_BLOCK_SIZE - dctx->buflen); + poly1305_simd_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); } - poly1305_integer_emit(&desc->h, digest); - - /* mac = (h + s) % (2^128) */ - f = (f >> 32) + le32_to_cpu(digest[0]) + desc->s[0]; - put_unaligned_le32(f, dst + 0); - f = (f >> 32) + le32_to_cpu(digest[1]) + desc->s[1]; - put_unaligned_le32(f, dst + 4); - f = (f >> 32) + le32_to_cpu(digest[2]) + desc->s[2]; - put_unaligned_le32(f, dst + 8); - f = (f >> 32) + le32_to_cpu(digest[3]) + desc->s[3]; - put_unaligned_le32(f, dst + 12); - - *desc = (struct poly1305_desc_ctx){}; + poly1305_simd_emit(&dctx->h, dst, dctx->s); + *dctx = (struct poly1305_desc_ctx){}; } EXPORT_SYMBOL(poly1305_final_arch); @@ -361,38 +224,34 @@ static int crypto_poly1305_init(struct shash_desc *desc) { struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - poly1305_core_init(&dctx->h); - dctx->buflen = 0; - dctx->rset = 0; - dctx->sset = false; - + *dctx = (struct poly1305_desc_ctx){}; return 0; } -static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst) +static int crypto_poly1305_update(struct shash_desc *desc, + const u8 *src, unsigned int srclen) { struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - if (unlikely(!dctx->sset)) - return -ENOKEY; - - poly1305_final_arch(dctx, dst); + poly1305_update_arch(dctx, src, srclen); return 0; } -static int poly1305_simd_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) +static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst) { struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - poly1305_update_arch(dctx, src, srclen); + if (unlikely(!dctx->sset)) + return -ENOKEY; + + poly1305_final_arch(dctx, dst); return 0; } static struct shash_alg alg = { .digestsize = POLY1305_DIGEST_SIZE, .init = crypto_poly1305_init, - .update = poly1305_simd_update, + .update = crypto_poly1305_update, .final = crypto_poly1305_final, .descsize = sizeof(struct poly1305_desc_ctx), .base = { @@ -406,17 +265,19 @@ static struct shash_alg alg = { static int __init poly1305_simd_mod_init(void) { - if (!boot_cpu_has(X86_FEATURE_XMM2)) - return 0; - - static_branch_enable(&poly1305_use_simd); - - if (IS_ENABLED(CONFIG_AS_AVX2) && - boot_cpu_has(X86_FEATURE_AVX) && + if (IS_ENABLED(CONFIG_AS_AVX) && boot_cpu_has(X86_FEATURE_AVX) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) + static_branch_enable(&poly1305_use_avx); + if (IS_ENABLED(CONFIG_AS_AVX2) && boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) && cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) static_branch_enable(&poly1305_use_avx2); - + if (IS_ENABLED(CONFIG_AS_AVX512) && boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX512F) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | XFEATURE_MASK_AVX512, NULL) && + /* Skylake downclocks unacceptably much when using zmm, but later generations are fast. */ + boot_cpu_data.x86_model != INTEL_FAM6_SKYLAKE_X) + static_branch_enable(&poly1305_use_avx512); return IS_REACHABLE(CONFIG_CRYPTO_HASH) ? crypto_register_shash(&alg) : 0; } @@ -430,7 +291,7 @@ module_init(poly1305_simd_mod_init); module_exit(poly1305_simd_mod_exit); MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Martin Willi "); +MODULE_AUTHOR("Jason A. Donenfeld "); MODULE_DESCRIPTION("Poly1305 authenticator"); MODULE_ALIAS_CRYPTO("poly1305"); MODULE_ALIAS_CRYPTO("poly1305-simd"); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 0b2c4fce26d9..14c032de276e 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -90,7 +90,7 @@ config CRYPTO_LIB_DES config CRYPTO_LIB_POLY1305_RSIZE int default 2 if MIPS - default 4 if X86_64 + default 11 if X86_64 default 9 if ARM || ARM64 default 1 -- cgit v1.2.3 From a8bdf2c42ee4d1ee42af1f3601f85de94e70a421 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 8 Jan 2020 12:37:35 +0800 Subject: crypto: curve25519 - Fix selftest build error If CRYPTO_CURVE25519 is y, CRYPTO_LIB_CURVE25519_GENERIC will be y, but CRYPTO_LIB_CURVE25519 may be set to m, this causes build errors: lib/crypto/curve25519-selftest.o: In function `curve25519': curve25519-selftest.c:(.text.unlikely+0xc): undefined reference to `curve25519_arch' lib/crypto/curve25519-selftest.o: In function `curve25519_selftest': curve25519-selftest.c:(.init.text+0x17e): undefined reference to `curve25519_base_arch' This is because the curve25519 self-test code is being controlled by the GENERIC option rather than the overall CURVE25519 option, as is the case with blake2s. To recap, the GENERIC and ARCH options for CURVE25519 are internal only and selected by users such as the Crypto API, or the externally visible CURVE25519 option which in turn is selected by wireguard. The self-test is specific to the the external CURVE25519 option and should not be enabled by the Crypto API. This patch fixes this by splitting the GENERIC module from the CURVE25519 module with the latter now containing just the self-test. Reported-by: Hulk Robot Fixes: aa127963f1ca ("crypto: lib/curve25519 - re-add selftests") Signed-off-by: Herbert Xu Reviewed-by: Jason A. Donenfeld Signed-off-by: Herbert Xu --- lib/crypto/Makefile | 9 ++++++--- lib/crypto/curve25519-generic.c | 24 ++++++++++++++++++++++++ lib/crypto/curve25519.c | 7 ------- 3 files changed, 30 insertions(+), 10 deletions(-) create mode 100644 lib/crypto/curve25519-generic.c (limited to 'lib') diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 6ecaf83a5a9a..3a435629d9ce 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -19,9 +19,12 @@ libblake2s-y += blake2s.o obj-$(CONFIG_CRYPTO_LIB_CHACHA20POLY1305) += libchacha20poly1305.o libchacha20poly1305-y += chacha20poly1305.o -obj-$(CONFIG_CRYPTO_LIB_CURVE25519_GENERIC) += libcurve25519.o -libcurve25519-y := curve25519-fiat32.o -libcurve25519-$(CONFIG_ARCH_SUPPORTS_INT128) := curve25519-hacl64.o +obj-$(CONFIG_CRYPTO_LIB_CURVE25519_GENERIC) += libcurve25519-generic.o +libcurve25519-generic-y := curve25519-fiat32.o +libcurve25519-generic-$(CONFIG_ARCH_SUPPORTS_INT128) := curve25519-hacl64.o +libcurve25519-generic-y += curve25519-generic.o + +obj-$(CONFIG_CRYPTO_LIB_CURVE25519) += libcurve25519.o libcurve25519-y += curve25519.o obj-$(CONFIG_CRYPTO_LIB_DES) += libdes.o diff --git a/lib/crypto/curve25519-generic.c b/lib/crypto/curve25519-generic.c new file mode 100644 index 000000000000..de7c99172fa2 --- /dev/null +++ b/lib/crypto/curve25519-generic.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is an implementation of the Curve25519 ECDH algorithm, using either + * a 32-bit implementation or a 64-bit implementation with 128-bit integers, + * depending on what is supported by the target compiler. + * + * Information: https://cr.yp.to/ecdh.html + */ + +#include +#include + +const u8 curve25519_null_point[CURVE25519_KEY_SIZE] __aligned(32) = { 0 }; +const u8 curve25519_base_point[CURVE25519_KEY_SIZE] __aligned(32) = { 9 }; + +EXPORT_SYMBOL(curve25519_null_point); +EXPORT_SYMBOL(curve25519_base_point); +EXPORT_SYMBOL(curve25519_generic); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Curve25519 scalar multiplication"); +MODULE_AUTHOR("Jason A. Donenfeld "); diff --git a/lib/crypto/curve25519.c b/lib/crypto/curve25519.c index c03ccdb99434..288a62cd29b2 100644 --- a/lib/crypto/curve25519.c +++ b/lib/crypto/curve25519.c @@ -15,13 +15,6 @@ bool curve25519_selftest(void); -const u8 curve25519_null_point[CURVE25519_KEY_SIZE] __aligned(32) = { 0 }; -const u8 curve25519_base_point[CURVE25519_KEY_SIZE] __aligned(32) = { 9 }; - -EXPORT_SYMBOL(curve25519_null_point); -EXPORT_SYMBOL(curve25519_base_point); -EXPORT_SYMBOL(curve25519_generic); - static int __init mod_init(void) { if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && -- cgit v1.2.3 From 49a101d7169c7729c7bab6b2f896faae34bd6c3d Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Thu, 16 Jan 2020 17:58:27 +0000 Subject: lib/vdso: Only read hrtimer_res when needed in __cvdso_clock_getres() Only perform READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res) for HRES and RAW clocks. Signed-off-by: Christophe Leroy Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/7ac2f0d21652f95e2bbdfa6bd514ae6c7caf53ab.1579196675.git.christophe.leroy@c-s.fr --- lib/vdso/gettimeofday.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index f342ac1fce77..f8b8ec5e63ac 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -312,7 +312,6 @@ static __maybe_unused int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) { const struct vdso_data *vd = __arch_get_vdso_data(); - u64 hrtimer_res; u32 msk; u64 ns; @@ -323,7 +322,6 @@ int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) if (IS_ENABLED(CONFIG_TIME_NS) && vd->clock_mode == VCLOCK_TIMENS) vd = __arch_get_timens_vdso_data(); - hrtimer_res = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res); /* * Convert the clockid to a bitmask and use it to check which * clocks are handled in the VDSO directly. @@ -333,7 +331,7 @@ int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) /* * Preserves the behaviour of posix_get_hrtimer_res(). */ - ns = hrtimer_res; + ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res); } else if (msk & VDSO_COARSE) { /* * Preserves the behaviour of posix_get_coarse_res(). -- cgit v1.2.3 From c24c57a4cc8a2f64de32084958920773c0906bc7 Mon Sep 17 00:00:00 2001 From: Petr Mladek Date: Thu, 16 Jan 2020 16:31:43 +0100 Subject: livepatch/selftest: Clean up shadow variable names and type The shadow variable selftest is quite tricky. Especially it is problematic to understand what values are stored, returned, and printed. Make it easier to understand by using "int *var, **sv" variables consistently everywhere instead of the generic "void *", "ret", and "ctor_data". Signed-off-by: Petr Mladek Reviewed-by: Joe Lawrence Acked-by: Miroslav Benes Reviewed-by: Kamalesh Babulal Signed-off-by: Jiri Kosina --- lib/livepatch/test_klp_shadow_vars.c | 93 ++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 41 deletions(-) (limited to 'lib') diff --git a/lib/livepatch/test_klp_shadow_vars.c b/lib/livepatch/test_klp_shadow_vars.c index fe5c413efe96..4e94f46234e8 100644 --- a/lib/livepatch/test_klp_shadow_vars.c +++ b/lib/livepatch/test_klp_shadow_vars.c @@ -60,36 +60,43 @@ static int ptr_id(void *ptr) */ static void *shadow_get(void *obj, unsigned long id) { - void *ret = klp_shadow_get(obj, id); + int **sv; + sv = klp_shadow_get(obj, id); pr_info("klp_%s(obj=PTR%d, id=0x%lx) = PTR%d\n", - __func__, ptr_id(obj), id, ptr_id(ret)); + __func__, ptr_id(obj), id, ptr_id(sv)); - return ret; + return sv; } static void *shadow_alloc(void *obj, unsigned long id, size_t size, gfp_t gfp_flags, klp_shadow_ctor_t ctor, void *ctor_data) { - void *ret = klp_shadow_alloc(obj, id, size, gfp_flags, ctor, - ctor_data); + int *var = ctor_data; + int **sv; + + sv = klp_shadow_alloc(obj, id, size, gfp_flags, ctor, var); pr_info("klp_%s(obj=PTR%d, id=0x%lx, size=%zx, gfp_flags=%pGg), ctor=PTR%d, ctor_data=PTR%d = PTR%d\n", __func__, ptr_id(obj), id, size, &gfp_flags, ptr_id(ctor), - ptr_id(ctor_data), ptr_id(ret)); - return ret; + ptr_id(var), ptr_id(sv)); + + return sv; } static void *shadow_get_or_alloc(void *obj, unsigned long id, size_t size, gfp_t gfp_flags, klp_shadow_ctor_t ctor, void *ctor_data) { - void *ret = klp_shadow_get_or_alloc(obj, id, size, gfp_flags, ctor, - ctor_data); + int *var = ctor_data; + int **sv; + + sv = klp_shadow_get_or_alloc(obj, id, size, gfp_flags, ctor, var); pr_info("klp_%s(obj=PTR%d, id=0x%lx, size=%zx, gfp_flags=%pGg), ctor=PTR%d, ctor_data=PTR%d = PTR%d\n", __func__, ptr_id(obj), id, size, &gfp_flags, ptr_id(ctor), - ptr_id(ctor_data), ptr_id(ret)); - return ret; + ptr_id(var), ptr_id(sv)); + + return sv; } static void shadow_free(void *obj, unsigned long id, klp_shadow_dtor_t dtor) @@ -110,18 +117,22 @@ static void shadow_free_all(unsigned long id, klp_shadow_dtor_t dtor) /* Shadow variable constructor - remember simple pointer data */ static int shadow_ctor(void *obj, void *shadow_data, void *ctor_data) { - int **shadow_int = shadow_data; - *shadow_int = ctor_data; + int **sv = shadow_data; + int *var = ctor_data; + + *sv = var; pr_info("%s: PTR%d -> PTR%d\n", - __func__, ptr_id(shadow_int), ptr_id(ctor_data)); + __func__, ptr_id(sv), ptr_id(var)); return 0; } static void shadow_dtor(void *obj, void *shadow_data) { + int **sv = shadow_data; + pr_info("%s(obj=PTR%d, shadow_data=PTR%d)\n", - __func__, ptr_id(obj), ptr_id(shadow_data)); + __func__, ptr_id(obj), ptr_id(sv)); } static int test_klp_shadow_vars_init(void) @@ -134,7 +145,7 @@ static int test_klp_shadow_vars_init(void) int var1, var2, var3, var4; int **sv1, **sv2, **sv3, **sv4; - void *ret; + int **sv; ptr_id(NULL); ptr_id(&var1); @@ -146,8 +157,8 @@ static int test_klp_shadow_vars_init(void) * With an empty shadow variable hash table, expect not to find * any matches. */ - ret = shadow_get(obj, id); - if (!ret) + sv = shadow_get(obj, id); + if (!sv) pr_info(" got expected NULL result\n"); /* @@ -169,23 +180,23 @@ static int test_klp_shadow_vars_init(void) * Verify we can find our new shadow variables and that they point * to expected data. */ - ret = shadow_get(obj, id); - if (!ret) + sv = shadow_get(obj, id); + if (!sv) return -EINVAL; - if (ret == sv1 && *sv1 == &var1) + if (sv == sv1 && *sv1 == &var1) pr_info(" got expected PTR%d -> PTR%d result\n", ptr_id(sv1), ptr_id(*sv1)); - ret = shadow_get(obj + 1, id); - if (!ret) + sv = shadow_get(obj + 1, id); + if (!sv) return -EINVAL; - if (ret == sv2 && *sv2 == &var2) + if (sv == sv2 && *sv2 == &var2) pr_info(" got expected PTR%d -> PTR%d result\n", ptr_id(sv2), ptr_id(*sv2)); - ret = shadow_get(obj, id + 1); - if (!ret) + sv = shadow_get(obj, id + 1); + if (!sv) return -EINVAL; - if (ret == sv3 && *sv3 == &var3) + if (sv == sv3 && *sv3 == &var3) pr_info(" got expected PTR%d -> PTR%d result\n", ptr_id(sv3), ptr_id(*sv3)); @@ -197,10 +208,10 @@ static int test_klp_shadow_vars_init(void) if (!sv4) return -ENOMEM; - ret = shadow_get_or_alloc(obj + 2, id, size, gfp_flags, shadow_ctor, &var4); - if (!ret) + sv = shadow_get_or_alloc(obj + 2, id, size, gfp_flags, shadow_ctor, &var4); + if (!sv) return -EINVAL; - if (ret == sv4 && *sv4 == &var4) + if (sv == sv4 && *sv4 == &var4) pr_info(" got expected PTR%d -> PTR%d result\n", ptr_id(sv4), ptr_id(*sv4)); @@ -209,27 +220,27 @@ static int test_klp_shadow_vars_init(void) * longer find them. */ shadow_free(obj, id, shadow_dtor); /* sv1 */ - ret = shadow_get(obj, id); - if (!ret) + sv = shadow_get(obj, id); + if (!sv) pr_info(" got expected NULL result\n"); shadow_free(obj + 1, id, shadow_dtor); /* sv2 */ - ret = shadow_get(obj + 1, id); - if (!ret) + sv = shadow_get(obj + 1, id); + if (!sv) pr_info(" got expected NULL result\n"); shadow_free(obj + 2, id, shadow_dtor); /* sv4 */ - ret = shadow_get(obj + 2, id); - if (!ret) + sv = shadow_get(obj + 2, id); + if (!sv) pr_info(" got expected NULL result\n"); /* * We should still find an variable. */ - ret = shadow_get(obj, id + 1); - if (!ret) + sv = shadow_get(obj, id + 1); + if (!sv) return -EINVAL; - if (ret == sv3 && *sv3 == &var3) + if (sv == sv3 && *sv3 == &var3) pr_info(" got expected PTR%d -> PTR%d result\n", ptr_id(sv3), ptr_id(*sv3)); @@ -237,8 +248,8 @@ static int test_klp_shadow_vars_init(void) * Free all the variables, too. */ shadow_free_all(id + 1, shadow_dtor); /* sv3 */ - ret = shadow_get(obj, id); - if (!ret) + sv = shadow_get(obj, id); + if (!sv) pr_info(" shadow_get() got expected NULL result\n"); -- cgit v1.2.3 From be6da98425b69388ed31b18bd2497f826116f29b Mon Sep 17 00:00:00 2001 From: Petr Mladek Date: Thu, 16 Jan 2020 16:31:44 +0100 Subject: livepatch/samples/selftest: Use klp_shadow_alloc() API correctly The commit e91c2518a5d22a ("livepatch: Initialize shadow variables safely by a custom callback") leads to the following static checker warning: samples/livepatch/livepatch-shadow-fix1.c:86 livepatch_fix1_dummy_alloc() error: 'klp_shadow_alloc()' 'leak' too small (4 vs 8) It is because klp_shadow_alloc() is used a wrong way: int *leak; shadow_leak = klp_shadow_alloc(d, SV_LEAK, sizeof(leak), GFP_KERNEL, shadow_leak_ctor, leak); The code is supposed to store the "leak" pointer into the shadow variable. 3rd parameter correctly passes size of the data (size of pointer). But the 5th parameter is wrong. It should pass pointer to the data (pointer to the pointer) but it passes the pointer directly. It works because shadow_leak_ctor() handle "ctor_data" as the data instead of pointer to the data. But it is semantically wrong and confusing. The same problem is also in the module used by selftests. In this case, "pvX" variables are introduced. They represent the data stored in the shadow variables. Reported-by: Dan Carpenter Signed-off-by: Petr Mladek Reviewed-by: Joe Lawrence Acked-by: Miroslav Benes Reviewed-by: Kamalesh Babulal Signed-off-by: Jiri Kosina --- lib/livepatch/test_klp_shadow_vars.c | 52 ++++++++++++++++++------------- samples/livepatch/livepatch-shadow-fix1.c | 9 ++++-- 2 files changed, 36 insertions(+), 25 deletions(-) (limited to 'lib') diff --git a/lib/livepatch/test_klp_shadow_vars.c b/lib/livepatch/test_klp_shadow_vars.c index 4e94f46234e8..f0b5a1d24e55 100644 --- a/lib/livepatch/test_klp_shadow_vars.c +++ b/lib/livepatch/test_klp_shadow_vars.c @@ -73,13 +73,13 @@ static void *shadow_alloc(void *obj, unsigned long id, size_t size, gfp_t gfp_flags, klp_shadow_ctor_t ctor, void *ctor_data) { - int *var = ctor_data; + int **var = ctor_data; int **sv; sv = klp_shadow_alloc(obj, id, size, gfp_flags, ctor, var); pr_info("klp_%s(obj=PTR%d, id=0x%lx, size=%zx, gfp_flags=%pGg), ctor=PTR%d, ctor_data=PTR%d = PTR%d\n", __func__, ptr_id(obj), id, size, &gfp_flags, ptr_id(ctor), - ptr_id(var), ptr_id(sv)); + ptr_id(*var), ptr_id(sv)); return sv; } @@ -88,13 +88,13 @@ static void *shadow_get_or_alloc(void *obj, unsigned long id, size_t size, gfp_t gfp_flags, klp_shadow_ctor_t ctor, void *ctor_data) { - int *var = ctor_data; + int **var = ctor_data; int **sv; sv = klp_shadow_get_or_alloc(obj, id, size, gfp_flags, ctor, var); pr_info("klp_%s(obj=PTR%d, id=0x%lx, size=%zx, gfp_flags=%pGg), ctor=PTR%d, ctor_data=PTR%d = PTR%d\n", __func__, ptr_id(obj), id, size, &gfp_flags, ptr_id(ctor), - ptr_id(var), ptr_id(sv)); + ptr_id(*var), ptr_id(sv)); return sv; } @@ -118,11 +118,14 @@ static void shadow_free_all(unsigned long id, klp_shadow_dtor_t dtor) static int shadow_ctor(void *obj, void *shadow_data, void *ctor_data) { int **sv = shadow_data; - int *var = ctor_data; + int **var = ctor_data; - *sv = var; + if (!var) + return -EINVAL; + + *sv = *var; pr_info("%s: PTR%d -> PTR%d\n", - __func__, ptr_id(sv), ptr_id(var)); + __func__, ptr_id(sv), ptr_id(*var)); return 0; } @@ -139,19 +142,24 @@ static int test_klp_shadow_vars_init(void) { void *obj = THIS_MODULE; int id = 0x1234; - size_t size = sizeof(int *); gfp_t gfp_flags = GFP_KERNEL; int var1, var2, var3, var4; + int *pv1, *pv2, *pv3, *pv4; int **sv1, **sv2, **sv3, **sv4; int **sv; + pv1 = &var1; + pv2 = &var2; + pv3 = &var3; + pv4 = &var4; + ptr_id(NULL); - ptr_id(&var1); - ptr_id(&var2); - ptr_id(&var3); - ptr_id(&var4); + ptr_id(pv1); + ptr_id(pv2); + ptr_id(pv3); + ptr_id(pv4); /* * With an empty shadow variable hash table, expect not to find @@ -164,15 +172,15 @@ static int test_klp_shadow_vars_init(void) /* * Allocate a few shadow variables with different and . */ - sv1 = shadow_alloc(obj, id, size, gfp_flags, shadow_ctor, &var1); + sv1 = shadow_alloc(obj, id, sizeof(pv1), gfp_flags, shadow_ctor, &pv1); if (!sv1) return -ENOMEM; - sv2 = shadow_alloc(obj + 1, id, size, gfp_flags, shadow_ctor, &var2); + sv2 = shadow_alloc(obj + 1, id, sizeof(pv2), gfp_flags, shadow_ctor, &pv2); if (!sv2) return -ENOMEM; - sv3 = shadow_alloc(obj, id + 1, size, gfp_flags, shadow_ctor, &var3); + sv3 = shadow_alloc(obj, id + 1, sizeof(pv3), gfp_flags, shadow_ctor, &pv3); if (!sv3) return -ENOMEM; @@ -183,20 +191,20 @@ static int test_klp_shadow_vars_init(void) sv = shadow_get(obj, id); if (!sv) return -EINVAL; - if (sv == sv1 && *sv1 == &var1) + if (sv == sv1 && *sv1 == pv1) pr_info(" got expected PTR%d -> PTR%d result\n", ptr_id(sv1), ptr_id(*sv1)); sv = shadow_get(obj + 1, id); if (!sv) return -EINVAL; - if (sv == sv2 && *sv2 == &var2) + if (sv == sv2 && *sv2 == pv2) pr_info(" got expected PTR%d -> PTR%d result\n", ptr_id(sv2), ptr_id(*sv2)); sv = shadow_get(obj, id + 1); if (!sv) return -EINVAL; - if (sv == sv3 && *sv3 == &var3) + if (sv == sv3 && *sv3 == pv3) pr_info(" got expected PTR%d -> PTR%d result\n", ptr_id(sv3), ptr_id(*sv3)); @@ -204,14 +212,14 @@ static int test_klp_shadow_vars_init(void) * Allocate or get a few more, this time with the same , . * The second invocation should return the same shadow var. */ - sv4 = shadow_get_or_alloc(obj + 2, id, size, gfp_flags, shadow_ctor, &var4); + sv4 = shadow_get_or_alloc(obj + 2, id, sizeof(pv4), gfp_flags, shadow_ctor, &pv4); if (!sv4) return -ENOMEM; - sv = shadow_get_or_alloc(obj + 2, id, size, gfp_flags, shadow_ctor, &var4); + sv = shadow_get_or_alloc(obj + 2, id, sizeof(pv4), gfp_flags, shadow_ctor, &pv4); if (!sv) return -EINVAL; - if (sv == sv4 && *sv4 == &var4) + if (sv == sv4 && *sv4 == pv4) pr_info(" got expected PTR%d -> PTR%d result\n", ptr_id(sv4), ptr_id(*sv4)); @@ -240,7 +248,7 @@ static int test_klp_shadow_vars_init(void) sv = shadow_get(obj, id + 1); if (!sv) return -EINVAL; - if (sv == sv3 && *sv3 == &var3) + if (sv == sv3 && *sv3 == pv3) pr_info(" got expected PTR%d -> PTR%d result\n", ptr_id(sv3), ptr_id(*sv3)); diff --git a/samples/livepatch/livepatch-shadow-fix1.c b/samples/livepatch/livepatch-shadow-fix1.c index bab12bdb753f..de0363b288a7 100644 --- a/samples/livepatch/livepatch-shadow-fix1.c +++ b/samples/livepatch/livepatch-shadow-fix1.c @@ -53,9 +53,12 @@ struct dummy { static int shadow_leak_ctor(void *obj, void *shadow_data, void *ctor_data) { int **shadow_leak = shadow_data; - int *leak = ctor_data; + int **leak = ctor_data; - *shadow_leak = leak; + if (!ctor_data) + return -EINVAL; + + *shadow_leak = *leak; return 0; } @@ -83,7 +86,7 @@ static struct dummy *livepatch_fix1_dummy_alloc(void) } klp_shadow_alloc(d, SV_LEAK, sizeof(leak), GFP_KERNEL, - shadow_leak_ctor, leak); + shadow_leak_ctor, &leak); pr_info("%s: dummy @ %p, expires @ %lx\n", __func__, d, d->jiffies_expire); -- cgit v1.2.3 From 35fd7a637c42bb54ba4608f4d40ae6e55fc88781 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Thu, 16 Jan 2020 19:55:29 +0100 Subject: debugobjects: Fix various data races The counters obj_pool_free, and obj_nr_tofree, and the flag obj_freeing are read locklessly outside the pool_lock critical sections. If read with plain accesses, this would result in data races. This is addressed as follows: * reads outside critical sections become READ_ONCE()s (pairing with WRITE_ONCE()s added); * writes become WRITE_ONCE()s (pairing with READ_ONCE()s added); since writes happen inside critical sections, only the write and not the read of RMWs needs to be atomic, thus WRITE_ONCE(var, var +/- X) is sufficient. The data races were reported by KCSAN: BUG: KCSAN: data-race in __free_object / fill_pool write to 0xffffffff8beb04f8 of 4 bytes by interrupt on cpu 1: __free_object+0x1ee/0x8e0 lib/debugobjects.c:404 __debug_check_no_obj_freed+0x199/0x330 lib/debugobjects.c:969 debug_check_no_obj_freed+0x3c/0x44 lib/debugobjects.c:994 slab_free_hook mm/slub.c:1422 [inline] read to 0xffffffff8beb04f8 of 4 bytes by task 1 on cpu 2: fill_pool+0x3d/0x520 lib/debugobjects.c:135 __debug_object_init+0x3c/0x810 lib/debugobjects.c:536 debug_object_init lib/debugobjects.c:591 [inline] debug_object_activate+0x228/0x320 lib/debugobjects.c:677 debug_rcu_head_queue kernel/rcu/rcu.h:176 [inline] BUG: KCSAN: data-race in __debug_object_init / fill_pool read to 0xffffffff8beb04f8 of 4 bytes by task 10 on cpu 6: fill_pool+0x3d/0x520 lib/debugobjects.c:135 __debug_object_init+0x3c/0x810 lib/debugobjects.c:536 debug_object_init_on_stack+0x39/0x50 lib/debugobjects.c:606 init_timer_on_stack_key kernel/time/timer.c:742 [inline] write to 0xffffffff8beb04f8 of 4 bytes by task 1 on cpu 3: alloc_object lib/debugobjects.c:258 [inline] __debug_object_init+0x717/0x810 lib/debugobjects.c:544 debug_object_init lib/debugobjects.c:591 [inline] debug_object_activate+0x228/0x320 lib/debugobjects.c:677 debug_rcu_head_queue kernel/rcu/rcu.h:176 [inline] BUG: KCSAN: data-race in free_obj_work / free_object read to 0xffffffff9140c190 of 4 bytes by task 10 on cpu 6: free_object+0x4b/0xd0 lib/debugobjects.c:426 debug_object_free+0x190/0x210 lib/debugobjects.c:824 destroy_timer_on_stack kernel/time/timer.c:749 [inline] write to 0xffffffff9140c190 of 4 bytes by task 93 on cpu 1: free_obj_work+0x24f/0x480 lib/debugobjects.c:313 process_one_work+0x454/0x8d0 kernel/workqueue.c:2264 worker_thread+0x9a/0x780 kernel/workqueue.c:2410 Reported-by: Qian Cai Signed-off-by: Marco Elver Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20200116185529.11026-1-elver@google.com --- lib/debugobjects.c | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) (limited to 'lib') diff --git a/lib/debugobjects.c b/lib/debugobjects.c index 61261195f5b6..48054dbf1b51 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -132,14 +132,18 @@ static void fill_pool(void) struct debug_obj *obj; unsigned long flags; - if (likely(obj_pool_free >= debug_objects_pool_min_level)) + if (likely(READ_ONCE(obj_pool_free) >= debug_objects_pool_min_level)) return; /* * Reuse objs from the global free list; they will be reinitialized * when allocating. + * + * Both obj_nr_tofree and obj_pool_free are checked locklessly; the + * READ_ONCE()s pair with the WRITE_ONCE()s in pool_lock critical + * sections. */ - while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) { + while (READ_ONCE(obj_nr_tofree) && (READ_ONCE(obj_pool_free) < obj_pool_min_free)) { raw_spin_lock_irqsave(&pool_lock, flags); /* * Recheck with the lock held as the worker thread might have @@ -148,9 +152,9 @@ static void fill_pool(void) while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) { obj = hlist_entry(obj_to_free.first, typeof(*obj), node); hlist_del(&obj->node); - obj_nr_tofree--; + WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1); hlist_add_head(&obj->node, &obj_pool); - obj_pool_free++; + WRITE_ONCE(obj_pool_free, obj_pool_free + 1); } raw_spin_unlock_irqrestore(&pool_lock, flags); } @@ -158,7 +162,7 @@ static void fill_pool(void) if (unlikely(!obj_cache)) return; - while (obj_pool_free < debug_objects_pool_min_level) { + while (READ_ONCE(obj_pool_free) < debug_objects_pool_min_level) { struct debug_obj *new[ODEBUG_BATCH_SIZE]; int cnt; @@ -174,7 +178,7 @@ static void fill_pool(void) while (cnt) { hlist_add_head(&new[--cnt]->node, &obj_pool); debug_objects_allocated++; - obj_pool_free++; + WRITE_ONCE(obj_pool_free, obj_pool_free + 1); } raw_spin_unlock_irqrestore(&pool_lock, flags); } @@ -236,7 +240,7 @@ alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr) obj = __alloc_object(&obj_pool); if (obj) { obj_pool_used++; - obj_pool_free--; + WRITE_ONCE(obj_pool_free, obj_pool_free - 1); /* * Looking ahead, allocate one batch of debug objects and @@ -255,7 +259,7 @@ alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr) &percpu_pool->free_objs); percpu_pool->obj_free++; obj_pool_used++; - obj_pool_free--; + WRITE_ONCE(obj_pool_free, obj_pool_free - 1); } } @@ -309,8 +313,8 @@ static void free_obj_work(struct work_struct *work) obj = hlist_entry(obj_to_free.first, typeof(*obj), node); hlist_del(&obj->node); hlist_add_head(&obj->node, &obj_pool); - obj_pool_free++; - obj_nr_tofree--; + WRITE_ONCE(obj_pool_free, obj_pool_free + 1); + WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1); } raw_spin_unlock_irqrestore(&pool_lock, flags); return; @@ -324,7 +328,7 @@ free_objs: if (obj_nr_tofree) { hlist_move_list(&obj_to_free, &tofree); debug_objects_freed += obj_nr_tofree; - obj_nr_tofree = 0; + WRITE_ONCE(obj_nr_tofree, 0); } raw_spin_unlock_irqrestore(&pool_lock, flags); @@ -375,10 +379,10 @@ free_to_obj_pool: obj_pool_used--; if (work) { - obj_nr_tofree++; + WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1); hlist_add_head(&obj->node, &obj_to_free); if (lookahead_count) { - obj_nr_tofree += lookahead_count; + WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + lookahead_count); obj_pool_used -= lookahead_count; while (lookahead_count) { hlist_add_head(&objs[--lookahead_count]->node, @@ -396,15 +400,15 @@ free_to_obj_pool: for (i = 0; i < ODEBUG_BATCH_SIZE; i++) { obj = __alloc_object(&obj_pool); hlist_add_head(&obj->node, &obj_to_free); - obj_pool_free--; - obj_nr_tofree++; + WRITE_ONCE(obj_pool_free, obj_pool_free - 1); + WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1); } } } else { - obj_pool_free++; + WRITE_ONCE(obj_pool_free, obj_pool_free + 1); hlist_add_head(&obj->node, &obj_pool); if (lookahead_count) { - obj_pool_free += lookahead_count; + WRITE_ONCE(obj_pool_free, obj_pool_free + lookahead_count); obj_pool_used -= lookahead_count; while (lookahead_count) { hlist_add_head(&objs[--lookahead_count]->node, @@ -423,7 +427,7 @@ free_to_obj_pool: static void free_object(struct debug_obj *obj) { __free_object(obj); - if (!obj_freeing && obj_nr_tofree) { + if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) { WRITE_ONCE(obj_freeing, true); schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY); } @@ -982,7 +986,7 @@ repeat: debug_objects_maxchecked = objs_checked; /* Schedule work to actually kmem_cache_free() objects */ - if (!obj_freeing && obj_nr_tofree) { + if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) { WRITE_ONCE(obj_freeing, true); schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY); } @@ -1008,12 +1012,12 @@ static int debug_stats_show(struct seq_file *m, void *v) seq_printf(m, "max_checked :%d\n", debug_objects_maxchecked); seq_printf(m, "warnings :%d\n", debug_objects_warnings); seq_printf(m, "fixups :%d\n", debug_objects_fixups); - seq_printf(m, "pool_free :%d\n", obj_pool_free + obj_percpu_free); + seq_printf(m, "pool_free :%d\n", READ_ONCE(obj_pool_free) + obj_percpu_free); seq_printf(m, "pool_pcp_free :%d\n", obj_percpu_free); seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free); seq_printf(m, "pool_used :%d\n", obj_pool_used - obj_percpu_free); seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used); - seq_printf(m, "on_free_list :%d\n", obj_nr_tofree); + seq_printf(m, "on_free_list :%d\n", READ_ONCE(obj_nr_tofree)); seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated); seq_printf(m, "objs_freed :%d\n", debug_objects_freed); return 0; -- cgit v1.2.3 From 430f24f94c8a174d411a550d7b5529301922e67a Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 17 Jan 2020 17:45:12 -0500 Subject: XArray: Fix infinite loop with entry at ULONG_MAX If there is an entry at ULONG_MAX, xa_for_each() will overflow the 'index + 1' in xa_find_after() and wrap around to 0. Catch this case and terminate the loop by returning NULL. Signed-off-by: Matthew Wilcox (Oracle) Cc: stable@vger.kernel.org --- lib/test_xarray.c | 17 +++++++++++++++++ lib/xarray.c | 3 +++ 2 files changed, 20 insertions(+) (limited to 'lib') diff --git a/lib/test_xarray.c b/lib/test_xarray.c index c477f22a95cd..90584c63ca39 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -1046,11 +1046,28 @@ static noinline void check_find_3(struct xarray *xa) xa_destroy(xa); } +static noinline void check_find_4(struct xarray *xa) +{ + unsigned long index = 0; + void *entry; + + xa_store_index(xa, ULONG_MAX, GFP_KERNEL); + + entry = xa_find_after(xa, &index, ULONG_MAX, XA_PRESENT); + XA_BUG_ON(xa, entry != xa_mk_index(ULONG_MAX)); + + entry = xa_find_after(xa, &index, ULONG_MAX, XA_PRESENT); + XA_BUG_ON(xa, entry); + + xa_erase_index(xa, ULONG_MAX); +} + static noinline void check_find(struct xarray *xa) { check_find_1(xa); check_find_2(xa); check_find_3(xa); + check_find_4(xa); check_multi_find(xa); check_multi_find_2(xa); } diff --git a/lib/xarray.c b/lib/xarray.c index bfaaa2c1f3fe..6ecf35c2e1da 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -1849,6 +1849,9 @@ void *xa_find_after(struct xarray *xa, unsigned long *indexp, XA_STATE(xas, xa, *indexp + 1); void *entry; + if (xas.xa_index == 0) + return NULL; + rcu_read_lock(); for (;;) { if ((__force unsigned int)filter < XA_MAX_MARKS) -- cgit v1.2.3 From 19c30f4dd0923ef191f35c652ee4058e91e89056 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 17 Jan 2020 22:00:41 -0500 Subject: XArray: Fix xa_find_after with multi-index entries If the entry is of an order which is a multiple of XA_CHUNK_SIZE, the current detection of sibling entries does not work. Factor out an xas_sibling() function to make xa_find_after() a little more understandable, and write a new implementation that doesn't suffer from the same bug. Signed-off-by: Matthew Wilcox (Oracle) Cc: stable@vger.kernel.org --- lib/test_xarray.c | 32 +++++++++++++++++++------------- lib/xarray.c | 20 +++++++++++++------- 2 files changed, 32 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/test_xarray.c b/lib/test_xarray.c index 90584c63ca39..c8cc97ded0fa 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -902,28 +902,30 @@ static noinline void check_store_iter(struct xarray *xa) XA_BUG_ON(xa, !xa_empty(xa)); } -static noinline void check_multi_find(struct xarray *xa) +static noinline void check_multi_find_1(struct xarray *xa, unsigned order) { #ifdef CONFIG_XARRAY_MULTI + unsigned long multi = 3 << order; + unsigned long next = 4 << order; unsigned long index; - xa_store_order(xa, 12, 2, xa_mk_value(12), GFP_KERNEL); - XA_BUG_ON(xa, xa_store_index(xa, 16, GFP_KERNEL) != NULL); + xa_store_order(xa, multi, order, xa_mk_value(multi), GFP_KERNEL); + XA_BUG_ON(xa, xa_store_index(xa, next, GFP_KERNEL) != NULL); index = 0; XA_BUG_ON(xa, xa_find(xa, &index, ULONG_MAX, XA_PRESENT) != - xa_mk_value(12)); - XA_BUG_ON(xa, index != 12); - index = 13; + xa_mk_value(multi)); + XA_BUG_ON(xa, index != multi); + index = multi + 1; XA_BUG_ON(xa, xa_find(xa, &index, ULONG_MAX, XA_PRESENT) != - xa_mk_value(12)); - XA_BUG_ON(xa, (index < 12) || (index >= 16)); + xa_mk_value(multi)); + XA_BUG_ON(xa, (index < multi) || (index >= next)); XA_BUG_ON(xa, xa_find_after(xa, &index, ULONG_MAX, XA_PRESENT) != - xa_mk_value(16)); - XA_BUG_ON(xa, index != 16); + xa_mk_value(next)); + XA_BUG_ON(xa, index != next); - xa_erase_index(xa, 12); - xa_erase_index(xa, 16); + xa_erase_index(xa, multi); + xa_erase_index(xa, next); XA_BUG_ON(xa, !xa_empty(xa)); #endif } @@ -1064,11 +1066,15 @@ static noinline void check_find_4(struct xarray *xa) static noinline void check_find(struct xarray *xa) { + unsigned i; + check_find_1(xa); check_find_2(xa); check_find_3(xa); check_find_4(xa); - check_multi_find(xa); + + for (i = 2; i < 10; i++) + check_multi_find_1(xa, i); check_multi_find_2(xa); } diff --git a/lib/xarray.c b/lib/xarray.c index 6ecf35c2e1da..03300a0d6425 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -1826,6 +1826,17 @@ void *xa_find(struct xarray *xa, unsigned long *indexp, } EXPORT_SYMBOL(xa_find); +static bool xas_sibling(struct xa_state *xas) +{ + struct xa_node *node = xas->xa_node; + unsigned long mask; + + if (!node) + return false; + mask = (XA_CHUNK_SIZE << node->shift) - 1; + return (xas->xa_index & mask) > (xas->xa_offset << node->shift); +} + /** * xa_find_after() - Search the XArray for a present entry. * @xa: XArray. @@ -1860,13 +1871,8 @@ void *xa_find_after(struct xarray *xa, unsigned long *indexp, entry = xas_find(&xas, max); if (xas.xa_node == XAS_BOUNDS) break; - if (xas.xa_shift) { - if (xas.xa_index & ((1UL << xas.xa_shift) - 1)) - continue; - } else { - if (xas.xa_offset < (xas.xa_index & XA_CHUNK_MASK)) - continue; - } + if (xas_sibling(&xas)) + continue; if (!xas_retry(&xas, entry)) break; } -- cgit v1.2.3 From c44aa5e8ab58b5f4cf473970ec784c3333496a2e Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 17 Jan 2020 22:13:21 -0500 Subject: XArray: Fix xas_find returning too many entries If you call xas_find() with the initial index > max, it should have returned NULL but was returning the entry at index. Signed-off-by: Matthew Wilcox (Oracle) Cc: stable@vger.kernel.org --- lib/test_xarray.c | 5 +++++ lib/xarray.c | 10 ++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/test_xarray.c b/lib/test_xarray.c index c8cc97ded0fa..55c14e8c8859 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -2,6 +2,7 @@ /* * test_xarray.c: Test the XArray API * Copyright (c) 2017-2018 Microsoft Corporation + * Copyright (c) 2019-2020 Oracle * Author: Matthew Wilcox */ @@ -911,6 +912,7 @@ static noinline void check_multi_find_1(struct xarray *xa, unsigned order) xa_store_order(xa, multi, order, xa_mk_value(multi), GFP_KERNEL); XA_BUG_ON(xa, xa_store_index(xa, next, GFP_KERNEL) != NULL); + XA_BUG_ON(xa, xa_store_index(xa, next + 1, GFP_KERNEL) != NULL); index = 0; XA_BUG_ON(xa, xa_find(xa, &index, ULONG_MAX, XA_PRESENT) != @@ -923,9 +925,12 @@ static noinline void check_multi_find_1(struct xarray *xa, unsigned order) XA_BUG_ON(xa, xa_find_after(xa, &index, ULONG_MAX, XA_PRESENT) != xa_mk_value(next)); XA_BUG_ON(xa, index != next); + XA_BUG_ON(xa, xa_find_after(xa, &index, next, XA_PRESENT) != NULL); + XA_BUG_ON(xa, index != next); xa_erase_index(xa, multi); xa_erase_index(xa, next); + xa_erase_index(xa, next + 1); XA_BUG_ON(xa, !xa_empty(xa)); #endif } diff --git a/lib/xarray.c b/lib/xarray.c index 03300a0d6425..1d9fab7db8da 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -1,7 +1,8 @@ // SPDX-License-Identifier: GPL-2.0+ /* * XArray implementation - * Copyright (c) 2017 Microsoft Corporation + * Copyright (c) 2017-2018 Microsoft Corporation + * Copyright (c) 2018-2020 Oracle * Author: Matthew Wilcox */ @@ -1083,6 +1084,8 @@ void *xas_find(struct xa_state *xas, unsigned long max) if (xas_error(xas) || xas->xa_node == XAS_BOUNDS) return NULL; + if (xas->xa_index > max) + return set_bounds(xas); if (!xas->xa_node) { xas->xa_index = 1; @@ -1152,6 +1155,8 @@ void *xas_find_marked(struct xa_state *xas, unsigned long max, xa_mark_t mark) if (xas_error(xas)) return NULL; + if (xas->xa_index > max) + goto max; if (!xas->xa_node) { xas->xa_index = 1; @@ -1869,7 +1874,8 @@ void *xa_find_after(struct xarray *xa, unsigned long *indexp, entry = xas_find_marked(&xas, max, filter); else entry = xas_find(&xas, max); - if (xas.xa_node == XAS_BOUNDS) + + if (xas_invalid(&xas)) break; if (xas_sibling(&xas)) continue; -- cgit v1.2.3 From 72c7943792c9e7788ddd182337bcf8f650cf56f5 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Thu, 16 Jan 2020 21:26:34 +0100 Subject: crypto: chacha20poly1305 - add back missing test vectors and test chunking When this was originally ported, the 12-byte nonce vectors were left out to keep things simple. I agree that we don't need nor want a library interface for 12-byte nonces. But these test vectors were specially crafted to look at issues in the underlying primitives and related interactions. Therefore, we actually want to keep around all of the test vectors, and simply have a helper function to test them with. Secondly, the sglist-based chunking code in the library interface is rather complicated, so this adds a developer-only test for ensuring that all the book keeping is correct, across a wide array of possibilities. Signed-off-by: Jason A. Donenfeld Signed-off-by: Herbert Xu --- lib/crypto/chacha20poly1305-selftest.c | 1712 +++++++++++++++++++++++++++++++- 1 file changed, 1698 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/crypto/chacha20poly1305-selftest.c b/lib/crypto/chacha20poly1305-selftest.c index 465de46dbdef..c391a91364e9 100644 --- a/lib/crypto/chacha20poly1305-selftest.c +++ b/lib/crypto/chacha20poly1305-selftest.c @@ -4,6 +4,7 @@ */ #include +#include #include #include @@ -1926,6 +1927,1104 @@ static const u8 enc_key012[] __initconst = { 0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64 }; +/* wycheproof - rfc7539 */ +static const u8 enc_input013[] __initconst = { + 0x4c, 0x61, 0x64, 0x69, 0x65, 0x73, 0x20, 0x61, + 0x6e, 0x64, 0x20, 0x47, 0x65, 0x6e, 0x74, 0x6c, + 0x65, 0x6d, 0x65, 0x6e, 0x20, 0x6f, 0x66, 0x20, + 0x74, 0x68, 0x65, 0x20, 0x63, 0x6c, 0x61, 0x73, + 0x73, 0x20, 0x6f, 0x66, 0x20, 0x27, 0x39, 0x39, + 0x3a, 0x20, 0x49, 0x66, 0x20, 0x49, 0x20, 0x63, + 0x6f, 0x75, 0x6c, 0x64, 0x20, 0x6f, 0x66, 0x66, + 0x65, 0x72, 0x20, 0x79, 0x6f, 0x75, 0x20, 0x6f, + 0x6e, 0x6c, 0x79, 0x20, 0x6f, 0x6e, 0x65, 0x20, + 0x74, 0x69, 0x70, 0x20, 0x66, 0x6f, 0x72, 0x20, + 0x74, 0x68, 0x65, 0x20, 0x66, 0x75, 0x74, 0x75, + 0x72, 0x65, 0x2c, 0x20, 0x73, 0x75, 0x6e, 0x73, + 0x63, 0x72, 0x65, 0x65, 0x6e, 0x20, 0x77, 0x6f, + 0x75, 0x6c, 0x64, 0x20, 0x62, 0x65, 0x20, 0x69, + 0x74, 0x2e +}; +static const u8 enc_output013[] __initconst = { + 0xd3, 0x1a, 0x8d, 0x34, 0x64, 0x8e, 0x60, 0xdb, + 0x7b, 0x86, 0xaf, 0xbc, 0x53, 0xef, 0x7e, 0xc2, + 0xa4, 0xad, 0xed, 0x51, 0x29, 0x6e, 0x08, 0xfe, + 0xa9, 0xe2, 0xb5, 0xa7, 0x36, 0xee, 0x62, 0xd6, + 0x3d, 0xbe, 0xa4, 0x5e, 0x8c, 0xa9, 0x67, 0x12, + 0x82, 0xfa, 0xfb, 0x69, 0xda, 0x92, 0x72, 0x8b, + 0x1a, 0x71, 0xde, 0x0a, 0x9e, 0x06, 0x0b, 0x29, + 0x05, 0xd6, 0xa5, 0xb6, 0x7e, 0xcd, 0x3b, 0x36, + 0x92, 0xdd, 0xbd, 0x7f, 0x2d, 0x77, 0x8b, 0x8c, + 0x98, 0x03, 0xae, 0xe3, 0x28, 0x09, 0x1b, 0x58, + 0xfa, 0xb3, 0x24, 0xe4, 0xfa, 0xd6, 0x75, 0x94, + 0x55, 0x85, 0x80, 0x8b, 0x48, 0x31, 0xd7, 0xbc, + 0x3f, 0xf4, 0xde, 0xf0, 0x8e, 0x4b, 0x7a, 0x9d, + 0xe5, 0x76, 0xd2, 0x65, 0x86, 0xce, 0xc6, 0x4b, + 0x61, 0x16, 0x1a, 0xe1, 0x0b, 0x59, 0x4f, 0x09, + 0xe2, 0x6a, 0x7e, 0x90, 0x2e, 0xcb, 0xd0, 0x60, + 0x06, 0x91 +}; +static const u8 enc_assoc013[] __initconst = { + 0x50, 0x51, 0x52, 0x53, 0xc0, 0xc1, 0xc2, 0xc3, + 0xc4, 0xc5, 0xc6, 0xc7 +}; +static const u8 enc_nonce013[] __initconst = { + 0x07, 0x00, 0x00, 0x00, 0x40, 0x41, 0x42, 0x43, + 0x44, 0x45, 0x46, 0x47 +}; +static const u8 enc_key013[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input014[] __initconst = { }; +static const u8 enc_output014[] __initconst = { + 0x76, 0xac, 0xb3, 0x42, 0xcf, 0x31, 0x66, 0xa5, + 0xb6, 0x3c, 0x0c, 0x0e, 0xa1, 0x38, 0x3c, 0x8d +}; +static const u8 enc_assoc014[] __initconst = { }; +static const u8 enc_nonce014[] __initconst = { + 0x4d, 0xa5, 0xbf, 0x8d, 0xfd, 0x58, 0x52, 0xc1, + 0xea, 0x12, 0x37, 0x9d +}; +static const u8 enc_key014[] __initconst = { + 0x80, 0xba, 0x31, 0x92, 0xc8, 0x03, 0xce, 0x96, + 0x5e, 0xa3, 0x71, 0xd5, 0xff, 0x07, 0x3c, 0xf0, + 0xf4, 0x3b, 0x6a, 0x2a, 0xb5, 0x76, 0xb2, 0x08, + 0x42, 0x6e, 0x11, 0x40, 0x9c, 0x09, 0xb9, 0xb0 +}; + +/* wycheproof - misc */ +static const u8 enc_input015[] __initconst = { }; +static const u8 enc_output015[] __initconst = { + 0x90, 0x6f, 0xa6, 0x28, 0x4b, 0x52, 0xf8, 0x7b, + 0x73, 0x59, 0xcb, 0xaa, 0x75, 0x63, 0xc7, 0x09 +}; +static const u8 enc_assoc015[] __initconst = { + 0xbd, 0x50, 0x67, 0x64, 0xf2, 0xd2, 0xc4, 0x10 +}; +static const u8 enc_nonce015[] __initconst = { + 0xa9, 0x2e, 0xf0, 0xac, 0x99, 0x1d, 0xd5, 0x16, + 0xa3, 0xc6, 0xf6, 0x89 +}; +static const u8 enc_key015[] __initconst = { + 0x7a, 0x4c, 0xd7, 0x59, 0x17, 0x2e, 0x02, 0xeb, + 0x20, 0x4d, 0xb2, 0xc3, 0xf5, 0xc7, 0x46, 0x22, + 0x7d, 0xf5, 0x84, 0xfc, 0x13, 0x45, 0x19, 0x63, + 0x91, 0xdb, 0xb9, 0x57, 0x7a, 0x25, 0x07, 0x42 +}; + +/* wycheproof - misc */ +static const u8 enc_input016[] __initconst = { + 0x2a +}; +static const u8 enc_output016[] __initconst = { + 0x3a, 0xca, 0xc2, 0x7d, 0xec, 0x09, 0x68, 0x80, + 0x1e, 0x9f, 0x6e, 0xde, 0xd6, 0x9d, 0x80, 0x75, + 0x22 +}; +static const u8 enc_assoc016[] __initconst = { }; +static const u8 enc_nonce016[] __initconst = { + 0x99, 0xe2, 0x3e, 0xc4, 0x89, 0x85, 0xbc, 0xcd, + 0xee, 0xab, 0x60, 0xf1 +}; +static const u8 enc_key016[] __initconst = { + 0xcc, 0x56, 0xb6, 0x80, 0x55, 0x2e, 0xb7, 0x50, + 0x08, 0xf5, 0x48, 0x4b, 0x4c, 0xb8, 0x03, 0xfa, + 0x50, 0x63, 0xeb, 0xd6, 0xea, 0xb9, 0x1f, 0x6a, + 0xb6, 0xae, 0xf4, 0x91, 0x6a, 0x76, 0x62, 0x73 +}; + +/* wycheproof - misc */ +static const u8 enc_input017[] __initconst = { + 0x51 +}; +static const u8 enc_output017[] __initconst = { + 0xc4, 0x16, 0x83, 0x10, 0xca, 0x45, 0xb1, 0xf7, + 0xc6, 0x6c, 0xad, 0x4e, 0x99, 0xe4, 0x3f, 0x72, + 0xb9 +}; +static const u8 enc_assoc017[] __initconst = { + 0x91, 0xca, 0x6c, 0x59, 0x2c, 0xbc, 0xca, 0x53 +}; +static const u8 enc_nonce017[] __initconst = { + 0xab, 0x0d, 0xca, 0x71, 0x6e, 0xe0, 0x51, 0xd2, + 0x78, 0x2f, 0x44, 0x03 +}; +static const u8 enc_key017[] __initconst = { + 0x46, 0xf0, 0x25, 0x49, 0x65, 0xf7, 0x69, 0xd5, + 0x2b, 0xdb, 0x4a, 0x70, 0xb4, 0x43, 0x19, 0x9f, + 0x8e, 0xf2, 0x07, 0x52, 0x0d, 0x12, 0x20, 0xc5, + 0x5e, 0x4b, 0x70, 0xf0, 0xfd, 0xa6, 0x20, 0xee +}; + +/* wycheproof - misc */ +static const u8 enc_input018[] __initconst = { + 0x5c, 0x60 +}; +static const u8 enc_output018[] __initconst = { + 0x4d, 0x13, 0x91, 0xe8, 0xb6, 0x1e, 0xfb, 0x39, + 0xc1, 0x22, 0x19, 0x54, 0x53, 0x07, 0x7b, 0x22, + 0xe5, 0xe2 +}; +static const u8 enc_assoc018[] __initconst = { }; +static const u8 enc_nonce018[] __initconst = { + 0x46, 0x1a, 0xf1, 0x22, 0xe9, 0xf2, 0xe0, 0x34, + 0x7e, 0x03, 0xf2, 0xdb +}; +static const u8 enc_key018[] __initconst = { + 0x2f, 0x7f, 0x7e, 0x4f, 0x59, 0x2b, 0xb3, 0x89, + 0x19, 0x49, 0x89, 0x74, 0x35, 0x07, 0xbf, 0x3e, + 0xe9, 0xcb, 0xde, 0x17, 0x86, 0xb6, 0x69, 0x5f, + 0xe6, 0xc0, 0x25, 0xfd, 0x9b, 0xa4, 0xc1, 0x00 +}; + +/* wycheproof - misc */ +static const u8 enc_input019[] __initconst = { + 0xdd, 0xf2 +}; +static const u8 enc_output019[] __initconst = { + 0xb6, 0x0d, 0xea, 0xd0, 0xfd, 0x46, 0x97, 0xec, + 0x2e, 0x55, 0x58, 0x23, 0x77, 0x19, 0xd0, 0x24, + 0x37, 0xa2 +}; +static const u8 enc_assoc019[] __initconst = { + 0x88, 0x36, 0x4f, 0xc8, 0x06, 0x05, 0x18, 0xbf +}; +static const u8 enc_nonce019[] __initconst = { + 0x61, 0x54, 0x6b, 0xa5, 0xf1, 0x72, 0x05, 0x90, + 0xb6, 0x04, 0x0a, 0xc6 +}; +static const u8 enc_key019[] __initconst = { + 0xc8, 0x83, 0x3d, 0xce, 0x5e, 0xa9, 0xf2, 0x48, + 0xaa, 0x20, 0x30, 0xea, 0xcf, 0xe7, 0x2b, 0xff, + 0xe6, 0x9a, 0x62, 0x0c, 0xaf, 0x79, 0x33, 0x44, + 0xe5, 0x71, 0x8f, 0xe0, 0xd7, 0xab, 0x1a, 0x58 +}; + +/* wycheproof - misc */ +static const u8 enc_input020[] __initconst = { + 0xab, 0x85, 0xe9, 0xc1, 0x57, 0x17, 0x31 +}; +static const u8 enc_output020[] __initconst = { + 0x5d, 0xfe, 0x34, 0x40, 0xdb, 0xb3, 0xc3, 0xed, + 0x7a, 0x43, 0x4e, 0x26, 0x02, 0xd3, 0x94, 0x28, + 0x1e, 0x0a, 0xfa, 0x9f, 0xb7, 0xaa, 0x42 +}; +static const u8 enc_assoc020[] __initconst = { }; +static const u8 enc_nonce020[] __initconst = { + 0x3c, 0x4e, 0x65, 0x4d, 0x66, 0x3f, 0xa4, 0x59, + 0x6d, 0xc5, 0x5b, 0xb7 +}; +static const u8 enc_key020[] __initconst = { + 0x55, 0x56, 0x81, 0x58, 0xd3, 0xa6, 0x48, 0x3f, + 0x1f, 0x70, 0x21, 0xea, 0xb6, 0x9b, 0x70, 0x3f, + 0x61, 0x42, 0x51, 0xca, 0xdc, 0x1a, 0xf5, 0xd3, + 0x4a, 0x37, 0x4f, 0xdb, 0xfc, 0x5a, 0xda, 0xc7 +}; + +/* wycheproof - misc */ +static const u8 enc_input021[] __initconst = { + 0x4e, 0xe5, 0xcd, 0xa2, 0x0d, 0x42, 0x90 +}; +static const u8 enc_output021[] __initconst = { + 0x4b, 0xd4, 0x72, 0x12, 0x94, 0x1c, 0xe3, 0x18, + 0x5f, 0x14, 0x08, 0xee, 0x7f, 0xbf, 0x18, 0xf5, + 0xab, 0xad, 0x6e, 0x22, 0x53, 0xa1, 0xba +}; +static const u8 enc_assoc021[] __initconst = { + 0x84, 0xe4, 0x6b, 0xe8, 0xc0, 0x91, 0x90, 0x53 +}; +static const u8 enc_nonce021[] __initconst = { + 0x58, 0x38, 0x93, 0x75, 0xc6, 0x9e, 0xe3, 0x98, + 0xde, 0x94, 0x83, 0x96 +}; +static const u8 enc_key021[] __initconst = { + 0xe3, 0xc0, 0x9e, 0x7f, 0xab, 0x1a, 0xef, 0xb5, + 0x16, 0xda, 0x6a, 0x33, 0x02, 0x2a, 0x1d, 0xd4, + 0xeb, 0x27, 0x2c, 0x80, 0xd5, 0x40, 0xc5, 0xda, + 0x52, 0xa7, 0x30, 0xf3, 0x4d, 0x84, 0x0d, 0x7f +}; + +/* wycheproof - misc */ +static const u8 enc_input022[] __initconst = { + 0xbe, 0x33, 0x08, 0xf7, 0x2a, 0x2c, 0x6a, 0xed +}; +static const u8 enc_output022[] __initconst = { + 0x8e, 0x94, 0x39, 0xa5, 0x6e, 0xee, 0xc8, 0x17, + 0xfb, 0xe8, 0xa6, 0xed, 0x8f, 0xab, 0xb1, 0x93, + 0x75, 0x39, 0xdd, 0x6c, 0x00, 0xe9, 0x00, 0x21 +}; +static const u8 enc_assoc022[] __initconst = { }; +static const u8 enc_nonce022[] __initconst = { + 0x4f, 0x07, 0xaf, 0xed, 0xfd, 0xc3, 0xb6, 0xc2, + 0x36, 0x18, 0x23, 0xd3 +}; +static const u8 enc_key022[] __initconst = { + 0x51, 0xe4, 0xbf, 0x2b, 0xad, 0x92, 0xb7, 0xaf, + 0xf1, 0xa4, 0xbc, 0x05, 0x55, 0x0b, 0xa8, 0x1d, + 0xf4, 0xb9, 0x6f, 0xab, 0xf4, 0x1c, 0x12, 0xc7, + 0xb0, 0x0e, 0x60, 0xe4, 0x8d, 0xb7, 0xe1, 0x52 +}; + +/* wycheproof - misc */ +static const u8 enc_input023[] __initconst = { + 0xa4, 0xc9, 0xc2, 0x80, 0x1b, 0x71, 0xf7, 0xdf +}; +static const u8 enc_output023[] __initconst = { + 0xb9, 0xb9, 0x10, 0x43, 0x3a, 0xf0, 0x52, 0xb0, + 0x45, 0x30, 0xf5, 0x1a, 0xee, 0xe0, 0x24, 0xe0, + 0xa4, 0x45, 0xa6, 0x32, 0x8f, 0xa6, 0x7a, 0x18 +}; +static const u8 enc_assoc023[] __initconst = { + 0x66, 0xc0, 0xae, 0x70, 0x07, 0x6c, 0xb1, 0x4d +}; +static const u8 enc_nonce023[] __initconst = { + 0xb4, 0xea, 0x66, 0x6e, 0xe1, 0x19, 0x56, 0x33, + 0x66, 0x48, 0x4a, 0x78 +}; +static const u8 enc_key023[] __initconst = { + 0x11, 0x31, 0xc1, 0x41, 0x85, 0x77, 0xa0, 0x54, + 0xde, 0x7a, 0x4a, 0xc5, 0x51, 0x95, 0x0f, 0x1a, + 0x05, 0x3f, 0x9a, 0xe4, 0x6e, 0x5b, 0x75, 0xfe, + 0x4a, 0xbd, 0x56, 0x08, 0xd7, 0xcd, 0xda, 0xdd +}; + +/* wycheproof - misc */ +static const u8 enc_input024[] __initconst = { + 0x42, 0xba, 0xae, 0x59, 0x78, 0xfe, 0xaf, 0x5c, + 0x36, 0x8d, 0x14, 0xe0 +}; +static const u8 enc_output024[] __initconst = { + 0xff, 0x7d, 0xc2, 0x03, 0xb2, 0x6c, 0x46, 0x7a, + 0x6b, 0x50, 0xdb, 0x33, 0x57, 0x8c, 0x0f, 0x27, + 0x58, 0xc2, 0xe1, 0x4e, 0x36, 0xd4, 0xfc, 0x10, + 0x6d, 0xcb, 0x29, 0xb4 +}; +static const u8 enc_assoc024[] __initconst = { }; +static const u8 enc_nonce024[] __initconst = { + 0x9a, 0x59, 0xfc, 0xe2, 0x6d, 0xf0, 0x00, 0x5e, + 0x07, 0x53, 0x86, 0x56 +}; +static const u8 enc_key024[] __initconst = { + 0x99, 0xb6, 0x2b, 0xd5, 0xaf, 0xbe, 0x3f, 0xb0, + 0x15, 0xbd, 0xe9, 0x3f, 0x0a, 0xbf, 0x48, 0x39, + 0x57, 0xa1, 0xc3, 0xeb, 0x3c, 0xa5, 0x9c, 0xb5, + 0x0b, 0x39, 0xf7, 0xf8, 0xa9, 0xcc, 0x51, 0xbe +}; + +/* wycheproof - misc */ +static const u8 enc_input025[] __initconst = { + 0xfd, 0xc8, 0x5b, 0x94, 0xa4, 0xb2, 0xa6, 0xb7, + 0x59, 0xb1, 0xa0, 0xda +}; +static const u8 enc_output025[] __initconst = { + 0x9f, 0x88, 0x16, 0xde, 0x09, 0x94, 0xe9, 0x38, + 0xd9, 0xe5, 0x3f, 0x95, 0xd0, 0x86, 0xfc, 0x6c, + 0x9d, 0x8f, 0xa9, 0x15, 0xfd, 0x84, 0x23, 0xa7, + 0xcf, 0x05, 0x07, 0x2f +}; +static const u8 enc_assoc025[] __initconst = { + 0xa5, 0x06, 0xe1, 0xa5, 0xc6, 0x90, 0x93, 0xf9 +}; +static const u8 enc_nonce025[] __initconst = { + 0x58, 0xdb, 0xd4, 0xad, 0x2c, 0x4a, 0xd3, 0x5d, + 0xd9, 0x06, 0xe9, 0xce +}; +static const u8 enc_key025[] __initconst = { + 0x85, 0xf3, 0x5b, 0x62, 0x82, 0xcf, 0xf4, 0x40, + 0xbc, 0x10, 0x20, 0xc8, 0x13, 0x6f, 0xf2, 0x70, + 0x31, 0x11, 0x0f, 0xa6, 0x3e, 0xc1, 0x6f, 0x1e, + 0x82, 0x51, 0x18, 0xb0, 0x06, 0xb9, 0x12, 0x57 +}; + +/* wycheproof - misc */ +static const u8 enc_input026[] __initconst = { + 0x51, 0xf8, 0xc1, 0xf7, 0x31, 0xea, 0x14, 0xac, + 0xdb, 0x21, 0x0a, 0x6d, 0x97, 0x3e, 0x07 +}; +static const u8 enc_output026[] __initconst = { + 0x0b, 0x29, 0x63, 0x8e, 0x1f, 0xbd, 0xd6, 0xdf, + 0x53, 0x97, 0x0b, 0xe2, 0x21, 0x00, 0x42, 0x2a, + 0x91, 0x34, 0x08, 0x7d, 0x67, 0xa4, 0x6e, 0x79, + 0x17, 0x8d, 0x0a, 0x93, 0xf5, 0xe1, 0xd2 +}; +static const u8 enc_assoc026[] __initconst = { }; +static const u8 enc_nonce026[] __initconst = { + 0x68, 0xab, 0x7f, 0xdb, 0xf6, 0x19, 0x01, 0xda, + 0xd4, 0x61, 0xd2, 0x3c +}; +static const u8 enc_key026[] __initconst = { + 0x67, 0x11, 0x96, 0x27, 0xbd, 0x98, 0x8e, 0xda, + 0x90, 0x62, 0x19, 0xe0, 0x8c, 0x0d, 0x0d, 0x77, + 0x9a, 0x07, 0xd2, 0x08, 0xce, 0x8a, 0x4f, 0xe0, + 0x70, 0x9a, 0xf7, 0x55, 0xee, 0xec, 0x6d, 0xcb +}; + +/* wycheproof - misc */ +static const u8 enc_input027[] __initconst = { + 0x97, 0x46, 0x9d, 0xa6, 0x67, 0xd6, 0x11, 0x0f, + 0x9c, 0xbd, 0xa1, 0xd1, 0xa2, 0x06, 0x73 +}; +static const u8 enc_output027[] __initconst = { + 0x32, 0xdb, 0x66, 0xc4, 0xa3, 0x81, 0x9d, 0x81, + 0x55, 0x74, 0x55, 0xe5, 0x98, 0x0f, 0xed, 0xfe, + 0xae, 0x30, 0xde, 0xc9, 0x4e, 0x6a, 0xd3, 0xa9, + 0xee, 0xa0, 0x6a, 0x0d, 0x70, 0x39, 0x17 +}; +static const u8 enc_assoc027[] __initconst = { + 0x64, 0x53, 0xa5, 0x33, 0x84, 0x63, 0x22, 0x12 +}; +static const u8 enc_nonce027[] __initconst = { + 0xd9, 0x5b, 0x32, 0x43, 0xaf, 0xae, 0xf7, 0x14, + 0xc5, 0x03, 0x5b, 0x6a +}; +static const u8 enc_key027[] __initconst = { + 0xe6, 0xf1, 0x11, 0x8d, 0x41, 0xe4, 0xb4, 0x3f, + 0xb5, 0x82, 0x21, 0xb7, 0xed, 0x79, 0x67, 0x38, + 0x34, 0xe0, 0xd8, 0xac, 0x5c, 0x4f, 0xa6, 0x0b, + 0xbc, 0x8b, 0xc4, 0x89, 0x3a, 0x58, 0x89, 0x4d +}; + +/* wycheproof - misc */ +static const u8 enc_input028[] __initconst = { + 0x54, 0x9b, 0x36, 0x5a, 0xf9, 0x13, 0xf3, 0xb0, + 0x81, 0x13, 0x1c, 0xcb, 0x6b, 0x82, 0x55, 0x88 +}; +static const u8 enc_output028[] __initconst = { + 0xe9, 0x11, 0x0e, 0x9f, 0x56, 0xab, 0x3c, 0xa4, + 0x83, 0x50, 0x0c, 0xea, 0xba, 0xb6, 0x7a, 0x13, + 0x83, 0x6c, 0xca, 0xbf, 0x15, 0xa6, 0xa2, 0x2a, + 0x51, 0xc1, 0x07, 0x1c, 0xfa, 0x68, 0xfa, 0x0c +}; +static const u8 enc_assoc028[] __initconst = { }; +static const u8 enc_nonce028[] __initconst = { + 0x2f, 0xcb, 0x1b, 0x38, 0xa9, 0x9e, 0x71, 0xb8, + 0x47, 0x40, 0xad, 0x9b +}; +static const u8 enc_key028[] __initconst = { + 0x59, 0xd4, 0xea, 0xfb, 0x4d, 0xe0, 0xcf, 0xc7, + 0xd3, 0xdb, 0x99, 0xa8, 0xf5, 0x4b, 0x15, 0xd7, + 0xb3, 0x9f, 0x0a, 0xcc, 0x8d, 0xa6, 0x97, 0x63, + 0xb0, 0x19, 0xc1, 0x69, 0x9f, 0x87, 0x67, 0x4a +}; + +/* wycheproof - misc */ +static const u8 enc_input029[] __initconst = { + 0x55, 0xa4, 0x65, 0x64, 0x4f, 0x5b, 0x65, 0x09, + 0x28, 0xcb, 0xee, 0x7c, 0x06, 0x32, 0x14, 0xd6 +}; +static const u8 enc_output029[] __initconst = { + 0xe4, 0xb1, 0x13, 0xcb, 0x77, 0x59, 0x45, 0xf3, + 0xd3, 0xa8, 0xae, 0x9e, 0xc1, 0x41, 0xc0, 0x0c, + 0x7c, 0x43, 0xf1, 0x6c, 0xe0, 0x96, 0xd0, 0xdc, + 0x27, 0xc9, 0x58, 0x49, 0xdc, 0x38, 0x3b, 0x7d +}; +static const u8 enc_assoc029[] __initconst = { + 0x03, 0x45, 0x85, 0x62, 0x1a, 0xf8, 0xd7, 0xff +}; +static const u8 enc_nonce029[] __initconst = { + 0x11, 0x8a, 0x69, 0x64, 0xc2, 0xd3, 0xe3, 0x80, + 0x07, 0x1f, 0x52, 0x66 +}; +static const u8 enc_key029[] __initconst = { + 0xb9, 0x07, 0xa4, 0x50, 0x75, 0x51, 0x3f, 0xe8, + 0xa8, 0x01, 0x9e, 0xde, 0xe3, 0xf2, 0x59, 0x14, + 0x87, 0xb2, 0xa0, 0x30, 0xb0, 0x3c, 0x6e, 0x1d, + 0x77, 0x1c, 0x86, 0x25, 0x71, 0xd2, 0xea, 0x1e +}; + +/* wycheproof - misc */ +static const u8 enc_input030[] __initconst = { + 0x3f, 0xf1, 0x51, 0x4b, 0x1c, 0x50, 0x39, 0x15, + 0x91, 0x8f, 0x0c, 0x0c, 0x31, 0x09, 0x4a, 0x6e, + 0x1f +}; +static const u8 enc_output030[] __initconst = { + 0x02, 0xcc, 0x3a, 0xcb, 0x5e, 0xe1, 0xfc, 0xdd, + 0x12, 0xa0, 0x3b, 0xb8, 0x57, 0x97, 0x64, 0x74, + 0xd3, 0xd8, 0x3b, 0x74, 0x63, 0xa2, 0xc3, 0x80, + 0x0f, 0xe9, 0x58, 0xc2, 0x8e, 0xaa, 0x29, 0x08, + 0x13 +}; +static const u8 enc_assoc030[] __initconst = { }; +static const u8 enc_nonce030[] __initconst = { + 0x45, 0xaa, 0xa3, 0xe5, 0xd1, 0x6d, 0x2d, 0x42, + 0xdc, 0x03, 0x44, 0x5d +}; +static const u8 enc_key030[] __initconst = { + 0x3b, 0x24, 0x58, 0xd8, 0x17, 0x6e, 0x16, 0x21, + 0xc0, 0xcc, 0x24, 0xc0, 0xc0, 0xe2, 0x4c, 0x1e, + 0x80, 0xd7, 0x2f, 0x7e, 0xe9, 0x14, 0x9a, 0x4b, + 0x16, 0x61, 0x76, 0x62, 0x96, 0x16, 0xd0, 0x11 +}; + +/* wycheproof - misc */ +static const u8 enc_input031[] __initconst = { + 0x63, 0x85, 0x8c, 0xa3, 0xe2, 0xce, 0x69, 0x88, + 0x7b, 0x57, 0x8a, 0x3c, 0x16, 0x7b, 0x42, 0x1c, + 0x9c +}; +static const u8 enc_output031[] __initconst = { + 0x35, 0x76, 0x64, 0x88, 0xd2, 0xbc, 0x7c, 0x2b, + 0x8d, 0x17, 0xcb, 0xbb, 0x9a, 0xbf, 0xad, 0x9e, + 0x6d, 0x1f, 0x39, 0x1e, 0x65, 0x7b, 0x27, 0x38, + 0xdd, 0xa0, 0x84, 0x48, 0xcb, 0xa2, 0x81, 0x1c, + 0xeb +}; +static const u8 enc_assoc031[] __initconst = { + 0x9a, 0xaf, 0x29, 0x9e, 0xee, 0xa7, 0x8f, 0x79 +}; +static const u8 enc_nonce031[] __initconst = { + 0xf0, 0x38, 0x4f, 0xb8, 0x76, 0x12, 0x14, 0x10, + 0x63, 0x3d, 0x99, 0x3d +}; +static const u8 enc_key031[] __initconst = { + 0xf6, 0x0c, 0x6a, 0x1b, 0x62, 0x57, 0x25, 0xf7, + 0x6c, 0x70, 0x37, 0xb4, 0x8f, 0xe3, 0x57, 0x7f, + 0xa7, 0xf7, 0xb8, 0x7b, 0x1b, 0xd5, 0xa9, 0x82, + 0x17, 0x6d, 0x18, 0x23, 0x06, 0xff, 0xb8, 0x70 +}; + +/* wycheproof - misc */ +static const u8 enc_input032[] __initconst = { + 0x10, 0xf1, 0xec, 0xf9, 0xc6, 0x05, 0x84, 0x66, + 0x5d, 0x9a, 0xe5, 0xef, 0xe2, 0x79, 0xe7, 0xf7, + 0x37, 0x7e, 0xea, 0x69, 0x16, 0xd2, 0xb1, 0x11 +}; +static const u8 enc_output032[] __initconst = { + 0x42, 0xf2, 0x6c, 0x56, 0xcb, 0x4b, 0xe2, 0x1d, + 0x9d, 0x8d, 0x0c, 0x80, 0xfc, 0x99, 0xdd, 0xe0, + 0x0d, 0x75, 0xf3, 0x80, 0x74, 0xbf, 0xe7, 0x64, + 0x54, 0xaa, 0x7e, 0x13, 0xd4, 0x8f, 0xff, 0x7d, + 0x75, 0x57, 0x03, 0x94, 0x57, 0x04, 0x0a, 0x3a +}; +static const u8 enc_assoc032[] __initconst = { }; +static const u8 enc_nonce032[] __initconst = { + 0xe6, 0xb1, 0xad, 0xf2, 0xfd, 0x58, 0xa8, 0x76, + 0x2c, 0x65, 0xf3, 0x1b +}; +static const u8 enc_key032[] __initconst = { + 0x02, 0x12, 0xa8, 0xde, 0x50, 0x07, 0xed, 0x87, + 0xb3, 0x3f, 0x1a, 0x70, 0x90, 0xb6, 0x11, 0x4f, + 0x9e, 0x08, 0xce, 0xfd, 0x96, 0x07, 0xf2, 0xc2, + 0x76, 0xbd, 0xcf, 0xdb, 0xc5, 0xce, 0x9c, 0xd7 +}; + +/* wycheproof - misc */ +static const u8 enc_input033[] __initconst = { + 0x92, 0x22, 0xf9, 0x01, 0x8e, 0x54, 0xfd, 0x6d, + 0xe1, 0x20, 0x08, 0x06, 0xa9, 0xee, 0x8e, 0x4c, + 0xc9, 0x04, 0xd2, 0x9f, 0x25, 0xcb, 0xa1, 0x93 +}; +static const u8 enc_output033[] __initconst = { + 0x12, 0x30, 0x32, 0x43, 0x7b, 0x4b, 0xfd, 0x69, + 0x20, 0xe8, 0xf7, 0xe7, 0xe0, 0x08, 0x7a, 0xe4, + 0x88, 0x9e, 0xbe, 0x7a, 0x0a, 0xd0, 0xe9, 0x00, + 0x3c, 0xf6, 0x8f, 0x17, 0x95, 0x50, 0xda, 0x63, + 0xd3, 0xb9, 0x6c, 0x2d, 0x55, 0x41, 0x18, 0x65 +}; +static const u8 enc_assoc033[] __initconst = { + 0x3e, 0x8b, 0xc5, 0xad, 0xe1, 0x82, 0xff, 0x08 +}; +static const u8 enc_nonce033[] __initconst = { + 0x6b, 0x28, 0x2e, 0xbe, 0xcc, 0x54, 0x1b, 0xcd, + 0x78, 0x34, 0xed, 0x55 +}; +static const u8 enc_key033[] __initconst = { + 0xc5, 0xbc, 0x09, 0x56, 0x56, 0x46, 0xe7, 0xed, + 0xda, 0x95, 0x4f, 0x1f, 0x73, 0x92, 0x23, 0xda, + 0xda, 0x20, 0xb9, 0x5c, 0x44, 0xab, 0x03, 0x3d, + 0x0f, 0xae, 0x4b, 0x02, 0x83, 0xd1, 0x8b, 0xe3 +}; + +/* wycheproof - misc */ +static const u8 enc_input034[] __initconst = { + 0xb0, 0x53, 0x99, 0x92, 0x86, 0xa2, 0x82, 0x4f, + 0x42, 0xcc, 0x8c, 0x20, 0x3a, 0xb2, 0x4e, 0x2c, + 0x97, 0xa6, 0x85, 0xad, 0xcc, 0x2a, 0xd3, 0x26, + 0x62, 0x55, 0x8e, 0x55, 0xa5, 0xc7, 0x29 +}; +static const u8 enc_output034[] __initconst = { + 0x45, 0xc7, 0xd6, 0xb5, 0x3a, 0xca, 0xd4, 0xab, + 0xb6, 0x88, 0x76, 0xa6, 0xe9, 0x6a, 0x48, 0xfb, + 0x59, 0x52, 0x4d, 0x2c, 0x92, 0xc9, 0xd8, 0xa1, + 0x89, 0xc9, 0xfd, 0x2d, 0xb9, 0x17, 0x46, 0x56, + 0x6d, 0x3c, 0xa1, 0x0e, 0x31, 0x1b, 0x69, 0x5f, + 0x3e, 0xae, 0x15, 0x51, 0x65, 0x24, 0x93 +}; +static const u8 enc_assoc034[] __initconst = { }; +static const u8 enc_nonce034[] __initconst = { + 0x04, 0xa9, 0xbe, 0x03, 0x50, 0x8a, 0x5f, 0x31, + 0x37, 0x1a, 0x6f, 0xd2 +}; +static const u8 enc_key034[] __initconst = { + 0x2e, 0xb5, 0x1c, 0x46, 0x9a, 0xa8, 0xeb, 0x9e, + 0x6c, 0x54, 0xa8, 0x34, 0x9b, 0xae, 0x50, 0xa2, + 0x0f, 0x0e, 0x38, 0x27, 0x11, 0xbb, 0xa1, 0x15, + 0x2c, 0x42, 0x4f, 0x03, 0xb6, 0x67, 0x1d, 0x71 +}; + +/* wycheproof - misc */ +static const u8 enc_input035[] __initconst = { + 0xf4, 0x52, 0x06, 0xab, 0xc2, 0x55, 0x52, 0xb2, + 0xab, 0xc9, 0xab, 0x7f, 0xa2, 0x43, 0x03, 0x5f, + 0xed, 0xaa, 0xdd, 0xc3, 0xb2, 0x29, 0x39, 0x56, + 0xf1, 0xea, 0x6e, 0x71, 0x56, 0xe7, 0xeb +}; +static const u8 enc_output035[] __initconst = { + 0x46, 0xa8, 0x0c, 0x41, 0x87, 0x02, 0x47, 0x20, + 0x08, 0x46, 0x27, 0x58, 0x00, 0x80, 0xdd, 0xe5, + 0xa3, 0xf4, 0xa1, 0x10, 0x93, 0xa7, 0x07, 0x6e, + 0xd6, 0xf3, 0xd3, 0x26, 0xbc, 0x7b, 0x70, 0x53, + 0x4d, 0x4a, 0xa2, 0x83, 0x5a, 0x52, 0xe7, 0x2d, + 0x14, 0xdf, 0x0e, 0x4f, 0x47, 0xf2, 0x5f +}; +static const u8 enc_assoc035[] __initconst = { + 0x37, 0x46, 0x18, 0xa0, 0x6e, 0xa9, 0x8a, 0x48 +}; +static const u8 enc_nonce035[] __initconst = { + 0x47, 0x0a, 0x33, 0x9e, 0xcb, 0x32, 0x19, 0xb8, + 0xb8, 0x1a, 0x1f, 0x8b +}; +static const u8 enc_key035[] __initconst = { + 0x7f, 0x5b, 0x74, 0xc0, 0x7e, 0xd1, 0xb4, 0x0f, + 0xd1, 0x43, 0x58, 0xfe, 0x2f, 0xf2, 0xa7, 0x40, + 0xc1, 0x16, 0xc7, 0x70, 0x65, 0x10, 0xe6, 0xa4, + 0x37, 0xf1, 0x9e, 0xa4, 0x99, 0x11, 0xce, 0xc4 +}; + +/* wycheproof - misc */ +static const u8 enc_input036[] __initconst = { + 0xb9, 0xc5, 0x54, 0xcb, 0xc3, 0x6a, 0xc1, 0x8a, + 0xe8, 0x97, 0xdf, 0x7b, 0xee, 0xca, 0xc1, 0xdb, + 0xeb, 0x4e, 0xaf, 0xa1, 0x56, 0xbb, 0x60, 0xce, + 0x2e, 0x5d, 0x48, 0xf0, 0x57, 0x15, 0xe6, 0x78 +}; +static const u8 enc_output036[] __initconst = { + 0xea, 0x29, 0xaf, 0xa4, 0x9d, 0x36, 0xe8, 0x76, + 0x0f, 0x5f, 0xe1, 0x97, 0x23, 0xb9, 0x81, 0x1e, + 0xd5, 0xd5, 0x19, 0x93, 0x4a, 0x44, 0x0f, 0x50, + 0x81, 0xac, 0x43, 0x0b, 0x95, 0x3b, 0x0e, 0x21, + 0x22, 0x25, 0x41, 0xaf, 0x46, 0xb8, 0x65, 0x33, + 0xc6, 0xb6, 0x8d, 0x2f, 0xf1, 0x08, 0xa7, 0xea +}; +static const u8 enc_assoc036[] __initconst = { }; +static const u8 enc_nonce036[] __initconst = { + 0x72, 0xcf, 0xd9, 0x0e, 0xf3, 0x02, 0x6c, 0xa2, + 0x2b, 0x7e, 0x6e, 0x6a +}; +static const u8 enc_key036[] __initconst = { + 0xe1, 0x73, 0x1d, 0x58, 0x54, 0xe1, 0xb7, 0x0c, + 0xb3, 0xff, 0xe8, 0xb7, 0x86, 0xa2, 0xb3, 0xeb, + 0xf0, 0x99, 0x43, 0x70, 0x95, 0x47, 0x57, 0xb9, + 0xdc, 0x8c, 0x7b, 0xc5, 0x35, 0x46, 0x34, 0xa3 +}; + +/* wycheproof - misc */ +static const u8 enc_input037[] __initconst = { + 0x6b, 0x26, 0x04, 0x99, 0x6c, 0xd3, 0x0c, 0x14, + 0xa1, 0x3a, 0x52, 0x57, 0xed, 0x6c, 0xff, 0xd3, + 0xbc, 0x5e, 0x29, 0xd6, 0xb9, 0x7e, 0xb1, 0x79, + 0x9e, 0xb3, 0x35, 0xe2, 0x81, 0xea, 0x45, 0x1e +}; +static const u8 enc_output037[] __initconst = { + 0x6d, 0xad, 0x63, 0x78, 0x97, 0x54, 0x4d, 0x8b, + 0xf6, 0xbe, 0x95, 0x07, 0xed, 0x4d, 0x1b, 0xb2, + 0xe9, 0x54, 0xbc, 0x42, 0x7e, 0x5d, 0xe7, 0x29, + 0xda, 0xf5, 0x07, 0x62, 0x84, 0x6f, 0xf2, 0xf4, + 0x7b, 0x99, 0x7d, 0x93, 0xc9, 0x82, 0x18, 0x9d, + 0x70, 0x95, 0xdc, 0x79, 0x4c, 0x74, 0x62, 0x32 +}; +static const u8 enc_assoc037[] __initconst = { + 0x23, 0x33, 0xe5, 0xce, 0x0f, 0x93, 0xb0, 0x59 +}; +static const u8 enc_nonce037[] __initconst = { + 0x26, 0x28, 0x80, 0xd4, 0x75, 0xf3, 0xda, 0xc5, + 0x34, 0x0d, 0xd1, 0xb8 +}; +static const u8 enc_key037[] __initconst = { + 0x27, 0xd8, 0x60, 0x63, 0x1b, 0x04, 0x85, 0xa4, + 0x10, 0x70, 0x2f, 0xea, 0x61, 0xbc, 0x87, 0x3f, + 0x34, 0x42, 0x26, 0x0c, 0xad, 0xed, 0x4a, 0xbd, + 0xe2, 0x5b, 0x78, 0x6a, 0x2d, 0x97, 0xf1, 0x45 +}; + +/* wycheproof - misc */ +static const u8 enc_input038[] __initconst = { + 0x97, 0x3d, 0x0c, 0x75, 0x38, 0x26, 0xba, 0xe4, + 0x66, 0xcf, 0x9a, 0xbb, 0x34, 0x93, 0x15, 0x2e, + 0x9d, 0xe7, 0x81, 0x9e, 0x2b, 0xd0, 0xc7, 0x11, + 0x71, 0x34, 0x6b, 0x4d, 0x2c, 0xeb, 0xf8, 0x04, + 0x1a, 0xa3, 0xce, 0xdc, 0x0d, 0xfd, 0x7b, 0x46, + 0x7e, 0x26, 0x22, 0x8b, 0xc8, 0x6c, 0x9a +}; +static const u8 enc_output038[] __initconst = { + 0xfb, 0xa7, 0x8a, 0xe4, 0xf9, 0xd8, 0x08, 0xa6, + 0x2e, 0x3d, 0xa4, 0x0b, 0xe2, 0xcb, 0x77, 0x00, + 0xc3, 0x61, 0x3d, 0x9e, 0xb2, 0xc5, 0x29, 0xc6, + 0x52, 0xe7, 0x6a, 0x43, 0x2c, 0x65, 0x8d, 0x27, + 0x09, 0x5f, 0x0e, 0xb8, 0xf9, 0x40, 0xc3, 0x24, + 0x98, 0x1e, 0xa9, 0x35, 0xe5, 0x07, 0xf9, 0x8f, + 0x04, 0x69, 0x56, 0xdb, 0x3a, 0x51, 0x29, 0x08, + 0xbd, 0x7a, 0xfc, 0x8f, 0x2a, 0xb0, 0xa9 +}; +static const u8 enc_assoc038[] __initconst = { }; +static const u8 enc_nonce038[] __initconst = { + 0xe7, 0x4a, 0x51, 0x5e, 0x7e, 0x21, 0x02, 0xb9, + 0x0b, 0xef, 0x55, 0xd2 +}; +static const u8 enc_key038[] __initconst = { + 0xcf, 0x0d, 0x40, 0xa4, 0x64, 0x4e, 0x5f, 0x51, + 0x81, 0x51, 0x65, 0xd5, 0x30, 0x1b, 0x22, 0x63, + 0x1f, 0x45, 0x44, 0xc4, 0x9a, 0x18, 0x78, 0xe3, + 0xa0, 0xa5, 0xe8, 0xe1, 0xaa, 0xe0, 0xf2, 0x64 +}; + +/* wycheproof - misc */ +static const u8 enc_input039[] __initconst = { + 0xa9, 0x89, 0x95, 0x50, 0x4d, 0xf1, 0x6f, 0x74, + 0x8b, 0xfb, 0x77, 0x85, 0xff, 0x91, 0xee, 0xb3, + 0xb6, 0x60, 0xea, 0x9e, 0xd3, 0x45, 0x0c, 0x3d, + 0x5e, 0x7b, 0x0e, 0x79, 0xef, 0x65, 0x36, 0x59, + 0xa9, 0x97, 0x8d, 0x75, 0x54, 0x2e, 0xf9, 0x1c, + 0x45, 0x67, 0x62, 0x21, 0x56, 0x40, 0xb9 +}; +static const u8 enc_output039[] __initconst = { + 0xa1, 0xff, 0xed, 0x80, 0x76, 0x18, 0x29, 0xec, + 0xce, 0x24, 0x2e, 0x0e, 0x88, 0xb1, 0x38, 0x04, + 0x90, 0x16, 0xbc, 0xa0, 0x18, 0xda, 0x2b, 0x6e, + 0x19, 0x98, 0x6b, 0x3e, 0x31, 0x8c, 0xae, 0x8d, + 0x80, 0x61, 0x98, 0xfb, 0x4c, 0x52, 0x7c, 0xc3, + 0x93, 0x50, 0xeb, 0xdd, 0xea, 0xc5, 0x73, 0xc4, + 0xcb, 0xf0, 0xbe, 0xfd, 0xa0, 0xb7, 0x02, 0x42, + 0xc6, 0x40, 0xd7, 0xcd, 0x02, 0xd7, 0xa3 +}; +static const u8 enc_assoc039[] __initconst = { + 0xb3, 0xe4, 0x06, 0x46, 0x83, 0xb0, 0x2d, 0x84 +}; +static const u8 enc_nonce039[] __initconst = { + 0xd4, 0xd8, 0x07, 0x34, 0x16, 0x83, 0x82, 0x5b, + 0x31, 0xcd, 0x4d, 0x95 +}; +static const u8 enc_key039[] __initconst = { + 0x6c, 0xbf, 0xd7, 0x1c, 0x64, 0x5d, 0x18, 0x4c, + 0xf5, 0xd2, 0x3c, 0x40, 0x2b, 0xdb, 0x0d, 0x25, + 0xec, 0x54, 0x89, 0x8c, 0x8a, 0x02, 0x73, 0xd4, + 0x2e, 0xb5, 0xbe, 0x10, 0x9f, 0xdc, 0xb2, 0xac +}; + +/* wycheproof - misc */ +static const u8 enc_input040[] __initconst = { + 0xd0, 0x96, 0x80, 0x31, 0x81, 0xbe, 0xef, 0x9e, + 0x00, 0x8f, 0xf8, 0x5d, 0x5d, 0xdc, 0x38, 0xdd, + 0xac, 0xf0, 0xf0, 0x9e, 0xe5, 0xf7, 0xe0, 0x7f, + 0x1e, 0x40, 0x79, 0xcb, 0x64, 0xd0, 0xdc, 0x8f, + 0x5e, 0x67, 0x11, 0xcd, 0x49, 0x21, 0xa7, 0x88, + 0x7d, 0xe7, 0x6e, 0x26, 0x78, 0xfd, 0xc6, 0x76, + 0x18, 0xf1, 0x18, 0x55, 0x86, 0xbf, 0xea, 0x9d, + 0x4c, 0x68, 0x5d, 0x50, 0xe4, 0xbb, 0x9a, 0x82 +}; +static const u8 enc_output040[] __initconst = { + 0x9a, 0x4e, 0xf2, 0x2b, 0x18, 0x16, 0x77, 0xb5, + 0x75, 0x5c, 0x08, 0xf7, 0x47, 0xc0, 0xf8, 0xd8, + 0xe8, 0xd4, 0xc1, 0x8a, 0x9c, 0xc2, 0x40, 0x5c, + 0x12, 0xbb, 0x51, 0xbb, 0x18, 0x72, 0xc8, 0xe8, + 0xb8, 0x77, 0x67, 0x8b, 0xec, 0x44, 0x2c, 0xfc, + 0xbb, 0x0f, 0xf4, 0x64, 0xa6, 0x4b, 0x74, 0x33, + 0x2c, 0xf0, 0x72, 0x89, 0x8c, 0x7e, 0x0e, 0xdd, + 0xf6, 0x23, 0x2e, 0xa6, 0xe2, 0x7e, 0xfe, 0x50, + 0x9f, 0xf3, 0x42, 0x7a, 0x0f, 0x32, 0xfa, 0x56, + 0x6d, 0x9c, 0xa0, 0xa7, 0x8a, 0xef, 0xc0, 0x13 +}; +static const u8 enc_assoc040[] __initconst = { }; +static const u8 enc_nonce040[] __initconst = { + 0xd6, 0x10, 0x40, 0xa3, 0x13, 0xed, 0x49, 0x28, + 0x23, 0xcc, 0x06, 0x5b +}; +static const u8 enc_key040[] __initconst = { + 0x5b, 0x1d, 0x10, 0x35, 0xc0, 0xb1, 0x7e, 0xe0, + 0xb0, 0x44, 0x47, 0x67, 0xf8, 0x0a, 0x25, 0xb8, + 0xc1, 0xb7, 0x41, 0xf4, 0xb5, 0x0a, 0x4d, 0x30, + 0x52, 0x22, 0x6b, 0xaa, 0x1c, 0x6f, 0xb7, 0x01 +}; + +/* wycheproof - misc */ +static const u8 enc_input041[] __initconst = { + 0x94, 0xee, 0x16, 0x6d, 0x6d, 0x6e, 0xcf, 0x88, + 0x32, 0x43, 0x71, 0x36, 0xb4, 0xae, 0x80, 0x5d, + 0x42, 0x88, 0x64, 0x35, 0x95, 0x86, 0xd9, 0x19, + 0x3a, 0x25, 0x01, 0x62, 0x93, 0xed, 0xba, 0x44, + 0x3c, 0x58, 0xe0, 0x7e, 0x7b, 0x71, 0x95, 0xec, + 0x5b, 0xd8, 0x45, 0x82, 0xa9, 0xd5, 0x6c, 0x8d, + 0x4a, 0x10, 0x8c, 0x7d, 0x7c, 0xe3, 0x4e, 0x6c, + 0x6f, 0x8e, 0xa1, 0xbe, 0xc0, 0x56, 0x73, 0x17 +}; +static const u8 enc_output041[] __initconst = { + 0x5f, 0xbb, 0xde, 0xcc, 0x34, 0xbe, 0x20, 0x16, + 0x14, 0xf6, 0x36, 0x03, 0x1e, 0xeb, 0x42, 0xf1, + 0xca, 0xce, 0x3c, 0x79, 0xa1, 0x2c, 0xff, 0xd8, + 0x71, 0xee, 0x8e, 0x73, 0x82, 0x0c, 0x82, 0x97, + 0x49, 0xf1, 0xab, 0xb4, 0x29, 0x43, 0x67, 0x84, + 0x9f, 0xb6, 0xc2, 0xaa, 0x56, 0xbd, 0xa8, 0xa3, + 0x07, 0x8f, 0x72, 0x3d, 0x7c, 0x1c, 0x85, 0x20, + 0x24, 0xb0, 0x17, 0xb5, 0x89, 0x73, 0xfb, 0x1e, + 0x09, 0x26, 0x3d, 0xa7, 0xb4, 0xcb, 0x92, 0x14, + 0x52, 0xf9, 0x7d, 0xca, 0x40, 0xf5, 0x80, 0xec +}; +static const u8 enc_assoc041[] __initconst = { + 0x71, 0x93, 0xf6, 0x23, 0x66, 0x33, 0x21, 0xa2 +}; +static const u8 enc_nonce041[] __initconst = { + 0xd3, 0x1c, 0x21, 0xab, 0xa1, 0x75, 0xb7, 0x0d, + 0xe4, 0xeb, 0xb1, 0x9c +}; +static const u8 enc_key041[] __initconst = { + 0x97, 0xd6, 0x35, 0xc4, 0xf4, 0x75, 0x74, 0xd9, + 0x99, 0x8a, 0x90, 0x87, 0x5d, 0xa1, 0xd3, 0xa2, + 0x84, 0xb7, 0x55, 0xb2, 0xd3, 0x92, 0x97, 0xa5, + 0x72, 0x52, 0x35, 0x19, 0x0e, 0x10, 0xa9, 0x7e +}; + +/* wycheproof - misc */ +static const u8 enc_input042[] __initconst = { + 0xb4, 0x29, 0xeb, 0x80, 0xfb, 0x8f, 0xe8, 0xba, + 0xed, 0xa0, 0xc8, 0x5b, 0x9c, 0x33, 0x34, 0x58, + 0xe7, 0xc2, 0x99, 0x2e, 0x55, 0x84, 0x75, 0x06, + 0x9d, 0x12, 0xd4, 0x5c, 0x22, 0x21, 0x75, 0x64, + 0x12, 0x15, 0x88, 0x03, 0x22, 0x97, 0xef, 0xf5, + 0x67, 0x83, 0x74, 0x2a, 0x5f, 0xc2, 0x2d, 0x74, + 0x10, 0xff, 0xb2, 0x9d, 0x66, 0x09, 0x86, 0x61, + 0xd7, 0x6f, 0x12, 0x6c, 0x3c, 0x27, 0x68, 0x9e, + 0x43, 0xb3, 0x72, 0x67, 0xca, 0xc5, 0xa3, 0xa6, + 0xd3, 0xab, 0x49, 0xe3, 0x91, 0xda, 0x29, 0xcd, + 0x30, 0x54, 0xa5, 0x69, 0x2e, 0x28, 0x07, 0xe4, + 0xc3, 0xea, 0x46, 0xc8, 0x76, 0x1d, 0x50, 0xf5, + 0x92 +}; +static const u8 enc_output042[] __initconst = { + 0xd0, 0x10, 0x2f, 0x6c, 0x25, 0x8b, 0xf4, 0x97, + 0x42, 0xce, 0xc3, 0x4c, 0xf2, 0xd0, 0xfe, 0xdf, + 0x23, 0xd1, 0x05, 0xfb, 0x4c, 0x84, 0xcf, 0x98, + 0x51, 0x5e, 0x1b, 0xc9, 0xa6, 0x4f, 0x8a, 0xd5, + 0xbe, 0x8f, 0x07, 0x21, 0xbd, 0xe5, 0x06, 0x45, + 0xd0, 0x00, 0x83, 0xc3, 0xa2, 0x63, 0xa3, 0x10, + 0x53, 0xb7, 0x60, 0x24, 0x5f, 0x52, 0xae, 0x28, + 0x66, 0xa5, 0xec, 0x83, 0xb1, 0x9f, 0x61, 0xbe, + 0x1d, 0x30, 0xd5, 0xc5, 0xd9, 0xfe, 0xcc, 0x4c, + 0xbb, 0xe0, 0x8f, 0xd3, 0x85, 0x81, 0x3a, 0x2a, + 0xa3, 0x9a, 0x00, 0xff, 0x9c, 0x10, 0xf7, 0xf2, + 0x37, 0x02, 0xad, 0xd1, 0xe4, 0xb2, 0xff, 0xa3, + 0x1c, 0x41, 0x86, 0x5f, 0xc7, 0x1d, 0xe1, 0x2b, + 0x19, 0x61, 0x21, 0x27, 0xce, 0x49, 0x99, 0x3b, + 0xb0 +}; +static const u8 enc_assoc042[] __initconst = { }; +static const u8 enc_nonce042[] __initconst = { + 0x17, 0xc8, 0x6a, 0x8a, 0xbb, 0xb7, 0xe0, 0x03, + 0xac, 0xde, 0x27, 0x99 +}; +static const u8 enc_key042[] __initconst = { + 0xfe, 0x6e, 0x55, 0xbd, 0xae, 0xd1, 0xf7, 0x28, + 0x4c, 0xa5, 0xfc, 0x0f, 0x8c, 0x5f, 0x2b, 0x8d, + 0xf5, 0x6d, 0xc0, 0xf4, 0x9e, 0x8c, 0xa6, 0x6a, + 0x41, 0x99, 0x5e, 0x78, 0x33, 0x51, 0xf9, 0x01 +}; + +/* wycheproof - misc */ +static const u8 enc_input043[] __initconst = { + 0xce, 0xb5, 0x34, 0xce, 0x50, 0xdc, 0x23, 0xff, + 0x63, 0x8a, 0xce, 0x3e, 0xf6, 0x3a, 0xb2, 0xcc, + 0x29, 0x73, 0xee, 0xad, 0xa8, 0x07, 0x85, 0xfc, + 0x16, 0x5d, 0x06, 0xc2, 0xf5, 0x10, 0x0f, 0xf5, + 0xe8, 0xab, 0x28, 0x82, 0xc4, 0x75, 0xaf, 0xcd, + 0x05, 0xcc, 0xd4, 0x9f, 0x2e, 0x7d, 0x8f, 0x55, + 0xef, 0x3a, 0x72, 0xe3, 0xdc, 0x51, 0xd6, 0x85, + 0x2b, 0x8e, 0x6b, 0x9e, 0x7a, 0xec, 0xe5, 0x7b, + 0xe6, 0x55, 0x6b, 0x0b, 0x6d, 0x94, 0x13, 0xe3, + 0x3f, 0xc5, 0xfc, 0x24, 0xa9, 0xa2, 0x05, 0xad, + 0x59, 0x57, 0x4b, 0xb3, 0x9d, 0x94, 0x4a, 0x92, + 0xdc, 0x47, 0x97, 0x0d, 0x84, 0xa6, 0xad, 0x31, + 0x76 +}; +static const u8 enc_output043[] __initconst = { + 0x75, 0x45, 0x39, 0x1b, 0x51, 0xde, 0x01, 0xd5, + 0xc5, 0x3d, 0xfa, 0xca, 0x77, 0x79, 0x09, 0x06, + 0x3e, 0x58, 0xed, 0xee, 0x4b, 0xb1, 0x22, 0x7e, + 0x71, 0x10, 0xac, 0x4d, 0x26, 0x20, 0xc2, 0xae, + 0xc2, 0xf8, 0x48, 0xf5, 0x6d, 0xee, 0xb0, 0x37, + 0xa8, 0xdc, 0xed, 0x75, 0xaf, 0xa8, 0xa6, 0xc8, + 0x90, 0xe2, 0xde, 0xe4, 0x2f, 0x95, 0x0b, 0xb3, + 0x3d, 0x9e, 0x24, 0x24, 0xd0, 0x8a, 0x50, 0x5d, + 0x89, 0x95, 0x63, 0x97, 0x3e, 0xd3, 0x88, 0x70, + 0xf3, 0xde, 0x6e, 0xe2, 0xad, 0xc7, 0xfe, 0x07, + 0x2c, 0x36, 0x6c, 0x14, 0xe2, 0xcf, 0x7c, 0xa6, + 0x2f, 0xb3, 0xd3, 0x6b, 0xee, 0x11, 0x68, 0x54, + 0x61, 0xb7, 0x0d, 0x44, 0xef, 0x8c, 0x66, 0xc5, + 0xc7, 0xbb, 0xf1, 0x0d, 0xca, 0xdd, 0x7f, 0xac, + 0xf6 +}; +static const u8 enc_assoc043[] __initconst = { + 0xa1, 0x1c, 0x40, 0xb6, 0x03, 0x76, 0x73, 0x30 +}; +static const u8 enc_nonce043[] __initconst = { + 0x46, 0x36, 0x2f, 0x45, 0xd6, 0x37, 0x9e, 0x63, + 0xe5, 0x22, 0x94, 0x60 +}; +static const u8 enc_key043[] __initconst = { + 0xaa, 0xbc, 0x06, 0x34, 0x74, 0xe6, 0x5c, 0x4c, + 0x3e, 0x9b, 0xdc, 0x48, 0x0d, 0xea, 0x97, 0xb4, + 0x51, 0x10, 0xc8, 0x61, 0x88, 0x46, 0xff, 0x6b, + 0x15, 0xbd, 0xd2, 0xa4, 0xa5, 0x68, 0x2c, 0x4e +}; + +/* wycheproof - misc */ +static const u8 enc_input044[] __initconst = { + 0xe5, 0xcc, 0xaa, 0x44, 0x1b, 0xc8, 0x14, 0x68, + 0x8f, 0x8f, 0x6e, 0x8f, 0x28, 0xb5, 0x00, 0xb2 +}; +static const u8 enc_output044[] __initconst = { + 0x7e, 0x72, 0xf5, 0xa1, 0x85, 0xaf, 0x16, 0xa6, + 0x11, 0x92, 0x1b, 0x43, 0x8f, 0x74, 0x9f, 0x0b, + 0x12, 0x42, 0xc6, 0x70, 0x73, 0x23, 0x34, 0x02, + 0x9a, 0xdf, 0xe1, 0xc5, 0x00, 0x16, 0x51, 0xe4 +}; +static const u8 enc_assoc044[] __initconst = { + 0x02 +}; +static const u8 enc_nonce044[] __initconst = { + 0x87, 0x34, 0x5f, 0x10, 0x55, 0xfd, 0x9e, 0x21, + 0x02, 0xd5, 0x06, 0x56 +}; +static const u8 enc_key044[] __initconst = { + 0x7d, 0x00, 0xb4, 0x80, 0x95, 0xad, 0xfa, 0x32, + 0x72, 0x05, 0x06, 0x07, 0xb2, 0x64, 0x18, 0x50, + 0x02, 0xba, 0x99, 0x95, 0x7c, 0x49, 0x8b, 0xe0, + 0x22, 0x77, 0x0f, 0x2c, 0xe2, 0xf3, 0x14, 0x3c +}; + +/* wycheproof - misc */ +static const u8 enc_input045[] __initconst = { + 0x02, 0xcd, 0xe1, 0x68, 0xfb, 0xa3, 0xf5, 0x44, + 0xbb, 0xd0, 0x33, 0x2f, 0x7a, 0xde, 0xad, 0xa8 +}; +static const u8 enc_output045[] __initconst = { + 0x85, 0xf2, 0x9a, 0x71, 0x95, 0x57, 0xcd, 0xd1, + 0x4d, 0x1f, 0x8f, 0xff, 0xab, 0x6d, 0x9e, 0x60, + 0x73, 0x2c, 0xa3, 0x2b, 0xec, 0xd5, 0x15, 0xa1, + 0xed, 0x35, 0x3f, 0x54, 0x2e, 0x99, 0x98, 0x58 +}; +static const u8 enc_assoc045[] __initconst = { + 0xb6, 0x48 +}; +static const u8 enc_nonce045[] __initconst = { + 0x87, 0xa3, 0x16, 0x3e, 0xc0, 0x59, 0x8a, 0xd9, + 0x5b, 0x3a, 0xa7, 0x13 +}; +static const u8 enc_key045[] __initconst = { + 0x64, 0x32, 0x71, 0x7f, 0x1d, 0xb8, 0x5e, 0x41, + 0xac, 0x78, 0x36, 0xbc, 0xe2, 0x51, 0x85, 0xa0, + 0x80, 0xd5, 0x76, 0x2b, 0x9e, 0x2b, 0x18, 0x44, + 0x4b, 0x6e, 0xc7, 0x2c, 0x3b, 0xd8, 0xe4, 0xdc +}; + +/* wycheproof - misc */ +static const u8 enc_input046[] __initconst = { + 0x16, 0xdd, 0xd2, 0x3f, 0xf5, 0x3f, 0x3d, 0x23, + 0xc0, 0x63, 0x34, 0x48, 0x70, 0x40, 0xeb, 0x47 +}; +static const u8 enc_output046[] __initconst = { + 0xc1, 0xb2, 0x95, 0x93, 0x6d, 0x56, 0xfa, 0xda, + 0xc0, 0x3e, 0x5f, 0x74, 0x2b, 0xff, 0x73, 0xa1, + 0x39, 0xc4, 0x57, 0xdb, 0xab, 0x66, 0x38, 0x2b, + 0xab, 0xb3, 0xb5, 0x58, 0x00, 0xcd, 0xa5, 0xb8 +}; +static const u8 enc_assoc046[] __initconst = { + 0xbd, 0x4c, 0xd0, 0x2f, 0xc7, 0x50, 0x2b, 0xbd, + 0xbd, 0xf6, 0xc9, 0xa3, 0xcb, 0xe8, 0xf0 +}; +static const u8 enc_nonce046[] __initconst = { + 0x6f, 0x57, 0x3a, 0xa8, 0x6b, 0xaa, 0x49, 0x2b, + 0xa4, 0x65, 0x96, 0xdf +}; +static const u8 enc_key046[] __initconst = { + 0x8e, 0x34, 0xcf, 0x73, 0xd2, 0x45, 0xa1, 0x08, + 0x2a, 0x92, 0x0b, 0x86, 0x36, 0x4e, 0xb8, 0x96, + 0xc4, 0x94, 0x64, 0x67, 0xbc, 0xb3, 0xd5, 0x89, + 0x29, 0xfc, 0xb3, 0x66, 0x90, 0xe6, 0x39, 0x4f +}; + +/* wycheproof - misc */ +static const u8 enc_input047[] __initconst = { + 0x62, 0x3b, 0x78, 0x50, 0xc3, 0x21, 0xe2, 0xcf, + 0x0c, 0x6f, 0xbc, 0xc8, 0xdf, 0xd1, 0xaf, 0xf2 +}; +static const u8 enc_output047[] __initconst = { + 0xc8, 0x4c, 0x9b, 0xb7, 0xc6, 0x1c, 0x1b, 0xcb, + 0x17, 0x77, 0x2a, 0x1c, 0x50, 0x0c, 0x50, 0x95, + 0xdb, 0xad, 0xf7, 0xa5, 0x13, 0x8c, 0xa0, 0x34, + 0x59, 0xa2, 0xcd, 0x65, 0x83, 0x1e, 0x09, 0x2f +}; +static const u8 enc_assoc047[] __initconst = { + 0x89, 0xcc, 0xe9, 0xfb, 0x47, 0x44, 0x1d, 0x07, + 0xe0, 0x24, 0x5a, 0x66, 0xfe, 0x8b, 0x77, 0x8b +}; +static const u8 enc_nonce047[] __initconst = { + 0x1a, 0x65, 0x18, 0xf0, 0x2e, 0xde, 0x1d, 0xa6, + 0x80, 0x92, 0x66, 0xd9 +}; +static const u8 enc_key047[] __initconst = { + 0xcb, 0x55, 0x75, 0xf5, 0xc7, 0xc4, 0x5c, 0x91, + 0xcf, 0x32, 0x0b, 0x13, 0x9f, 0xb5, 0x94, 0x23, + 0x75, 0x60, 0xd0, 0xa3, 0xe6, 0xf8, 0x65, 0xa6, + 0x7d, 0x4f, 0x63, 0x3f, 0x2c, 0x08, 0xf0, 0x16 +}; + +/* wycheproof - misc */ +static const u8 enc_input048[] __initconst = { + 0x87, 0xb3, 0xa4, 0xd7, 0xb2, 0x6d, 0x8d, 0x32, + 0x03, 0xa0, 0xde, 0x1d, 0x64, 0xef, 0x82, 0xe3 +}; +static const u8 enc_output048[] __initconst = { + 0x94, 0xbc, 0x80, 0x62, 0x1e, 0xd1, 0xe7, 0x1b, + 0x1f, 0xd2, 0xb5, 0xc3, 0xa1, 0x5e, 0x35, 0x68, + 0x33, 0x35, 0x11, 0x86, 0x17, 0x96, 0x97, 0x84, + 0x01, 0x59, 0x8b, 0x96, 0x37, 0x22, 0xf5, 0xb3 +}; +static const u8 enc_assoc048[] __initconst = { + 0xd1, 0x9f, 0x2d, 0x98, 0x90, 0x95, 0xf7, 0xab, + 0x03, 0xa5, 0xfd, 0xe8, 0x44, 0x16, 0xe0, 0x0c, + 0x0e +}; +static const u8 enc_nonce048[] __initconst = { + 0x56, 0x4d, 0xee, 0x49, 0xab, 0x00, 0xd2, 0x40, + 0xfc, 0x10, 0x68, 0xc3 +}; +static const u8 enc_key048[] __initconst = { + 0xa5, 0x56, 0x9e, 0x72, 0x9a, 0x69, 0xb2, 0x4b, + 0xa6, 0xe0, 0xff, 0x15, 0xc4, 0x62, 0x78, 0x97, + 0x43, 0x68, 0x24, 0xc9, 0x41, 0xe9, 0xd0, 0x0b, + 0x2e, 0x93, 0xfd, 0xdc, 0x4b, 0xa7, 0x76, 0x57 +}; + +/* wycheproof - misc */ +static const u8 enc_input049[] __initconst = { + 0xe6, 0x01, 0xb3, 0x85, 0x57, 0x79, 0x7d, 0xa2, + 0xf8, 0xa4, 0x10, 0x6a, 0x08, 0x9d, 0x1d, 0xa6 +}; +static const u8 enc_output049[] __initconst = { + 0x29, 0x9b, 0x5d, 0x3f, 0x3d, 0x03, 0xc0, 0x87, + 0x20, 0x9a, 0x16, 0xe2, 0x85, 0x14, 0x31, 0x11, + 0x4b, 0x45, 0x4e, 0xd1, 0x98, 0xde, 0x11, 0x7e, + 0x83, 0xec, 0x49, 0xfa, 0x8d, 0x85, 0x08, 0xd6 +}; +static const u8 enc_assoc049[] __initconst = { + 0x5e, 0x64, 0x70, 0xfa, 0xcd, 0x99, 0xc1, 0xd8, + 0x1e, 0x37, 0xcd, 0x44, 0x01, 0x5f, 0xe1, 0x94, + 0x80, 0xa2, 0xa4, 0xd3, 0x35, 0x2a, 0x4f, 0xf5, + 0x60, 0xc0, 0x64, 0x0f, 0xdb, 0xda +}; +static const u8 enc_nonce049[] __initconst = { + 0xdf, 0x87, 0x13, 0xe8, 0x7e, 0xc3, 0xdb, 0xcf, + 0xad, 0x14, 0xd5, 0x3e +}; +static const u8 enc_key049[] __initconst = { + 0x56, 0x20, 0x74, 0x65, 0xb4, 0xe4, 0x8e, 0x6d, + 0x04, 0x63, 0x0f, 0x4a, 0x42, 0xf3, 0x5c, 0xfc, + 0x16, 0x3a, 0xb2, 0x89, 0xc2, 0x2a, 0x2b, 0x47, + 0x84, 0xf6, 0xf9, 0x29, 0x03, 0x30, 0xbe, 0xe0 +}; + +/* wycheproof - misc */ +static const u8 enc_input050[] __initconst = { + 0xdc, 0x9e, 0x9e, 0xaf, 0x11, 0xe3, 0x14, 0x18, + 0x2d, 0xf6, 0xa4, 0xeb, 0xa1, 0x7a, 0xec, 0x9c +}; +static const u8 enc_output050[] __initconst = { + 0x60, 0x5b, 0xbf, 0x90, 0xae, 0xb9, 0x74, 0xf6, + 0x60, 0x2b, 0xc7, 0x78, 0x05, 0x6f, 0x0d, 0xca, + 0x38, 0xea, 0x23, 0xd9, 0x90, 0x54, 0xb4, 0x6b, + 0x42, 0xff, 0xe0, 0x04, 0x12, 0x9d, 0x22, 0x04 +}; +static const u8 enc_assoc050[] __initconst = { + 0xba, 0x44, 0x6f, 0x6f, 0x9a, 0x0c, 0xed, 0x22, + 0x45, 0x0f, 0xeb, 0x10, 0x73, 0x7d, 0x90, 0x07, + 0xfd, 0x69, 0xab, 0xc1, 0x9b, 0x1d, 0x4d, 0x90, + 0x49, 0xa5, 0x55, 0x1e, 0x86, 0xec, 0x2b, 0x37 +}; +static const u8 enc_nonce050[] __initconst = { + 0x8d, 0xf4, 0xb1, 0x5a, 0x88, 0x8c, 0x33, 0x28, + 0x6a, 0x7b, 0x76, 0x51 +}; +static const u8 enc_key050[] __initconst = { + 0x39, 0x37, 0x98, 0x6a, 0xf8, 0x6d, 0xaf, 0xc1, + 0xba, 0x0c, 0x46, 0x72, 0xd8, 0xab, 0xc4, 0x6c, + 0x20, 0x70, 0x62, 0x68, 0x2d, 0x9c, 0x26, 0x4a, + 0xb0, 0x6d, 0x6c, 0x58, 0x07, 0x20, 0x51, 0x30 +}; + +/* wycheproof - misc */ +static const u8 enc_input051[] __initconst = { + 0x81, 0xce, 0x84, 0xed, 0xe9, 0xb3, 0x58, 0x59, + 0xcc, 0x8c, 0x49, 0xa8, 0xf6, 0xbe, 0x7d, 0xc6 +}; +static const u8 enc_output051[] __initconst = { + 0x7b, 0x7c, 0xe0, 0xd8, 0x24, 0x80, 0x9a, 0x70, + 0xde, 0x32, 0x56, 0x2c, 0xcf, 0x2c, 0x2b, 0xbd, + 0x15, 0xd4, 0x4a, 0x00, 0xce, 0x0d, 0x19, 0xb4, + 0x23, 0x1f, 0x92, 0x1e, 0x22, 0xbc, 0x0a, 0x43 +}; +static const u8 enc_assoc051[] __initconst = { + 0xd4, 0x1a, 0x82, 0x8d, 0x5e, 0x71, 0x82, 0x92, + 0x47, 0x02, 0x19, 0x05, 0x40, 0x2e, 0xa2, 0x57, + 0xdc, 0xcb, 0xc3, 0xb8, 0x0f, 0xcd, 0x56, 0x75, + 0x05, 0x6b, 0x68, 0xbb, 0x59, 0xe6, 0x2e, 0x88, + 0x73 +}; +static const u8 enc_nonce051[] __initconst = { + 0xbe, 0x40, 0xe5, 0xf1, 0xa1, 0x18, 0x17, 0xa0, + 0xa8, 0xfa, 0x89, 0x49 +}; +static const u8 enc_key051[] __initconst = { + 0x36, 0x37, 0x2a, 0xbc, 0xdb, 0x78, 0xe0, 0x27, + 0x96, 0x46, 0xac, 0x3d, 0x17, 0x6b, 0x96, 0x74, + 0xe9, 0x15, 0x4e, 0xec, 0xf0, 0xd5, 0x46, 0x9c, + 0x65, 0x1e, 0xc7, 0xe1, 0x6b, 0x4c, 0x11, 0x99 +}; + +/* wycheproof - misc */ +static const u8 enc_input052[] __initconst = { + 0xa6, 0x67, 0x47, 0xc8, 0x9e, 0x85, 0x7a, 0xf3, + 0xa1, 0x8e, 0x2c, 0x79, 0x50, 0x00, 0x87, 0xed +}; +static const u8 enc_output052[] __initconst = { + 0xca, 0x82, 0xbf, 0xf3, 0xe2, 0xf3, 0x10, 0xcc, + 0xc9, 0x76, 0x67, 0x2c, 0x44, 0x15, 0xe6, 0x9b, + 0x57, 0x63, 0x8c, 0x62, 0xa5, 0xd8, 0x5d, 0xed, + 0x77, 0x4f, 0x91, 0x3c, 0x81, 0x3e, 0xa0, 0x32 +}; +static const u8 enc_assoc052[] __initconst = { + 0x3f, 0x2d, 0xd4, 0x9b, 0xbf, 0x09, 0xd6, 0x9a, + 0x78, 0xa3, 0xd8, 0x0e, 0xa2, 0x56, 0x66, 0x14, + 0xfc, 0x37, 0x94, 0x74, 0x19, 0x6c, 0x1a, 0xae, + 0x84, 0x58, 0x3d, 0xa7, 0x3d, 0x7f, 0xf8, 0x5c, + 0x6f, 0x42, 0xca, 0x42, 0x05, 0x6a, 0x97, 0x92, + 0xcc, 0x1b, 0x9f, 0xb3, 0xc7, 0xd2, 0x61 +}; +static const u8 enc_nonce052[] __initconst = { + 0x84, 0xc8, 0x7d, 0xae, 0x4e, 0xee, 0x27, 0x73, + 0x0e, 0xc3, 0x5d, 0x12 +}; +static const u8 enc_key052[] __initconst = { + 0x9f, 0x14, 0x79, 0xed, 0x09, 0x7d, 0x7f, 0xe5, + 0x29, 0xc1, 0x1f, 0x2f, 0x5a, 0xdd, 0x9a, 0xaf, + 0xf4, 0xa1, 0xca, 0x0b, 0x68, 0x99, 0x7a, 0x2c, + 0xb7, 0xf7, 0x97, 0x49, 0xbd, 0x90, 0xaa, 0xf4 +}; + /* wycheproof - misc */ static const u8 enc_input053[] __initconst = { 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, @@ -2759,6 +3858,126 @@ static const u8 enc_key073[] __initconst = { 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; +/* wycheproof - checking for int overflows */ +static const u8 enc_input074[] __initconst = { + 0xd4, 0x50, 0x0b, 0xf0, 0x09, 0x49, 0x35, 0x51, + 0xc3, 0x80, 0xad, 0xf5, 0x2c, 0x57, 0x3a, 0x69, + 0xdf, 0x7e, 0x8b, 0x76, 0x24, 0x63, 0x33, 0x0f, + 0xac, 0xc1, 0x6a, 0x57, 0x26, 0xbe, 0x71, 0x90, + 0xc6, 0x3c, 0x5a, 0x1c, 0x92, 0x65, 0x84, 0xa0, + 0x96, 0x75, 0x68, 0x28, 0xdc, 0xdc, 0x64, 0xac, + 0xdf, 0x96, 0x3d, 0x93, 0x1b, 0xf1, 0xda, 0xe2, + 0x38, 0xf3, 0xf1, 0x57, 0x22, 0x4a, 0xc4, 0xb5, + 0x42, 0xd7, 0x85, 0xb0, 0xdd, 0x84, 0xdb, 0x6b, + 0xe3, 0xbc, 0x5a, 0x36, 0x63, 0xe8, 0x41, 0x49, + 0xff, 0xbe, 0xd0, 0x9e, 0x54, 0xf7, 0x8f, 0x16, + 0xa8, 0x22, 0x3b, 0x24, 0xcb, 0x01, 0x9f, 0x58, + 0xb2, 0x1b, 0x0e, 0x55, 0x1e, 0x7a, 0xa0, 0x73, + 0x27, 0x62, 0x95, 0x51, 0x37, 0x6c, 0xcb, 0xc3, + 0x93, 0x76, 0x71, 0xa0, 0x62, 0x9b, 0xd9, 0x5c, + 0x99, 0x15, 0xc7, 0x85, 0x55, 0x77, 0x1e, 0x7a +}; +static const u8 enc_output074[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x0b, 0x30, 0x0d, 0x8d, 0xa5, 0x6c, 0x21, 0x85, + 0x75, 0x52, 0x79, 0x55, 0x3c, 0x4c, 0x82, 0xca +}; +static const u8 enc_assoc074[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce074[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x00, 0x02, 0x50, 0x6e +}; +static const u8 enc_key074[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input075[] __initconst = { + 0x7d, 0xe8, 0x7f, 0x67, 0x29, 0x94, 0x52, 0x75, + 0xd0, 0x65, 0x5d, 0xa4, 0xc7, 0xfd, 0xe4, 0x56, + 0x9e, 0x16, 0xf1, 0x11, 0xb5, 0xeb, 0x26, 0xc2, + 0x2d, 0x85, 0x9e, 0x3f, 0xf8, 0x22, 0xec, 0xed, + 0x3a, 0x6d, 0xd9, 0xa6, 0x0f, 0x22, 0x95, 0x7f, + 0x7b, 0x7c, 0x85, 0x7e, 0x88, 0x22, 0xeb, 0x9f, + 0xe0, 0xb8, 0xd7, 0x02, 0x21, 0x41, 0xf2, 0xd0, + 0xb4, 0x8f, 0x4b, 0x56, 0x12, 0xd3, 0x22, 0xa8, + 0x8d, 0xd0, 0xfe, 0x0b, 0x4d, 0x91, 0x79, 0x32, + 0x4f, 0x7c, 0x6c, 0x9e, 0x99, 0x0e, 0xfb, 0xd8, + 0x0e, 0x5e, 0xd6, 0x77, 0x58, 0x26, 0x49, 0x8b, + 0x1e, 0xfe, 0x0f, 0x71, 0xa0, 0xf3, 0xec, 0x5b, + 0x29, 0xcb, 0x28, 0xc2, 0x54, 0x0a, 0x7d, 0xcd, + 0x51, 0xb7, 0xda, 0xae, 0xe0, 0xff, 0x4a, 0x7f, + 0x3a, 0xc1, 0xee, 0x54, 0xc2, 0x9e, 0xe4, 0xc1, + 0x70, 0xde, 0x40, 0x8f, 0x66, 0x69, 0x21, 0x94 +}; +static const u8 enc_output075[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xc5, 0x78, 0xe2, 0xaa, 0x44, 0xd3, 0x09, 0xb7, + 0xb6, 0xa5, 0x19, 0x3b, 0xdc, 0x61, 0x18, 0xf5 +}; +static const u8 enc_assoc075[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce075[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x00, 0x03, 0x18, 0xa5 +}; +static const u8 enc_key075[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + /* wycheproof - checking for int overflows */ static const u8 enc_input076[] __initconst = { 0x1b, 0x99, 0x6f, 0x9a, 0x3c, 0xcc, 0x67, 0x85, @@ -3349,6 +4568,286 @@ static const u8 enc_key085[] __initconst = { 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; +/* wycheproof - special case tag */ +static const u8 enc_input086[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output086[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f +}; +static const u8 enc_assoc086[] __initconst = { + 0x85, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xa6, 0x90, 0x2f, 0xcb, 0xc8, 0x83, 0xbb, 0xc1, + 0x80, 0xb2, 0x56, 0xae, 0x34, 0xad, 0x7f, 0x00 +}; +static const u8 enc_nonce086[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key086[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input087[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output087[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_assoc087[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x24, 0x7e, 0x50, 0x64, 0x2a, 0x1c, 0x0a, 0x2f, + 0x8f, 0x77, 0x21, 0x96, 0x09, 0xdb, 0xa9, 0x58 +}; +static const u8 enc_nonce087[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key087[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input088[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output088[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_assoc088[] __initconst = { + 0x7c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xd9, 0xe7, 0x2c, 0x06, 0x4a, 0xc8, 0x96, 0x1f, + 0x3f, 0xa5, 0x85, 0xe0, 0xe2, 0xab, 0xd6, 0x00 +}; +static const u8 enc_nonce088[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key088[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input089[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output089[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 +}; +static const u8 enc_assoc089[] __initconst = { + 0x65, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x95, 0xaf, 0x0f, 0x4d, 0x0b, 0x68, 0x6e, 0xae, + 0xcc, 0xca, 0x43, 0x07, 0xd5, 0x96, 0xf5, 0x02 +}; +static const u8 enc_nonce089[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key089[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input090[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output090[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f +}; +static const u8 enc_assoc090[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x85, 0x40, 0xb4, 0x64, 0x35, 0x77, 0x07, 0xbe, + 0x3a, 0x39, 0xd5, 0x5c, 0x34, 0xf8, 0xbc, 0xb3 +}; +static const u8 enc_nonce090[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key090[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input091[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output091[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00 +}; +static const u8 enc_assoc091[] __initconst = { + 0x4f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x66, 0x23, 0xd9, 0x90, 0xb8, 0x98, 0xd8, 0x30, + 0xd2, 0x12, 0xaf, 0x23, 0x83, 0x33, 0x07, 0x01 +}; +static const u8 enc_nonce091[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key091[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input092[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output092[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_assoc092[] __initconst = { + 0x83, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x5f, 0x16, 0xd0, 0x9f, 0x17, 0x78, 0x72, 0x11, + 0xb7, 0xd4, 0x84, 0xe0, 0x24, 0xf8, 0x97, 0x01 +}; +static const u8 enc_nonce092[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key092[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + /* wycheproof - edge case intermediate sums in poly1305 */ static const u8 enc_input093[] __initconst = { 0x00, 0x52, 0x35, 0xd2, 0xa9, 0x19, 0xf2, 0x8d, @@ -4455,6 +5954,86 @@ chacha20poly1305_enc_vectors[] __initconst = { sizeof(enc_input011), sizeof(enc_assoc011), sizeof(enc_nonce011) }, { enc_input012, enc_output012, enc_assoc012, enc_nonce012, enc_key012, sizeof(enc_input012), sizeof(enc_assoc012), sizeof(enc_nonce012) }, + { enc_input013, enc_output013, enc_assoc013, enc_nonce013, enc_key013, + sizeof(enc_input013), sizeof(enc_assoc013), sizeof(enc_nonce013) }, + { enc_input014, enc_output014, enc_assoc014, enc_nonce014, enc_key014, + sizeof(enc_input014), sizeof(enc_assoc014), sizeof(enc_nonce014) }, + { enc_input015, enc_output015, enc_assoc015, enc_nonce015, enc_key015, + sizeof(enc_input015), sizeof(enc_assoc015), sizeof(enc_nonce015) }, + { enc_input016, enc_output016, enc_assoc016, enc_nonce016, enc_key016, + sizeof(enc_input016), sizeof(enc_assoc016), sizeof(enc_nonce016) }, + { enc_input017, enc_output017, enc_assoc017, enc_nonce017, enc_key017, + sizeof(enc_input017), sizeof(enc_assoc017), sizeof(enc_nonce017) }, + { enc_input018, enc_output018, enc_assoc018, enc_nonce018, enc_key018, + sizeof(enc_input018), sizeof(enc_assoc018), sizeof(enc_nonce018) }, + { enc_input019, enc_output019, enc_assoc019, enc_nonce019, enc_key019, + sizeof(enc_input019), sizeof(enc_assoc019), sizeof(enc_nonce019) }, + { enc_input020, enc_output020, enc_assoc020, enc_nonce020, enc_key020, + sizeof(enc_input020), sizeof(enc_assoc020), sizeof(enc_nonce020) }, + { enc_input021, enc_output021, enc_assoc021, enc_nonce021, enc_key021, + sizeof(enc_input021), sizeof(enc_assoc021), sizeof(enc_nonce021) }, + { enc_input022, enc_output022, enc_assoc022, enc_nonce022, enc_key022, + sizeof(enc_input022), sizeof(enc_assoc022), sizeof(enc_nonce022) }, + { enc_input023, enc_output023, enc_assoc023, enc_nonce023, enc_key023, + sizeof(enc_input023), sizeof(enc_assoc023), sizeof(enc_nonce023) }, + { enc_input024, enc_output024, enc_assoc024, enc_nonce024, enc_key024, + sizeof(enc_input024), sizeof(enc_assoc024), sizeof(enc_nonce024) }, + { enc_input025, enc_output025, enc_assoc025, enc_nonce025, enc_key025, + sizeof(enc_input025), sizeof(enc_assoc025), sizeof(enc_nonce025) }, + { enc_input026, enc_output026, enc_assoc026, enc_nonce026, enc_key026, + sizeof(enc_input026), sizeof(enc_assoc026), sizeof(enc_nonce026) }, + { enc_input027, enc_output027, enc_assoc027, enc_nonce027, enc_key027, + sizeof(enc_input027), sizeof(enc_assoc027), sizeof(enc_nonce027) }, + { enc_input028, enc_output028, enc_assoc028, enc_nonce028, enc_key028, + sizeof(enc_input028), sizeof(enc_assoc028), sizeof(enc_nonce028) }, + { enc_input029, enc_output029, enc_assoc029, enc_nonce029, enc_key029, + sizeof(enc_input029), sizeof(enc_assoc029), sizeof(enc_nonce029) }, + { enc_input030, enc_output030, enc_assoc030, enc_nonce030, enc_key030, + sizeof(enc_input030), sizeof(enc_assoc030), sizeof(enc_nonce030) }, + { enc_input031, enc_output031, enc_assoc031, enc_nonce031, enc_key031, + sizeof(enc_input031), sizeof(enc_assoc031), sizeof(enc_nonce031) }, + { enc_input032, enc_output032, enc_assoc032, enc_nonce032, enc_key032, + sizeof(enc_input032), sizeof(enc_assoc032), sizeof(enc_nonce032) }, + { enc_input033, enc_output033, enc_assoc033, enc_nonce033, enc_key033, + sizeof(enc_input033), sizeof(enc_assoc033), sizeof(enc_nonce033) }, + { enc_input034, enc_output034, enc_assoc034, enc_nonce034, enc_key034, + sizeof(enc_input034), sizeof(enc_assoc034), sizeof(enc_nonce034) }, + { enc_input035, enc_output035, enc_assoc035, enc_nonce035, enc_key035, + sizeof(enc_input035), sizeof(enc_assoc035), sizeof(enc_nonce035) }, + { enc_input036, enc_output036, enc_assoc036, enc_nonce036, enc_key036, + sizeof(enc_input036), sizeof(enc_assoc036), sizeof(enc_nonce036) }, + { enc_input037, enc_output037, enc_assoc037, enc_nonce037, enc_key037, + sizeof(enc_input037), sizeof(enc_assoc037), sizeof(enc_nonce037) }, + { enc_input038, enc_output038, enc_assoc038, enc_nonce038, enc_key038, + sizeof(enc_input038), sizeof(enc_assoc038), sizeof(enc_nonce038) }, + { enc_input039, enc_output039, enc_assoc039, enc_nonce039, enc_key039, + sizeof(enc_input039), sizeof(enc_assoc039), sizeof(enc_nonce039) }, + { enc_input040, enc_output040, enc_assoc040, enc_nonce040, enc_key040, + sizeof(enc_input040), sizeof(enc_assoc040), sizeof(enc_nonce040) }, + { enc_input041, enc_output041, enc_assoc041, enc_nonce041, enc_key041, + sizeof(enc_input041), sizeof(enc_assoc041), sizeof(enc_nonce041) }, + { enc_input042, enc_output042, enc_assoc042, enc_nonce042, enc_key042, + sizeof(enc_input042), sizeof(enc_assoc042), sizeof(enc_nonce042) }, + { enc_input043, enc_output043, enc_assoc043, enc_nonce043, enc_key043, + sizeof(enc_input043), sizeof(enc_assoc043), sizeof(enc_nonce043) }, + { enc_input044, enc_output044, enc_assoc044, enc_nonce044, enc_key044, + sizeof(enc_input044), sizeof(enc_assoc044), sizeof(enc_nonce044) }, + { enc_input045, enc_output045, enc_assoc045, enc_nonce045, enc_key045, + sizeof(enc_input045), sizeof(enc_assoc045), sizeof(enc_nonce045) }, + { enc_input046, enc_output046, enc_assoc046, enc_nonce046, enc_key046, + sizeof(enc_input046), sizeof(enc_assoc046), sizeof(enc_nonce046) }, + { enc_input047, enc_output047, enc_assoc047, enc_nonce047, enc_key047, + sizeof(enc_input047), sizeof(enc_assoc047), sizeof(enc_nonce047) }, + { enc_input048, enc_output048, enc_assoc048, enc_nonce048, enc_key048, + sizeof(enc_input048), sizeof(enc_assoc048), sizeof(enc_nonce048) }, + { enc_input049, enc_output049, enc_assoc049, enc_nonce049, enc_key049, + sizeof(enc_input049), sizeof(enc_assoc049), sizeof(enc_nonce049) }, + { enc_input050, enc_output050, enc_assoc050, enc_nonce050, enc_key050, + sizeof(enc_input050), sizeof(enc_assoc050), sizeof(enc_nonce050) }, + { enc_input051, enc_output051, enc_assoc051, enc_nonce051, enc_key051, + sizeof(enc_input051), sizeof(enc_assoc051), sizeof(enc_nonce051) }, + { enc_input052, enc_output052, enc_assoc052, enc_nonce052, enc_key052, + sizeof(enc_input052), sizeof(enc_assoc052), sizeof(enc_nonce052) }, { enc_input053, enc_output053, enc_assoc053, enc_nonce053, enc_key053, sizeof(enc_input053), sizeof(enc_assoc053), sizeof(enc_nonce053) }, { enc_input054, enc_output054, enc_assoc054, enc_nonce054, enc_key054, @@ -4497,6 +6076,10 @@ chacha20poly1305_enc_vectors[] __initconst = { sizeof(enc_input072), sizeof(enc_assoc072), sizeof(enc_nonce072) }, { enc_input073, enc_output073, enc_assoc073, enc_nonce073, enc_key073, sizeof(enc_input073), sizeof(enc_assoc073), sizeof(enc_nonce073) }, + { enc_input074, enc_output074, enc_assoc074, enc_nonce074, enc_key074, + sizeof(enc_input074), sizeof(enc_assoc074), sizeof(enc_nonce074) }, + { enc_input075, enc_output075, enc_assoc075, enc_nonce075, enc_key075, + sizeof(enc_input075), sizeof(enc_assoc075), sizeof(enc_nonce075) }, { enc_input076, enc_output076, enc_assoc076, enc_nonce076, enc_key076, sizeof(enc_input076), sizeof(enc_assoc076), sizeof(enc_nonce076) }, { enc_input077, enc_output077, enc_assoc077, enc_nonce077, enc_key077, @@ -4517,6 +6100,20 @@ chacha20poly1305_enc_vectors[] __initconst = { sizeof(enc_input084), sizeof(enc_assoc084), sizeof(enc_nonce084) }, { enc_input085, enc_output085, enc_assoc085, enc_nonce085, enc_key085, sizeof(enc_input085), sizeof(enc_assoc085), sizeof(enc_nonce085) }, + { enc_input086, enc_output086, enc_assoc086, enc_nonce086, enc_key086, + sizeof(enc_input086), sizeof(enc_assoc086), sizeof(enc_nonce086) }, + { enc_input087, enc_output087, enc_assoc087, enc_nonce087, enc_key087, + sizeof(enc_input087), sizeof(enc_assoc087), sizeof(enc_nonce087) }, + { enc_input088, enc_output088, enc_assoc088, enc_nonce088, enc_key088, + sizeof(enc_input088), sizeof(enc_assoc088), sizeof(enc_nonce088) }, + { enc_input089, enc_output089, enc_assoc089, enc_nonce089, enc_key089, + sizeof(enc_input089), sizeof(enc_assoc089), sizeof(enc_nonce089) }, + { enc_input090, enc_output090, enc_assoc090, enc_nonce090, enc_key090, + sizeof(enc_input090), sizeof(enc_assoc090), sizeof(enc_nonce090) }, + { enc_input091, enc_output091, enc_assoc091, enc_nonce091, enc_key091, + sizeof(enc_input091), sizeof(enc_assoc091), sizeof(enc_nonce091) }, + { enc_input092, enc_output092, enc_assoc092, enc_nonce092, enc_key092, + sizeof(enc_input092), sizeof(enc_assoc092), sizeof(enc_nonce092) }, { enc_input093, enc_output093, enc_assoc093, enc_nonce093, enc_key093, sizeof(enc_input093), sizeof(enc_assoc093), sizeof(enc_nonce093) }, { enc_input094, enc_output094, enc_assoc094, enc_nonce094, enc_key094, @@ -7224,6 +8821,43 @@ xchacha20poly1305_dec_vectors[] __initconst = { sizeof(xdec_input001), sizeof(xdec_assoc001), sizeof(xdec_nonce001) } }; +/* This is for the selftests-only, since it is only useful for the purpose of + * testing the underlying primitives and interactions. + */ +static void __init +chacha20poly1305_encrypt_bignonce(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u8 nonce[12], + const u8 key[CHACHA20POLY1305_KEY_SIZE]) +{ + const u8 *pad0 = page_address(ZERO_PAGE(0)); + struct poly1305_desc_ctx poly1305_state; + u32 chacha20_state[CHACHA_STATE_WORDS]; + union { + u8 block0[POLY1305_KEY_SIZE]; + __le64 lens[2]; + } b = {{ 0 }}; + u8 bottom_row[16] = { 0 }; + u32 le_key[8]; + int i; + + memcpy(&bottom_row[4], nonce, 12); + for (i = 0; i < 8; ++i) + le_key[i] = get_unaligned_le32(key + sizeof(le_key[i]) * i); + chacha_init(chacha20_state, le_key, bottom_row); + chacha20_crypt(chacha20_state, b.block0, b.block0, sizeof(b.block0)); + poly1305_init(&poly1305_state, b.block0); + poly1305_update(&poly1305_state, ad, ad_len); + poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf); + chacha20_crypt(chacha20_state, dst, src, src_len); + poly1305_update(&poly1305_state, dst, src_len); + poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf); + b.lens[0] = cpu_to_le64(ad_len); + b.lens[1] = cpu_to_le64(src_len); + poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); + poly1305_final(&poly1305_state, dst + src_len); +} + static void __init chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len, const u8 *ad, const size_t ad_len, @@ -7233,6 +8867,9 @@ chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len, if (nonce_len == 8) chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, get_unaligned_le64(nonce), key); + else if (nonce_len == 12) + chacha20poly1305_encrypt_bignonce(dst, src, src_len, ad, + ad_len, nonce, key); else BUG(); } @@ -7248,14 +8885,14 @@ decryption_success(bool func_ret, bool expect_failure, int memcmp_result) bool __init chacha20poly1305_selftest(void) { enum { MAXIMUM_TEST_BUFFER_LEN = 1UL << 12 }; - size_t i; - u8 *computed_output = NULL, *heap_src = NULL; - struct scatterlist sg_src; + size_t i, j, k, total_len; + u8 *computed_output = NULL, *input = NULL; bool success = true, ret; + struct scatterlist sg_src[3]; - heap_src = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); computed_output = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); - if (!heap_src || !computed_output) { + input = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); + if (!computed_output || !input) { pr_err("chacha20poly1305 self-test malloc: FAIL\n"); success = false; goto out; @@ -7284,17 +8921,17 @@ bool __init chacha20poly1305_selftest(void) for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) { if (chacha20poly1305_enc_vectors[i].nlen != 8) continue; - memcpy(heap_src, chacha20poly1305_enc_vectors[i].input, + memcpy(computed_output, chacha20poly1305_enc_vectors[i].input, chacha20poly1305_enc_vectors[i].ilen); - sg_init_one(&sg_src, heap_src, + sg_init_one(sg_src, computed_output, chacha20poly1305_enc_vectors[i].ilen + POLY1305_DIGEST_SIZE); - chacha20poly1305_encrypt_sg_inplace(&sg_src, + ret = chacha20poly1305_encrypt_sg_inplace(sg_src, chacha20poly1305_enc_vectors[i].ilen, chacha20poly1305_enc_vectors[i].assoc, chacha20poly1305_enc_vectors[i].alen, get_unaligned_le64(chacha20poly1305_enc_vectors[i].nonce), chacha20poly1305_enc_vectors[i].key); - if (memcmp(heap_src, + if (!ret || memcmp(computed_output, chacha20poly1305_enc_vectors[i].output, chacha20poly1305_enc_vectors[i].ilen + POLY1305_DIGEST_SIZE)) { @@ -7326,11 +8963,11 @@ bool __init chacha20poly1305_selftest(void) } for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) { - memcpy(heap_src, chacha20poly1305_dec_vectors[i].input, + memcpy(computed_output, chacha20poly1305_dec_vectors[i].input, chacha20poly1305_dec_vectors[i].ilen); - sg_init_one(&sg_src, heap_src, + sg_init_one(sg_src, computed_output, chacha20poly1305_dec_vectors[i].ilen); - ret = chacha20poly1305_decrypt_sg_inplace(&sg_src, + ret = chacha20poly1305_decrypt_sg_inplace(sg_src, chacha20poly1305_dec_vectors[i].ilen, chacha20poly1305_dec_vectors[i].assoc, chacha20poly1305_dec_vectors[i].alen, @@ -7338,7 +8975,7 @@ bool __init chacha20poly1305_selftest(void) chacha20poly1305_dec_vectors[i].key); if (!decryption_success(ret, chacha20poly1305_dec_vectors[i].failure, - memcmp(heap_src, chacha20poly1305_dec_vectors[i].output, + memcmp(computed_output, chacha20poly1305_dec_vectors[i].output, chacha20poly1305_dec_vectors[i].ilen - POLY1305_DIGEST_SIZE))) { pr_err("chacha20poly1305 sg decryption self-test %zu: FAIL\n", @@ -7365,6 +9002,7 @@ bool __init chacha20poly1305_selftest(void) success = false; } } + for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_dec_vectors); ++i) { memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); ret = xchacha20poly1305_decrypt(computed_output, @@ -7386,8 +9024,54 @@ bool __init chacha20poly1305_selftest(void) } } + for (total_len = POLY1305_DIGEST_SIZE; IS_ENABLED(DEBUG_CHACHA20POLY1305_SLOW_CHUNK_TEST) + && total_len <= 1 << 10; ++total_len) { + for (i = 0; i <= total_len; ++i) { + for (j = i; j <= total_len; ++j) { + sg_init_table(sg_src, 3); + sg_set_buf(&sg_src[0], input, i); + sg_set_buf(&sg_src[1], input + i, j - i); + sg_set_buf(&sg_src[2], input + j, total_len - j); + memset(computed_output, 0, total_len); + memset(input, 0, total_len); + + if (!chacha20poly1305_encrypt_sg_inplace(sg_src, + total_len - POLY1305_DIGEST_SIZE, NULL, 0, + 0, enc_key001)) + goto chunkfail; + chacha20poly1305_encrypt(computed_output, + computed_output, + total_len - POLY1305_DIGEST_SIZE, NULL, 0, 0, + enc_key001); + if (memcmp(computed_output, input, total_len)) + goto chunkfail; + if (!chacha20poly1305_decrypt(computed_output, + input, total_len, NULL, 0, 0, enc_key001)) + goto chunkfail; + for (k = 0; k < total_len - POLY1305_DIGEST_SIZE; ++k) { + if (computed_output[k]) + goto chunkfail; + } + if (!chacha20poly1305_decrypt_sg_inplace(sg_src, + total_len, NULL, 0, 0, enc_key001)) + goto chunkfail; + for (k = 0; k < total_len - POLY1305_DIGEST_SIZE; ++k) { + if (input[k]) + goto chunkfail; + } + continue; + + chunkfail: + pr_err("chacha20poly1305 chunked self-test %zu/%zu/%zu: FAIL\n", + total_len, i, j); + success = false; + } + + } + } + out: - kfree(heap_src); kfree(computed_output); + kfree(input); return success; } -- cgit v1.2.3 From 0e0c12316d8a645e7b1880e135837ee78d18aed9 Mon Sep 17 00:00:00 2001 From: "Ben Dooks (Codethink)" Date: Fri, 24 Jan 2020 01:01:35 +0800 Subject: lib: crc64: include for 'crc64_be' The crc64_be() is declared in so include this where the symbol is defined to avoid the following warning: lib/crc64.c:43:12: warning: symbol 'crc64_be' was not declared. Should it be static? Signed-off-by: Ben Dooks (Codethink) Reviewed-by: Andy Shevchenko Signed-off-by: Coly Li Signed-off-by: Jens Axboe --- lib/crc64.c | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/crc64.c b/lib/crc64.c index 0ef8ae6ac047..f8928ce28280 100644 --- a/lib/crc64.c +++ b/lib/crc64.c @@ -28,6 +28,7 @@ #include #include +#include #include "crc64table.h" MODULE_DESCRIPTION("CRC64 calculations"); -- cgit v1.2.3 From ab10ae1c3bef56c29bac61e1201c752221b87b41 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Thu, 23 Jan 2020 08:34:18 +0000 Subject: lib: Reduce user_access_begin() boundaries in strncpy_from_user() and strnlen_user() The range passed to user_access_begin() by strncpy_from_user() and strnlen_user() starts at 'src' and goes up to the limit of userspace although reads will be limited by the 'count' param. On 32 bits powerpc (book3s/32) access has to be granted for each 256Mbytes segment and the cost increases with the number of segments to unlock. Limit the range with 'count' param. Fixes: 594cc251fdd0 ("make 'user_access_begin()' do 'access_ok()'") Signed-off-by: Christophe Leroy Signed-off-by: Linus Torvalds --- lib/strncpy_from_user.c | 14 +++++++------- lib/strnlen_user.c | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/strncpy_from_user.c b/lib/strncpy_from_user.c index dccb95af6003..706020b06617 100644 --- a/lib/strncpy_from_user.c +++ b/lib/strncpy_from_user.c @@ -30,13 +30,6 @@ static inline long do_strncpy_from_user(char *dst, const char __user *src, const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS; unsigned long res = 0; - /* - * Truncate 'max' to the user-specified limit, so that - * we only have one limit we need to check in the loop - */ - if (max > count) - max = count; - if (IS_UNALIGNED(src, dst)) goto byte_at_a_time; @@ -114,6 +107,13 @@ long strncpy_from_user(char *dst, const char __user *src, long count) unsigned long max = max_addr - src_addr; long retval; + /* + * Truncate 'max' to the user-specified limit, so that + * we only have one limit we need to check in the loop + */ + if (max > count) + max = count; + kasan_check_write(dst, count); check_object_size(dst, count, false); if (user_access_begin(src, max)) { diff --git a/lib/strnlen_user.c b/lib/strnlen_user.c index 6c0005d5dd5c..41670d4a5816 100644 --- a/lib/strnlen_user.c +++ b/lib/strnlen_user.c @@ -26,13 +26,6 @@ static inline long do_strnlen_user(const char __user *src, unsigned long count, unsigned long align, res = 0; unsigned long c; - /* - * Truncate 'max' to the user-specified limit, so that - * we only have one limit we need to check in the loop - */ - if (max > count) - max = count; - /* * Do everything aligned. But that means that we * need to also expand the maximum.. @@ -109,6 +102,13 @@ long strnlen_user(const char __user *str, long count) unsigned long max = max_addr - src_addr; long retval; + /* + * Truncate 'max' to the user-specified limit, so that + * we only have one limit we need to check in the loop + */ + if (max > count) + max = count; + if (user_access_begin(str, max)) { retval = do_strnlen_user(str, count, max); user_access_end(); -- cgit v1.2.3 From 2092767168f0681aa03727448b801600a364c013 Mon Sep 17 00:00:00 2001 From: Stefano Brivio Date: Wed, 22 Jan 2020 00:17:54 +0100 Subject: bitmap: Introduce bitmap_cut(): cut bits and shift remaining The new bitmap function bitmap_cut() copies bits from source to destination by removing the region specified by parameters first and cut, and remapping the bits above the cut region by right shifting them. Signed-off-by: Stefano Brivio Signed-off-by: Pablo Neira Ayuso --- include/linux/bitmap.h | 4 +++ lib/bitmap.c | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) (limited to 'lib') diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index ff335b22f23c..f0f3a9fffa6a 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -53,6 +53,7 @@ * bitmap_find_next_zero_area_off(buf, len, pos, n, mask) as above * bitmap_shift_right(dst, src, n, nbits) *dst = *src >> n * bitmap_shift_left(dst, src, n, nbits) *dst = *src << n + * bitmap_cut(dst, src, first, n, nbits) Cut n bits from first, copy rest * bitmap_replace(dst, old, new, mask, nbits) *dst = (*old & ~(*mask)) | (*new & *mask) * bitmap_remap(dst, src, old, new, nbits) *dst = map(old, new)(src) * bitmap_bitremap(oldbit, old, new, nbits) newbit = map(old, new)(oldbit) @@ -133,6 +134,9 @@ extern void __bitmap_shift_right(unsigned long *dst, const unsigned long *src, unsigned int shift, unsigned int nbits); extern void __bitmap_shift_left(unsigned long *dst, const unsigned long *src, unsigned int shift, unsigned int nbits); +extern void bitmap_cut(unsigned long *dst, const unsigned long *src, + unsigned int first, unsigned int cut, + unsigned int nbits); extern int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); extern void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1, diff --git a/lib/bitmap.c b/lib/bitmap.c index 4250519d7d1c..6e175fbd69a9 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -168,6 +168,72 @@ void __bitmap_shift_left(unsigned long *dst, const unsigned long *src, } EXPORT_SYMBOL(__bitmap_shift_left); +/** + * bitmap_cut() - remove bit region from bitmap and right shift remaining bits + * @dst: destination bitmap, might overlap with src + * @src: source bitmap + * @first: start bit of region to be removed + * @cut: number of bits to remove + * @nbits: bitmap size, in bits + * + * Set the n-th bit of @dst iff the n-th bit of @src is set and + * n is less than @first, or the m-th bit of @src is set for any + * m such that @first <= n < nbits, and m = n + @cut. + * + * In pictures, example for a big-endian 32-bit architecture: + * + * @src: + * 31 63 + * | | + * 10000000 11000001 11110010 00010101 10000000 11000001 01110010 00010101 + * | | | | + * 16 14 0 32 + * + * if @cut is 3, and @first is 14, bits 14-16 in @src are cut and @dst is: + * + * 31 63 + * | | + * 10110000 00011000 00110010 00010101 00010000 00011000 00101110 01000010 + * | | | + * 14 (bit 17 0 32 + * from @src) + * + * Note that @dst and @src might overlap partially or entirely. + * + * This is implemented in the obvious way, with a shift and carry + * step for each moved bit. Optimisation is left as an exercise + * for the compiler. + */ +void bitmap_cut(unsigned long *dst, const unsigned long *src, + unsigned int first, unsigned int cut, unsigned int nbits) +{ + unsigned int len = BITS_TO_LONGS(nbits); + unsigned long keep = 0, carry; + int i; + + memmove(dst, src, len * sizeof(*dst)); + + if (first % BITS_PER_LONG) { + keep = src[first / BITS_PER_LONG] & + (~0UL >> (BITS_PER_LONG - first % BITS_PER_LONG)); + } + + while (cut--) { + for (i = first / BITS_PER_LONG; i < len; i++) { + if (i < len - 1) + carry = dst[i + 1] & 1UL; + else + carry = 0; + + dst[i] = (dst[i] >> 1) | (carry << (BITS_PER_LONG - 1)); + } + } + + dst[first / BITS_PER_LONG] &= ~0UL << (first % BITS_PER_LONG); + dst[first / BITS_PER_LONG] |= keep; +} +EXPORT_SYMBOL(bitmap_cut); + int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int bits) { -- cgit v1.2.3 From 69334ca530da80c1563ac6a3bd32afa40884ccd3 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 30 Jan 2020 22:11:01 -0800 Subject: lib/test_bitmap: correct test data offsets for 32-bit On 32-bit platform the size of long is only 32 bits which makes wrong offset in the array of 64 bit size. Calculate offset based on BITS_PER_LONG. Link: http://lkml.kernel.org/r/20200109103601.45929-1-andriy.shevchenko@linux.intel.com Fixes: 30544ed5de43 ("lib/bitmap: introduce bitmap_replace() helper") Signed-off-by: Andy Shevchenko Reported-by: Guenter Roeck Cc: Rasmus Villemoes Cc: Yury Norov Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test_bitmap.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c index e14a15ac250b..71ec3afe1681 100644 --- a/lib/test_bitmap.c +++ b/lib/test_bitmap.c @@ -275,22 +275,23 @@ static void __init test_copy(void) static void __init test_replace(void) { unsigned int nbits = 64; + unsigned int nlongs = DIV_ROUND_UP(nbits, BITS_PER_LONG); DECLARE_BITMAP(bmap, 1024); bitmap_zero(bmap, 1024); - bitmap_replace(bmap, &exp2[0], &exp2[1], exp2_to_exp3_mask, nbits); + bitmap_replace(bmap, &exp2[0 * nlongs], &exp2[1 * nlongs], exp2_to_exp3_mask, nbits); expect_eq_bitmap(bmap, exp3_0_1, nbits); bitmap_zero(bmap, 1024); - bitmap_replace(bmap, &exp2[1], &exp2[0], exp2_to_exp3_mask, nbits); + bitmap_replace(bmap, &exp2[1 * nlongs], &exp2[0 * nlongs], exp2_to_exp3_mask, nbits); expect_eq_bitmap(bmap, exp3_1_0, nbits); bitmap_fill(bmap, 1024); - bitmap_replace(bmap, &exp2[0], &exp2[1], exp2_to_exp3_mask, nbits); + bitmap_replace(bmap, &exp2[0 * nlongs], &exp2[1 * nlongs], exp2_to_exp3_mask, nbits); expect_eq_bitmap(bmap, exp3_0_1, nbits); bitmap_fill(bmap, 1024); - bitmap_replace(bmap, &exp2[1], &exp2[0], exp2_to_exp3_mask, nbits); + bitmap_replace(bmap, &exp2[1 * nlongs], &exp2[0 * nlongs], exp2_to_exp3_mask, nbits); expect_eq_bitmap(bmap, exp3_1_0, nbits); } -- cgit v1.2.3 From 3e21d9a501bf99aee2e5835d7f34d8c823f115b5 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Thu, 30 Jan 2020 22:13:51 -0800 Subject: lib/test_kasan.c: fix memory leak in kmalloc_oob_krealloc_more() In case memory resources for _ptr2_ were allocated, release them before return. Notice that in case _ptr1_ happens to be NULL, krealloc() behaves exactly like kmalloc(). Addresses-Coverity-ID: 1490594 ("Resource leak") Link: http://lkml.kernel.org/r/20200123160115.GA4202@embeddedor Fixes: 3f15801cdc23 ("lib: add kasan test module") Signed-off-by: Gustavo A. R. Silva Reviewed-by: Dmitry Vyukov Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test_kasan.c | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/test_kasan.c b/lib/test_kasan.c index 328d33beae36..3872d250ed2c 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -158,6 +158,7 @@ static noinline void __init kmalloc_oob_krealloc_more(void) if (!ptr1 || !ptr2) { pr_err("Allocation failed\n"); kfree(ptr1); + kfree(ptr2); return; } -- cgit v1.2.3 From aa5b395b69b65450e008b95ec623b4fc4b175f9f Mon Sep 17 00:00:00 2001 From: Mikhail Zaslonko Date: Thu, 30 Jan 2020 22:16:17 -0800 Subject: lib/zlib: add s390 hardware support for kernel zlib_deflate Patch series "S390 hardware support for kernel zlib", v3. With IBM z15 mainframe the new DFLTCC instruction is available. It implements deflate algorithm in hardware (Nest Acceleration Unit - NXU) with estimated compression and decompression performance orders of magnitude faster than the current zlib. This patchset adds s390 hardware compression support to kernel zlib. The code is based on the userspace zlib implementation: https://github.com/madler/zlib/pull/410 The coding style is also preserved for future maintainability. There is only limited set of userspace zlib functions represented in kernel. Apart from that, all the memory allocation should be performed in advance. Thus, the workarea structures are extended with the parameter lists required for the DEFLATE CONVENTION CALL instruction. Since kernel zlib itself does not support gzip headers, only Adler-32 checksum is processed (also can be produced by DFLTCC facility). Like it was implemented for userspace, kernel zlib will compress in hardware on level 1, and in software on all other levels. Decompression will always happen in hardware (when enabled). Two DFLTCC compression calls produce the same results only when they both are made on machines of the same generation, and when the respective buffers have the same offset relative to the start of the page. Therefore care should be taken when using hardware compression when reproducible results are desired. However it does always produce the standard conform output which can be inflated anyway. The new kernel command line parameter 'dfltcc' is introduced to configure s390 zlib hardware support: Format: { on | off | def_only | inf_only | always } on: s390 zlib hardware support for compression on level 1 and decompression (default) off: No s390 zlib hardware support def_only: s390 zlib hardware support for deflate only (compression on level 1) inf_only: s390 zlib hardware support for inflate only (decompression) always: Same as 'on' but ignores the selected compression level always using hardware support (used for debugging) The main purpose of the integration of the NXU support into the kernel zlib is the use of hardware deflate in btrfs filesystem with on-the-fly compression enabled. Apart from that, hardware support can also be used during boot for decompressing the kernel or the ramdisk image With the patch for btrfs expanding zlib buffer from 1 to 4 pages (patch 6) the following performance results have been achieved using the ramdisk with btrfs. These are relative numbers based on throughput rate and compression ratio for zlib level 1: Input data Deflate rate Inflate rate Compression ratio NXU/Software NXU/Software NXU/Software stream of zeroes 1.46 1.02 1.00 random ASCII data 10.44 3.00 0.96 ASCII text (dickens) 6,21 3.33 0.94 binary data (vmlinux) 8,37 3.90 1.02 This means that s390 hardware deflate can provide up to 10 times faster compression (on level 1) and up to 4 times faster decompression (refers to all compression levels) for btrfs zlib. Disclaimer: Performance results are based on IBM internal tests using DD command-line utility on btrfs on a Fedora 30 based internal driver in native LPAR on a z15 system. Results may vary based on individual workload, configuration and software levels. This patch (of 9): Create zlib_dfltcc library with the s390 DEFLATE CONVERSION CALL implementation and related compression functions. Update zlib_deflate functions with the hooks for s390 hardware support and adjust workspace structures with extra parameter lists required for hardware deflate. Link: http://lkml.kernel.org/r/20200103223334.20669-2-zaslonko@linux.ibm.com Signed-off-by: Ilya Leoshkevich Signed-off-by: Mikhail Zaslonko Co-developed-by: Ilya Leoshkevich Cc: Chris Mason Cc: Christian Borntraeger Cc: David Sterba Cc: Eduard Shishkin Cc: Heiko Carstens Cc: Josef Bacik Cc: Richard Purdie Cc: Vasily Gorbik Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/Kconfig | 7 + lib/Makefile | 1 + lib/zlib_deflate/deflate.c | 79 +++++------ lib/zlib_deflate/deftree.c | 54 -------- lib/zlib_deflate/defutil.h | 134 +++++++++++++++++-- lib/zlib_dfltcc/Makefile | 11 ++ lib/zlib_dfltcc/dfltcc.c | 52 ++++++++ lib/zlib_dfltcc/dfltcc.h | 115 +++++++++++++++++ lib/zlib_dfltcc/dfltcc_deflate.c | 273 +++++++++++++++++++++++++++++++++++++++ lib/zlib_dfltcc/dfltcc_syms.c | 17 +++ lib/zlib_dfltcc/dfltcc_util.h | 110 ++++++++++++++++ 11 files changed, 751 insertions(+), 102 deletions(-) create mode 100644 lib/zlib_dfltcc/Makefile create mode 100644 lib/zlib_dfltcc/dfltcc.c create mode 100644 lib/zlib_dfltcc/dfltcc.h create mode 100644 lib/zlib_dfltcc/dfltcc_deflate.c create mode 100644 lib/zlib_dfltcc/dfltcc_syms.c create mode 100644 lib/zlib_dfltcc/dfltcc_util.h (limited to 'lib') diff --git a/lib/Kconfig b/lib/Kconfig index 6e790dc55c5b..bc7e56370129 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -278,6 +278,13 @@ config ZLIB_DEFLATE tristate select BITREVERSE +config ZLIB_DFLTCC + def_bool y + depends on S390 + prompt "Enable s390x DEFLATE CONVERSION CALL support for kernel zlib" + help + Enable s390x hardware support for zlib in the kernel. + config LZO_COMPRESS tristate diff --git a/lib/Makefile b/lib/Makefile index c20b1debe9b4..5fb02b93b1dc 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -140,6 +140,7 @@ obj-$(CONFIG_842_COMPRESS) += 842/ obj-$(CONFIG_842_DECOMPRESS) += 842/ obj-$(CONFIG_ZLIB_INFLATE) += zlib_inflate/ obj-$(CONFIG_ZLIB_DEFLATE) += zlib_deflate/ +obj-$(CONFIG_ZLIB_DFLTCC) += zlib_dfltcc/ obj-$(CONFIG_REED_SOLOMON) += reed_solomon/ obj-$(CONFIG_BCH) += bch.o obj-$(CONFIG_LZO_COMPRESS) += lzo/ diff --git a/lib/zlib_deflate/deflate.c b/lib/zlib_deflate/deflate.c index d20ef458f137..0d60d81a2637 100644 --- a/lib/zlib_deflate/deflate.c +++ b/lib/zlib_deflate/deflate.c @@ -52,16 +52,18 @@ #include #include "defutil.h" +/* architecture-specific bits */ +#ifdef CONFIG_ZLIB_DFLTCC +# include "../zlib_dfltcc/dfltcc.h" +#else +#define DEFLATE_RESET_HOOK(strm) do {} while (0) +#define DEFLATE_HOOK(strm, flush, bstate) 0 +#define DEFLATE_NEED_CHECKSUM(strm) 1 +#endif /* =========================================================================== * Function prototypes. */ -typedef enum { - need_more, /* block not completed, need more input or more output */ - block_done, /* block flush performed */ - finish_started, /* finish started, need only more output at next deflate */ - finish_done /* finish done, accept no more input or output */ -} block_state; typedef block_state (*compress_func) (deflate_state *s, int flush); /* Compression function. Returns the block state after the call. */ @@ -72,7 +74,6 @@ static block_state deflate_fast (deflate_state *s, int flush); static block_state deflate_slow (deflate_state *s, int flush); static void lm_init (deflate_state *s); static void putShortMSB (deflate_state *s, uInt b); -static void flush_pending (z_streamp strm); static int read_buf (z_streamp strm, Byte *buf, unsigned size); static uInt longest_match (deflate_state *s, IPos cur_match); @@ -98,6 +99,25 @@ static void check_match (deflate_state *s, IPos start, IPos match, * See deflate.c for comments about the MIN_MATCH+1. */ +/* Workspace to be allocated for deflate processing */ +typedef struct deflate_workspace { + /* State memory for the deflator */ + deflate_state deflate_memory; +#ifdef CONFIG_ZLIB_DFLTCC + /* State memory for s390 hardware deflate */ + struct dfltcc_state dfltcc_memory; +#endif + Byte *window_memory; + Pos *prev_memory; + Pos *head_memory; + char *overlay_memory; +} deflate_workspace; + +#ifdef CONFIG_ZLIB_DFLTCC +/* dfltcc_state must be doubleword aligned for DFLTCC call */ +static_assert(offsetof(struct deflate_workspace, dfltcc_memory) % 8 == 0); +#endif + /* Values for max_lazy_match, good_match and max_chain_length, depending on * the desired pack level (0..9). The values given below have been tuned to * exclude worst case performance for pathological files. Better values may be @@ -207,7 +227,15 @@ int zlib_deflateInit2( */ next = (char *) mem; next += sizeof(*mem); +#ifdef CONFIG_ZLIB_DFLTCC + /* + * DFLTCC requires the window to be page aligned. + * Thus, we overallocate and take the aligned portion of the buffer. + */ + mem->window_memory = (Byte *) PTR_ALIGN(next, PAGE_SIZE); +#else mem->window_memory = (Byte *) next; +#endif next += zlib_deflate_window_memsize(windowBits); mem->prev_memory = (Pos *) next; next += zlib_deflate_prev_memsize(windowBits); @@ -277,6 +305,8 @@ int zlib_deflateReset( zlib_tr_init(s); lm_init(s); + DEFLATE_RESET_HOOK(strm); + return Z_OK; } @@ -294,35 +324,6 @@ static void putShortMSB( put_byte(s, (Byte)(b & 0xff)); } -/* ========================================================================= - * Flush as much pending output as possible. All deflate() output goes - * through this function so some applications may wish to modify it - * to avoid allocating a large strm->next_out buffer and copying into it. - * (See also read_buf()). - */ -static void flush_pending( - z_streamp strm -) -{ - deflate_state *s = (deflate_state *) strm->state; - unsigned len = s->pending; - - if (len > strm->avail_out) len = strm->avail_out; - if (len == 0) return; - - if (strm->next_out != NULL) { - memcpy(strm->next_out, s->pending_out, len); - strm->next_out += len; - } - s->pending_out += len; - strm->total_out += len; - strm->avail_out -= len; - s->pending -= len; - if (s->pending == 0) { - s->pending_out = s->pending_buf; - } -} - /* ========================================================================= */ int zlib_deflate( z_streamp strm, @@ -404,7 +405,8 @@ int zlib_deflate( (flush != Z_NO_FLUSH && s->status != FINISH_STATE)) { block_state bstate; - bstate = (*(configuration_table[s->level].func))(s, flush); + bstate = DEFLATE_HOOK(strm, flush, &bstate) ? bstate : + (*(configuration_table[s->level].func))(s, flush); if (bstate == finish_started || bstate == finish_done) { s->status = FINISH_STATE; @@ -503,7 +505,8 @@ static int read_buf( strm->avail_in -= len; - if (!((deflate_state *)(strm->state))->noheader) { + if (!DEFLATE_NEED_CHECKSUM(strm)) {} + else if (!((deflate_state *)(strm->state))->noheader) { strm->adler = zlib_adler32(strm->adler, strm->next_in, len); } memcpy(buf, strm->next_in, len); diff --git a/lib/zlib_deflate/deftree.c b/lib/zlib_deflate/deftree.c index 9b1756b12743..a4a34da512fe 100644 --- a/lib/zlib_deflate/deftree.c +++ b/lib/zlib_deflate/deftree.c @@ -76,11 +76,6 @@ static const uch bl_order[BL_CODES] * probability, to avoid transmitting the lengths for unused bit length codes. */ -#define Buf_size (8 * 2*sizeof(char)) -/* Number of bits used within bi_buf. (bi_buf might be implemented on - * more than 16 bits on some systems.) - */ - /* =========================================================================== * Local data. These are initialized only once. */ @@ -147,7 +142,6 @@ static void send_all_trees (deflate_state *s, int lcodes, int dcodes, static void compress_block (deflate_state *s, ct_data *ltree, ct_data *dtree); static void set_data_type (deflate_state *s); -static void bi_windup (deflate_state *s); static void bi_flush (deflate_state *s); static void copy_block (deflate_state *s, char *buf, unsigned len, int header); @@ -169,54 +163,6 @@ static void copy_block (deflate_state *s, char *buf, unsigned len, * used. */ -/* =========================================================================== - * Send a value on a given number of bits. - * IN assertion: length <= 16 and value fits in length bits. - */ -#ifdef DEBUG_ZLIB -static void send_bits (deflate_state *s, int value, int length); - -static void send_bits( - deflate_state *s, - int value, /* value to send */ - int length /* number of bits */ -) -{ - Tracevv((stderr," l %2d v %4x ", length, value)); - Assert(length > 0 && length <= 15, "invalid length"); - s->bits_sent += (ulg)length; - - /* If not enough room in bi_buf, use (valid) bits from bi_buf and - * (16 - bi_valid) bits from value, leaving (width - (16-bi_valid)) - * unused bits in value. - */ - if (s->bi_valid > (int)Buf_size - length) { - s->bi_buf |= (value << s->bi_valid); - put_short(s, s->bi_buf); - s->bi_buf = (ush)value >> (Buf_size - s->bi_valid); - s->bi_valid += length - Buf_size; - } else { - s->bi_buf |= value << s->bi_valid; - s->bi_valid += length; - } -} -#else /* !DEBUG_ZLIB */ - -#define send_bits(s, value, length) \ -{ int len = length;\ - if (s->bi_valid > (int)Buf_size - len) {\ - int val = value;\ - s->bi_buf |= (val << s->bi_valid);\ - put_short(s, s->bi_buf);\ - s->bi_buf = (ush)val >> (Buf_size - s->bi_valid);\ - s->bi_valid += len - Buf_size;\ - } else {\ - s->bi_buf |= (value) << s->bi_valid;\ - s->bi_valid += len;\ - }\ -} -#endif /* DEBUG_ZLIB */ - /* =========================================================================== * Initialize the various 'constant' tables. In a multi-threaded environment, * this function may be called by two threads concurrently, but this is diff --git a/lib/zlib_deflate/defutil.h b/lib/zlib_deflate/defutil.h index a8c370897c9f..385333b22ec6 100644 --- a/lib/zlib_deflate/defutil.h +++ b/lib/zlib_deflate/defutil.h @@ -1,5 +1,7 @@ +#ifndef DEFUTIL_H +#define DEFUTIL_H - +#include #define Assert(err, str) #define Trace(dummy) @@ -238,17 +240,13 @@ typedef struct deflate_state { } deflate_state; -typedef struct deflate_workspace { - /* State memory for the deflator */ - deflate_state deflate_memory; - Byte *window_memory; - Pos *prev_memory; - Pos *head_memory; - char *overlay_memory; -} deflate_workspace; - +#ifdef CONFIG_ZLIB_DFLTCC +#define zlib_deflate_window_memsize(windowBits) \ + (2 * (1 << (windowBits)) * sizeof(Byte) + PAGE_SIZE) +#else #define zlib_deflate_window_memsize(windowBits) \ (2 * (1 << (windowBits)) * sizeof(Byte)) +#endif #define zlib_deflate_prev_memsize(windowBits) \ ((1 << (windowBits)) * sizeof(Pos)) #define zlib_deflate_head_memsize(memLevel) \ @@ -292,6 +290,24 @@ void zlib_tr_stored_type_only (deflate_state *); put_byte(s, (uch)((ush)(w) >> 8)); \ } +/* =========================================================================== + * Reverse the first len bits of a code, using straightforward code (a faster + * method would use a table) + * IN assertion: 1 <= len <= 15 + */ +static inline unsigned bi_reverse( + unsigned code, /* the value to invert */ + int len /* its bit length */ +) +{ + register unsigned res = 0; + do { + res |= code & 1; + code >>= 1, res <<= 1; + } while (--len > 0); + return res >> 1; +} + /* =========================================================================== * Flush the bit buffer, keeping at most 7 bits in it. */ @@ -325,3 +341,101 @@ static inline void bi_windup(deflate_state *s) #endif } +typedef enum { + need_more, /* block not completed, need more input or more output */ + block_done, /* block flush performed */ + finish_started, /* finish started, need only more output at next deflate */ + finish_done /* finish done, accept no more input or output */ +} block_state; + +#define Buf_size (8 * 2*sizeof(char)) +/* Number of bits used within bi_buf. (bi_buf might be implemented on + * more than 16 bits on some systems.) + */ + +/* =========================================================================== + * Send a value on a given number of bits. + * IN assertion: length <= 16 and value fits in length bits. + */ +#ifdef DEBUG_ZLIB +static void send_bits (deflate_state *s, int value, int length); + +static void send_bits( + deflate_state *s, + int value, /* value to send */ + int length /* number of bits */ +) +{ + Tracevv((stderr," l %2d v %4x ", length, value)); + Assert(length > 0 && length <= 15, "invalid length"); + s->bits_sent += (ulg)length; + + /* If not enough room in bi_buf, use (valid) bits from bi_buf and + * (16 - bi_valid) bits from value, leaving (width - (16-bi_valid)) + * unused bits in value. + */ + if (s->bi_valid > (int)Buf_size - length) { + s->bi_buf |= (value << s->bi_valid); + put_short(s, s->bi_buf); + s->bi_buf = (ush)value >> (Buf_size - s->bi_valid); + s->bi_valid += length - Buf_size; + } else { + s->bi_buf |= value << s->bi_valid; + s->bi_valid += length; + } +} +#else /* !DEBUG_ZLIB */ + +#define send_bits(s, value, length) \ +{ int len = length;\ + if (s->bi_valid > (int)Buf_size - len) {\ + int val = value;\ + s->bi_buf |= (val << s->bi_valid);\ + put_short(s, s->bi_buf);\ + s->bi_buf = (ush)val >> (Buf_size - s->bi_valid);\ + s->bi_valid += len - Buf_size;\ + } else {\ + s->bi_buf |= (value) << s->bi_valid;\ + s->bi_valid += len;\ + }\ +} +#endif /* DEBUG_ZLIB */ + +static inline void zlib_tr_send_bits( + deflate_state *s, + int value, + int length +) +{ + send_bits(s, value, length); +} + +/* ========================================================================= + * Flush as much pending output as possible. All deflate() output goes + * through this function so some applications may wish to modify it + * to avoid allocating a large strm->next_out buffer and copying into it. + * (See also read_buf()). + */ +static inline void flush_pending( + z_streamp strm +) +{ + deflate_state *s = (deflate_state *) strm->state; + unsigned len = s->pending; + + if (len > strm->avail_out) len = strm->avail_out; + if (len == 0) return; + + if (strm->next_out != NULL) { + memcpy(strm->next_out, s->pending_out, len); + strm->next_out += len; + } + s->pending_out += len; + strm->total_out += len; + strm->avail_out -= len; + s->pending -= len; + if (s->pending == 0) { + s->pending_out = s->pending_buf; + } +} +#endif /* DEFUTIL_H */ diff --git a/lib/zlib_dfltcc/Makefile b/lib/zlib_dfltcc/Makefile new file mode 100644 index 000000000000..863d3b37e09d --- /dev/null +++ b/lib/zlib_dfltcc/Makefile @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This is a modified version of zlib, which does all memory +# allocation ahead of time. +# +# This is the code for s390 zlib hardware support. +# + +obj-$(CONFIG_ZLIB_DFLTCC) += zlib_dfltcc.o + +zlib_dfltcc-objs := dfltcc.o dfltcc_deflate.o dfltcc_syms.o diff --git a/lib/zlib_dfltcc/dfltcc.c b/lib/zlib_dfltcc/dfltcc.c new file mode 100644 index 000000000000..7f77e5bb01c6 --- /dev/null +++ b/lib/zlib_dfltcc/dfltcc.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Zlib +/* dfltcc.c - SystemZ DEFLATE CONVERSION CALL support. */ + +#include +#include "dfltcc_util.h" +#include "dfltcc.h" + +char *oesc_msg( + char *buf, + int oesc +) +{ + if (oesc == 0x00) + return NULL; /* Successful completion */ + else { +#ifdef STATIC + return NULL; /* Ignore for pre-boot decompressor */ +#else + sprintf(buf, "Operation-Ending-Supplemental Code is 0x%.2X", oesc); + return buf; +#endif + } +} + +void dfltcc_reset( + z_streamp strm, + uInt size +) +{ + struct dfltcc_state *dfltcc_state = + (struct dfltcc_state *)((char *)strm->state + size); + struct dfltcc_qaf_param *param = + (struct dfltcc_qaf_param *)&dfltcc_state->param; + + /* Initialize available functions */ + if (is_dfltcc_enabled()) { + dfltcc(DFLTCC_QAF, param, NULL, NULL, NULL, NULL, NULL); + memmove(&dfltcc_state->af, param, sizeof(dfltcc_state->af)); + } else + memset(&dfltcc_state->af, 0, sizeof(dfltcc_state->af)); + + /* Initialize parameter block */ + memset(&dfltcc_state->param, 0, sizeof(dfltcc_state->param)); + dfltcc_state->param.nt = 1; + + /* Initialize tuning parameters */ + dfltcc_state->level_mask = DFLTCC_LEVEL_MASK; + dfltcc_state->block_size = DFLTCC_BLOCK_SIZE; + dfltcc_state->block_threshold = DFLTCC_FIRST_FHT_BLOCK_SIZE; + dfltcc_state->dht_threshold = DFLTCC_DHT_MIN_SAMPLE_SIZE; + dfltcc_state->param.ribm = DFLTCC_RIBM; +} diff --git a/lib/zlib_dfltcc/dfltcc.h b/lib/zlib_dfltcc/dfltcc.h new file mode 100644 index 000000000000..18fed7a444bc --- /dev/null +++ b/lib/zlib_dfltcc/dfltcc.h @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: Zlib +#ifndef DFLTCC_H +#define DFLTCC_H + +#include "../zlib_deflate/defutil.h" + +/* + * Tuning parameters. + */ +#define DFLTCC_LEVEL_MASK 0x2 /* DFLTCC compression for level 1 only */ +#define DFLTCC_BLOCK_SIZE 1048576 +#define DFLTCC_FIRST_FHT_BLOCK_SIZE 4096 +#define DFLTCC_DHT_MIN_SAMPLE_SIZE 4096 +#define DFLTCC_RIBM 0 + +/* + * Parameter Block for Query Available Functions. + */ +struct dfltcc_qaf_param { + char fns[16]; + char reserved1[8]; + char fmts[2]; + char reserved2[6]; +}; + +static_assert(sizeof(struct dfltcc_qaf_param) == 32); + +#define DFLTCC_FMT0 0 + +/* + * Parameter Block for Generate Dynamic-Huffman Table, Compress and Expand. + */ +struct dfltcc_param_v0 { + uint16_t pbvn; /* Parameter-Block-Version Number */ + uint8_t mvn; /* Model-Version Number */ + uint8_t ribm; /* Reserved for IBM use */ + unsigned reserved32 : 31; + unsigned cf : 1; /* Continuation Flag */ + uint8_t reserved64[8]; + unsigned nt : 1; /* New Task */ + unsigned reserved129 : 1; + unsigned cvt : 1; /* Check Value Type */ + unsigned reserved131 : 1; + unsigned htt : 1; /* Huffman-Table Type */ + unsigned bcf : 1; /* Block-Continuation Flag */ + unsigned bcc : 1; /* Block Closing Control */ + unsigned bhf : 1; /* Block Header Final */ + unsigned reserved136 : 1; + unsigned reserved137 : 1; + unsigned dhtgc : 1; /* DHT Generation Control */ + unsigned reserved139 : 5; + unsigned reserved144 : 5; + unsigned sbb : 3; /* Sub-Byte Boundary */ + uint8_t oesc; /* Operation-Ending-Supplemental Code */ + unsigned reserved160 : 12; + unsigned ifs : 4; /* Incomplete-Function Status */ + uint16_t ifl; /* Incomplete-Function Length */ + uint8_t reserved192[8]; + uint8_t reserved256[8]; + uint8_t reserved320[4]; + uint16_t hl; /* History Length */ + unsigned reserved368 : 1; + uint16_t ho : 15; /* History Offset */ + uint32_t cv; /* Check Value */ + unsigned eobs : 15; /* End-of-block Symbol */ + unsigned reserved431: 1; + uint8_t eobl : 4; /* End-of-block Length */ + unsigned reserved436 : 12; + unsigned reserved448 : 4; + uint16_t cdhtl : 12; /* Compressed-Dynamic-Huffman Table + Length */ + uint8_t reserved464[6]; + uint8_t cdht[288]; + uint8_t reserved[32]; + uint8_t csb[1152]; +}; + +static_assert(sizeof(struct dfltcc_param_v0) == 1536); + +#define CVT_CRC32 0 +#define CVT_ADLER32 1 +#define HTT_FIXED 0 +#define HTT_DYNAMIC 1 + +/* + * Extension of inflate_state and deflate_state for DFLTCC. + */ +struct dfltcc_state { + struct dfltcc_param_v0 param; /* Parameter block */ + struct dfltcc_qaf_param af; /* Available functions */ + uLong level_mask; /* Levels on which to use DFLTCC */ + uLong block_size; /* New block each X bytes */ + uLong block_threshold; /* New block after total_in > X */ + uLong dht_threshold; /* New block only if avail_in >= X */ + char msg[64]; /* Buffer for strm->msg */ +}; + +/* Resides right after inflate_state or deflate_state */ +#define GET_DFLTCC_STATE(state) ((struct dfltcc_state *)((state) + 1)) + +/* External functions */ +int dfltcc_can_deflate(z_streamp strm); +int dfltcc_deflate(z_streamp strm, + int flush, + block_state *result); +void dfltcc_reset(z_streamp strm, uInt size); + +#define DEFLATE_RESET_HOOK(strm) \ + dfltcc_reset((strm), sizeof(deflate_state)) + +#define DEFLATE_HOOK dfltcc_deflate + +#define DEFLATE_NEED_CHECKSUM(strm) (!dfltcc_can_deflate((strm))) + +#endif /* DFLTCC_H */ diff --git a/lib/zlib_dfltcc/dfltcc_deflate.c b/lib/zlib_dfltcc/dfltcc_deflate.c new file mode 100644 index 000000000000..9f0f1c90c4e0 --- /dev/null +++ b/lib/zlib_dfltcc/dfltcc_deflate.c @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: Zlib + +#include "../zlib_deflate/defutil.h" +#include "dfltcc_util.h" +#include "dfltcc.h" +#include + +/* + * Compress. + */ +int dfltcc_can_deflate( + z_streamp strm +) +{ + deflate_state *state = (deflate_state *)strm->state; + struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + + /* Unsupported compression settings */ + if (!dfltcc_are_params_ok(state->level, state->w_bits, state->strategy, + dfltcc_state->level_mask)) + return 0; + + /* Unsupported hardware */ + if (!is_bit_set(dfltcc_state->af.fns, DFLTCC_GDHT) || + !is_bit_set(dfltcc_state->af.fns, DFLTCC_CMPR) || + !is_bit_set(dfltcc_state->af.fmts, DFLTCC_FMT0)) + return 0; + + return 1; +} + +static void dfltcc_gdht( + z_streamp strm +) +{ + deflate_state *state = (deflate_state *)strm->state; + struct dfltcc_param_v0 *param = &GET_DFLTCC_STATE(state)->param; + size_t avail_in = avail_in = strm->avail_in; + + dfltcc(DFLTCC_GDHT, + param, NULL, NULL, + &strm->next_in, &avail_in, NULL); +} + +static dfltcc_cc dfltcc_cmpr( + z_streamp strm +) +{ + deflate_state *state = (deflate_state *)strm->state; + struct dfltcc_param_v0 *param = &GET_DFLTCC_STATE(state)->param; + size_t avail_in = strm->avail_in; + size_t avail_out = strm->avail_out; + dfltcc_cc cc; + + cc = dfltcc(DFLTCC_CMPR | HBT_CIRCULAR, + param, &strm->next_out, &avail_out, + &strm->next_in, &avail_in, state->window); + strm->total_in += (strm->avail_in - avail_in); + strm->total_out += (strm->avail_out - avail_out); + strm->avail_in = avail_in; + strm->avail_out = avail_out; + return cc; +} + +static void send_eobs( + z_streamp strm, + const struct dfltcc_param_v0 *param +) +{ + deflate_state *state = (deflate_state *)strm->state; + + zlib_tr_send_bits( + state, + bi_reverse(param->eobs >> (15 - param->eobl), param->eobl), + param->eobl); + flush_pending(strm); + if (state->pending != 0) { + /* The remaining data is located in pending_out[0:pending]. If someone + * calls put_byte() - this might happen in deflate() - the byte will be + * placed into pending_buf[pending], which is incorrect. Move the + * remaining data to the beginning of pending_buf so that put_byte() is + * usable again. + */ + memmove(state->pending_buf, state->pending_out, state->pending); + state->pending_out = state->pending_buf; + } +#ifdef ZLIB_DEBUG + state->compressed_len += param->eobl; +#endif +} + +int dfltcc_deflate( + z_streamp strm, + int flush, + block_state *result +) +{ + deflate_state *state = (deflate_state *)strm->state; + struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + struct dfltcc_param_v0 *param = &dfltcc_state->param; + uInt masked_avail_in; + dfltcc_cc cc; + int need_empty_block; + int soft_bcc; + int no_flush; + + if (!dfltcc_can_deflate(strm)) + return 0; + +again: + masked_avail_in = 0; + soft_bcc = 0; + no_flush = flush == Z_NO_FLUSH; + + /* Trailing empty block. Switch to software, except when Continuation Flag + * is set, which means that DFLTCC has buffered some output in the + * parameter block and needs to be called again in order to flush it. + */ + if (flush == Z_FINISH && strm->avail_in == 0 && !param->cf) { + if (param->bcf) { + /* A block is still open, and the hardware does not support closing + * blocks without adding data. Thus, close it manually. + */ + send_eobs(strm, param); + param->bcf = 0; + } + return 0; + } + + if (strm->avail_in == 0 && !param->cf) { + *result = need_more; + return 1; + } + + /* There is an open non-BFINAL block, we are not going to close it just + * yet, we have compressed more than DFLTCC_BLOCK_SIZE bytes and we see + * more than DFLTCC_DHT_MIN_SAMPLE_SIZE bytes. Open a new block with a new + * DHT in order to adapt to a possibly changed input data distribution. + */ + if (param->bcf && no_flush && + strm->total_in > dfltcc_state->block_threshold && + strm->avail_in >= dfltcc_state->dht_threshold) { + if (param->cf) { + /* We need to flush the DFLTCC buffer before writing the + * End-of-block Symbol. Mask the input data and proceed as usual. + */ + masked_avail_in += strm->avail_in; + strm->avail_in = 0; + no_flush = 0; + } else { + /* DFLTCC buffer is empty, so we can manually write the + * End-of-block Symbol right away. + */ + send_eobs(strm, param); + param->bcf = 0; + dfltcc_state->block_threshold = + strm->total_in + dfltcc_state->block_size; + if (strm->avail_out == 0) { + *result = need_more; + return 1; + } + } + } + + /* The caller gave us too much data. Pass only one block worth of + * uncompressed data to DFLTCC and mask the rest, so that on the next + * iteration we start a new block. + */ + if (no_flush && strm->avail_in > dfltcc_state->block_size) { + masked_avail_in += (strm->avail_in - dfltcc_state->block_size); + strm->avail_in = dfltcc_state->block_size; + } + + /* When we have an open non-BFINAL deflate block and caller indicates that + * the stream is ending, we need to close an open deflate block and open a + * BFINAL one. + */ + need_empty_block = flush == Z_FINISH && param->bcf && !param->bhf; + + /* Translate stream to parameter block */ + param->cvt = CVT_ADLER32; + if (!no_flush) + /* We need to close a block. Always do this in software - when there is + * no input data, the hardware will not nohor BCC. */ + soft_bcc = 1; + if (flush == Z_FINISH && !param->bcf) + /* We are about to open a BFINAL block, set Block Header Final bit + * until the stream ends. + */ + param->bhf = 1; + /* DFLTCC-CMPR will write to next_out, so make sure that buffers with + * higher precedence are empty. + */ + Assert(state->pending == 0, "There must be no pending bytes"); + Assert(state->bi_valid < 8, "There must be less than 8 pending bits"); + param->sbb = (unsigned int)state->bi_valid; + if (param->sbb > 0) + *strm->next_out = (Byte)state->bi_buf; + if (param->hl) + param->nt = 0; /* Honor history */ + param->cv = strm->adler; + + /* When opening a block, choose a Huffman-Table Type */ + if (!param->bcf) { + if (strm->total_in == 0 && dfltcc_state->block_threshold > 0) { + param->htt = HTT_FIXED; + } + else { + param->htt = HTT_DYNAMIC; + dfltcc_gdht(strm); + } + } + + /* Deflate */ + do { + cc = dfltcc_cmpr(strm); + if (strm->avail_in < 4096 && masked_avail_in > 0) + /* We are about to call DFLTCC with a small input buffer, which is + * inefficient. Since there is masked data, there will be at least + * one more DFLTCC call, so skip the current one and make the next + * one handle more data. + */ + break; + } while (cc == DFLTCC_CC_AGAIN); + + /* Translate parameter block to stream */ + strm->msg = oesc_msg(dfltcc_state->msg, param->oesc); + state->bi_valid = param->sbb; + if (state->bi_valid == 0) + state->bi_buf = 0; /* Avoid accessing next_out */ + else + state->bi_buf = *strm->next_out & ((1 << state->bi_valid) - 1); + strm->adler = param->cv; + + /* Unmask the input data */ + strm->avail_in += masked_avail_in; + masked_avail_in = 0; + + /* If we encounter an error, it means there is a bug in DFLTCC call */ + Assert(cc != DFLTCC_CC_OP2_CORRUPT || param->oesc == 0, "BUG"); + + /* Update Block-Continuation Flag. It will be used to check whether to call + * GDHT the next time. + */ + if (cc == DFLTCC_CC_OK) { + if (soft_bcc) { + send_eobs(strm, param); + param->bcf = 0; + dfltcc_state->block_threshold = + strm->total_in + dfltcc_state->block_size; + } else + param->bcf = 1; + if (flush == Z_FINISH) { + if (need_empty_block) + /* Make the current deflate() call also close the stream */ + return 0; + else { + bi_windup(state); + *result = finish_done; + } + } else { + if (flush == Z_FULL_FLUSH) + param->hl = 0; /* Clear history */ + *result = flush == Z_NO_FLUSH ? need_more : block_done; + } + } else { + param->bcf = 1; + *result = need_more; + } + if (strm->avail_in != 0 && strm->avail_out != 0) + goto again; /* deflate() must use all input or all output */ + return 1; +} diff --git a/lib/zlib_dfltcc/dfltcc_syms.c b/lib/zlib_dfltcc/dfltcc_syms.c new file mode 100644 index 000000000000..6f23481804c1 --- /dev/null +++ b/lib/zlib_dfltcc/dfltcc_syms.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/lib/zlib_dfltcc/dfltcc_syms.c + * + * Exported symbols for the s390 zlib dfltcc support. + * + */ + +#include +#include +#include +#include "dfltcc.h" + +EXPORT_SYMBOL(dfltcc_can_deflate); +EXPORT_SYMBOL(dfltcc_deflate); +EXPORT_SYMBOL(dfltcc_reset); +MODULE_LICENSE("GPL"); diff --git a/lib/zlib_dfltcc/dfltcc_util.h b/lib/zlib_dfltcc/dfltcc_util.h new file mode 100644 index 000000000000..82cd1950c416 --- /dev/null +++ b/lib/zlib_dfltcc/dfltcc_util.h @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: Zlib +#ifndef DFLTCC_UTIL_H +#define DFLTCC_UTIL_H + +#include +#include + +/* + * C wrapper for the DEFLATE CONVERSION CALL instruction. + */ +typedef enum { + DFLTCC_CC_OK = 0, + DFLTCC_CC_OP1_TOO_SHORT = 1, + DFLTCC_CC_OP2_TOO_SHORT = 2, + DFLTCC_CC_OP2_CORRUPT = 2, + DFLTCC_CC_AGAIN = 3, +} dfltcc_cc; + +#define DFLTCC_QAF 0 +#define DFLTCC_GDHT 1 +#define DFLTCC_CMPR 2 +#define DFLTCC_XPND 4 +#define HBT_CIRCULAR (1 << 7) +#define HB_BITS 15 +#define HB_SIZE (1 << HB_BITS) +#define DFLTCC_FACILITY 151 + +static inline dfltcc_cc dfltcc( + int fn, + void *param, + Byte **op1, + size_t *len1, + const Byte **op2, + size_t *len2, + void *hist +) +{ + Byte *t2 = op1 ? *op1 : NULL; + size_t t3 = len1 ? *len1 : 0; + const Byte *t4 = op2 ? *op2 : NULL; + size_t t5 = len2 ? *len2 : 0; + register int r0 __asm__("r0") = fn; + register void *r1 __asm__("r1") = param; + register Byte *r2 __asm__("r2") = t2; + register size_t r3 __asm__("r3") = t3; + register const Byte *r4 __asm__("r4") = t4; + register size_t r5 __asm__("r5") = t5; + int cc; + + __asm__ volatile( + ".insn rrf,0xb9390000,%[r2],%[r4],%[hist],0\n" + "ipm %[cc]\n" + : [r2] "+r" (r2) + , [r3] "+r" (r3) + , [r4] "+r" (r4) + , [r5] "+r" (r5) + , [cc] "=r" (cc) + : [r0] "r" (r0) + , [r1] "r" (r1) + , [hist] "r" (hist) + : "cc", "memory"); + t2 = r2; t3 = r3; t4 = r4; t5 = r5; + + if (op1) + *op1 = t2; + if (len1) + *len1 = t3; + if (op2) + *op2 = t4; + if (len2) + *len2 = t5; + return (cc >> 28) & 3; +} + +static inline int is_bit_set( + const char *bits, + int n +) +{ + return bits[n / 8] & (1 << (7 - (n % 8))); +} + +static inline void turn_bit_off( + char *bits, + int n +) +{ + bits[n / 8] &= ~(1 << (7 - (n % 8))); +} + +static inline int dfltcc_are_params_ok( + int level, + uInt window_bits, + int strategy, + uLong level_mask +) +{ + return (level_mask & (1 << level)) != 0 && + (window_bits == HB_BITS) && + (strategy == Z_DEFAULT_STRATEGY); +} + +static inline int is_dfltcc_enabled(void) +{ + return test_facility(DFLTCC_FACILITY); +} + +char *oesc_msg(char *buf, int oesc); + +#endif /* DFLTCC_UTIL_H */ -- cgit v1.2.3 From 12619610006371bfc30159937d4546e731d7b297 Mon Sep 17 00:00:00 2001 From: Mikhail Zaslonko Date: Thu, 30 Jan 2020 22:16:23 -0800 Subject: lib/zlib: add s390 hardware support for kernel zlib_inflate Add decompression functions to zlib_dfltcc library. Update zlib_inflate functions with the hooks for s390 hardware support and adjust workspace structures with extra parameter lists required for hardware inflate decompression. Link: http://lkml.kernel.org/r/20200103223334.20669-4-zaslonko@linux.ibm.com Signed-off-by: Ilya Leoshkevich Signed-off-by: Mikhail Zaslonko Co-developed-by: Ilya Leoshkevich Cc: Chris Mason Cc: Christian Borntraeger Cc: David Sterba Cc: Eduard Shishkin Cc: Heiko Carstens Cc: Josef Bacik Cc: Richard Purdie Cc: Vasily Gorbik Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/decompress_inflate.c | 13 ++++ lib/zlib_dfltcc/Makefile | 2 +- lib/zlib_dfltcc/dfltcc.h | 28 ++++++++ lib/zlib_dfltcc/dfltcc_inflate.c | 143 +++++++++++++++++++++++++++++++++++++++ lib/zlib_inflate/inflate.c | 32 ++++++--- lib/zlib_inflate/inflate.h | 8 +++ lib/zlib_inflate/infutil.h | 18 ++++- 7 files changed, 233 insertions(+), 11 deletions(-) create mode 100644 lib/zlib_dfltcc/dfltcc_inflate.c (limited to 'lib') diff --git a/lib/decompress_inflate.c b/lib/decompress_inflate.c index 63b4b7eee138..6130c42b8e59 100644 --- a/lib/decompress_inflate.c +++ b/lib/decompress_inflate.c @@ -10,6 +10,10 @@ #include "zlib_inflate/inftrees.c" #include "zlib_inflate/inffast.c" #include "zlib_inflate/inflate.c" +#ifdef CONFIG_ZLIB_DFLTCC +#include "zlib_dfltcc/dfltcc.c" +#include "zlib_dfltcc/dfltcc_inflate.c" +#endif #else /* STATIC */ /* initramfs et al: linked */ @@ -76,7 +80,12 @@ STATIC int INIT __gunzip(unsigned char *buf, long len, } strm->workspace = malloc(flush ? zlib_inflate_workspacesize() : +#ifdef CONFIG_ZLIB_DFLTCC + /* Always allocate the full workspace for DFLTCC */ + zlib_inflate_workspacesize()); +#else sizeof(struct inflate_state)); +#endif if (strm->workspace == NULL) { error("Out of memory while allocating workspace"); goto gunzip_nomem4; @@ -123,10 +132,14 @@ STATIC int INIT __gunzip(unsigned char *buf, long len, rc = zlib_inflateInit2(strm, -MAX_WBITS); +#ifdef CONFIG_ZLIB_DFLTCC + /* Always keep the window for DFLTCC */ +#else if (!flush) { WS(strm)->inflate_state.wsize = 0; WS(strm)->inflate_state.window = NULL; } +#endif while (rc == Z_OK) { if (strm->avail_in == 0) { diff --git a/lib/zlib_dfltcc/Makefile b/lib/zlib_dfltcc/Makefile index 863d3b37e09d..8e4d5afbbb10 100644 --- a/lib/zlib_dfltcc/Makefile +++ b/lib/zlib_dfltcc/Makefile @@ -8,4 +8,4 @@ obj-$(CONFIG_ZLIB_DFLTCC) += zlib_dfltcc.o -zlib_dfltcc-objs := dfltcc.o dfltcc_deflate.o dfltcc_syms.o +zlib_dfltcc-objs := dfltcc.o dfltcc_deflate.o dfltcc_inflate.o dfltcc_syms.o diff --git a/lib/zlib_dfltcc/dfltcc.h b/lib/zlib_dfltcc/dfltcc.h index 18fed7a444bc..4782c92bb2ff 100644 --- a/lib/zlib_dfltcc/dfltcc.h +++ b/lib/zlib_dfltcc/dfltcc.h @@ -104,6 +104,14 @@ int dfltcc_deflate(z_streamp strm, int flush, block_state *result); void dfltcc_reset(z_streamp strm, uInt size); +int dfltcc_can_inflate(z_streamp strm); +typedef enum { + DFLTCC_INFLATE_CONTINUE, + DFLTCC_INFLATE_BREAK, + DFLTCC_INFLATE_SOFTWARE, +} dfltcc_inflate_action; +dfltcc_inflate_action dfltcc_inflate(z_streamp strm, + int flush, int *ret); #define DEFLATE_RESET_HOOK(strm) \ dfltcc_reset((strm), sizeof(deflate_state)) @@ -112,4 +120,24 @@ void dfltcc_reset(z_streamp strm, uInt size); #define DEFLATE_NEED_CHECKSUM(strm) (!dfltcc_can_deflate((strm))) +#define INFLATE_RESET_HOOK(strm) \ + dfltcc_reset((strm), sizeof(struct inflate_state)) + +#define INFLATE_TYPEDO_HOOK(strm, flush) \ + if (dfltcc_can_inflate((strm))) { \ + dfltcc_inflate_action action; \ +\ + RESTORE(); \ + action = dfltcc_inflate((strm), (flush), &ret); \ + LOAD(); \ + if (action == DFLTCC_INFLATE_CONTINUE) \ + break; \ + else if (action == DFLTCC_INFLATE_BREAK) \ + goto inf_leave; \ + } + +#define INFLATE_NEED_CHECKSUM(strm) (!dfltcc_can_inflate((strm))) + +#define INFLATE_NEED_UPDATEWINDOW(strm) (!dfltcc_can_inflate((strm))) + #endif /* DFLTCC_H */ diff --git a/lib/zlib_dfltcc/dfltcc_inflate.c b/lib/zlib_dfltcc/dfltcc_inflate.c new file mode 100644 index 000000000000..12a93a06bd61 --- /dev/null +++ b/lib/zlib_dfltcc/dfltcc_inflate.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: Zlib + +#include "../zlib_inflate/inflate.h" +#include "dfltcc_util.h" +#include "dfltcc.h" +#include + +/* + * Expand. + */ +int dfltcc_can_inflate( + z_streamp strm +) +{ + struct inflate_state *state = (struct inflate_state *)strm->state; + struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + + /* Unsupported compression settings */ + if (state->wbits != HB_BITS) + return 0; + + /* Unsupported hardware */ + return is_bit_set(dfltcc_state->af.fns, DFLTCC_XPND) && + is_bit_set(dfltcc_state->af.fmts, DFLTCC_FMT0); +} + +static int dfltcc_was_inflate_used( + z_streamp strm +) +{ + struct inflate_state *state = (struct inflate_state *)strm->state; + struct dfltcc_param_v0 *param = &GET_DFLTCC_STATE(state)->param; + + return !param->nt; +} + +static int dfltcc_inflate_disable( + z_streamp strm +) +{ + struct inflate_state *state = (struct inflate_state *)strm->state; + struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + + if (!dfltcc_can_inflate(strm)) + return 0; + if (dfltcc_was_inflate_used(strm)) + /* DFLTCC has already decompressed some data. Since there is not + * enough information to resume decompression in software, the call + * must fail. + */ + return 1; + /* DFLTCC was not used yet - decompress in software */ + memset(&dfltcc_state->af, 0, sizeof(dfltcc_state->af)); + return 0; +} + +static dfltcc_cc dfltcc_xpnd( + z_streamp strm +) +{ + struct inflate_state *state = (struct inflate_state *)strm->state; + struct dfltcc_param_v0 *param = &GET_DFLTCC_STATE(state)->param; + size_t avail_in = strm->avail_in; + size_t avail_out = strm->avail_out; + dfltcc_cc cc; + + cc = dfltcc(DFLTCC_XPND | HBT_CIRCULAR, + param, &strm->next_out, &avail_out, + &strm->next_in, &avail_in, state->window); + strm->avail_in = avail_in; + strm->avail_out = avail_out; + return cc; +} + +dfltcc_inflate_action dfltcc_inflate( + z_streamp strm, + int flush, + int *ret +) +{ + struct inflate_state *state = (struct inflate_state *)strm->state; + struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + struct dfltcc_param_v0 *param = &dfltcc_state->param; + dfltcc_cc cc; + + if (flush == Z_BLOCK) { + /* DFLTCC does not support stopping on block boundaries */ + if (dfltcc_inflate_disable(strm)) { + *ret = Z_STREAM_ERROR; + return DFLTCC_INFLATE_BREAK; + } else + return DFLTCC_INFLATE_SOFTWARE; + } + + if (state->last) { + if (state->bits != 0) { + strm->next_in++; + strm->avail_in--; + state->bits = 0; + } + state->mode = CHECK; + return DFLTCC_INFLATE_CONTINUE; + } + + if (strm->avail_in == 0 && !param->cf) + return DFLTCC_INFLATE_BREAK; + + if (!state->window || state->wsize == 0) { + state->mode = MEM; + return DFLTCC_INFLATE_CONTINUE; + } + + /* Translate stream to parameter block */ + param->cvt = CVT_ADLER32; + param->sbb = state->bits; + param->hl = state->whave; /* Software and hardware history formats match */ + param->ho = (state->write - state->whave) & ((1 << HB_BITS) - 1); + if (param->hl) + param->nt = 0; /* Honor history for the first block */ + param->cv = state->flags ? REVERSE(state->check) : state->check; + + /* Inflate */ + do { + cc = dfltcc_xpnd(strm); + } while (cc == DFLTCC_CC_AGAIN); + + /* Translate parameter block to stream */ + strm->msg = oesc_msg(dfltcc_state->msg, param->oesc); + state->last = cc == DFLTCC_CC_OK; + state->bits = param->sbb; + state->whave = param->hl; + state->write = (param->ho + param->hl) & ((1 << HB_BITS) - 1); + state->check = state->flags ? REVERSE(param->cv) : param->cv; + if (cc == DFLTCC_CC_OP2_CORRUPT && param->oesc != 0) { + /* Report an error if stream is corrupted */ + state->mode = BAD; + return DFLTCC_INFLATE_CONTINUE; + } + state->mode = TYPEDO; + /* Break if operands are exhausted, otherwise continue looping */ + return (cc == DFLTCC_CC_OP1_TOO_SHORT || cc == DFLTCC_CC_OP2_TOO_SHORT) ? + DFLTCC_INFLATE_BREAK : DFLTCC_INFLATE_CONTINUE; +} diff --git a/lib/zlib_inflate/inflate.c b/lib/zlib_inflate/inflate.c index 48f14cd58c77..67cc9b08ae9d 100644 --- a/lib/zlib_inflate/inflate.c +++ b/lib/zlib_inflate/inflate.c @@ -15,6 +15,16 @@ #include "inffast.h" #include "infutil.h" +/* architecture-specific bits */ +#ifdef CONFIG_ZLIB_DFLTCC +# include "../zlib_dfltcc/dfltcc.h" +#else +#define INFLATE_RESET_HOOK(strm) do {} while (0) +#define INFLATE_TYPEDO_HOOK(strm, flush) do {} while (0) +#define INFLATE_NEED_UPDATEWINDOW(strm) 1 +#define INFLATE_NEED_CHECKSUM(strm) 1 +#endif + int zlib_inflate_workspacesize(void) { return sizeof(struct inflate_workspace); @@ -42,6 +52,7 @@ int zlib_inflateReset(z_streamp strm) state->write = 0; state->whave = 0; + INFLATE_RESET_HOOK(strm); return Z_OK; } @@ -66,7 +77,15 @@ int zlib_inflateInit2(z_streamp strm, int windowBits) return Z_STREAM_ERROR; } state->wbits = (unsigned)windowBits; +#ifdef CONFIG_ZLIB_DFLTCC + /* + * DFLTCC requires the window to be page aligned. + * Thus, we overallocate and take the aligned portion of the buffer. + */ + state->window = PTR_ALIGN(&WS(strm)->working_window[0], PAGE_SIZE); +#else state->window = &WS(strm)->working_window[0]; +#endif return zlib_inflateReset(strm); } @@ -227,11 +246,6 @@ static int zlib_inflateSyncPacket(z_streamp strm) bits -= bits & 7; \ } while (0) -/* Reverse the bytes in a 32-bit value */ -#define REVERSE(q) \ - ((((q) >> 24) & 0xff) + (((q) >> 8) & 0xff00) + \ - (((q) & 0xff00) << 8) + (((q) & 0xff) << 24)) - /* inflate() uses a state machine to process as much input data and generate as much output data as possible before returning. The state machine is @@ -395,6 +409,7 @@ int zlib_inflate(z_streamp strm, int flush) if (flush == Z_BLOCK) goto inf_leave; /* fall through */ case TYPEDO: + INFLATE_TYPEDO_HOOK(strm, flush); if (state->last) { BYTEBITS(); state->mode = CHECK; @@ -692,7 +707,7 @@ int zlib_inflate(z_streamp strm, int flush) out -= left; strm->total_out += out; state->total += out; - if (out) + if (INFLATE_NEED_CHECKSUM(strm) && out) strm->adler = state->check = UPDATE(state->check, put - out, out); out = left; @@ -726,7 +741,8 @@ int zlib_inflate(z_streamp strm, int flush) */ inf_leave: RESTORE(); - if (state->wsize || (state->mode < CHECK && out != strm->avail_out)) + if (INFLATE_NEED_UPDATEWINDOW(strm) && + (state->wsize || (state->mode < CHECK && out != strm->avail_out))) zlib_updatewindow(strm, out); in -= strm->avail_in; @@ -734,7 +750,7 @@ int zlib_inflate(z_streamp strm, int flush) strm->total_in += in; strm->total_out += out; state->total += out; - if (state->wrap && out) + if (INFLATE_NEED_CHECKSUM(strm) && state->wrap && out) strm->adler = state->check = UPDATE(state->check, strm->next_out - out, out); diff --git a/lib/zlib_inflate/inflate.h b/lib/zlib_inflate/inflate.h index 3d17b3d1b21f..f79337ddf98c 100644 --- a/lib/zlib_inflate/inflate.h +++ b/lib/zlib_inflate/inflate.h @@ -11,6 +11,8 @@ subject to change. Applications should only use zlib.h. */ +#include "inftrees.h" + /* Possible inflate modes between inflate() calls */ typedef enum { HEAD, /* i: waiting for magic header */ @@ -108,4 +110,10 @@ struct inflate_state { unsigned short work[288]; /* work area for code table building */ code codes[ENOUGH]; /* space for code tables */ }; + +/* Reverse the bytes in a 32-bit value */ +#define REVERSE(q) \ + ((((q) >> 24) & 0xff) + (((q) >> 8) & 0xff00) + \ + (((q) & 0xff00) << 8) + (((q) & 0xff) << 24)) + #endif diff --git a/lib/zlib_inflate/infutil.h b/lib/zlib_inflate/infutil.h index eb1a9007bd86..784ab33b7842 100644 --- a/lib/zlib_inflate/infutil.h +++ b/lib/zlib_inflate/infutil.h @@ -12,14 +12,28 @@ #define _INFUTIL_H #include +#ifdef CONFIG_ZLIB_DFLTCC +#include "../zlib_dfltcc/dfltcc.h" +#include +#endif /* memory allocation for inflation */ struct inflate_workspace { struct inflate_state inflate_state; - unsigned char working_window[1 << MAX_WBITS]; +#ifdef CONFIG_ZLIB_DFLTCC + struct dfltcc_state dfltcc_state; + unsigned char working_window[(1 << MAX_WBITS) + PAGE_SIZE]; +#else + unsigned char working_window[(1 << MAX_WBITS)]; +#endif }; -#define WS(z) ((struct inflate_workspace *)(z->workspace)) +#ifdef CONFIG_ZLIB_DFLTCC +/* dfltcc_state must be doubleword aligned for DFLTCC call */ +static_assert(offsetof(struct inflate_workspace, dfltcc_state) % 8 == 0); +#endif + +#define WS(strm) ((struct inflate_workspace *)(strm->workspace)) #endif -- cgit v1.2.3 From c65e6815db1c2e28d5554bd99d3a6e522ab599d1 Mon Sep 17 00:00:00 2001 From: Mikhail Zaslonko Date: Thu, 30 Jan 2020 22:16:27 -0800 Subject: s390/boot: add dfltcc= kernel command line parameter Add the new kernel command line parameter 'dfltcc=' to configure s390 zlib hardware support. Format: { on | off | def_only | inf_only | always } on: s390 zlib hardware support for compression on level 1 and decompression (default) off: No s390 zlib hardware support def_only: s390 zlib hardware support for deflate only (compression on level 1) inf_only: s390 zlib hardware support for inflate only (decompression) always: Same as 'on' but ignores the selected compression level always using hardware support (used for debugging) Link: http://lkml.kernel.org/r/20200103223334.20669-5-zaslonko@linux.ibm.com Signed-off-by: Mikhail Zaslonko Cc: Chris Mason Cc: Christian Borntraeger Cc: David Sterba Cc: Eduard Shishkin Cc: Heiko Carstens Cc: Ilya Leoshkevich Cc: Josef Bacik Cc: Richard Purdie Cc: Vasily Gorbik Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/admin-guide/kernel-parameters.txt | 12 ++++++++++++ arch/s390/boot/ipl_parm.c | 14 ++++++++++++++ arch/s390/include/asm/setup.h | 7 +++++++ arch/s390/kernel/setup.c | 2 ++ lib/zlib_dfltcc/dfltcc.c | 5 ++++- lib/zlib_dfltcc/dfltcc.h | 1 + lib/zlib_dfltcc/dfltcc_deflate.c | 6 ++++++ lib/zlib_dfltcc/dfltcc_inflate.c | 6 ++++++ lib/zlib_dfltcc/dfltcc_util.h | 4 +++- 9 files changed, 55 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index ec92120a7952..ddc5ccdd4cd1 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -834,6 +834,18 @@ dump out devices still on the deferred probe list after retrying. + dfltcc= [HW,S390] + Format: { on | off | def_only | inf_only | always } + on: s390 zlib hardware support for compression on + level 1 and decompression (default) + off: No s390 zlib hardware support + def_only: s390 zlib hardware support for deflate + only (compression on level 1) + inf_only: s390 zlib hardware support for inflate + only (decompression) + always: Same as 'on' but ignores the selected compression + level always using hardware support (used for debugging) + dhash_entries= [KNL] Set number of hash buckets for dentry cache. diff --git a/arch/s390/boot/ipl_parm.c b/arch/s390/boot/ipl_parm.c index 24ef67eb1cef..357adad991d2 100644 --- a/arch/s390/boot/ipl_parm.c +++ b/arch/s390/boot/ipl_parm.c @@ -14,6 +14,7 @@ char __bootdata(early_command_line)[COMMAND_LINE_SIZE]; struct ipl_parameter_block __bootdata_preserved(ipl_block); int __bootdata_preserved(ipl_block_valid); +unsigned int __bootdata_preserved(zlib_dfltcc_support) = ZLIB_DFLTCC_FULL; unsigned long __bootdata(vmalloc_size) = VMALLOC_DEFAULT_SIZE; unsigned long __bootdata(memory_end); @@ -229,6 +230,19 @@ void parse_boot_command_line(void) if (!strcmp(param, "vmalloc") && val) vmalloc_size = round_up(memparse(val, NULL), PAGE_SIZE); + if (!strcmp(param, "dfltcc")) { + if (!strcmp(val, "off")) + zlib_dfltcc_support = ZLIB_DFLTCC_DISABLED; + else if (!strcmp(val, "on")) + zlib_dfltcc_support = ZLIB_DFLTCC_FULL; + else if (!strcmp(val, "def_only")) + zlib_dfltcc_support = ZLIB_DFLTCC_DEFLATE_ONLY; + else if (!strcmp(val, "inf_only")) + zlib_dfltcc_support = ZLIB_DFLTCC_INFLATE_ONLY; + else if (!strcmp(val, "always")) + zlib_dfltcc_support = ZLIB_DFLTCC_FULL_DEBUG; + } + if (!strcmp(param, "noexec")) { rc = kstrtobool(val, &enabled); if (!rc && !enabled) diff --git a/arch/s390/include/asm/setup.h b/arch/s390/include/asm/setup.h index 69289e99cabd..b241ddb67caf 100644 --- a/arch/s390/include/asm/setup.h +++ b/arch/s390/include/asm/setup.h @@ -79,6 +79,13 @@ struct parmarea { char command_line[ARCH_COMMAND_LINE_SIZE]; /* 0x10480 */ }; +extern unsigned int zlib_dfltcc_support; +#define ZLIB_DFLTCC_DISABLED 0 +#define ZLIB_DFLTCC_FULL 1 +#define ZLIB_DFLTCC_DEFLATE_ONLY 2 +#define ZLIB_DFLTCC_INFLATE_ONLY 3 +#define ZLIB_DFLTCC_FULL_DEBUG 4 + extern int noexec_disabled; extern int memory_end_set; extern unsigned long memory_end; diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 7d113e208f65..b2c2f75860e8 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -111,6 +111,8 @@ unsigned long __bootdata_preserved(__etext_dma); unsigned long __bootdata_preserved(__sdma); unsigned long __bootdata_preserved(__edma); unsigned long __bootdata_preserved(__kaslr_offset); +unsigned int __bootdata_preserved(zlib_dfltcc_support); +EXPORT_SYMBOL(zlib_dfltcc_support); unsigned long VMALLOC_START; EXPORT_SYMBOL(VMALLOC_START); diff --git a/lib/zlib_dfltcc/dfltcc.c b/lib/zlib_dfltcc/dfltcc.c index 7f77e5bb01c6..c30de430b30c 100644 --- a/lib/zlib_dfltcc/dfltcc.c +++ b/lib/zlib_dfltcc/dfltcc.c @@ -44,7 +44,10 @@ void dfltcc_reset( dfltcc_state->param.nt = 1; /* Initialize tuning parameters */ - dfltcc_state->level_mask = DFLTCC_LEVEL_MASK; + if (zlib_dfltcc_support == ZLIB_DFLTCC_FULL_DEBUG) + dfltcc_state->level_mask = DFLTCC_LEVEL_MASK_DEBUG; + else + dfltcc_state->level_mask = DFLTCC_LEVEL_MASK; dfltcc_state->block_size = DFLTCC_BLOCK_SIZE; dfltcc_state->block_threshold = DFLTCC_FIRST_FHT_BLOCK_SIZE; dfltcc_state->dht_threshold = DFLTCC_DHT_MIN_SAMPLE_SIZE; diff --git a/lib/zlib_dfltcc/dfltcc.h b/lib/zlib_dfltcc/dfltcc.h index 4782c92bb2ff..be70c807b62f 100644 --- a/lib/zlib_dfltcc/dfltcc.h +++ b/lib/zlib_dfltcc/dfltcc.h @@ -8,6 +8,7 @@ * Tuning parameters. */ #define DFLTCC_LEVEL_MASK 0x2 /* DFLTCC compression for level 1 only */ +#define DFLTCC_LEVEL_MASK_DEBUG 0x3fe /* DFLTCC compression for all levels */ #define DFLTCC_BLOCK_SIZE 1048576 #define DFLTCC_FIRST_FHT_BLOCK_SIZE 4096 #define DFLTCC_DHT_MIN_SAMPLE_SIZE 4096 diff --git a/lib/zlib_dfltcc/dfltcc_deflate.c b/lib/zlib_dfltcc/dfltcc_deflate.c index 9f0f1c90c4e0..00c185101c6d 100644 --- a/lib/zlib_dfltcc/dfltcc_deflate.c +++ b/lib/zlib_dfltcc/dfltcc_deflate.c @@ -3,6 +3,7 @@ #include "../zlib_deflate/defutil.h" #include "dfltcc_util.h" #include "dfltcc.h" +#include #include /* @@ -15,6 +16,11 @@ int dfltcc_can_deflate( deflate_state *state = (deflate_state *)strm->state; struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + /* Check for kernel dfltcc command line parameter */ + if (zlib_dfltcc_support == ZLIB_DFLTCC_DISABLED || + zlib_dfltcc_support == ZLIB_DFLTCC_INFLATE_ONLY) + return 0; + /* Unsupported compression settings */ if (!dfltcc_are_params_ok(state->level, state->w_bits, state->strategy, dfltcc_state->level_mask)) diff --git a/lib/zlib_dfltcc/dfltcc_inflate.c b/lib/zlib_dfltcc/dfltcc_inflate.c index 12a93a06bd61..aa9ef23474df 100644 --- a/lib/zlib_dfltcc/dfltcc_inflate.c +++ b/lib/zlib_dfltcc/dfltcc_inflate.c @@ -3,6 +3,7 @@ #include "../zlib_inflate/inflate.h" #include "dfltcc_util.h" #include "dfltcc.h" +#include #include /* @@ -15,6 +16,11 @@ int dfltcc_can_inflate( struct inflate_state *state = (struct inflate_state *)strm->state; struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + /* Check for kernel dfltcc command line parameter */ + if (zlib_dfltcc_support == ZLIB_DFLTCC_DISABLED || + zlib_dfltcc_support == ZLIB_DFLTCC_DEFLATE_ONLY) + return 0; + /* Unsupported compression settings */ if (state->wbits != HB_BITS) return 0; diff --git a/lib/zlib_dfltcc/dfltcc_util.h b/lib/zlib_dfltcc/dfltcc_util.h index 82cd1950c416..7c0d3bdc50a9 100644 --- a/lib/zlib_dfltcc/dfltcc_util.h +++ b/lib/zlib_dfltcc/dfltcc_util.h @@ -4,6 +4,7 @@ #include #include +#include /* * C wrapper for the DEFLATE CONVERSION CALL instruction. @@ -102,7 +103,8 @@ static inline int dfltcc_are_params_ok( static inline int is_dfltcc_enabled(void) { - return test_facility(DFLTCC_FACILITY); + return (zlib_dfltcc_support != ZLIB_DFLTCC_DISABLED && + test_facility(DFLTCC_FACILITY)); } char *oesc_msg(char *buf, int oesc); -- cgit v1.2.3 From 803521b149c8c71a712445a32bd9316e074df76a Mon Sep 17 00:00:00 2001 From: Mikhail Zaslonko Date: Thu, 30 Jan 2020 22:16:30 -0800 Subject: lib/zlib: add zlib_deflate_dfltcc_enabled() function Add a new function to zlib.h checking if s390 Deflate-Conversion facility is installed and enabled. Link: http://lkml.kernel.org/r/20200103223334.20669-6-zaslonko@linux.ibm.com Signed-off-by: Mikhail Zaslonko Cc: Chris Mason Cc: Christian Borntraeger Cc: David Sterba Cc: Eduard Shishkin Cc: Heiko Carstens Cc: Ilya Leoshkevich Cc: Josef Bacik Cc: Richard Purdie Cc: Vasily Gorbik Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/zlib.h | 6 ++++++ lib/zlib_deflate/deflate.c | 6 ++++++ lib/zlib_deflate/deflate_syms.c | 1 + lib/zlib_dfltcc/dfltcc.h | 11 +++++++++++ lib/zlib_dfltcc/dfltcc_util.h | 9 --------- 5 files changed, 24 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/include/linux/zlib.h b/include/linux/zlib.h index 92dbbd3f6c75..c757d848a758 100644 --- a/include/linux/zlib.h +++ b/include/linux/zlib.h @@ -191,6 +191,12 @@ extern int zlib_deflate_workspacesize (int windowBits, int memLevel); exceed those passed here. */ +extern int zlib_deflate_dfltcc_enabled (void); +/* + Returns 1 if Deflate-Conversion facility is installed and enabled, + otherwise 0. +*/ + /* extern int deflateInit (z_streamp strm, int level); diff --git a/lib/zlib_deflate/deflate.c b/lib/zlib_deflate/deflate.c index 0d60d81a2637..8a878d0d892c 100644 --- a/lib/zlib_deflate/deflate.c +++ b/lib/zlib_deflate/deflate.c @@ -59,6 +59,7 @@ #define DEFLATE_RESET_HOOK(strm) do {} while (0) #define DEFLATE_HOOK(strm, flush, bstate) 0 #define DEFLATE_NEED_CHECKSUM(strm) 1 +#define DEFLATE_DFLTCC_ENABLED() 0 #endif /* =========================================================================== @@ -1138,3 +1139,8 @@ int zlib_deflate_workspacesize(int windowBits, int memLevel) + zlib_deflate_head_memsize(memLevel) + zlib_deflate_overlay_memsize(memLevel); } + +int zlib_deflate_dfltcc_enabled(void) +{ + return DEFLATE_DFLTCC_ENABLED(); +} diff --git a/lib/zlib_deflate/deflate_syms.c b/lib/zlib_deflate/deflate_syms.c index 72fe4b73be53..24b740b99678 100644 --- a/lib/zlib_deflate/deflate_syms.c +++ b/lib/zlib_deflate/deflate_syms.c @@ -12,6 +12,7 @@ #include EXPORT_SYMBOL(zlib_deflate_workspacesize); +EXPORT_SYMBOL(zlib_deflate_dfltcc_enabled); EXPORT_SYMBOL(zlib_deflate); EXPORT_SYMBOL(zlib_deflateInit2); EXPORT_SYMBOL(zlib_deflateEnd); diff --git a/lib/zlib_dfltcc/dfltcc.h b/lib/zlib_dfltcc/dfltcc.h index be70c807b62f..2a2fac1d050a 100644 --- a/lib/zlib_dfltcc/dfltcc.h +++ b/lib/zlib_dfltcc/dfltcc.h @@ -3,6 +3,8 @@ #define DFLTCC_H #include "../zlib_deflate/defutil.h" +#include +#include /* * Tuning parameters. @@ -14,6 +16,8 @@ #define DFLTCC_DHT_MIN_SAMPLE_SIZE 4096 #define DFLTCC_RIBM 0 +#define DFLTCC_FACILITY 151 + /* * Parameter Block for Query Available Functions. */ @@ -113,6 +117,11 @@ typedef enum { } dfltcc_inflate_action; dfltcc_inflate_action dfltcc_inflate(z_streamp strm, int flush, int *ret); +static inline int is_dfltcc_enabled(void) +{ +return (zlib_dfltcc_support != ZLIB_DFLTCC_DISABLED && + test_facility(DFLTCC_FACILITY)); +} #define DEFLATE_RESET_HOOK(strm) \ dfltcc_reset((strm), sizeof(deflate_state)) @@ -121,6 +130,8 @@ dfltcc_inflate_action dfltcc_inflate(z_streamp strm, #define DEFLATE_NEED_CHECKSUM(strm) (!dfltcc_can_deflate((strm))) +#define DEFLATE_DFLTCC_ENABLED() is_dfltcc_enabled() + #define INFLATE_RESET_HOOK(strm) \ dfltcc_reset((strm), sizeof(struct inflate_state)) diff --git a/lib/zlib_dfltcc/dfltcc_util.h b/lib/zlib_dfltcc/dfltcc_util.h index 7c0d3bdc50a9..4a46b5009f0d 100644 --- a/lib/zlib_dfltcc/dfltcc_util.h +++ b/lib/zlib_dfltcc/dfltcc_util.h @@ -3,8 +3,6 @@ #define DFLTCC_UTIL_H #include -#include -#include /* * C wrapper for the DEFLATE CONVERSION CALL instruction. @@ -24,7 +22,6 @@ typedef enum { #define HBT_CIRCULAR (1 << 7) #define HB_BITS 15 #define HB_SIZE (1 << HB_BITS) -#define DFLTCC_FACILITY 151 static inline dfltcc_cc dfltcc( int fn, @@ -101,12 +98,6 @@ static inline int dfltcc_are_params_ok( (strategy == Z_DEFAULT_STRATEGY); } -static inline int is_dfltcc_enabled(void) -{ - return (zlib_dfltcc_support != ZLIB_DFLTCC_DISABLED && - test_facility(DFLTCC_FACILITY)); -} - char *oesc_msg(char *buf, int oesc); #endif /* DFLTCC_UTIL_H */ -- cgit v1.2.3 From 4e456fee215677584cafa7f67298a76917e89c64 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Thu, 30 Jan 2020 22:16:37 -0800 Subject: lib/scatterlist.c: adjust indentation in __sg_alloc_table Clang warns: ../lib/scatterlist.c:314:5: warning: misleading indentation; statement is not part of the previous 'if' [-Wmisleading-indentation] return -ENOMEM; ^ ../lib/scatterlist.c:311:4: note: previous statement is here if (prv) ^ 1 warning generated. This warning occurs because there is a space before the tab on this line. Remove it so that the indentation is consistent with the Linux kernel coding style and clang no longer warns. Link: http://lkml.kernel.org/r/20191218033606.11942-1-natechancellor@gmail.com Link: https://github.com/ClangBuiltLinux/linux/issues/830 Fixes: edce6820a9fd ("scatterlist: prevent invalid free when alloc fails") Signed-off-by: Nathan Chancellor Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/scatterlist.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/scatterlist.c b/lib/scatterlist.c index c2cf2c311b7d..5813072bc589 100644 --- a/lib/scatterlist.c +++ b/lib/scatterlist.c @@ -311,7 +311,7 @@ int __sg_alloc_table(struct sg_table *table, unsigned int nents, if (prv) table->nents = ++table->orig_nents; - return -ENOMEM; + return -ENOMEM; } sg_init_table(sg, alloc_size); -- cgit v1.2.3 From d5767057c9a76a29f073dad66b7fa12a90e8c748 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Thu, 30 Jan 2020 22:16:40 -0800 Subject: uapi: rename ext2_swab() to swab() and share globally in swab.h ext2_swab() is defined locally in lib/find_bit.c However it is not specific to ext2, neither to bitmaps. There are many potential users of it, so rename it to just swab() and move to include/uapi/linux/swab.h ABI guarantees that size of unsigned long corresponds to BITS_PER_LONG, therefore drop unneeded cast. Link: http://lkml.kernel.org/r/20200103202846.21616-1-yury.norov@gmail.com Signed-off-by: Yury Norov Cc: Allison Randal Cc: Joe Perches Cc: Thomas Gleixner Cc: William Breathitt Gray Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/swab.h | 1 + include/uapi/linux/swab.h | 10 ++++++++++ lib/find_bit.c | 16 ++-------------- 3 files changed, 13 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/include/linux/swab.h b/include/linux/swab.h index e466fd159c85..bcff5149861a 100644 --- a/include/linux/swab.h +++ b/include/linux/swab.h @@ -7,6 +7,7 @@ # define swab16 __swab16 # define swab32 __swab32 # define swab64 __swab64 +# define swab __swab # define swahw32 __swahw32 # define swahb32 __swahb32 # define swab16p __swab16p diff --git a/include/uapi/linux/swab.h b/include/uapi/linux/swab.h index 23cd84868cc3..fa7f97da5b76 100644 --- a/include/uapi/linux/swab.h +++ b/include/uapi/linux/swab.h @@ -4,6 +4,7 @@ #include #include +#include #include /* @@ -132,6 +133,15 @@ static inline __attribute_const__ __u32 __fswahb32(__u32 val) __fswab64(x)) #endif +static __always_inline unsigned long __swab(const unsigned long y) +{ +#if BITS_PER_LONG == 64 + return __swab64(y); +#else /* BITS_PER_LONG == 32 */ + return __swab32(y); +#endif +} + /** * __swahw32 - return a word-swapped 32-bit value * @x: value to wordswap diff --git a/lib/find_bit.c b/lib/find_bit.c index e35a76b291e6..06e503c339f3 100644 --- a/lib/find_bit.c +++ b/lib/find_bit.c @@ -149,18 +149,6 @@ EXPORT_SYMBOL(find_last_bit); #ifdef __BIG_ENDIAN -/* include/linux/byteorder does not support "unsigned long" type */ -static inline unsigned long ext2_swab(const unsigned long y) -{ -#if BITS_PER_LONG == 64 - return (unsigned long) __swab64((u64) y); -#elif BITS_PER_LONG == 32 - return (unsigned long) __swab32((u32) y); -#else -#error BITS_PER_LONG not defined -#endif -} - #if !defined(find_next_bit_le) || !defined(find_next_zero_bit_le) static inline unsigned long _find_next_bit_le(const unsigned long *addr1, const unsigned long *addr2, unsigned long nbits, @@ -177,7 +165,7 @@ static inline unsigned long _find_next_bit_le(const unsigned long *addr1, tmp ^= invert; /* Handle 1st word. */ - tmp &= ext2_swab(BITMAP_FIRST_WORD_MASK(start)); + tmp &= swab(BITMAP_FIRST_WORD_MASK(start)); start = round_down(start, BITS_PER_LONG); while (!tmp) { @@ -191,7 +179,7 @@ static inline unsigned long _find_next_bit_le(const unsigned long *addr1, tmp ^= invert; } - return min(start + __ffs(ext2_swab(tmp)), nbits); + return min(start + __ffs(swab(tmp)), nbits); } #endif -- cgit v1.2.3 From b78c57135d470cfc5b461f2abde4eaeb07d100fb Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Thu, 30 Jan 2020 22:16:43 -0800 Subject: lib/find_bit.c: join _find_next_bit{_le} _find_next_bit and _find_next_bit_le are very similar functions. It's possible to join them by adding 1 parameter and a couple of simple checks. It's simplify maintenance and make possible to shrink the size of .text by un-inlining the unified function (in the following patch). Link: http://lkml.kernel.org/r/20200103202846.21616-2-yury.norov@gmail.com Signed-off-by: Yury Norov Cc: Allison Randal Cc: Joe Perches Cc: Thomas Gleixner Cc: William Breathitt Gray Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/find_bit.c | 64 +++++++++++++++++----------------------------------------- 1 file changed, 19 insertions(+), 45 deletions(-) (limited to 'lib') diff --git a/lib/find_bit.c b/lib/find_bit.c index 06e503c339f3..c897c127316d 100644 --- a/lib/find_bit.c +++ b/lib/find_bit.c @@ -17,9 +17,9 @@ #include #include -#if !defined(find_next_bit) || !defined(find_next_zero_bit) || \ - !defined(find_next_and_bit) - +#if !defined(find_next_bit) || !defined(find_next_zero_bit) || \ + !defined(find_next_bit_le) || !defined(find_next_zero_bit_le) || \ + !defined(find_next_and_bit) /* * This is a common helper function for find_next_bit, find_next_zero_bit, and * find_next_and_bit. The differences are: @@ -29,9 +29,9 @@ */ static inline unsigned long _find_next_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long nbits, - unsigned long start, unsigned long invert) + unsigned long start, unsigned long invert, unsigned long le) { - unsigned long tmp; + unsigned long tmp, mask; if (unlikely(start >= nbits)) return nbits; @@ -42,7 +42,12 @@ static inline unsigned long _find_next_bit(const unsigned long *addr1, tmp ^= invert; /* Handle 1st word. */ - tmp &= BITMAP_FIRST_WORD_MASK(start); + mask = BITMAP_FIRST_WORD_MASK(start); + if (le) + mask = swab(mask); + + tmp &= mask; + start = round_down(start, BITS_PER_LONG); while (!tmp) { @@ -56,6 +61,9 @@ static inline unsigned long _find_next_bit(const unsigned long *addr1, tmp ^= invert; } + if (le) + tmp = swab(tmp); + return min(start + __ffs(tmp), nbits); } #endif @@ -67,7 +75,7 @@ static inline unsigned long _find_next_bit(const unsigned long *addr1, unsigned long find_next_bit(const unsigned long *addr, unsigned long size, unsigned long offset) { - return _find_next_bit(addr, NULL, size, offset, 0UL); + return _find_next_bit(addr, NULL, size, offset, 0UL, 0); } EXPORT_SYMBOL(find_next_bit); #endif @@ -76,7 +84,7 @@ EXPORT_SYMBOL(find_next_bit); unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size, unsigned long offset) { - return _find_next_bit(addr, NULL, size, offset, ~0UL); + return _find_next_bit(addr, NULL, size, offset, ~0UL, 0); } EXPORT_SYMBOL(find_next_zero_bit); #endif @@ -86,7 +94,7 @@ unsigned long find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long size, unsigned long offset) { - return _find_next_bit(addr1, addr2, size, offset, 0UL); + return _find_next_bit(addr1, addr2, size, offset, 0UL, 0); } EXPORT_SYMBOL(find_next_and_bit); #endif @@ -149,45 +157,11 @@ EXPORT_SYMBOL(find_last_bit); #ifdef __BIG_ENDIAN -#if !defined(find_next_bit_le) || !defined(find_next_zero_bit_le) -static inline unsigned long _find_next_bit_le(const unsigned long *addr1, - const unsigned long *addr2, unsigned long nbits, - unsigned long start, unsigned long invert) -{ - unsigned long tmp; - - if (unlikely(start >= nbits)) - return nbits; - - tmp = addr1[start / BITS_PER_LONG]; - if (addr2) - tmp &= addr2[start / BITS_PER_LONG]; - tmp ^= invert; - - /* Handle 1st word. */ - tmp &= swab(BITMAP_FIRST_WORD_MASK(start)); - start = round_down(start, BITS_PER_LONG); - - while (!tmp) { - start += BITS_PER_LONG; - if (start >= nbits) - return nbits; - - tmp = addr1[start / BITS_PER_LONG]; - if (addr2) - tmp &= addr2[start / BITS_PER_LONG]; - tmp ^= invert; - } - - return min(start + __ffs(swab(tmp)), nbits); -} -#endif - #ifndef find_next_zero_bit_le unsigned long find_next_zero_bit_le(const void *addr, unsigned long size, unsigned long offset) { - return _find_next_bit_le(addr, NULL, size, offset, ~0UL); + return _find_next_bit(addr, NULL, size, offset, ~0UL, 1); } EXPORT_SYMBOL(find_next_zero_bit_le); #endif @@ -196,7 +170,7 @@ EXPORT_SYMBOL(find_next_zero_bit_le); unsigned long find_next_bit_le(const void *addr, unsigned long size, unsigned long offset) { - return _find_next_bit_le(addr, NULL, size, offset, 0UL); + return _find_next_bit(addr, NULL, size, offset, 0UL, 1); } EXPORT_SYMBOL(find_next_bit_le); #endif -- cgit v1.2.3 From 7dfaa98f646bbc29c3575bc184f2b37aa2dd62c5 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Thu, 30 Jan 2020 22:16:47 -0800 Subject: lib/find_bit.c: uninline helper _find_next_bit() It saves 25% of .text for arm64, and more for BE architectures. Before: $ size lib/find_bit.o text data bss dec hex filename 1012 56 0 1068 42c lib/find_bit.o After: $ size lib/find_bit.o text data bss dec hex filename 776 56 0 832 340 lib/find_bit.o Link: http://lkml.kernel.org/r/20200103202846.21616-3-yury.norov@gmail.com Signed-off-by: Yury Norov Cc: Thomas Gleixner Cc: Allison Randal Cc: William Breathitt Gray Cc: Joe Perches Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/find_bit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/find_bit.c b/lib/find_bit.c index c897c127316d..49f875f1baf7 100644 --- a/lib/find_bit.c +++ b/lib/find_bit.c @@ -27,7 +27,7 @@ * searching it for one bits. * - The optional "addr2", which is anded with "addr1" if present. */ -static inline unsigned long _find_next_bit(const unsigned long *addr1, +static unsigned long _find_next_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long nbits, unsigned long start, unsigned long invert, unsigned long le) { -- cgit v1.2.3 From 43e76af85fa7e75ac9b71fc2fcc250abb1889bff Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Thu, 30 Jan 2020 22:17:35 -0800 Subject: kcov: ignore fault-inject and stacktrace Don't instrument 3 more files that contain debugging facilities and produce large amounts of uninteresting coverage for every syscall. The following snippets are sprinkled all over the place in kcov traces in a debugging kernel. We already try to disable instrumentation of stack unwinding code and of most debug facilities. I guess we did not use fault-inject.c at the time, and stacktrace.c was somehow missed (or something has changed in kernel/configs). This change both speeds up kcov (kernel doesn't need to store these PCs, user-space doesn't need to process them) and frees trace buffer capacity for more useful coverage. should_fail lib/fault-inject.c:149 fail_dump lib/fault-inject.c:45 stack_trace_save kernel/stacktrace.c:124 stack_trace_consume_entry kernel/stacktrace.c:86 stack_trace_consume_entry kernel/stacktrace.c:89 ... a hundred frames skipped ... stack_trace_consume_entry kernel/stacktrace.c:93 stack_trace_consume_entry kernel/stacktrace.c:86 Link: http://lkml.kernel.org/r/20200116111449.217744-1-dvyukov@gmail.com Signed-off-by: Dmitry Vyukov Reviewed-by: Andrey Konovalov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/Makefile | 1 + lib/Makefile | 1 + mm/Makefile | 1 + 3 files changed, 3 insertions(+) (limited to 'lib') diff --git a/kernel/Makefile b/kernel/Makefile index f2cc0d118a0b..4cb4130ced32 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -27,6 +27,7 @@ KCOV_INSTRUMENT_softirq.o := n # and produce insane amounts of uninteresting coverage. KCOV_INSTRUMENT_module.o := n KCOV_INSTRUMENT_extable.o := n +KCOV_INSTRUMENT_stacktrace.o := n # Don't self-instrument. KCOV_INSTRUMENT_kcov.o := n KASAN_SANITIZE_kcov.o := n diff --git a/lib/Makefile b/lib/Makefile index 5fb02b93b1dc..23ca78d43d24 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -16,6 +16,7 @@ KCOV_INSTRUMENT_rbtree.o := n KCOV_INSTRUMENT_list_debug.o := n KCOV_INSTRUMENT_debugobjects.o := n KCOV_INSTRUMENT_dynamic_debug.o := n +KCOV_INSTRUMENT_fault-inject.o := n # Early boot use of cmdline, don't instrument it ifdef CONFIG_AMD_MEM_ENCRYPT diff --git a/mm/Makefile b/mm/Makefile index 1937cc251883..32f08e22e824 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -20,6 +20,7 @@ KCOV_INSTRUMENT_kmemleak.o := n KCOV_INSTRUMENT_memcontrol.o := n KCOV_INSTRUMENT_mmzone.o := n KCOV_INSTRUMENT_vmstat.o := n +KCOV_INSTRUMENT_failslab.o := n CFLAGS_init-mm.o += $(call cc-disable-warning, override-init) CFLAGS_init-mm.o += $(call cc-disable-warning, initializer-overrides) -- cgit v1.2.3 From 5f2fb52fac15a8a8e10ce020dd532504a8abfc4e Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 2 Feb 2020 01:49:24 +0900 Subject: kbuild: rename hostprogs-y/always to hostprogs/always-y In old days, the "host-progs" syntax was used for specifying host programs. It was renamed to the current "hostprogs-y" in 2004. It is typically useful in scripts/Makefile because it allows Kbuild to selectively compile host programs based on the kernel configuration. This commit renames like follows: always -> always-y hostprogs-y -> hostprogs So, scripts/Makefile will look like this: always-$(CONFIG_BUILD_BIN2C) += ... always-$(CONFIG_KALLSYMS) += ... ... hostprogs := $(always-y) $(always-m) I think this makes more sense because a host program is always a host program, irrespective of the kernel configuration. We want to specify which ones to compile by CONFIG options, so always-y will be handier. The "always", "hostprogs-y", "hostprogs-m" will be kept for backward compatibility for a while. Signed-off-by: Masahiro Yamada --- Documentation/kbuild/makefiles.rst | 49 +++++---------- Kbuild | 8 +-- arch/alpha/boot/Makefile | 2 +- arch/arm/vdso/Makefile | 2 +- arch/arm64/kernel/vdso32/Makefile | 4 +- arch/mips/boot/Makefile | 2 +- arch/mips/boot/compressed/Makefile | 4 +- arch/mips/boot/tools/Makefile | 2 +- arch/mips/tools/Makefile | 4 +- arch/mips/vdso/Makefile | 2 +- arch/powerpc/boot/Makefile | 4 +- arch/s390/tools/Makefile | 4 +- arch/sparc/boot/Makefile | 2 +- arch/sparc/vdso/Makefile | 2 +- arch/x86/boot/Makefile | 4 +- arch/x86/boot/compressed/Makefile | 2 +- arch/x86/entry/vdso/Makefile | 2 +- arch/x86/realmode/rm/Makefile | 2 +- arch/x86/tools/Makefile | 4 +- drivers/gpu/drm/radeon/Makefile | 2 +- drivers/tty/vt/Makefile | 2 +- drivers/video/logo/Makefile | 2 +- drivers/zorro/Makefile | 2 +- fs/unicode/Makefile | 2 +- lib/Makefile | 4 +- lib/raid6/Makefile | 2 +- net/bpfilter/Makefile | 2 +- samples/bpf/Makefile | 118 ++++++++++++++++++------------------ samples/connector/Makefile | 8 +-- samples/hidraw/Makefile | 6 +- samples/mei/Makefile | 4 +- samples/pidfd/Makefile | 4 +- samples/seccomp/Makefile | 4 +- samples/uhid/Makefile | 4 +- samples/vfs/Makefile | 5 +- scripts/Makefile | 22 +++---- scripts/Makefile.build | 8 ++- scripts/Makefile.clean | 4 +- scripts/Makefile.host | 8 +-- scripts/Makefile.lib | 6 +- scripts/basic/Makefile | 4 +- scripts/dtc/Makefile | 4 +- scripts/gcc-plugins/Makefile | 2 +- scripts/genksyms/Makefile | 4 +- scripts/kconfig/Makefile | 10 +-- scripts/mod/Makefile | 4 +- scripts/selinux/genheaders/Makefile | 4 +- scripts/selinux/mdp/Makefile | 4 +- usr/Makefile | 2 +- 49 files changed, 172 insertions(+), 190 deletions(-) (limited to 'lib') diff --git a/Documentation/kbuild/makefiles.rst b/Documentation/kbuild/makefiles.rst index b1733b877025..0e0eb2c8da7d 100644 --- a/Documentation/kbuild/makefiles.rst +++ b/Documentation/kbuild/makefiles.rst @@ -28,7 +28,6 @@ This document describes the Linux kernel Makefiles. --- 4.3 Using C++ for host programs --- 4.4 Controlling compiler options for host programs --- 4.5 When host programs are actually built - --- 4.6 Using hostprogs-$(CONFIG_FOO) === 5 Kbuild clean infrastructure @@ -595,11 +594,11 @@ compilation stage. Two steps are required in order to use a host executable. The first step is to tell kbuild that a host program exists. This is -done utilising the variable hostprogs-y. +done utilising the variable "hostprogs". The second step is to add an explicit dependency to the executable. This can be done in two ways. Either add the dependency in a rule, -or utilise the variable $(always). +or utilise the variable "always-y". Both possibilities are described in the following. 4.1 Simple Host Program @@ -612,7 +611,7 @@ Both possibilities are described in the following. Example:: - hostprogs-y := bin2hex + hostprogs := bin2hex Kbuild assumes in the above example that bin2hex is made from a single c-source file named bin2hex.c located in the same directory as @@ -630,7 +629,7 @@ Both possibilities are described in the following. Example:: #scripts/lxdialog/Makefile - hostprogs-y := lxdialog + hostprogs := lxdialog lxdialog-objs := checklist.o lxdialog.o Objects with extension .o are compiled from the corresponding .c @@ -650,7 +649,7 @@ Both possibilities are described in the following. Example:: #scripts/kconfig/Makefile - hostprogs-y := qconf + hostprogs := qconf qconf-cxxobjs := qconf.o In the example above the executable is composed of the C++ file @@ -662,7 +661,7 @@ Both possibilities are described in the following. Example:: #scripts/kconfig/Makefile - hostprogs-y := qconf + hostprogs := qconf qconf-cxxobjs := qconf.o qconf-objs := check.o @@ -710,7 +709,7 @@ Both possibilities are described in the following. Example:: #drivers/pci/Makefile - hostprogs-y := gen-devlist + hostprogs := gen-devlist $(obj)/devlist.h: $(src)/pci.ids $(obj)/gen-devlist ( cd $(obj); ./gen-devlist ) < $< @@ -718,47 +717,31 @@ Both possibilities are described in the following. $(obj)/gen-devlist is updated. Note that references to the host programs in special rules must be prefixed with $(obj). - (2) Use $(always) + (2) Use always-y When there is no suitable special rule, and the host program - shall be built when a makefile is entered, the $(always) + shall be built when a makefile is entered, the always-y variable shall be used. Example:: #scripts/lxdialog/Makefile - hostprogs-y := lxdialog - always := $(hostprogs-y) + hostprogs := lxdialog + always-y := $(hostprogs) This will tell kbuild to build lxdialog even if not referenced in any rule. -4.6 Using hostprogs-$(CONFIG_FOO) ---------------------------------- - - A typical pattern in a Kbuild file looks like this: - - Example:: - - #scripts/Makefile - hostprogs-$(CONFIG_KALLSYMS) += kallsyms - - Kbuild knows about both 'y' for built-in and 'm' for module. - So if a config symbol evaluates to 'm', kbuild will still build - the binary. In other words, Kbuild handles hostprogs-m exactly - like hostprogs-y. But only hostprogs-y is recommended to be used - when no CONFIG symbols are involved. - 5 Kbuild clean infrastructure ============================= "make clean" deletes most generated files in the obj tree where the kernel is compiled. This includes generated files such as host programs. -Kbuild knows targets listed in $(hostprogs-y), $(hostprogs-m), $(always), -$(extra-y) and $(targets). They are all deleted during "make clean". -Files matching the patterns "*.[oas]", "*.ko", plus some additional files -generated by kbuild are deleted all over the kernel src tree when -"make clean" is executed. +Kbuild knows targets listed in $(hostprogs), $(always-y), $(always-m), +$(always-), $(extra-y), $(extra-) and $(targets). They are all deleted +during "make clean". Files matching the patterns "*.[oas]", "*.ko", plus +some additional files generated by kbuild are deleted all over the kernel +source tree when "make clean" is executed. Additional files or directories can be specified in kbuild makefiles by use of $(clean-files). diff --git a/Kbuild b/Kbuild index 3109ac786e76..fa441b98c9f6 100644 --- a/Kbuild +++ b/Kbuild @@ -7,7 +7,7 @@ bounds-file := include/generated/bounds.h -always := $(bounds-file) +always-y := $(bounds-file) targets := kernel/bounds.s $(bounds-file): kernel/bounds.s FORCE @@ -28,7 +28,7 @@ $(timeconst-file): kernel/time/timeconst.bc FORCE offsets-file := include/generated/asm-offsets.h -always += $(offsets-file) +always-y += $(offsets-file) targets += arch/$(SRCARCH)/kernel/asm-offsets.s arch/$(SRCARCH)/kernel/asm-offsets.s: $(timeconst-file) $(bounds-file) @@ -39,7 +39,7 @@ $(offsets-file): arch/$(SRCARCH)/kernel/asm-offsets.s FORCE ##### # Check for missing system calls -always += missing-syscalls +always-y += missing-syscalls quiet_cmd_syscalls = CALL $< cmd_syscalls = $(CONFIG_SHELL) $< $(CC) $(c_flags) $(missing_syscalls_flags) @@ -50,7 +50,7 @@ missing-syscalls: scripts/checksyscalls.sh $(offsets-file) FORCE ##### # Check atomic headers are up-to-date -always += old-atomics +always-y += old-atomics quiet_cmd_atomics = CALL $< cmd_atomics = $(CONFIG_SHELL) $< diff --git a/arch/alpha/boot/Makefile b/arch/alpha/boot/Makefile index 991e023a6fc4..d8dba85e606c 100644 --- a/arch/alpha/boot/Makefile +++ b/arch/alpha/boot/Makefile @@ -8,7 +8,7 @@ # Copyright (C) 1994 by Linus Torvalds # -hostprogs-y := tools/mkbb tools/objstrip +hostprogs := tools/mkbb tools/objstrip targets := vmlinux.gz vmlinux \ vmlinux.nh tools/lxboot tools/bootlx tools/bootph \ tools/bootpzh bootloader bootpheader bootpzheader diff --git a/arch/arm/vdso/Makefile b/arch/arm/vdso/Makefile index 1babb392e70a..d3c9f03e7e79 100644 --- a/arch/arm/vdso/Makefile +++ b/arch/arm/vdso/Makefile @@ -5,7 +5,7 @@ ARCH_REL_TYPE_ABS := R_ARM_JUMP_SLOT|R_ARM_GLOB_DAT|R_ARM_ABS32 include $(srctree)/lib/vdso/Makefile -hostprogs-y := vdsomunge +hostprogs := vdsomunge obj-vdso := vgettimeofday.o datapage.o note.o diff --git a/arch/arm64/kernel/vdso32/Makefile b/arch/arm64/kernel/vdso32/Makefile index 76b327f88fbb..04df57b43cb1 100644 --- a/arch/arm64/kernel/vdso32/Makefile +++ b/arch/arm64/kernel/vdso32/Makefile @@ -115,9 +115,9 @@ VDSO_LDFLAGS += $(call cc32-ldoption,-fuse-ld=bfd) # Borrow vdsomunge.c from the arm vDSO # We have to use a relative path because scripts/Makefile.host prefixes -# $(hostprogs-y) with $(obj) +# $(hostprogs) with $(obj) munge := ../../../arm/vdso/vdsomunge -hostprogs-y := $(munge) +hostprogs := $(munge) c-obj-vdso := note.o c-obj-vdso-gettimeofday := vgettimeofday.o diff --git a/arch/mips/boot/Makefile b/arch/mips/boot/Makefile index 4ed45ade32a1..a3da2c5d63c2 100644 --- a/arch/mips/boot/Makefile +++ b/arch/mips/boot/Makefile @@ -21,7 +21,7 @@ endif drop-sections := .reginfo .mdebug .comment .note .pdr .options .MIPS.options strip-flags := $(addprefix --remove-section=,$(drop-sections)) -hostprogs-y := elf2ecoff +hostprogs := elf2ecoff suffix-y := bin suffix-$(CONFIG_KERNEL_BZIP2) := bz2 diff --git a/arch/mips/boot/compressed/Makefile b/arch/mips/boot/compressed/Makefile index d859f079b771..0df0ee8a298d 100644 --- a/arch/mips/boot/compressed/Makefile +++ b/arch/mips/boot/compressed/Makefile @@ -84,7 +84,7 @@ $(obj)/piggy.o: $(obj)/dummy.o $(obj)/vmlinux.bin.z FORCE HOSTCFLAGS_calc_vmlinuz_load_addr.o += $(LINUXINCLUDE) # Calculate the load address of the compressed kernel image -hostprogs-y := calc_vmlinuz_load_addr +hostprogs := calc_vmlinuz_load_addr ifneq ($(zload-y),) VMLINUZ_LOAD_ADDRESS := $(zload-y) @@ -112,7 +112,7 @@ ifdef CONFIG_MACH_DECSTATION endif # elf2ecoff can only handle 32bit image -hostprogs-y += ../elf2ecoff +hostprogs += ../elf2ecoff ifdef CONFIG_32BIT VMLINUZ = vmlinuz diff --git a/arch/mips/boot/tools/Makefile b/arch/mips/boot/tools/Makefile index 5f8e737348eb..592e05a51a4a 100644 --- a/arch/mips/boot/tools/Makefile +++ b/arch/mips/boot/tools/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -hostprogs-y += relocs +hostprogs += relocs relocs-objs += relocs_32.o relocs-objs += relocs_64.o relocs-objs += relocs_main.o diff --git a/arch/mips/tools/Makefile b/arch/mips/tools/Makefile index aaef688749f5..b851e5dcc65a 100644 --- a/arch/mips/tools/Makefile +++ b/arch/mips/tools/Makefile @@ -1,10 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 -hostprogs-y := elf-entry +hostprogs := elf-entry PHONY += elf-entry elf-entry: $(obj)/elf-entry @: -hostprogs-$(CONFIG_CPU_LOONGSON3_WORKAROUNDS) += loongson3-llsc-check +hostprogs += loongson3-llsc-check PHONY += loongson3-llsc-check loongson3-llsc-check: $(obj)/loongson3-llsc-check @: diff --git a/arch/mips/vdso/Makefile b/arch/mips/vdso/Makefile index b2a2e032dc99..aa89a41dc5dd 100644 --- a/arch/mips/vdso/Makefile +++ b/arch/mips/vdso/Makefile @@ -100,7 +100,7 @@ $(obj)/%.so.raw: OBJCOPYFLAGS := -S $(obj)/%.so.raw: $(obj)/%.so.dbg.raw FORCE $(call if_changed,objcopy) -hostprogs-y := genvdso +hostprogs := genvdso quiet_cmd_genvdso = GENVDSO $@ define cmd_genvdso diff --git a/arch/powerpc/boot/Makefile b/arch/powerpc/boot/Makefile index dfbd7f22eef5..0556bf4fc9e9 100644 --- a/arch/powerpc/boot/Makefile +++ b/arch/powerpc/boot/Makefile @@ -224,7 +224,7 @@ $(patsubst %.S,%.o, $(filter %.S, $(src-boot))): %.o: %.S FORCE $(obj)/wrapper.a: $(obj-wlib) FORCE $(call if_changed,bootar) -hostprogs-y := addnote hack-coff mktree +hostprogs := addnote hack-coff mktree targets += $(patsubst $(obj)/%,%,$(obj-boot) wrapper.a) extra-y := $(obj)/wrapper.a $(obj-plat) $(obj)/empty.o \ @@ -464,7 +464,7 @@ WRAPPER_BINDIR := /usr/sbin INSTALL := install extra-installed := $(patsubst $(obj)/%, $(DESTDIR)$(WRAPPER_OBJDIR)/%, $(extra-y)) -hostprogs-installed := $(patsubst %, $(DESTDIR)$(WRAPPER_BINDIR)/%, $(hostprogs-y)) +hostprogs-installed := $(patsubst %, $(DESTDIR)$(WRAPPER_BINDIR)/%, $(hostprogs)) wrapper-installed := $(DESTDIR)$(WRAPPER_BINDIR)/wrapper dts-installed := $(patsubst $(dtstree)/%, $(DESTDIR)$(WRAPPER_DTSDIR)/%, $(wildcard $(dtstree)/*.dts)) diff --git a/arch/s390/tools/Makefile b/arch/s390/tools/Makefile index b5e35e8f999a..f9dd47ff9ac4 100644 --- a/arch/s390/tools/Makefile +++ b/arch/s390/tools/Makefile @@ -10,8 +10,8 @@ PHONY += kapi kapi: $(kapi-hdrs-y) -hostprogs-y += gen_facilities -hostprogs-y += gen_opcode_table +hostprogs += gen_facilities +hostprogs += gen_opcode_table HOSTCFLAGS_gen_facilities.o += $(LINUXINCLUDE) diff --git a/arch/sparc/boot/Makefile b/arch/sparc/boot/Makefile index ec8cd703b708..380e2b018992 100644 --- a/arch/sparc/boot/Makefile +++ b/arch/sparc/boot/Makefile @@ -7,7 +7,7 @@ ROOT_IMG := /usr/src/root.img ELFTOAOUT := elftoaout -hostprogs-y := piggyback +hostprogs := piggyback targets := tftpboot.img image zImage vmlinux.aout clean-files := System.map diff --git a/arch/sparc/vdso/Makefile b/arch/sparc/vdso/Makefile index 997ffe46e953..708cb6304c2d 100644 --- a/arch/sparc/vdso/Makefile +++ b/arch/sparc/vdso/Makefile @@ -41,7 +41,7 @@ $(obj)/vdso64.so.dbg: $(obj)/vdso.lds $(vobjs) FORCE $(call if_changed,vdso) HOST_EXTRACFLAGS += -I$(srctree)/tools/include -hostprogs-y += vdso2c +hostprogs += vdso2c quiet_cmd_vdso2c = VDSO2C $@ cmd_vdso2c = $(obj)/vdso2c $< $(<:%.dbg=%) $@ diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile index 748b6d28a91d..012b82fc8617 100644 --- a/arch/x86/boot/Makefile +++ b/arch/x86/boot/Makefile @@ -45,8 +45,8 @@ setup-y += video-vesa.o setup-y += video-bios.o targets += $(setup-y) -hostprogs-y := tools/build -hostprogs-$(CONFIG_X86_FEATURE_NAMES) += mkcpustr +hostprogs := tools/build +hostprogs += mkcpustr HOST_EXTRACFLAGS += -I$(srctree)/tools/include \ -include include/generated/autoconf.h \ diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 56aa5fa0a66b..26050ae0b27e 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -58,7 +58,7 @@ KBUILD_LDFLAGS += $(shell $(LD) --help 2>&1 | grep -q "\-z noreloc-overflow" \ endif LDFLAGS_vmlinux := -T -hostprogs-y := mkpiggy +hostprogs := mkpiggy HOST_EXTRACFLAGS += -I$(srctree)/tools/include sed-voffset := -e 's/^\([0-9a-fA-F]*\) [ABCDGRSTVW] \(_text\|__bss_start\|_end\)$$/\#define VO_\2 _AC(0x\1,UL)/p' diff --git a/arch/x86/entry/vdso/Makefile b/arch/x86/entry/vdso/Makefile index 2b75e80f6b41..433a1259f61d 100644 --- a/arch/x86/entry/vdso/Makefile +++ b/arch/x86/entry/vdso/Makefile @@ -59,7 +59,7 @@ $(obj)/vdso64.so.dbg: $(obj)/vdso.lds $(vobjs) FORCE $(call if_changed,vdso_and_check) HOST_EXTRACFLAGS += -I$(srctree)/tools/include -I$(srctree)/include/uapi -I$(srctree)/arch/$(SUBARCH)/include/uapi -hostprogs-y += vdso2c +hostprogs += vdso2c quiet_cmd_vdso2c = VDSO2C $@ cmd_vdso2c = $(obj)/vdso2c $< $(<:%.dbg=%) $@ diff --git a/arch/x86/realmode/rm/Makefile b/arch/x86/realmode/rm/Makefile index f60501a384f9..99b6332ba540 100644 --- a/arch/x86/realmode/rm/Makefile +++ b/arch/x86/realmode/rm/Makefile @@ -12,7 +12,7 @@ OBJECT_FILES_NON_STANDARD := y # Prevents link failures: __sanitizer_cov_trace_pc() is not linked in. KCOV_INSTRUMENT := n -always := realmode.bin realmode.relocs +always-y := realmode.bin realmode.relocs wakeup-objs := wakeup_asm.o wakemain.o video-mode.o wakeup-objs += copy.o bioscall.o regs.o diff --git a/arch/x86/tools/Makefile b/arch/x86/tools/Makefile index 09af7ff53044..55b1ab378974 100644 --- a/arch/x86/tools/Makefile +++ b/arch/x86/tools/Makefile @@ -26,7 +26,7 @@ posttest: $(obj)/insn_decoder_test vmlinux $(obj)/insn_sanity $(call cmd,posttest) $(call cmd,sanitytest) -hostprogs-y += insn_decoder_test insn_sanity +hostprogs += insn_decoder_test insn_sanity # -I needed for generated C source and C source which in the kernel tree. HOSTCFLAGS_insn_decoder_test.o := -Wall -I$(objtree)/arch/x86/lib/ -I$(srctree)/arch/x86/include/uapi/ -I$(srctree)/arch/x86/include/ -I$(srctree)/arch/x86/lib/ -I$(srctree)/include/uapi/ @@ -39,7 +39,7 @@ $(obj)/insn_decoder_test.o: $(srctree)/arch/x86/lib/insn.c $(srctree)/arch/x86/l $(obj)/insn_sanity.o: $(srctree)/arch/x86/lib/insn.c $(srctree)/arch/x86/lib/inat.c $(srctree)/arch/x86/include/asm/inat_types.h $(srctree)/arch/x86/include/asm/inat.h $(srctree)/arch/x86/include/asm/insn.h $(objtree)/arch/x86/lib/inat-tables.c HOST_EXTRACFLAGS += -I$(srctree)/tools/include -hostprogs-y += relocs +hostprogs += relocs relocs-objs := relocs_32.o relocs_64.o relocs_common.o PHONY += relocs relocs: $(obj)/relocs diff --git a/drivers/gpu/drm/radeon/Makefile b/drivers/gpu/drm/radeon/Makefile index 92ccd7aed0d4..c693b2ca0329 100644 --- a/drivers/gpu/drm/radeon/Makefile +++ b/drivers/gpu/drm/radeon/Makefile @@ -5,7 +5,7 @@ ccflags-y := -Idrivers/gpu/drm/amd/include -hostprogs-y := mkregtable +hostprogs := mkregtable clean-files := rn50_reg_safe.h r100_reg_safe.h r200_reg_safe.h rv515_reg_safe.h r300_reg_safe.h r420_reg_safe.h rs600_reg_safe.h r600_reg_safe.h evergreen_reg_safe.h cayman_reg_safe.h quiet_cmd_mkregtable = MKREGTABLE $@ diff --git a/drivers/tty/vt/Makefile b/drivers/tty/vt/Makefile index 329ca336b8ee..fe30ce512819 100644 --- a/drivers/tty/vt/Makefile +++ b/drivers/tty/vt/Makefile @@ -12,7 +12,7 @@ obj-$(CONFIG_HW_CONSOLE) += vt.o defkeymap.o # Files generated that shall be removed upon make clean clean-files := consolemap_deftbl.c defkeymap.c -hostprogs-y += conmakehash +hostprogs += conmakehash quiet_cmd_conmk = CONMK $@ cmd_conmk = $(obj)/conmakehash $< > $@ diff --git a/drivers/video/logo/Makefile b/drivers/video/logo/Makefile index bcda657493a4..895c60b8402e 100644 --- a/drivers/video/logo/Makefile +++ b/drivers/video/logo/Makefile @@ -18,7 +18,7 @@ obj-$(CONFIG_SPU_BASE) += logo_spe_clut224.o # How to generate logo's -hostprogs-y := pnmtologo +hostprogs := pnmtologo # Create commands like "pnmtologo -t mono -n logo_mac_mono -o ..." quiet_cmd_logo = LOGO $@ diff --git a/drivers/zorro/Makefile b/drivers/zorro/Makefile index b360ac4ea846..91ba82e633e7 100644 --- a/drivers/zorro/Makefile +++ b/drivers/zorro/Makefile @@ -7,7 +7,7 @@ obj-$(CONFIG_ZORRO) += zorro.o zorro-driver.o zorro-sysfs.o obj-$(CONFIG_PROC_FS) += proc.o obj-$(CONFIG_ZORRO_NAMES) += names.o -hostprogs-y := gen-devlist +hostprogs := gen-devlist # Files generated that shall be removed upon make clean clean-files := devlist.h diff --git a/fs/unicode/Makefile b/fs/unicode/Makefile index d46e9baee285..b88aecc86550 100644 --- a/fs/unicode/Makefile +++ b/fs/unicode/Makefile @@ -35,4 +35,4 @@ $(obj)/utf8data.h: $(src)/utf8data.h_shipped FORCE endif targets += utf8data.h -hostprogs-y += mkutf8data +hostprogs += mkutf8data diff --git a/lib/Makefile b/lib/Makefile index 23ca78d43d24..8508143d3b76 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -239,8 +239,8 @@ obj-$(CONFIG_ASN1) += asn1_decoder.o obj-$(CONFIG_FONT_SUPPORT) += fonts/ -hostprogs-y := gen_crc32table -hostprogs-y += gen_crc64table +hostprogs := gen_crc32table +hostprogs += gen_crc64table clean-files := crc32table.h clean-files += crc64table.h diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile index 0083b5cc646c..b4c0df6d706d 100644 --- a/lib/raid6/Makefile +++ b/lib/raid6/Makefile @@ -10,7 +10,7 @@ raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o \ raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o -hostprogs-y += mktables +hostprogs += mktables ifeq ($(CONFIG_ALTIVEC),y) altivec_flags := -maltivec $(call cc-option,-mabi=altivec) diff --git a/net/bpfilter/Makefile b/net/bpfilter/Makefile index aa945ab5b655..36580301da70 100644 --- a/net/bpfilter/Makefile +++ b/net/bpfilter/Makefile @@ -3,7 +3,7 @@ # Makefile for the Linux BPFILTER layer. # -hostprogs-y := bpfilter_umh +hostprogs := bpfilter_umh bpfilter_umh-objs := main.o KBUILD_HOSTCFLAGS += -I $(srctree)/tools/include/ -I $(srctree)/tools/include/uapi HOSTCC := $(CC) diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile index b0e8adf7eb01..79b0fee6943b 100644 --- a/samples/bpf/Makefile +++ b/samples/bpf/Makefile @@ -111,65 +111,65 @@ ibumad-objs := bpf_load.o ibumad_user.o $(TRACE_HELPERS) hbm-objs := bpf_load.o hbm.o $(CGROUP_HELPERS) # Tell kbuild to always build the programs -always := $(tprogs-y) -always += sockex1_kern.o -always += sockex2_kern.o -always += sockex3_kern.o -always += tracex1_kern.o -always += tracex2_kern.o -always += tracex3_kern.o -always += tracex4_kern.o -always += tracex5_kern.o -always += tracex6_kern.o -always += tracex7_kern.o -always += sock_flags_kern.o -always += test_probe_write_user_kern.o -always += trace_output_kern.o -always += tcbpf1_kern.o -always += tc_l2_redirect_kern.o -always += lathist_kern.o -always += offwaketime_kern.o -always += spintest_kern.o -always += map_perf_test_kern.o -always += test_overhead_tp_kern.o -always += test_overhead_raw_tp_kern.o -always += test_overhead_kprobe_kern.o -always += parse_varlen.o parse_simple.o parse_ldabs.o -always += test_cgrp2_tc_kern.o -always += xdp1_kern.o -always += xdp2_kern.o -always += xdp_router_ipv4_kern.o -always += test_current_task_under_cgroup_kern.o -always += trace_event_kern.o -always += sampleip_kern.o -always += lwt_len_hist_kern.o -always += xdp_tx_iptunnel_kern.o -always += test_map_in_map_kern.o -always += tcp_synrto_kern.o -always += tcp_rwnd_kern.o -always += tcp_bufs_kern.o -always += tcp_cong_kern.o -always += tcp_iw_kern.o -always += tcp_clamp_kern.o -always += tcp_basertt_kern.o -always += tcp_tos_reflect_kern.o -always += tcp_dumpstats_kern.o -always += xdp_redirect_kern.o -always += xdp_redirect_map_kern.o -always += xdp_redirect_cpu_kern.o -always += xdp_monitor_kern.o -always += xdp_rxq_info_kern.o -always += xdp2skb_meta_kern.o -always += syscall_tp_kern.o -always += cpustat_kern.o -always += xdp_adjust_tail_kern.o -always += xdp_fwd_kern.o -always += task_fd_query_kern.o -always += xdp_sample_pkts_kern.o -always += ibumad_kern.o -always += hbm_out_kern.o -always += hbm_edt_kern.o -always += xdpsock_kern.o +always-y := $(tprogs-y) +always-y += sockex1_kern.o +always-y += sockex2_kern.o +always-y += sockex3_kern.o +always-y += tracex1_kern.o +always-y += tracex2_kern.o +always-y += tracex3_kern.o +always-y += tracex4_kern.o +always-y += tracex5_kern.o +always-y += tracex6_kern.o +always-y += tracex7_kern.o +always-y += sock_flags_kern.o +always-y += test_probe_write_user_kern.o +always-y += trace_output_kern.o +always-y += tcbpf1_kern.o +always-y += tc_l2_redirect_kern.o +always-y += lathist_kern.o +always-y += offwaketime_kern.o +always-y += spintest_kern.o +always-y += map_perf_test_kern.o +always-y += test_overhead_tp_kern.o +always-y += test_overhead_raw_tp_kern.o +always-y += test_overhead_kprobe_kern.o +always-y += parse_varlen.o parse_simple.o parse_ldabs.o +always-y += test_cgrp2_tc_kern.o +always-y += xdp1_kern.o +always-y += xdp2_kern.o +always-y += xdp_router_ipv4_kern.o +always-y += test_current_task_under_cgroup_kern.o +always-y += trace_event_kern.o +always-y += sampleip_kern.o +always-y += lwt_len_hist_kern.o +always-y += xdp_tx_iptunnel_kern.o +always-y += test_map_in_map_kern.o +always-y += tcp_synrto_kern.o +always-y += tcp_rwnd_kern.o +always-y += tcp_bufs_kern.o +always-y += tcp_cong_kern.o +always-y += tcp_iw_kern.o +always-y += tcp_clamp_kern.o +always-y += tcp_basertt_kern.o +always-y += tcp_tos_reflect_kern.o +always-y += tcp_dumpstats_kern.o +always-y += xdp_redirect_kern.o +always-y += xdp_redirect_map_kern.o +always-y += xdp_redirect_cpu_kern.o +always-y += xdp_monitor_kern.o +always-y += xdp_rxq_info_kern.o +always-y += xdp2skb_meta_kern.o +always-y += syscall_tp_kern.o +always-y += cpustat_kern.o +always-y += xdp_adjust_tail_kern.o +always-y += xdp_fwd_kern.o +always-y += task_fd_query_kern.o +always-y += xdp_sample_pkts_kern.o +always-y += ibumad_kern.o +always-y += hbm_out_kern.o +always-y += hbm_edt_kern.o +always-y += xdpsock_kern.o ifeq ($(ARCH), arm) # Strip all except -D__LINUX_ARM_ARCH__ option needed to handle linux diff --git a/samples/connector/Makefile b/samples/connector/Makefile index 6ad71620e503..b785cbde5ffa 100644 --- a/samples/connector/Makefile +++ b/samples/connector/Makefile @@ -2,12 +2,8 @@ obj-$(CONFIG_SAMPLE_CONNECTOR) += cn_test.o # List of programs to build -ifdef CONFIG_SAMPLE_CONNECTOR -hostprogs-y := ucon -endif - -# Tell kbuild to always build the programs -always := $(hostprogs-y) +hostprogs := ucon +always-y := $(hostprogs) HOSTCFLAGS_ucon.o += -I$(objtree)/usr/include diff --git a/samples/hidraw/Makefile b/samples/hidraw/Makefile index dec1b22adf54..8bd25f77671f 100644 --- a/samples/hidraw/Makefile +++ b/samples/hidraw/Makefile @@ -1,9 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 # List of programs to build -hostprogs-y := hid-example - -# Tell kbuild to always build the programs -always := $(hostprogs-y) +hostprogs := hid-example +always-y := $(hostprogs) HOSTCFLAGS_hid-example.o += -I$(objtree)/usr/include diff --git a/samples/mei/Makefile b/samples/mei/Makefile index 27f37efdadb4..f5b9d02be2cd 100644 --- a/samples/mei/Makefile +++ b/samples/mei/Makefile @@ -1,10 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 # Copyright (c) 2012-2019, Intel Corporation. All rights reserved. -hostprogs-y := mei-amt-version +hostprogs := mei-amt-version HOSTCFLAGS_mei-amt-version.o += -I$(objtree)/usr/include -always := $(hostprogs-y) +always-y := $(hostprogs) all: mei-amt-version diff --git a/samples/pidfd/Makefile b/samples/pidfd/Makefile index 0ff97784177a..ee2979849d92 100644 --- a/samples/pidfd/Makefile +++ b/samples/pidfd/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -hostprogs-y := pidfd-metadata -always := $(hostprogs-y) +hostprogs := pidfd-metadata +always-y := $(hostprogs) HOSTCFLAGS_pidfd-metadata.o += -I$(objtree)/usr/include all: pidfd-metadata diff --git a/samples/seccomp/Makefile b/samples/seccomp/Makefile index 009775b52538..89279e8b87df 100644 --- a/samples/seccomp/Makefile +++ b/samples/seccomp/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 ifndef CROSS_COMPILE -hostprogs-y := bpf-fancy dropper bpf-direct user-trap +hostprogs := bpf-fancy dropper bpf-direct user-trap HOSTCFLAGS_bpf-fancy.o += -I$(objtree)/usr/include HOSTCFLAGS_bpf-fancy.o += -idirafter $(objtree)/include @@ -40,5 +40,5 @@ HOSTLDLIBS_bpf-fancy += $(MFLAG) HOSTLDLIBS_dropper += $(MFLAG) HOSTLDLIBS_user-trap += $(MFLAG) endif -always := $(hostprogs-y) +always-y := $(hostprogs) endif diff --git a/samples/uhid/Makefile b/samples/uhid/Makefile index 8c9bc9f98d37..5f44ea40d6d5 100644 --- a/samples/uhid/Makefile +++ b/samples/uhid/Makefile @@ -1,8 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only # List of programs to build -hostprogs-y := uhid-example +hostprogs := uhid-example # Tell kbuild to always build the programs -always := $(hostprogs-y) +always-y := $(hostprogs) HOSTCFLAGS_uhid-example.o += -I$(objtree)/usr/include diff --git a/samples/vfs/Makefile b/samples/vfs/Makefile index e21c9f6fe9be..65acdde5c117 100644 --- a/samples/vfs/Makefile +++ b/samples/vfs/Makefile @@ -1,11 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only # List of programs to build -hostprogs-y := \ +hostprogs := \ test-fsmount \ test-statx -# Tell kbuild to always build the programs -always := $(hostprogs-y) +always-y := $(hostprogs) HOSTCFLAGS_test-fsmount.o += -I$(objtree)/usr/include HOSTCFLAGS_test-statx.o += -I$(objtree)/usr/include diff --git a/scripts/Makefile b/scripts/Makefile index 4d41f48e7376..5e75802b1a44 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -7,14 +7,14 @@ HOST_EXTRACFLAGS += -I$(srctree)/tools/include -hostprogs-$(CONFIG_BUILD_BIN2C) += bin2c -hostprogs-$(CONFIG_KALLSYMS) += kallsyms -hostprogs-$(BUILD_C_RECORDMCOUNT) += recordmcount -hostprogs-$(CONFIG_BUILDTIME_TABLE_SORT) += sorttable -hostprogs-$(CONFIG_ASN1) += asn1_compiler -hostprogs-$(CONFIG_MODULE_SIG_FORMAT) += sign-file -hostprogs-$(CONFIG_SYSTEM_TRUSTED_KEYRING) += extract-cert -hostprogs-$(CONFIG_SYSTEM_EXTRA_CERTIFICATE) += insert-sys-cert +always-$(CONFIG_BUILD_BIN2C) += bin2c +always-$(CONFIG_KALLSYMS) += kallsyms +always-$(BUILD_C_RECORDMCOUNT) += recordmcount +always-$(CONFIG_BUILDTIME_TABLE_SORT) += sorttable +always-$(CONFIG_ASN1) += asn1_compiler +always-$(CONFIG_MODULE_SIG_FORMAT) += sign-file +always-$(CONFIG_SYSTEM_TRUSTED_KEYRING) += extract-cert +always-$(CONFIG_SYSTEM_EXTRA_CERTIFICATE) += insert-sys-cert HOSTCFLAGS_sorttable.o = -I$(srctree)/tools/include HOSTCFLAGS_asn1_compiler.o = -I$(srctree)/include @@ -30,10 +30,10 @@ HOSTCFLAGS_sorttable.o += -DUNWINDER_ORC_ENABLED HOSTLDLIBS_sorttable = -lpthread endif -always := $(hostprogs-y) $(hostprogs-m) +hostprogs := $(always-y) $(always-m) -# The following hostprogs-y programs are only build on demand -hostprogs-y += unifdef +# The following programs are only built on demand +hostprogs += unifdef subdir-$(CONFIG_GCC_PLUGINS) += gcc-plugins subdir-$(CONFIG_MODVERSIONS) += genksyms diff --git a/scripts/Makefile.build b/scripts/Makefile.build index a562d695f0fa..a1730d42e5f3 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -16,6 +16,8 @@ obj-m := lib-y := lib-m := always := +always-y := +always-m := targets := subdir-y := subdir-m := @@ -44,7 +46,7 @@ include $(kbuild-file) include scripts/Makefile.lib # Do not include host rules unless needed -ifneq ($(hostprogs-y)$(hostprogs-m)$(hostlibs-y)$(hostlibs-m)$(hostcxxlibs-y)$(hostcxxlibs-m),) +ifneq ($(hostprogs)$(hostlibs-y)$(hostlibs-m)$(hostcxxlibs-y)$(hostcxxlibs-m),) include scripts/Makefile.host endif @@ -348,7 +350,7 @@ $(obj)/%.o: $(src)/%.S $(objtool_dep) FORCE $(call if_changed_rule,as_o_S) targets += $(filter-out $(subdir-obj-y), $(real-obj-y)) $(real-obj-m) $(lib-y) -targets += $(extra-y) $(MAKECMDGOALS) $(always) +targets += $(extra-y) $(always-y) $(MAKECMDGOALS) # Linker scripts preprocessor (.lds.S -> .lds) # --------------------------------------------------------------------------- @@ -490,7 +492,7 @@ else __build: $(if $(KBUILD_BUILTIN),$(builtin-target) $(lib-target) $(extra-y)) \ $(if $(KBUILD_MODULES),$(obj-m) $(mod-targets) $(modorder-target)) \ - $(subdir-ym) $(always) + $(subdir-ym) $(always-y) @: endif diff --git a/scripts/Makefile.clean b/scripts/Makefile.clean index e367eb95c5c0..1e4206566a82 100644 --- a/scripts/Makefile.clean +++ b/scripts/Makefile.clean @@ -28,8 +28,8 @@ subdir-ymn := $(addprefix $(obj)/,$(subdir-ymn)) # directory __clean-files := $(extra-y) $(extra-m) $(extra-) \ - $(always) $(targets) $(clean-files) \ - $(hostprogs-y) $(hostprogs-m) $(hostprogs-) \ + $(always) $(always-y) $(always-m) $(always-) $(targets) $(clean-files) \ + $(hostprogs) $(hostprogs-y) $(hostprogs-m) $(hostprogs-) \ $(hostlibs-y) $(hostlibs-m) $(hostlibs-) \ $(hostcxxlibs-y) $(hostcxxlibs-m) diff --git a/scripts/Makefile.host b/scripts/Makefile.host index 4c51c95d40f4..3b7121d43324 100644 --- a/scripts/Makefile.host +++ b/scripts/Makefile.host @@ -24,21 +24,21 @@ $(obj)/%.tab.c $(obj)/%.tab.h: $(src)/%.y FORCE # Both C and C++ are supported, but preferred language is C for such utilities. # # Sample syntax (see Documentation/kbuild/makefiles.rst for reference) -# hostprogs-y := bin2hex +# hostprogs := bin2hex # Will compile bin2hex.c and create an executable named bin2hex # -# hostprogs-y := lxdialog +# hostprogs := lxdialog # lxdialog-objs := checklist.o lxdialog.o # Will compile lxdialog.c and checklist.c, and then link the executable # lxdialog, based on checklist.o and lxdialog.o # -# hostprogs-y := qconf +# hostprogs := qconf # qconf-cxxobjs := qconf.o # qconf-objs := menu.o # Will compile qconf as a C++ program, and menu as a C program. # They are linked as C++ code to the executable qconf -__hostprogs := $(sort $(hostprogs-y) $(hostprogs-m)) +__hostprogs := $(sort $(hostprogs)) host-cshlib := $(sort $(hostlibs-y) $(hostlibs-m)) host-cxxshlib := $(sort $(hostcxxlibs-y) $(hostcxxlibs-m)) diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index d10f7a03e0ee..bae62549e3d2 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -4,6 +4,8 @@ asflags-y += $(EXTRA_AFLAGS) ccflags-y += $(EXTRA_CFLAGS) cppflags-y += $(EXTRA_CPPFLAGS) ldflags-y += $(EXTRA_LDFLAGS) +always-y += $(always) +hostprogs += $(hostprogs-y) $(hostprogs-m) # flags that take effect in current and sub directories KBUILD_AFLAGS += $(subdir-asflags-y) @@ -59,6 +61,8 @@ subdir-obj-y := $(filter %/built-in.a, $(obj-y)) real-obj-y := $(foreach m, $(obj-y), $(if $(strip $($(m:.o=-objs)) $($(m:.o=-y)) $($(m:.o=-))),$($(m:.o=-objs)) $($(m:.o=-y)),$(m))) real-obj-m := $(foreach m, $(obj-m), $(if $(strip $($(m:.o=-objs)) $($(m:.o=-y)) $($(m:.o=-m)) $($(m:.o=-))),$($(m:.o=-objs)) $($(m:.o=-y)) $($(m:.o=-m)),$(m))) +always-y += $(always-m) + # DTB # If CONFIG_OF_ALL_DTBS is enabled, all DT blobs are built extra-y += $(dtb-y) @@ -72,7 +76,7 @@ endif # Add subdir path extra-y := $(addprefix $(obj)/,$(extra-y)) -always := $(addprefix $(obj)/,$(always)) +always-y := $(addprefix $(obj)/,$(always-y)) targets := $(addprefix $(obj)/,$(targets)) modorder := $(addprefix $(obj)/,$(modorder)) obj-m := $(addprefix $(obj)/,$(obj-m)) diff --git a/scripts/basic/Makefile b/scripts/basic/Makefile index 7c9cb80d097b..290dd27d2809 100644 --- a/scripts/basic/Makefile +++ b/scripts/basic/Makefile @@ -2,5 +2,5 @@ # # fixdep: used to generate dependency information during build process -hostprogs-y := fixdep -always := $(hostprogs-y) +hostprogs := fixdep +always-y := $(hostprogs) diff --git a/scripts/dtc/Makefile b/scripts/dtc/Makefile index b5a5b1c548c9..3acbb410904c 100644 --- a/scripts/dtc/Makefile +++ b/scripts/dtc/Makefile @@ -1,8 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 # scripts/dtc makefile -hostprogs-$(CONFIG_DTC) := dtc -always := $(hostprogs-y) +hostprogs := dtc +always-$(CONFIG_DTC) := $(hostprogs) dtc-objs := dtc.o flattree.o fstree.o data.o livetree.o treesource.o \ srcpos.o checks.o util.o diff --git a/scripts/gcc-plugins/Makefile b/scripts/gcc-plugins/Makefile index aa0d0ec6936d..f2ee8bd7abc6 100644 --- a/scripts/gcc-plugins/Makefile +++ b/scripts/gcc-plugins/Makefile @@ -23,7 +23,7 @@ $(objtree)/$(obj)/randomize_layout_seed.h: FORCE targets = randomize_layout_seed.h randomize_layout_hash.h $(HOSTLIBS)-y := $(foreach p,$(GCC_PLUGIN),$(if $(findstring /,$(p)),,$(p))) -always := $($(HOSTLIBS)-y) +always-y := $($(HOSTLIBS)-y) $(foreach p,$($(HOSTLIBS)-y:%.so=%),$(eval $(p)-objs := $(p).o)) diff --git a/scripts/genksyms/Makefile b/scripts/genksyms/Makefile index 78629f515e78..d328de1e10ee 100644 --- a/scripts/genksyms/Makefile +++ b/scripts/genksyms/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 -hostprogs-y := genksyms -always := $(hostprogs-y) +hostprogs := genksyms +always-y := $(hostprogs) genksyms-objs := genksyms.o parse.tab.o lex.lex.o diff --git a/scripts/kconfig/Makefile b/scripts/kconfig/Makefile index fbeb62ae3401..5887ceb6229e 100644 --- a/scripts/kconfig/Makefile +++ b/scripts/kconfig/Makefile @@ -157,11 +157,11 @@ HOSTCFLAGS_lexer.lex.o := -I $(srctree)/$(src) HOSTCFLAGS_parser.tab.o := -I $(srctree)/$(src) # conf: Used for defconfig, oldconfig and related targets -hostprogs-y += conf +hostprogs += conf conf-objs := conf.o $(common-objs) # nconf: Used for the nconfig target based on ncurses -hostprogs-y += nconf +hostprogs += nconf nconf-objs := nconf.o nconf.gui.o $(common-objs) HOSTLDLIBS_nconf = $(shell . $(obj)/nconf-cfg && echo $$libs) @@ -171,7 +171,7 @@ HOSTCFLAGS_nconf.gui.o = $(shell . $(obj)/nconf-cfg && echo $$cflags) $(obj)/nconf.o $(obj)/nconf.gui.o: $(obj)/nconf-cfg # mconf: Used for the menuconfig target based on lxdialog -hostprogs-y += mconf +hostprogs += mconf lxdialog := $(addprefix lxdialog/, \ checklist.o inputbox.o menubox.o textbox.o util.o yesno.o) mconf-objs := mconf.o $(lxdialog) $(common-objs) @@ -183,7 +183,7 @@ $(foreach f, mconf.o $(lxdialog), \ $(addprefix $(obj)/, mconf.o $(lxdialog)): $(obj)/mconf-cfg # qconf: Used for the xconfig target based on Qt -hostprogs-y += qconf +hostprogs += qconf qconf-cxxobjs := qconf.o qconf-objs := images.o $(common-objs) @@ -199,7 +199,7 @@ $(obj)/%.moc: $(src)/%.h $(obj)/qconf-cfg $(call cmd,moc) # gconf: Used for the gconfig target based on GTK+ -hostprogs-y += gconf +hostprogs += gconf gconf-objs := gconf.o images.o $(common-objs) HOSTLDLIBS_gconf = $(shell . $(obj)/gconf-cfg && echo $$libs) diff --git a/scripts/mod/Makefile b/scripts/mod/Makefile index 42c5d50f2bcc..296b6a3878b2 100644 --- a/scripts/mod/Makefile +++ b/scripts/mod/Makefile @@ -1,8 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 OBJECT_FILES_NON_STANDARD := y -hostprogs-y := modpost mk_elfconfig -always := $(hostprogs-y) empty.o +hostprogs := modpost mk_elfconfig +always-y := $(hostprogs) empty.o modpost-objs := modpost.o file2alias.o sumversion.o diff --git a/scripts/selinux/genheaders/Makefile b/scripts/selinux/genheaders/Makefile index e8c533140981..70cf8d95d07c 100644 --- a/scripts/selinux/genheaders/Makefile +++ b/scripts/selinux/genheaders/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 -hostprogs-y := genheaders +hostprogs := genheaders HOST_EXTRACFLAGS += \ -I$(srctree)/include/uapi -I$(srctree)/include \ -I$(srctree)/security/selinux/include -always := $(hostprogs-y) +always-y := $(hostprogs) diff --git a/scripts/selinux/mdp/Makefile b/scripts/selinux/mdp/Makefile index 8a1269a9d0ba..3026f3c2aa2b 100644 --- a/scripts/selinux/mdp/Makefile +++ b/scripts/selinux/mdp/Makefile @@ -1,8 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 -hostprogs-y := mdp +hostprogs := mdp HOST_EXTRACFLAGS += \ -I$(srctree)/include/uapi -I$(srctree)/include \ -I$(srctree)/security/selinux/include -I$(objtree)/include -always := $(hostprogs-y) +always-y := $(hostprogs) clean-files := policy.* file_contexts diff --git a/usr/Makefile b/usr/Makefile index 244862bfb765..18aed2ab98da 100644 --- a/usr/Makefile +++ b/usr/Makefile @@ -52,7 +52,7 @@ ifeq ($(cpio-data),) cpio-data := $(obj)/initramfs_data.cpio -hostprogs-y := gen_init_cpio +hostprogs := gen_init_cpio # .initramfs_data.cpio.d is used to identify all files included # in initramfs and to detect if any files are added/removed. -- cgit v1.2.3 From 0bee0cece2a6a71ccc347fdc1d46cf638cd5fd1c Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Mon, 3 Feb 2020 17:37:20 -0800 Subject: lib/string: add strnchrnul() Patch series "lib: rework bitmap_parse", v5. Similarl to the recently revisited bitmap_parselist(), bitmap_parse() is ineffective and overcomplicated. This series reworks it, aligns its interface with bitmap_parselist() and makes it simpler to use. The series also adds a test for the function and fixes usage of it in cpumask_parse() according to the new design - drops the calculating of length of an input string. bitmap_parse() takes the array of numbers to be put into the map in the BE order which is reversed to the natural LE order for bitmaps. For example, to construct bitmap containing a bit on the position 42, we have to put a line '400,0'. Current implementation reads chunk one by one from the beginning ('400' before '0') and makes bitmap shift after each successful parse. It makes the complexity of the whole process as O(n^2). We can do it in reverse direction ('0' before '400') and avoid shifting, but it requires reverse parsing helpers. This patch (of 7): New function works like strchrnul() with a length limited string. Link: http://lkml.kernel.org/r/20200102043031.30357-2-yury.norov@gmail.com Signed-off-by: Yury Norov Reviewed-by: Andy Shevchenko Cc: Rasmus Villemoes Cc: Amritha Nambiar Cc: Willem de Bruijn Cc: Kees Cook Cc: Matthew Wilcox Cc: "Tobin C . Harding" Cc: Will Deacon Cc: Miklos Szeredi Cc: Vineet Gupta Cc: Chris Wilson Cc: Arnaldo Carvalho de Melo Cc: Steffen Klassert Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/string.h | 1 + lib/string.c | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) (limited to 'lib') diff --git a/include/linux/string.h b/include/linux/string.h index 02894e417565..6dfbb2efa815 100644 --- a/include/linux/string.h +++ b/include/linux/string.h @@ -62,6 +62,7 @@ extern char * strchr(const char *,int); #ifndef __HAVE_ARCH_STRCHRNUL extern char * strchrnul(const char *,int); #endif +extern char * strnchrnul(const char *, size_t, int); #ifndef __HAVE_ARCH_STRNCHR extern char * strnchr(const char *, size_t, int); #endif diff --git a/lib/string.c b/lib/string.c index 08ec58cc673b..f607b967d978 100644 --- a/lib/string.c +++ b/lib/string.c @@ -434,6 +434,23 @@ char *strchrnul(const char *s, int c) EXPORT_SYMBOL(strchrnul); #endif +/** + * strnchrnul - Find and return a character in a length limited string, + * or end of string + * @s: The string to be searched + * @count: The number of characters to be searched + * @c: The character to search for + * + * Returns pointer to the first occurrence of 'c' in s. If c is not found, + * then return a pointer to the last character of the string. + */ +char *strnchrnul(const char *s, size_t count, int c) +{ + while (count-- && *s && *s != (char)c) + s++; + return (char *)s; +} + #ifndef __HAVE_ARCH_STRRCHR /** * strrchr - Find the last occurrence of a character in a string -- cgit v1.2.3 From 7eb2e94e969435519cfd8e07dc654508b3a42d81 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Mon, 3 Feb 2020 17:37:27 -0800 Subject: lib: add test for bitmap_parse() The test is derived from bitmap_parselist() NO_LEN is reserved for use in following patches. [yury.norov@gmail.com: fix rebase issue] Link: http://lkml.kernel.org/r/20200102182659.6685-1-yury.norov@gmail.com [andriy.shevchenko@linux.intel.com: fix address space when test user buffer] Link: http://lkml.kernel.org/r/20200109103601.45929-2-andriy.shevchenko@linux.intel.com Link: http://lkml.kernel.org/r/20200102043031.30357-4-yury.norov@gmail.com Signed-off-by: Yury Norov Signed-off-by: Andy Shevchenko Reviewed-by: Andy Shevchenko Cc: Amritha Nambiar Cc: Arnaldo Carvalho de Melo Cc: Chris Wilson Cc: Kees Cook Cc: Matthew Wilcox Cc: Miklos Szeredi Cc: Rasmus Villemoes Cc: Steffen Klassert Cc: "Tobin C . Harding" Cc: Vineet Gupta Cc: Will Deacon Cc: Willem de Bruijn Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test_bitmap.c | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c index 71ec3afe1681..bde8ed8c0e79 100644 --- a/lib/test_bitmap.c +++ b/lib/test_bitmap.c @@ -295,7 +295,8 @@ static void __init test_replace(void) expect_eq_bitmap(bmap, exp3_1_0, nbits); } -#define PARSE_TIME 0x1 +#define PARSE_TIME 0x1 +#define NO_LEN 0x2 struct test_bitmap_parselist{ const int errno; @@ -349,7 +350,6 @@ static const struct test_bitmap_parselist parselist_tests[] __initconst = { {-EINVAL, "0-31:a/1", NULL, 8, 0}, {-EINVAL, "0-\n", NULL, 8, 0}, -#undef step }; static void __init __test_bitmap_parselist(int is_user) @@ -401,6 +401,87 @@ static void __init __test_bitmap_parselist(int is_user) } } +static const unsigned long parse_test[] __initconst = { + BITMAP_FROM_U64(0), + BITMAP_FROM_U64(1), + BITMAP_FROM_U64(0xdeadbeef), + BITMAP_FROM_U64(0x100000000ULL), +}; + +static const unsigned long parse_test2[] __initconst = { + BITMAP_FROM_U64(0x100000000ULL), BITMAP_FROM_U64(0xdeadbeef), + BITMAP_FROM_U64(0x100000000ULL), BITMAP_FROM_U64(0xbaadf00ddeadbeef), + BITMAP_FROM_U64(0x100000000ULL), BITMAP_FROM_U64(0x0badf00ddeadbeef), +}; + +static const struct test_bitmap_parselist parse_tests[] __initconst = { + {0, "0", &parse_test[0 * step], 32, 0}, + {0, "1", &parse_test[1 * step], 32, 0}, + {0, "deadbeef", &parse_test[2 * step], 32, 0}, + {0, "1,0", &parse_test[3 * step], 33, 0}, + + {0, "deadbeef,1,0", &parse_test2[0 * 2 * step], 96, 0}, + {0, "baadf00d,deadbeef,1,0", &parse_test2[1 * 2 * step], 128, 0}, + {0, "badf00d,deadbeef,1,0", &parse_test2[2 * 2 * step], 124, 0}, + + {-EINVAL, "goodfood,deadbeef,1,0", NULL, 128, 0}, + {-EOVERFLOW, "3,0", NULL, 33, 0}, + {-EOVERFLOW, "123badf00d,deadbeef,1,0", NULL, 128, 0}, + {-EOVERFLOW, "badf00d,deadbeef,1,0", NULL, 90, 0}, + {-EOVERFLOW, "fbadf00d,deadbeef,1,0", NULL, 95, 0}, + {-EOVERFLOW, "badf00d,deadbeef,1,0", NULL, 100, 0}, +#undef step +}; + +static void __init __test_bitmap_parse(int is_user) +{ + int i; + int err; + ktime_t time; + DECLARE_BITMAP(bmap, 2048); + char *mode = is_user ? "_user" : ""; + + for (i = 0; i < ARRAY_SIZE(parse_tests); i++) { + struct test_bitmap_parselist test = parse_tests[i]; + + if (is_user) { + size_t len = strlen(test.in); + mm_segment_t orig_fs = get_fs(); + + set_fs(KERNEL_DS); + time = ktime_get(); + err = bitmap_parse_user((__force const char __user *)test.in, len, + bmap, test.nbits); + time = ktime_get() - time; + set_fs(orig_fs); + } else { + size_t len = test.flags & NO_LEN ? + UINT_MAX : strlen(test.in); + time = ktime_get(); + err = bitmap_parse(test.in, len, bmap, test.nbits); + time = ktime_get() - time; + } + + if (err != test.errno) { + pr_err("parse%s: %d: input is %s, errno is %d, expected %d\n", + mode, i, test.in, err, test.errno); + continue; + } + + if (!err && test.expected + && !__bitmap_equal(bmap, test.expected, test.nbits)) { + pr_err("parse%s: %d: input is %s, result is 0x%lx, expected 0x%lx\n", + mode, i, test.in, bmap[0], + *test.expected); + continue; + } + + if (test.flags & PARSE_TIME) + pr_err("parse%s: %d: input is '%s' OK, Time: %llu\n", + mode, i, test.in, time); + } +} + static void __init test_bitmap_parselist(void) { __test_bitmap_parselist(0); @@ -411,6 +492,16 @@ static void __init test_bitmap_parselist_user(void) __test_bitmap_parselist(1); } +static void __init test_bitmap_parse(void) +{ + __test_bitmap_parse(0); +} + +static void __init test_bitmap_parse_user(void) +{ + __test_bitmap_parse(1); +} + #define EXP1_IN_BITS (sizeof(exp1) * 8) static void __init test_bitmap_arr32(void) @@ -516,6 +607,8 @@ static void __init selftest(void) test_copy(); test_replace(); test_bitmap_arr32(); + test_bitmap_parse(); + test_bitmap_parse_user(); test_bitmap_parselist(); test_bitmap_parselist_user(); test_mem_optimisations(); -- cgit v1.2.3 From e66eda0615c8767694150b3c595f3e9cf9ac4294 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Mon, 3 Feb 2020 17:37:31 -0800 Subject: lib: make bitmap_parse_user a wrapper on bitmap_parse Currently we parse user data byte after byte which leads to overcomplicating of parsing algorithm. There are no performance critical users of bitmap_parse_user(), and so we can duplicate user data to kernel buffer and simply call bitmap_parselist(). This rework lets us unify and simplify bitmap_parse() and bitmap_parse_user(), which is done in the following patch. Link: http://lkml.kernel.org/r/20200102043031.30357-5-yury.norov@gmail.com Signed-off-by: Yury Norov Reviewed-by: Andy Shevchenko Cc: Amritha Nambiar Cc: Arnaldo Carvalho de Melo Cc: Chris Wilson Cc: Kees Cook Cc: Matthew Wilcox Cc: Miklos Szeredi Cc: Rasmus Villemoes Cc: Steffen Klassert Cc: "Tobin C . Harding" Cc: Vineet Gupta Cc: Will Deacon Cc: Willem de Bruijn Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/bitmap.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/bitmap.c b/lib/bitmap.c index 6e175fbd69a9..0a24ce484e91 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -530,22 +530,22 @@ EXPORT_SYMBOL(__bitmap_parse); * then it must be terminated with a \0. * @maskp: pointer to bitmap array that will contain result. * @nmaskbits: size of bitmap, in bits. - * - * Wrapper for __bitmap_parse(), providing it with user buffer. - * - * We cannot have this as an inline function in bitmap.h because it needs - * linux/uaccess.h to get the access_ok() declaration and this causes - * cyclic dependencies. */ int bitmap_parse_user(const char __user *ubuf, unsigned int ulen, unsigned long *maskp, int nmaskbits) { - if (!access_ok(ubuf, ulen)) - return -EFAULT; - return __bitmap_parse((const char __force *)ubuf, - ulen, 1, maskp, nmaskbits); + char *buf; + int ret; + buf = memdup_user_nul(ubuf, ulen); + if (IS_ERR(buf)) + return PTR_ERR(buf); + + ret = bitmap_parse(buf, ulen, maskp, nmaskbits); + + kfree(buf); + return ret; } EXPORT_SYMBOL(bitmap_parse_user); -- cgit v1.2.3 From 2d6261583be005a91e4933aa53bbd678ef98e4c4 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Mon, 3 Feb 2020 17:37:34 -0800 Subject: lib: rework bitmap_parse() bitmap_parse() is ineffective and full of opaque variables and opencoded parts. It leads to hard understanding and usage of it. This rework includes: - remove bitmap_shift_left() call from the cycle. Now it makes the complexity of the algorithm as O(nbits^2). In the suggested approach the input string is parsed in reverse direction, so no shifts needed; - relax requirement on a single comma and no white spaces between chunks. It is considered useful in scripting, and it aligns with bitmap_parselist(); - split bitmap_parse() to small readable helpers; - make an explicit calculation of the end of input line at the beginning, so users of the bitmap_parse() won't bother doing this. Link: http://lkml.kernel.org/r/20200102043031.30357-6-yury.norov@gmail.com Signed-off-by: Yury Norov Cc: Amritha Nambiar Cc: Andy Shevchenko Cc: Arnaldo Carvalho de Melo Cc: Chris Wilson Cc: Kees Cook Cc: Matthew Wilcox Cc: Miklos Szeredi Cc: Rasmus Villemoes Cc: Steffen Klassert Cc: "Tobin C . Harding" Cc: Vineet Gupta Cc: Will Deacon Cc: Willem de Bruijn Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/bitmap.h | 8 +-- lib/bitmap.c | 175 +++++++++++++++++++++++-------------------------- 2 files changed, 84 insertions(+), 99 deletions(-) (limited to 'lib') diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index 80ad521116d7..e52ceb1a73d3 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -186,7 +186,7 @@ bitmap_find_next_zero_area(unsigned long *map, align_mask, 0); } -extern int __bitmap_parse(const char *buf, unsigned int buflen, int is_user, +extern int bitmap_parse(const char *buf, unsigned int buflen, unsigned long *dst, int nbits); extern int bitmap_parse_user(const char __user *ubuf, unsigned int ulen, unsigned long *dst, int nbits); @@ -454,12 +454,6 @@ static inline void bitmap_replace(unsigned long *dst, __bitmap_replace(dst, old, new, mask, nbits); } -static inline int bitmap_parse(const char *buf, unsigned int buflen, - unsigned long *maskp, int nmaskbits) -{ - return __bitmap_parse(buf, buflen, 0, maskp, nmaskbits); -} - static inline void bitmap_next_clear_region(unsigned long *bitmap, unsigned int *rs, unsigned int *re, unsigned int end) diff --git a/lib/bitmap.c b/lib/bitmap.c index 0a24ce484e91..89260aa342d6 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -431,97 +431,6 @@ EXPORT_SYMBOL(bitmap_find_next_zero_area_off); * second version by Paul Jackson, third by Joe Korty. */ -#define CHUNKSZ 32 -#define nbits_to_hold_value(val) fls(val) -#define BASEDEC 10 /* fancier cpuset lists input in decimal */ - -/** - * __bitmap_parse - convert an ASCII hex string into a bitmap. - * @buf: pointer to buffer containing string. - * @buflen: buffer size in bytes. If string is smaller than this - * then it must be terminated with a \0. - * @is_user: location of buffer, 0 indicates kernel space - * @maskp: pointer to bitmap array that will contain result. - * @nmaskbits: size of bitmap, in bits. - * - * Commas group hex digits into chunks. Each chunk defines exactly 32 - * bits of the resultant bitmask. No chunk may specify a value larger - * than 32 bits (%-EOVERFLOW), and if a chunk specifies a smaller value - * then leading 0-bits are prepended. %-EINVAL is returned for illegal - * characters and for grouping errors such as "1,,5", ",44", "," and "". - * Leading and trailing whitespace accepted, but not embedded whitespace. - */ -int __bitmap_parse(const char *buf, unsigned int buflen, - int is_user, unsigned long *maskp, - int nmaskbits) -{ - int c, old_c, totaldigits, ndigits, nchunks, nbits; - u32 chunk; - const char __user __force *ubuf = (const char __user __force *)buf; - - bitmap_zero(maskp, nmaskbits); - - nchunks = nbits = totaldigits = c = 0; - do { - chunk = 0; - ndigits = totaldigits; - - /* Get the next chunk of the bitmap */ - while (buflen) { - old_c = c; - if (is_user) { - if (__get_user(c, ubuf++)) - return -EFAULT; - } - else - c = *buf++; - buflen--; - if (isspace(c)) - continue; - - /* - * If the last character was a space and the current - * character isn't '\0', we've got embedded whitespace. - * This is a no-no, so throw an error. - */ - if (totaldigits && c && isspace(old_c)) - return -EINVAL; - - /* A '\0' or a ',' signal the end of the chunk */ - if (c == '\0' || c == ',') - break; - - if (!isxdigit(c)) - return -EINVAL; - - /* - * Make sure there are at least 4 free bits in 'chunk'. - * If not, this hexdigit will overflow 'chunk', so - * throw an error. - */ - if (chunk & ~((1UL << (CHUNKSZ - 4)) - 1)) - return -EOVERFLOW; - - chunk = (chunk << 4) | hex_to_bin(c); - totaldigits++; - } - if (ndigits == totaldigits) - return -EINVAL; - if (nchunks == 0 && chunk == 0) - continue; - - __bitmap_shift_left(maskp, maskp, CHUNKSZ, nmaskbits); - *maskp |= chunk; - nchunks++; - nbits += (nchunks == 1) ? nbits_to_hold_value(chunk) : CHUNKSZ; - if (nbits > nmaskbits) - return -EOVERFLOW; - } while (buflen && c == ','); - - return 0; -} -EXPORT_SYMBOL(__bitmap_parse); - /** * bitmap_parse_user - convert an ASCII hex string in a user buffer into a bitmap * @@ -542,7 +451,7 @@ int bitmap_parse_user(const char __user *ubuf, if (IS_ERR(buf)) return PTR_ERR(buf); - ret = bitmap_parse(buf, ulen, maskp, nmaskbits); + ret = bitmap_parse(buf, UINT_MAX, maskp, nmaskbits); kfree(buf); return ret; @@ -653,6 +562,14 @@ static const char *bitmap_find_region(const char *str) return end_of_str(*str) ? NULL : str; } +static const char *bitmap_find_region_reverse(const char *start, const char *end) +{ + while (start <= end && __end_of_region(*end)) + end--; + + return end; +} + static const char *bitmap_parse_region(const char *str, struct region *r) { str = bitmap_getnum(str, &r->start); @@ -776,6 +693,80 @@ int bitmap_parselist_user(const char __user *ubuf, } EXPORT_SYMBOL(bitmap_parselist_user); +static const char *bitmap_get_x32_reverse(const char *start, + const char *end, u32 *num) +{ + u32 ret = 0; + int c, i; + + for (i = 0; i < 32; i += 4) { + c = hex_to_bin(*end--); + if (c < 0) + return ERR_PTR(-EINVAL); + + ret |= c << i; + + if (start > end || __end_of_region(*end)) + goto out; + } + + if (hex_to_bin(*end--) >= 0) + return ERR_PTR(-EOVERFLOW); +out: + *num = ret; + return end; +} + +/** + * bitmap_parse - convert an ASCII hex string into a bitmap. + * @start: pointer to buffer containing string. + * @buflen: buffer size in bytes. If string is smaller than this + * then it must be terminated with a \0 or \n. In that case, + * UINT_MAX may be provided instead of string length. + * @maskp: pointer to bitmap array that will contain result. + * @nmaskbits: size of bitmap, in bits. + * + * Commas group hex digits into chunks. Each chunk defines exactly 32 + * bits of the resultant bitmask. No chunk may specify a value larger + * than 32 bits (%-EOVERFLOW), and if a chunk specifies a smaller value + * then leading 0-bits are prepended. %-EINVAL is returned for illegal + * characters. Grouping such as "1,,5", ",44", "," or "" is allowed. + * Leading, embedded and trailing whitespace accepted. + */ +int bitmap_parse(const char *start, unsigned int buflen, + unsigned long *maskp, int nmaskbits) +{ + const char *end = strnchrnul(start, buflen, '\n') - 1; + int chunks = BITS_TO_U32(nmaskbits); + u32 *bitmap = (u32 *)maskp; + int unset_bit; + + while (1) { + end = bitmap_find_region_reverse(start, end); + if (start > end) + break; + + if (!chunks--) + return -EOVERFLOW; + + end = bitmap_get_x32_reverse(start, end, bitmap++); + if (IS_ERR(end)) + return PTR_ERR(end); + } + + unset_bit = (BITS_TO_U32(nmaskbits) - chunks) * 32; + if (unset_bit < nmaskbits) { + bitmap_clear(maskp, unset_bit, nmaskbits - unset_bit); + return 0; + } + + if (find_next_bit(maskp, unset_bit, nmaskbits) != unset_bit) + return -EOVERFLOW; + + return 0; +} +EXPORT_SYMBOL(bitmap_parse); + #ifdef CONFIG_NUMA /** -- cgit v1.2.3 From 809e308f7f854933a792e4a14c4571320cd11f7e Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Mon, 3 Feb 2020 17:37:38 -0800 Subject: lib: new testcases for bitmap_parse{_user} New version of bitmap_parse() is unified with bitmap_parse_list(), and therefore: - weakens rules on whitespaces and commas between hex chunks; - in addition to - allows passing UINT_MAX or any other big number as the length of input string instead of actual string length. The patch covers the cases. Link: http://lkml.kernel.org/r/20200102043031.30357-7-yury.norov@gmail.com Signed-off-by: Yury Norov Reviewed-by: Andy Shevchenko Cc: Amritha Nambiar Cc: Arnaldo Carvalho de Melo Cc: Chris Wilson Cc: Kees Cook Cc: Matthew Wilcox Cc: Miklos Szeredi Cc: Rasmus Villemoes Cc: Steffen Klassert Cc: "Tobin C . Harding" Cc: Vineet Gupta Cc: Will Deacon Cc: Willem de Bruijn Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test_bitmap.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'lib') diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c index bde8ed8c0e79..61ed71c1daba 100644 --- a/lib/test_bitmap.c +++ b/lib/test_bitmap.c @@ -415,14 +415,22 @@ static const unsigned long parse_test2[] __initconst = { }; static const struct test_bitmap_parselist parse_tests[] __initconst = { + {0, "", &parse_test[0 * step], 32, 0}, + {0, " ", &parse_test[0 * step], 32, 0}, {0, "0", &parse_test[0 * step], 32, 0}, + {0, "0\n", &parse_test[0 * step], 32, 0}, {0, "1", &parse_test[1 * step], 32, 0}, {0, "deadbeef", &parse_test[2 * step], 32, 0}, {0, "1,0", &parse_test[3 * step], 33, 0}, + {0, "deadbeef,\n,0,1", &parse_test[2 * step], 96, 0}, {0, "deadbeef,1,0", &parse_test2[0 * 2 * step], 96, 0}, {0, "baadf00d,deadbeef,1,0", &parse_test2[1 * 2 * step], 128, 0}, {0, "badf00d,deadbeef,1,0", &parse_test2[2 * 2 * step], 124, 0}, + {0, "badf00d,deadbeef,1,0", &parse_test2[2 * 2 * step], 124, NO_LEN}, + {0, " badf00d,deadbeef,1,0 ", &parse_test2[2 * 2 * step], 124, 0}, + {0, " , badf00d,deadbeef,1,0 , ", &parse_test2[2 * 2 * step], 124, 0}, + {0, " , badf00d, ,, ,,deadbeef,1,0 , ", &parse_test2[2 * 2 * step], 124, 0}, {-EINVAL, "goodfood,deadbeef,1,0", NULL, 128, 0}, {-EOVERFLOW, "3,0", NULL, 33, 0}, -- cgit v1.2.3 From 597c0e3b4540972580ab36bcc8aa34394bd88cfb Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Wed, 5 Feb 2020 22:50:04 +0900 Subject: bootconfig: Add more parse error messages Add more error messages for following cases. - Exceeding max number of nodes - Config tree data is empty (e.g. comment only) - Config data is empty or exceeding max size - bootconfig is already initialized Link: http://lkml.kernel.org/r/158091060401.27924.9024818742827122764.stgit@devnote2 Signed-off-by: Masami Hiramatsu Signed-off-by: Steven Rostedt (VMware) --- lib/bootconfig.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/bootconfig.c b/lib/bootconfig.c index 055014e233a5..a98ae136529c 100644 --- a/lib/bootconfig.c +++ b/lib/bootconfig.c @@ -373,7 +373,8 @@ static struct xbc_node * __init xbc_add_sibling(char *data, u32 flag) sib->next = xbc_node_index(node); } } - } + } else + xbc_parse_error("Too many nodes", data); return node; } @@ -657,8 +658,10 @@ static int __init xbc_verify_tree(void) struct xbc_node *n, *m; /* Empty tree */ - if (xbc_node_num == 0) + if (xbc_node_num == 0) { + xbc_parse_error("Empty config", xbc_data); return -ENOENT; + } for (i = 0; i < xbc_node_num; i++) { if (xbc_nodes[i].next > xbc_node_num) { @@ -732,12 +735,17 @@ int __init xbc_init(char *buf) char *p, *q; int ret, c; - if (xbc_data) + if (xbc_data) { + pr_err("Error: bootconfig is already initialized.\n"); return -EBUSY; + } ret = strlen(buf); - if (ret > XBC_DATA_MAX - 1 || ret == 0) + if (ret > XBC_DATA_MAX - 1 || ret == 0) { + pr_err("Error: Config data is %s.\n", + ret ? "too big" : "empty"); return -ERANGE; + } xbc_data = buf; xbc_data_size = ret + 1; -- cgit v1.2.3 From 0f0d0a77ccbdc6bfe51c0d184f92d99ebcc2e809 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Wed, 5 Feb 2020 22:50:13 +0900 Subject: tools/bootconfig: Show the number of bootconfig nodes Show the number of bootconfig nodes when applying new bootconfig to initrd. Since there are limitations of bootconfig not only in its filesize, but also the number of nodes, the number should be shown when applying so that user can get the feeling of scale of current bootconfig. Link: http://lkml.kernel.org/r/158091061337.27924.10886706631693823982.stgit@devnote2 Signed-off-by: Masami Hiramatsu Signed-off-by: Steven Rostedt (VMware) --- lib/bootconfig.c | 5 ++++- tools/bootconfig/main.c | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/bootconfig.c b/lib/bootconfig.c index a98ae136529c..afb2e767e6fe 100644 --- a/lib/bootconfig.c +++ b/lib/bootconfig.c @@ -728,7 +728,8 @@ void __init xbc_destroy_all(void) * * This parses the boot config text in @buf. @buf must be a * null terminated string and smaller than XBC_DATA_MAX. - * Return 0 if succeeded, or -errno if there is any error. + * Return the number of stored nodes (>0) if succeeded, or -errno + * if there is any error. */ int __init xbc_init(char *buf) { @@ -788,6 +789,8 @@ int __init xbc_init(char *buf) if (ret < 0) xbc_destroy_all(); + else + ret = xbc_node_num; return ret; } diff --git a/tools/bootconfig/main.c b/tools/bootconfig/main.c index 91c9a5c0c499..47f488458328 100644 --- a/tools/bootconfig/main.c +++ b/tools/bootconfig/main.c @@ -268,6 +268,7 @@ int apply_xbc(const char *path, const char *xbc_path) return ret; } printf("Apply %s to %s\n", xbc_path, path); + printf("\tNumber of nodes: %d\n", ret); printf("\tSize: %u bytes\n", (unsigned int)size); printf("\tChecksum: %d\n", (unsigned int)csum); -- cgit v1.2.3 From 26445f98ead38b90423a2deffce2caa3b97a3d78 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 6 Feb 2020 20:14:53 +0900 Subject: bootconfig: Remove unneeded CONFIG_LIBXBC Since there is no user except CONFIG_BOOT_CONFIG and no plan to use it from other functions, CONFIG_LIBXBC can be removed and we can use CONFIG_BOOT_CONFIG directly. Link: http://lkml.kernel.org/r/158098769281.939.16293492056419481105.stgit@devnote2 Suggested-by: Geert Uytterhoeven Signed-off-by: Masami Hiramatsu Signed-off-by: Steven Rostedt (VMware) --- init/Kconfig | 1 - lib/Kconfig | 3 --- lib/Makefile | 2 +- 3 files changed, 1 insertion(+), 5 deletions(-) (limited to 'lib') diff --git a/init/Kconfig b/init/Kconfig index 9506299a53e3..4a672c6629d0 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1218,7 +1218,6 @@ endif config BOOT_CONFIG bool "Boot config support" depends on BLK_DEV_INITRD - select LIBXBC default y help Extra boot config allows system admin to pass a config file as diff --git a/lib/Kconfig b/lib/Kconfig index 10012b646009..6e790dc55c5b 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -566,9 +566,6 @@ config DIMLIB config LIBFDT bool -config LIBXBC - bool - config OID_REGISTRY tristate help diff --git a/lib/Makefile b/lib/Makefile index 75a64d2552a2..74c1223828c1 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -228,7 +228,7 @@ $(foreach file, $(libfdt_files), \ $(eval CFLAGS_$(file) = -I $(srctree)/scripts/dtc/libfdt)) lib-$(CONFIG_LIBFDT) += $(libfdt_files) -lib-$(CONFIG_LIBXBC) += bootconfig.o +lib-$(CONFIG_BOOT_CONFIG) += bootconfig.o obj-$(CONFIG_RBTREE_TEST) += rbtree_test.o obj-$(CONFIG_INTERVAL_TREE_TEST) += interval_tree_test.o -- cgit v1.2.3 From a91e4f12ffc42fa019b5a66eaa8702f5e0f08a6a Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Fri, 7 Feb 2020 23:28:17 +0900 Subject: bootconfig: Allocate xbc_nodes array dynamically To reduce the large static array from kernel data, allocate xbc_nodes array dynamically only if the kernel loads a bootconfig. Note that this also add dummy memblock.h for user-spacae bootconfig tool. Link: http://lkml.kernel.org/r/158108569699.3187.6512834527603883707.stgit@devnote2 Signed-off-by: Masami Hiramatsu Signed-off-by: Steven Rostedt (VMware) --- lib/bootconfig.c | 15 ++++++++++++--- tools/bootconfig/include/linux/memblock.h | 12 ++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 tools/bootconfig/include/linux/memblock.h (limited to 'lib') diff --git a/lib/bootconfig.c b/lib/bootconfig.c index afb2e767e6fe..3ea601a2eba5 100644 --- a/lib/bootconfig.c +++ b/lib/bootconfig.c @@ -6,12 +6,13 @@ #define pr_fmt(fmt) "bootconfig: " fmt +#include #include #include #include #include +#include #include -#include #include /* @@ -23,7 +24,7 @@ * node (for array). */ -static struct xbc_node xbc_nodes[XBC_NODE_MAX] __initdata; +static struct xbc_node *xbc_nodes __initdata; static int xbc_node_num __initdata; static char *xbc_data __initdata; static size_t xbc_data_size __initdata; @@ -719,7 +720,8 @@ void __init xbc_destroy_all(void) xbc_data = NULL; xbc_data_size = 0; xbc_node_num = 0; - memset(xbc_nodes, 0, sizeof(xbc_nodes)); + memblock_free(__pa(xbc_nodes), sizeof(struct xbc_node) * XBC_NODE_MAX); + xbc_nodes = NULL; } /** @@ -748,6 +750,13 @@ int __init xbc_init(char *buf) return -ERANGE; } + xbc_nodes = memblock_alloc(sizeof(struct xbc_node) * XBC_NODE_MAX, + SMP_CACHE_BYTES); + if (!xbc_nodes) { + pr_err("Failed to allocate memory for bootconfig nodes.\n"); + return -ENOMEM; + } + memset(xbc_nodes, 0, sizeof(struct xbc_node) * XBC_NODE_MAX); xbc_data = buf; xbc_data_size = ret + 1; last_parent = NULL; diff --git a/tools/bootconfig/include/linux/memblock.h b/tools/bootconfig/include/linux/memblock.h new file mode 100644 index 000000000000..7862f217d85d --- /dev/null +++ b/tools/bootconfig/include/linux/memblock.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _XBC_LINUX_MEMBLOCK_H +#define _XBC_LINUX_MEMBLOCK_H + +#include + +#define __pa(addr) (addr) +#define SMP_CACHE_BYTES 0 +#define memblock_alloc(size, align) malloc(size) +#define memblock_free(paddr, size) free(paddr) + +#endif -- cgit v1.2.3 From c9cc0517bba9f0213f1e55172feceb99e5512daf Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Thu, 6 Feb 2020 12:42:01 +0100 Subject: crypto: chacha20poly1305 - prevent integer overflow on large input This code assigns src_len (size_t) to sl (int), which causes problems when src_len is very large. Probably nobody in the kernel should be passing this much data to chacha20poly1305 all in one go anyway, so I don't think we need to change the algorithm or introduce larger types or anything. But we should at least error out early in this case and print a warning so that we get reports if this does happen and can look into why anybody is possibly passing it that much data or if they're accidently passing -1 or similar. Fixes: d95312a3ccc0 ("crypto: lib/chacha20poly1305 - reimplement crypt_from_sg() routine") Cc: Ard Biesheuvel Cc: stable@vger.kernel.org # 5.5+ Signed-off-by: Jason A. Donenfeld Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- lib/crypto/chacha20poly1305.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/crypto/chacha20poly1305.c b/lib/crypto/chacha20poly1305.c index 6d83cafebc69..ad0699ce702f 100644 --- a/lib/crypto/chacha20poly1305.c +++ b/lib/crypto/chacha20poly1305.c @@ -235,6 +235,9 @@ bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src, __le64 lens[2]; } b __aligned(16); + if (WARN_ON(src_len > INT_MAX)) + return false; + chacha_load_key(b.k, key); b.iv[0] = 0; -- cgit v1.2.3 From a24d286f36104ed45108a5a36f3868938434772f Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 20 Feb 2020 21:19:12 +0900 Subject: bootconfig: Reject subkey and value on same parent key Reject if a value node is mixed with subkey node on same parent key node. A value node can not co-exist with subkey node under some key node, e.g. key = value key.subkey = another-value This is not be allowed because bootconfig API is not designed to handle such case. Link: http://lkml.kernel.org/r/158220115232.26565.7792340045009731803.stgit@devnote2 Signed-off-by: Masami Hiramatsu Signed-off-by: Steven Rostedt (VMware) --- Documentation/admin-guide/bootconfig.rst | 7 +++++++ lib/bootconfig.c | 16 ++++++++++++---- tools/bootconfig/samples/bad-mixed-kv1.bconf | 3 +++ tools/bootconfig/samples/bad-mixed-kv2.bconf | 3 +++ 4 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 tools/bootconfig/samples/bad-mixed-kv1.bconf create mode 100644 tools/bootconfig/samples/bad-mixed-kv2.bconf (limited to 'lib') diff --git a/Documentation/admin-guide/bootconfig.rst b/Documentation/admin-guide/bootconfig.rst index 5e7609936507..dfeffa73dca3 100644 --- a/Documentation/admin-guide/bootconfig.rst +++ b/Documentation/admin-guide/bootconfig.rst @@ -62,6 +62,13 @@ Or more shorter, written as following:: In both styles, same key words are automatically merged when parsing it at boot time. So you can append similar trees or key-values. +Note that a sub-key and a value can not co-exist under a parent key. +For example, following config is NOT allowed.:: + + foo = value1 + foo.bar = value2 # !ERROR! subkey "bar" and value "value1" can NOT co-exist + + Comments -------- diff --git a/lib/bootconfig.c b/lib/bootconfig.c index 3ea601a2eba5..54ac623ca781 100644 --- a/lib/bootconfig.c +++ b/lib/bootconfig.c @@ -533,7 +533,7 @@ struct xbc_node *find_match_node(struct xbc_node *node, char *k) static int __init __xbc_add_key(char *k) { - struct xbc_node *node; + struct xbc_node *node, *child; if (!xbc_valid_keyword(k)) return xbc_parse_error("Invalid keyword", k); @@ -543,8 +543,12 @@ static int __init __xbc_add_key(char *k) if (!last_parent) /* the first level */ node = find_match_node(xbc_nodes, k); - else - node = find_match_node(xbc_node_get_child(last_parent), k); + else { + child = xbc_node_get_child(last_parent); + if (child && xbc_node_is_value(child)) + return xbc_parse_error("Subkey is mixed with value", k); + node = find_match_node(child, k); + } if (node) last_parent = node; @@ -577,7 +581,7 @@ static int __init __xbc_parse_keys(char *k) static int __init xbc_parse_kv(char **k, char *v) { struct xbc_node *prev_parent = last_parent; - struct xbc_node *node; + struct xbc_node *node, *child; char *next; int c, ret; @@ -585,6 +589,10 @@ static int __init xbc_parse_kv(char **k, char *v) if (ret) return ret; + child = xbc_node_get_child(last_parent); + if (child && xbc_node_is_key(child)) + return xbc_parse_error("Value is mixed with subkey", v); + c = __xbc_parse_value(&v, &next); if (c < 0) return c; diff --git a/tools/bootconfig/samples/bad-mixed-kv1.bconf b/tools/bootconfig/samples/bad-mixed-kv1.bconf new file mode 100644 index 000000000000..1761547dd05c --- /dev/null +++ b/tools/bootconfig/samples/bad-mixed-kv1.bconf @@ -0,0 +1,3 @@ +# value -> subkey pattern +key = value +key.subkey = another-value diff --git a/tools/bootconfig/samples/bad-mixed-kv2.bconf b/tools/bootconfig/samples/bad-mixed-kv2.bconf new file mode 100644 index 000000000000..6b32e0c3878c --- /dev/null +++ b/tools/bootconfig/samples/bad-mixed-kv2.bconf @@ -0,0 +1,3 @@ +# subkey -> value pattern +key.subkey = value +key = another-value -- cgit v1.2.3 From 4e4694d8729f7cd6381f6691e8f83e378fce3160 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Fri, 21 Feb 2020 17:13:42 +0900 Subject: bootconfig: Prohibit re-defining value on same key Currently, bootconfig adds a new value on the existing key to the tail of an array. But this looks a bit confusing because an admin can easily rewrite the original value in the same config file. This rejects the following value re-definition. key = value1 ... key = value2 You should rewrite value1 to value2 in this case. Link: http://lkml.kernel.org/r/158227282199.12842.10110929876059658601.stgit@devnote2 Suggested-by: Steven Rostedt (VMware) Signed-off-by: Masami Hiramatsu [ Fixed spelling of arraies to arrays ] Signed-off-by: Steven Rostedt (VMware) --- Documentation/admin-guide/bootconfig.rst | 11 ++++++++++- lib/bootconfig.c | 13 ++++++++----- tools/bootconfig/samples/bad-samekey.bconf | 6 ++++++ 3 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 tools/bootconfig/samples/bad-samekey.bconf (limited to 'lib') diff --git a/Documentation/admin-guide/bootconfig.rst b/Documentation/admin-guide/bootconfig.rst index dfeffa73dca3..57119fb69d36 100644 --- a/Documentation/admin-guide/bootconfig.rst +++ b/Documentation/admin-guide/bootconfig.rst @@ -62,7 +62,16 @@ Or more shorter, written as following:: In both styles, same key words are automatically merged when parsing it at boot time. So you can append similar trees or key-values. -Note that a sub-key and a value can not co-exist under a parent key. +Same-key Values +--------------- + +It is prohibited that two or more values or arrays share a same-key. +For example,:: + + foo = bar, baz + foo = qux # !ERROR! we can not re-define same key + +Also, a sub-key and a value can not co-exist under a parent key. For example, following config is NOT allowed.:: foo = value1 diff --git a/lib/bootconfig.c b/lib/bootconfig.c index 54ac623ca781..2ef304db31f2 100644 --- a/lib/bootconfig.c +++ b/lib/bootconfig.c @@ -581,7 +581,7 @@ static int __init __xbc_parse_keys(char *k) static int __init xbc_parse_kv(char **k, char *v) { struct xbc_node *prev_parent = last_parent; - struct xbc_node *node, *child; + struct xbc_node *child; char *next; int c, ret; @@ -590,15 +590,18 @@ static int __init xbc_parse_kv(char **k, char *v) return ret; child = xbc_node_get_child(last_parent); - if (child && xbc_node_is_key(child)) - return xbc_parse_error("Value is mixed with subkey", v); + if (child) { + if (xbc_node_is_key(child)) + return xbc_parse_error("Value is mixed with subkey", v); + else + return xbc_parse_error("Value is redefined", v); + } c = __xbc_parse_value(&v, &next); if (c < 0) return c; - node = xbc_add_sibling(v, XBC_VALUE); - if (!node) + if (!xbc_add_sibling(v, XBC_VALUE)) return -ENOMEM; if (c == ',') { /* Array */ diff --git a/tools/bootconfig/samples/bad-samekey.bconf b/tools/bootconfig/samples/bad-samekey.bconf new file mode 100644 index 000000000000..e8d983a4563c --- /dev/null +++ b/tools/bootconfig/samples/bad-samekey.bconf @@ -0,0 +1,6 @@ +# Same key value is not allowed +key { + foo = value + bar = value2 +} +key.foo = value -- cgit v1.2.3 From 5f811c57c99205e048926293bb812c750a6ea562 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Fri, 21 Feb 2020 17:13:52 +0900 Subject: bootconfig: Add append value operator support Add append value operator "+=" support to bootconfig syntax. With this operator, user can add new value to the key as an entry of array instead of overwriting. For example, foo = bar ... foo += baz Then the key "foo" has "bar" and "baz" values as an array. Link: http://lkml.kernel.org/r/158227283195.12842.8310503105963275584.stgit@devnote2 Signed-off-by: Masami Hiramatsu Signed-off-by: Steven Rostedt (VMware) --- Documentation/admin-guide/bootconfig.rst | 10 +++++++++- lib/bootconfig.c | 15 +++++++++++---- tools/bootconfig/test-bootconfig.sh | 16 ++++++++++++++-- 3 files changed, 34 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/Documentation/admin-guide/bootconfig.rst b/Documentation/admin-guide/bootconfig.rst index 57119fb69d36..cf2edcd09183 100644 --- a/Documentation/admin-guide/bootconfig.rst +++ b/Documentation/admin-guide/bootconfig.rst @@ -71,7 +71,15 @@ For example,:: foo = bar, baz foo = qux # !ERROR! we can not re-define same key -Also, a sub-key and a value can not co-exist under a parent key. +If you want to append the value to existing key as an array member, +you can use ``+=`` operator. For example:: + + foo = bar, baz + foo += qux + +In this case, the key ``foo`` has ``bar``, ``baz`` and ``qux``. + +However, a sub-key and a value can not co-exist under a parent key. For example, following config is NOT allowed.:: foo = value1 diff --git a/lib/bootconfig.c b/lib/bootconfig.c index 2ef304db31f2..ec3ce7fd299f 100644 --- a/lib/bootconfig.c +++ b/lib/bootconfig.c @@ -578,7 +578,7 @@ static int __init __xbc_parse_keys(char *k) return __xbc_add_key(k); } -static int __init xbc_parse_kv(char **k, char *v) +static int __init xbc_parse_kv(char **k, char *v, int op) { struct xbc_node *prev_parent = last_parent; struct xbc_node *child; @@ -593,7 +593,7 @@ static int __init xbc_parse_kv(char **k, char *v) if (child) { if (xbc_node_is_key(child)) return xbc_parse_error("Value is mixed with subkey", v); - else + else if (op == '=') return xbc_parse_error("Value is redefined", v); } @@ -774,7 +774,7 @@ int __init xbc_init(char *buf) p = buf; do { - q = strpbrk(p, "{}=;\n#"); + q = strpbrk(p, "{}=+;\n#"); if (!q) { p = skip_spaces(p); if (*p != '\0') @@ -785,8 +785,15 @@ int __init xbc_init(char *buf) c = *q; *q++ = '\0'; switch (c) { + case '+': + if (*q++ != '=') { + ret = xbc_parse_error("Wrong '+' operator", + q - 2); + break; + } + /* Fall through */ case '=': - ret = xbc_parse_kv(&p, q); + ret = xbc_parse_kv(&p, q, c); break; case '{': ret = xbc_open_brace(&p, q); diff --git a/tools/bootconfig/test-bootconfig.sh b/tools/bootconfig/test-bootconfig.sh index adafb7c50940..1411f4c3454f 100755 --- a/tools/bootconfig/test-bootconfig.sh +++ b/tools/bootconfig/test-bootconfig.sh @@ -9,7 +9,7 @@ TEMPCONF=`mktemp temp-XXXX.bconf` NG=0 cleanup() { - rm -f $INITRD $TEMPCONF + rm -f $INITRD $TEMPCONF $OUTFILE exit $NG } @@ -71,7 +71,6 @@ printf " \0\0\0 \0\0\0" >> $INITRD $BOOTCONF -a $TEMPCONF $INITRD > $OUTFILE 2>&1 xfail grep -i "failed" $OUTFILE xfail grep -i "error" $OUTFILE -rm $OUTFILE echo "Max node number check" @@ -96,6 +95,19 @@ truncate -s 32764 $TEMPCONF echo "\"" >> $TEMPCONF # add 2 bytes + terminal ('\"\n\0') xpass $BOOTCONF -a $TEMPCONF $INITRD +echo "Adding same-key values" +cat > $TEMPCONF << EOF +key = bar, baz +key += qux +EOF +echo > $INITRD + +xpass $BOOTCONF -a $TEMPCONF $INITRD +$BOOTCONF $INITRD > $OUTFILE +xpass grep -q "bar" $OUTFILE +xpass grep -q "baz" $OUTFILE +xpass grep -q "qux" $OUTFILE + echo "=== expected failure cases ===" for i in samples/bad-* ; do xfail $BOOTCONF -a $i $INITRD -- cgit v1.2.3 From c11d3fa0116a6bc832a9e387427caa16f8de5ef2 Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Thu, 20 Feb 2020 20:04:21 -0800 Subject: lib/string.c: update match_string() doc-strings with correct behavior There were a few attempts at changing behavior of the match_string() helpers (i.e. 'match_string()' & 'sysfs_match_string()'), to change & extend the behavior according to the doc-string. But the simplest approach is to just fix the doc-strings. The current behavior is fine as-is, and some bugs were introduced trying to fix it. As for extending the behavior, new helpers can always be introduced if needed. The match_string() helpers behave more like 'strncmp()' in the sense that they go up to n elements or until the first NULL element in the array of strings. This change updates the doc-strings with this info. Link: http://lkml.kernel.org/r/20200213072722.8249-1-alexandru.ardelean@analog.com Signed-off-by: Alexandru Ardelean Acked-by: Andy Shevchenko Cc: Kees Cook Cc: "Tobin C . Harding" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/string.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'lib') diff --git a/lib/string.c b/lib/string.c index f607b967d978..6012c385fb31 100644 --- a/lib/string.c +++ b/lib/string.c @@ -699,6 +699,14 @@ EXPORT_SYMBOL(sysfs_streq); * @n: number of strings in the array or -1 for NULL terminated arrays * @string: string to match with * + * This routine will look for a string in an array of strings up to the + * n-th element in the array or until the first NULL element. + * + * Historically the value of -1 for @n, was used to search in arrays that + * are NULL terminated. However, the function does not make a distinction + * when finishing the search: either @n elements have been compared OR + * the first NULL element was found. + * * Return: * index of a @string in the @array if matches, or %-EINVAL otherwise. */ @@ -727,6 +735,14 @@ EXPORT_SYMBOL(match_string); * * Returns index of @str in the @array or -EINVAL, just like match_string(). * Uses sysfs_streq instead of strcmp for matching. + * + * This routine will look for a string in an array of strings up to the + * n-th element in the array or until the first NULL element. + * + * Historically the value of -1 for @n, was used to search in arrays that + * are NULL terminated. However, the function does not make a distinction + * when finishing the search: either @n elements have been compared OR + * the first NULL element was found. */ int __sysfs_match_string(const char * const *array, size_t n, const char *str) { -- cgit v1.2.3 From 305e519ce48e935702c32241f07d393c3c8fed3e Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Thu, 20 Feb 2020 20:04:30 -0800 Subject: lib/stackdepot.c: fix global out-of-bounds in stack_slabs Walter Wu has reported a potential case in which init_stack_slab() is called after stack_slabs[STACK_ALLOC_MAX_SLABS - 1] has already been initialized. In that case init_stack_slab() will overwrite stack_slabs[STACK_ALLOC_MAX_SLABS], which may result in a memory corruption. Link: http://lkml.kernel.org/r/20200218102950.260263-1-glider@google.com Fixes: cd11016e5f521 ("mm, kasan: stackdepot implementation. Enable stackdepot for SLAB") Signed-off-by: Alexander Potapenko Reported-by: Walter Wu Cc: Dmitry Vyukov Cc: Matthias Brugger Cc: Thomas Gleixner Cc: Josh Poimboeuf Cc: Kate Stewart Cc: Greg Kroah-Hartman Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/stackdepot.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/stackdepot.c b/lib/stackdepot.c index ed717dd08ff3..81c69c08d1d1 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -83,15 +83,19 @@ static bool init_stack_slab(void **prealloc) return true; if (stack_slabs[depot_index] == NULL) { stack_slabs[depot_index] = *prealloc; + *prealloc = NULL; } else { - stack_slabs[depot_index + 1] = *prealloc; + /* If this is the last depot slab, do not touch the next one. */ + if (depot_index + 1 < STACK_ALLOC_MAX_SLABS) { + stack_slabs[depot_index + 1] = *prealloc; + *prealloc = NULL; + } /* * This smp_store_release pairs with smp_load_acquire() from * |next_slab_inited| above and in stack_depot_save(). */ smp_store_release(&next_slab_inited, 1); } - *prealloc = NULL; return true; } -- cgit v1.2.3 From c8cfcb78c65877313cda7bcbace624d3dbd1f3b3 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Wed, 18 Mar 2020 20:27:32 -0600 Subject: crypto: arm64/chacha - correctly walk through blocks Prior, passing in chunks of 2, 3, or 4, followed by any additional chunks would result in the chacha state counter getting out of sync, resulting in incorrect encryption/decryption, which is a pretty nasty crypto vuln: "why do images look weird on webpages?" WireGuard users never experienced this prior, because we have always, out of tree, used a different crypto library, until the recent Frankenzinc addition. This commit fixes the issue by advancing the pointers and state counter by the actual size processed. It also fixes up a bug in the (optional, costly) stride test that prevented it from running on arm64. Fixes: b3aad5bad26a ("crypto: arm64/chacha - expose arm64 ChaCha routine as library function") Reported-and-tested-by: Emil Renner Berthing Cc: Ard Biesheuvel Cc: stable@vger.kernel.org # v5.5+ Signed-off-by: Jason A. Donenfeld Reviewed-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm64/crypto/chacha-neon-glue.c | 8 ++++---- lib/crypto/chacha20poly1305-selftest.c | 11 ++++++++--- 2 files changed, 12 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/arch/arm64/crypto/chacha-neon-glue.c b/arch/arm64/crypto/chacha-neon-glue.c index c1f9660d104c..37ca3e889848 100644 --- a/arch/arm64/crypto/chacha-neon-glue.c +++ b/arch/arm64/crypto/chacha-neon-glue.c @@ -55,10 +55,10 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, break; } chacha_4block_xor_neon(state, dst, src, nrounds, l); - bytes -= CHACHA_BLOCK_SIZE * 5; - src += CHACHA_BLOCK_SIZE * 5; - dst += CHACHA_BLOCK_SIZE * 5; - state[12] += 5; + bytes -= l; + src += l; + dst += l; + state[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE); } } diff --git a/lib/crypto/chacha20poly1305-selftest.c b/lib/crypto/chacha20poly1305-selftest.c index c391a91364e9..fa43deda2660 100644 --- a/lib/crypto/chacha20poly1305-selftest.c +++ b/lib/crypto/chacha20poly1305-selftest.c @@ -9028,10 +9028,15 @@ bool __init chacha20poly1305_selftest(void) && total_len <= 1 << 10; ++total_len) { for (i = 0; i <= total_len; ++i) { for (j = i; j <= total_len; ++j) { + k = 0; sg_init_table(sg_src, 3); - sg_set_buf(&sg_src[0], input, i); - sg_set_buf(&sg_src[1], input + i, j - i); - sg_set_buf(&sg_src[2], input + j, total_len - j); + if (i) + sg_set_buf(&sg_src[k++], input, i); + if (j - i) + sg_set_buf(&sg_src[k++], input + i, j - i); + if (total_len - j) + sg_set_buf(&sg_src[k++], input + j, total_len - j); + sg_init_marker(sg_src, k); memset(computed_output, 0, total_len); memset(input, 0, total_len); -- cgit v1.2.3