From e8829ef1f73fa4051a936ab9f0204195dae4ef2b Mon Sep 17 00:00:00 2001
From: Joachim Vandersmissen <git@jvdsn.com>
Date: Sat, 3 Feb 2024 01:19:59 -0600
Subject: [PATCH] crypto: rsa - restrict plaintext/ciphertext values more
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

SP 800-56Br2, Section 7.1.1 [1] specifies that:
1. If m does not satisfy 1 < m < (n – 1), output an indication that m is
out of range, and exit without further processing.

Similarly, Section 7.1.2 of the same standard specifies that:
1. If the ciphertext c does not satisfy 1 < c < (n – 1), output an
indication that the ciphertext is out of range, and exit without further
processing.

This range is slightly more conservative than RFC3447, as it also
excludes RSA fixed points 0, 1, and n - 1.

[1] https://doi.org/10.6028/NIST.SP.800-56Br2

Signed-off-by: Joachim Vandersmissen <git@jvdsn.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
 crypto/rsa.c | 36 ++++++++++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)

diff --git a/crypto/rsa.c b/crypto/rsa.c
index b9cd11fb7d36..d9be9e86097e 100644
--- a/crypto/rsa.c
+++ b/crypto/rsa.c
@@ -24,14 +24,38 @@ struct rsa_mpi_key {
 	MPI qinv;
 };
 
+static int rsa_check_payload(MPI x, MPI n)
+{
+	MPI n1;
+
+	if (mpi_cmp_ui(x, 1) <= 0)
+		return -EINVAL;
+
+	n1 = mpi_alloc(0);
+	if (!n1)
+		return -ENOMEM;
+
+	if (mpi_sub_ui(n1, n, 1) || mpi_cmp(x, n1) >= 0) {
+		mpi_free(n1);
+		return -EINVAL;
+	}
+
+	mpi_free(n1);
+	return 0;
+}
+
 /*
  * RSAEP function [RFC3447 sec 5.1.1]
  * c = m^e mod n;
  */
 static int _rsa_enc(const struct rsa_mpi_key *key, MPI c, MPI m)
 {
-	/* (1) Validate 0 <= m < n */
-	if (mpi_cmp_ui(m, 0) < 0 || mpi_cmp(m, key->n) >= 0)
+	/*
+	 * Even though (1) in RFC3447 only requires 0 <= m <= n - 1, we are
+	 * slightly more conservative and require 1 < m < n - 1. This is in line
+	 * with SP 800-56Br2, Section 7.1.1.
+	 */
+	if (rsa_check_payload(m, key->n))
 		return -EINVAL;
 
 	/* (2) c = m^e mod n */
@@ -50,8 +74,12 @@ static int _rsa_dec_crt(const struct rsa_mpi_key *key, MPI m_or_m1_or_h, MPI c)
 	MPI m2, m12_or_qh;
 	int ret = -ENOMEM;
 
-	/* (1) Validate 0 <= c < n */
-	if (mpi_cmp_ui(c, 0) < 0 || mpi_cmp(c, key->n) >= 0)
+	/*
+	 * Even though (1) in RFC3447 only requires 0 <= c <= n - 1, we are
+	 * slightly more conservative and require 1 < c < n - 1. This is in line
+	 * with SP 800-56Br2, Section 7.1.2.
+	 */
+	if (rsa_check_payload(c, key->n))
 		return -EINVAL;
 
 	m2 = mpi_alloc(0);