libtommath: Fix possible integer overflow CVE-2023-36328

Cherry picked from libtommath 7bbc1f8e4fe6dce75055957645117180768efb15.

Vulnerability Detail:
  CVE Identifier: CVE-2023-36328
  Description: Integer Overflow vulnerability in mp_grow in libtom
    libtommath before commit beba892bc0d4e4ded4d667ab1d2a94f4d75109a9,
    allows attackers to execute arbitrary code and cause a denial of
    service (DoS).
  Reference: https://nvd.nist.gov/vuln/detail/CVE-2023-36328

Reported-by: https://github.com/Crispy-fried-chicken

lib/hcrypto/libtommath/bn_mp_2expt.c

@@ -12,6 +12,10 @@ mp_err mp_2expt(mp_int *a, int b)
 {
    mp_err    err;
 
+   if (b < 0) {
+      return MP_VAL;
+   }
+
    /* zero a as per default */
    mp_zero(a);
 

lib/hcrypto/libtommath/bn_mp_grow.c

@@ -9,6 +9,10 @@ mp_err mp_grow(mp_int *a, int size)
    int     i;
    mp_digit *tmp;
 
+   if (size < 0) {
+      return MP_VAL;
+   }
+
    /* if the alloc size is smaller alloc more ram */
    if (a->alloc < size) {
       /* reallocate the array a->dp

lib/hcrypto/libtommath/bn_mp_init_size.c

@@ -6,6 +6,11 @@
 /* init an mp_init for a given size */
 mp_err mp_init_size(mp_int *a, int size)
 {
+
+   if (size < 0) {
+      return MP_VAL;
+   }
+
    size = MP_MAX(MP_MIN_PREC, size);
 
    /* alloc mem */

lib/hcrypto/libtommath/bn_mp_mul_2d.c

@@ -9,6 +9,10 @@ mp_err mp_mul_2d(const mp_int *a, int b, mp_int *c)
    mp_digit d;
    mp_err   err;
 
+   if (b < 0) {
+      return MP_VAL;
+   }
+
    /* copy */
    if (a != c) {
       if ((err = mp_copy(a, c)) != MP_OKAY) {

lib/hcrypto/libtommath/bn_s_mp_mul_digs.c

@@ -16,6 +16,10 @@ mp_err s_mp_mul_digs(const mp_int *a, const mp_int *b, mp_int *c, int digs)
    mp_word r;
    mp_digit tmpx, *tmpt, *tmpy;
 
+   if (digs < 0) {
+      return MP_VAL;
+   }
+
    /* can we use the fast multiplier? */
    if ((digs < MP_WARRAY) &&
        (MP_MIN(a->used, b->used) < MP_MAXFAST)) {

lib/hcrypto/libtommath/bn_s_mp_mul_digs_fast.c

@@ -26,6 +26,10 @@ mp_err s_mp_mul_digs_fast(const mp_int *a, const mp_int *b, mp_int *c, int digs)
    mp_digit W[MP_WARRAY];
    mp_word  _W;
 
+   if (digs < 0) {
+      return MP_VAL;
+   }
+
    /* grow the destination as required */
    if (c->alloc < digs) {
       if ((err = mp_grow(c, digs)) != MP_OKAY) {

lib/hcrypto/libtommath/bn_s_mp_mul_high_digs.c

@@ -15,6 +15,10 @@ mp_err s_mp_mul_high_digs(const mp_int *a, const mp_int *b, mp_int *c, int digs)
    mp_word  r;
    mp_digit tmpx, *tmpt, *tmpy;
 
+   if (digs < 0) {
+      return MP_VAL;
+   }
+
    /* can we use the fast multiplier? */
    if (MP_HAS(S_MP_MUL_HIGH_DIGS_FAST)
        && ((a->used + b->used + 1) < MP_WARRAY)

lib/hcrypto/libtommath/bn_s_mp_mul_high_digs_fast.c

@@ -19,6 +19,10 @@ mp_err s_mp_mul_high_digs_fast(const mp_int *a, const mp_int *b, mp_int *c, int
    mp_digit W[MP_WARRAY];
    mp_word  _W;
 
+   if (digs < 0) {
+      return MP_VAL;
+   }
+
    /* grow the destination as required */
    pa = a->used + b->used;
    if (c->alloc < pa) {