Fix encoding of "unsigned" integers. If MSB is set, we need to pad

with a zero byte.


git-svn-id: svn://svn.h5l.se/heimdal/trunk/heimdal@12337 ec53bebd-3082-4978-b11e-865c3cabbd6b

lib/asn1/check-der.c

@@ -90,6 +90,43 @@ test_integer (void)
 			 cmp_integer);
 }
 
+static int
+cmp_unsigned (void *a, void *b)
+{
+    return *(unsigned int*)b - *(unsigned int*)a;
+}
+
+static int
+test_unsigned (void)
+{
+    struct test_case tests[] = {
+	{NULL, 3, "\x02\x01\x00"},
+	{NULL, 3, "\x02\x01\x7f"},
+	{NULL, 4, "\x02\x02\x00\x80"},
+	{NULL, 4, "\x02\x02\x01\x00"},
+	{NULL, 4, "\x02\x02\x02\x00"},
+	{NULL, 5, "\x02\x03\x00\x80\x00"},
+	{NULL, 7, "\x02\x05\x00\x80\x00\x00\x00"},
+	{NULL, 6, "\x02\x04\x7f\xff\xff\xff"}
+    };
+
+    unsigned int values[] = {0, 127, 128, 256, 512, 32768, 
+			     0x80000000, 0x7fffffff};
+    int i;
+    int ntests = sizeof(tests) / sizeof(*tests);
+
+    for (i = 0; i < ntests; ++i) {
+	tests[i].val = &values[i];
+	asprintf (&tests[i].name, "unsigned %u", values[i]);
+    }
+
+    return generic_test (tests, ntests, sizeof(int),
+			 (generic_encode)encode_unsigned,
+			 (generic_length) length_unsigned,
+			 (generic_decode)decode_unsigned,
+			 cmp_unsigned);
+}
+
 static int
 cmp_octet_string (void *a, void *b)
 {
@@ -189,6 +226,7 @@ main(int argc, char **argv)
     int ret = 0;
 
     ret += test_integer ();
+    ret += test_unsigned ();
     ret += test_octet_string ();
     ret += test_general_string ();
     ret += test_generalized_time ();

lib/asn1/der_length.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997 - 2001 Kungliga Tekniska H<>gskolan
+ * Copyright (c) 1997-2003 Kungliga Tekniska H<>gskolan
  * (Royal Institute of Technology, Stockholm, Sweden). 
  * All rights reserved. 
  *
@@ -38,33 +38,33 @@ RCSID("$Id$");
 static size_t
 len_unsigned (unsigned val)
 {
-  size_t ret = 0;
+    size_t ret = 0;
 
-  do {
-    ++ret;
-    val /= 256;
-  } while (val);
-  return ret;
+    do {
+	++ret;
+	val /= 256;
+    } while (val);
+    return ret;
 }
 
 static size_t
 len_int (int val)
 {
-  size_t ret = 0;
+    size_t ret = 0;
 
-  if (val == 0)
-    return 1;
-  while (val > 255 || val < -255) {
-    ++ret;
-    val /= 256;
-  }
-  if (val != 0) {
-    ++ret;
-    if ((signed char)val != val)
-      ++ret;
-    val /= 256;
-  }
-  return ret;
+    if (val == 0)
+	return 1;
+    while (val > 255 || val < -255) {
+	++ret;
+	val /= 256;
+    }
+    if (val != 0) {
+	++ret;
+	if ((signed char)val != val)
+	    ++ret;
+	val /= 256;
+    }
+    return ret;
 }
 
 static size_t
@@ -89,66 +89,74 @@ len_oid (const oid *oid)
 size_t
 length_len (size_t len)
 {
-  if (len < 128)
-    return 1;
-  else
-    return len_unsigned (len) + 1;
+    if (len < 128)
+	return 1;
+    else
+	return len_unsigned (len) + 1;
 }
 
 size_t
 length_integer (const int *data)
 {
-  size_t len = len_int (*data);
+    size_t len = len_int (*data);
 
-  return 1 + length_len(len) + len;
+    return 1 + length_len(len) + len;
 }
 
 size_t
 length_unsigned (const unsigned *data)
 {
-  size_t len = len_unsigned (*data);
-
-  return 1 + length_len(len) + len;
+    unsigned val = *data;
+    size_t len = 0;
+ 
+    while (val > 255) {
+	++len;
+	val /= 256;
+    }
+    len++;
+    if (val >= 128)
+	len++;
+    return 1 + length_len(len) + len;
 }
 
 size_t
 length_enumerated (const unsigned *data)
 {
-  size_t len = len_int (*data);
+    size_t len = len_int (*data);
 
-  return 1 + length_len(len) + len;
+    return 1 + length_len(len) + len;
 }
 
 size_t
 length_general_string (const general_string *data)
 {
-  char *str = *data;
-  size_t len = strlen(str);
-  return 1 + length_len(len) + len;
+    char *str = *data;
+    size_t len = strlen(str);
+    return 1 + length_len(len) + len;
 }
 
 size_t
 length_octet_string (const octet_string *k)
 {
-  return 1 + length_len(k->length) + k->length;
+    return 1 + length_len(k->length) + k->length;
 }
 
 size_t
 length_oid (const oid *k)
 {
-  size_t len = len_oid (k);
+    size_t len = len_oid (k);
 
-  return 1 + length_len(len) + len;
+    return 1 + length_len(len) + len;
 }
 
 size_t
 length_generalized_time (const time_t *t)
 {
-  octet_string k;
-  size_t ret;
+    octet_string k;
+    size_t ret;
 
-  time2generalizedtime (*t, &k);
-  ret = 1 + length_len(k.length) + k.length;
-  free (k.data);
-  return ret;
+    time2generalizedtime (*t, &k);
+    ret = 1 + length_len(k.length) + k.length;
+    free (k.data);
+    return ret;
 }

lib/asn1/der_put.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997 - 2001 Kungliga Tekniska H<>gskolan
+ * Copyright (c) 1997-2003 Kungliga Tekniska H<>gskolan
  * (Royal Institute of Technology, Stockholm, Sweden). 
  * All rights reserved. 
  *
@@ -264,6 +264,15 @@ encode_unsigned (unsigned char *p, size_t len, const unsigned *data,
     p -= l;
     len -= l;
     ret += l;
+    /* if first octet has msb set, we need to pad with a zero byte */
+    if(p[1] >= 128) {
+	if(len == 0)
+	    return ASN1_OVERFLOW;
+	*p-- = 0;
+	len--;
+	ret++;
+	l++;
+    }
     e = der_put_length_and_tag (p, len, l, UNIV, PRIM, UT_Integer, &l);
     if (e)
 	return e;