From 1d8f59cfa191b5134776e720ef79cf03171e8b08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Love=20H=C3=B6rnquist=20=C3=85strand?= Date: Tue, 28 Nov 2006 12:00:08 +0000 Subject: [PATCH] sprinkle more error strings git-svn-id: svn://svn.h5l.se/heimdal/trunk/heimdal@19171 ec53bebd-3082-4978-b11e-865c3cabbd6b --- lib/hx509/cert.c | 6 ++- lib/hx509/cms.c | 5 ++- lib/hx509/crypto.c | 96 ++++++++++++++++------------------------------ 3 files changed, 40 insertions(+), 67 deletions(-) diff --git a/lib/hx509/cert.c b/lib/hx509/cert.c index dffeecc88..f7ea786a5 100644 --- a/lib/hx509/cert.c +++ b/lib/hx509/cert.c @@ -1039,12 +1039,14 @@ _hx509_cert_private_decrypt(hx509_context context, } int -_hx509_cert_public_encrypt(const heim_octet_string *cleartext, +_hx509_cert_public_encrypt(hx509_context context, + const heim_octet_string *cleartext, const hx509_cert p, heim_oid *encryption_oid, heim_octet_string *ciphertext) { - return _hx509_public_encrypt(cleartext, p->data, + return _hx509_public_encrypt(context, + cleartext, p->data, encryption_oid, ciphertext); } diff --git a/lib/hx509/cms.c b/lib/hx509/cms.c index 818852662..8e25c97e2 100644 --- a/lib/hx509/cms.c +++ b/lib/hx509/cms.c @@ -517,11 +517,12 @@ hx509_cms_envelope_1(hx509_context context, goto out; } - ret = _hx509_cert_public_encrypt(&key, cert, + ret = _hx509_cert_public_encrypt(context, + &key, cert, &ri->keyEncryptionAlgorithm.algorithm, &ri->encryptedKey); if (ret) { - hx509_set_error_string(context, 0, ret, + hx509_set_error_string(context, HX509_ERROR_APPEND, ret, "Failed to encrypt transport key for " "EnvelopedData"); goto out; diff --git a/lib/hx509/crypto.c b/lib/hx509/crypto.c index 0a1fa5860..a11310885 100644 --- a/lib/hx509/crypto.c +++ b/lib/hx509/crypto.c @@ -184,9 +184,10 @@ rsa_verify_signature(hx509_context context, retsize = RSA_public_decrypt(sig->length, (unsigned char *)sig->data, to, rsa, RSA_PKCS1_PADDING); - if (retsize == -1) { + if (retsize <= 0) { ret = HX509_CRYPTO_SIG_INVALID_FORMAT; - hx509_set_error_string(context, 0, ret, "RSA public decrypt failed"); + hx509_set_error_string(context, 0, ret, + "RSA public decrypt failed: %d", retsize); free(to); goto out; } @@ -272,8 +273,10 @@ rsa_create_signature(hx509_context context, if (signatureAlgorithm) { ret = _hx509_set_digest_alg(signatureAlgorithm, sig_oid, "\x05\x00", 2); - if (ret) + if (ret) { + hx509_clear_error_string(context); return ret; + } } memset(&di, 0, sizeof(di)); @@ -284,6 +287,8 @@ rsa_create_signature(hx509_context context, data, &di.digestAlgorithm, &di.digest); + if (ret) + return ret; ASN1_MALLOC_ENCODE(DigestInfo, indata.data, indata.length, @@ -291,23 +296,31 @@ rsa_create_signature(hx509_context context, &size, ret); free_DigestInfo(&di); - if (ret) + if (ret) { + hx509_set_error_string(context, 0, ret, "out of memory"); return ret; + } if (indata.length != size) _hx509_abort("internal ASN.1 encoder error"); sig->length = RSA_size(signer->private_key.rsa); sig->data = malloc(sig->length); - if (sig->data == NULL) + if (sig->data == NULL) { + hx509_set_error_string(context, 0, ENOMEM, "out of memory"); return ENOMEM; + } ret = RSA_private_encrypt(indata.length, indata.data, sig->data, signer->private_key.rsa, RSA_PKCS1_PADDING); der_free_octet_string(&indata); - if (ret <= 0) - return HX509_CMS_FAILED_CREATE_SIGATURE; + if (ret <= 0) { + ret = HX509_CMS_FAILED_CREATE_SIGATURE; + hx509_set_error_string(context, 0, ret, + "RSA private decrypt failed: %d", ret); + return ret; + } if (ret < sig->length) _hx509_abort("RSA signature prelen shorter the output len"); @@ -316,58 +329,6 @@ rsa_create_signature(hx509_context context, return 0; } - -#if 0 -static int -create_signature(const struct signature_alg *sig_alg, - const hx509_private_key signer, - const AlgorithmIdentifier *alg, - const heim_octet_string *data, - AlgorithmIdentifier *signatureAlgorithm, - heim_octet_string *sig) -{ - const heim_oid *digest_oid, *sig_oid; - const EVP_MD *mdtype; - EVP_MD_CTX md; - unsigned len; - int ret; - - if (alg) - sig_oid = &alg->algorithm; - else - sig_oid = signer->signature_alg; - - if (der_heim_oid_cmp(sig_oid, oid_id_dsa_with_sha1()) == 0) { - mdtype = EVP_sha1(); - digest_oid = oid_id_secsig_sha_1(); - } else - return HX509_ALG_NOT_SUPP; - - if (signatureAlgorithm) { - ret = _hx509_set_digest_alg(signatureAlgorithm, - sig_oid, "\x05\x00", 2); - if (ret) - return ret; - } - - sig->data = malloc(EVP_PKEY_size(signer->private_key)); - if (sig->data == NULL) - return ENOMEM; - - EVP_SignInit(&md, mdtype); - EVP_SignUpdate(&md, data->data, data->length); - ret = EVP_SignFinal(&md, sig->data, &len, signer->private_key); - if (ret != 1) { - free(sig->data); - sig->data = NULL; - return HX509_CMS_FAILED_CREATE_SIGATURE; - } - sig->length = len; - - return 0; -} -#endif - static int rsa_parse_private_key(hx509_context context, const struct signature_alg *sig_alg, @@ -980,7 +941,8 @@ _hx509_create_signature(hx509_context context, } int -_hx509_public_encrypt(const heim_octet_string *cleartext, +_hx509_public_encrypt(hx509_context context, + const heim_octet_string *cleartext, const Certificate *cert, heim_oid *encryption_oid, heim_octet_string *ciphertext) @@ -999,15 +961,18 @@ _hx509_public_encrypt(const heim_octet_string *cleartext, spi = &cert->tbsCertificate.subjectPublicKeyInfo; rsa = RSA_new(); - if (rsa == NULL) + if (rsa == NULL) { + hx509_set_error_string(context, 0, ENOMEM, "out of memory"); return ENOMEM; + } ret = decode_RSAPublicKey(spi->subjectPublicKey.data, spi->subjectPublicKey.length / 8, &pk, &size); if (ret) { RSA_free(rsa); - return ENOMEM; + hx509_set_error_string(context, 0, ret, "RSAPublicKey decode failure"); + return ret; } rsa->n = heim_int2BN(&pk.modulus); rsa->e = heim_int2BN(&pk.publicExponent); @@ -1016,6 +981,7 @@ _hx509_public_encrypt(const heim_octet_string *cleartext, if (rsa->n == NULL || rsa->e == NULL) { RSA_free(rsa); + hx509_set_error_string(context, 0, ENOMEM, "out of memory"); return ENOMEM; } @@ -1023,6 +989,7 @@ _hx509_public_encrypt(const heim_octet_string *cleartext, to = malloc(tosize); if (to == NULL) { RSA_free(rsa); + hx509_set_error_string(context, 0, ENOMEM, "out of memory"); return ENOMEM; } @@ -1030,8 +997,10 @@ _hx509_public_encrypt(const heim_octet_string *cleartext, (unsigned char *)cleartext->data, to, rsa, RSA_PKCS1_PADDING); RSA_free(rsa); - if (ret < 0) { + if (ret <= 0) { free(to); + hx509_set_error_string(context, 0, HX509_CRYPTO_RSA_PUBLIC_ENCRYPT, + "RSA public encrypt failed with %d", ret); return HX509_CRYPTO_RSA_PUBLIC_ENCRYPT; } if (ret > tosize) @@ -1043,6 +1012,7 @@ _hx509_public_encrypt(const heim_octet_string *cleartext, ret = der_copy_oid(oid_id_pkcs1_rsaEncryption(), encryption_oid); if (ret) { der_free_octet_string(ciphertext); + hx509_set_error_string(context, 0, ENOMEM, "out of memory"); return ENOMEM; }