z/OS support: Proper bit amount calculation

src/misc.c

@@ -303,8 +303,7 @@ pgp_free_errors(pgp_error_t *errstack)
 static int
 hash_uint32(pgp_hash_t *hash, uint32_t n)
 {
-    uint8_t    ibuf[4];
-
+    uint8_t ibuf[4];
     ibuf[0] = (uint8_t)(n >> 24) & 0xff;
     ibuf[1] = (uint8_t)(n >> 16) & 0xff;
     ibuf[2] = (uint8_t)(n >> 8) & 0xff;
@@ -354,22 +353,38 @@ hash_bignum(pgp_hash_t *hash, BIGNUM *bignum)
     return (int)(sizeof(len) + len + padbyte);
 }
 #elif defined(HAVE_GSKSSL)
+inline int check_gsk_buf_zero(gsk_buffer *buf) {
+    int i;
+    for(i=0;i<buf->length;i++) {
+       if(((uint8_t*)buf->data)[i]) return 0;
+    }
+    return 1;
+}
+
 static int
-hash_bignum(pgp_hash_t *hash, gsk_buffer buf)
+hash_bignum(pgp_hash_t *hash, gsk_buffer *buf)
 {
-    unsigned char *bn;
-    size_t        len = buf.length;
+    uint8_t *bn;
+    size_t        len;
     int           padbyte;
     int i;
 
-    bn=malloc(len+1);
-
-    // filling the data into the char array
-    for(i=0; i<len; i++) {
-        bn[i+1]=((char*)buf.data)[i];
+    if (check_gsk_buf_zero(buf)) {
+        hash_uint32(hash, 0);
+        return sizeof(len);
+    }
+    if ((len = (size_t)buf->length) < 1) {
+        (void) fprintf(stderr, "hash_bignum: bad size\n");
+        return 0;
+    }
+    if ((bn = calloc(1, len)) == NULL) {
+        (void) fprintf(stderr, "hash_bignum: bad bn alloc\n");
+        return 0;
     }
 
-    bn[0] = 0x0;
+    bn=malloc(len+1);
+    memset(bn,0,len+1);
+    memcpy(bn+1,buf->data,len);
     padbyte = (bn[1] & 0x80) ? 1 : 0;
     hash_string(hash, bn + 1 - padbyte, (unsigned)(len + padbyte));
     free(bn);
@@ -389,9 +404,9 @@ hash_bignum(pgp_hash_t *hash, gsk_buffer buf)
 int 
 pgp_fingerprint(pgp_fingerprint_t *fp, const pgp_pubkey_t *key, pgp_hash_alg_t hashtype)
 {
-    pgp_memory_t    *mem;
-    pgp_hash_t     hash;
-    uint32_t     len;
+    pgp_memory_t *mem;
+    pgp_hash_t hash;
+    uint32_t len;
 
     mem = pgp_memory_new();
     if (key->version == 2 || key->version == 3) {
@@ -404,8 +419,7 @@ pgp_fingerprint(pgp_fingerprint_t *fp, const pgp_pubkey_t *key, pgp_hash_alg_t h
         }
         pgp_hash_md5(&hash);
         if (!hash.init(&hash)) {
-            (void) fprintf(stderr,
-                "pgp_fingerprint: bad md5 alloc\n");
+            (void) fprintf(stderr, "pgp_fingerprint: bad md5 alloc\n");
             return 0;
         }
         hash_bignum(&hash, key->key.rsa.n);
@@ -418,8 +432,7 @@ pgp_fingerprint(pgp_fingerprint_t *fp, const pgp_pubkey_t *key, pgp_hash_alg_t h
         pgp_build_pubkey(mem, key, 0);
         pgp_hash_sha1(&hash);
         if (!hash.init(&hash)) {
-            (void) fprintf(stderr,
-                "pgp_fingerprint: bad sha1 alloc\n");
+            (void) fprintf(stderr, "pgp_fingerprint: bad sha1 alloc\n");
             return 0;
         }
         len = (unsigned)pgp_mem_len(mem);
@@ -454,7 +467,7 @@ pgp_keyid(uint8_t *keyid, const size_t idlen, const pgp_pubkey_t *key, pgp_hash_
 #if defined(HAVE_OPENSSL_BN_H)
         n = (unsigned) BN_num_bytes(key->key.rsa.n);
 #elif defined(HAVE_GSKSSL)
-        n = (unsigned) key->key.rsa.n.length;
+        n = (unsigned) key->key.rsa.n->length;
 #endif
         if (n > sizeof(bn)) {
             (void) fprintf(stderr, "pgp_keyid: bad num bytes\n");
@@ -469,14 +482,12 @@ pgp_keyid(uint8_t *keyid, const size_t idlen, const pgp_pubkey_t *key, pgp_hash_
 #if defined(HAVE_OPENSSL_BN_H)
         BN_bn2bin(key->key.rsa.n, bn);
 #elif defined(HAVE_GSKSSL)
-	memcpy(bn,key->key.rsa.n.data,n);
+    memcpy(bn, key->key.rsa.n->data, n);
 #endif
         (void) memcpy(keyid, bn + n - idlen, idlen);
     } else {
         pgp_fingerprint(&finger, key, hashtype);
-        (void) memcpy(keyid,
-                finger.fingerprint + finger.length - idlen,
-                idlen);
+        memcpy(keyid, finger.fingerprint + finger.length - idlen, idlen);
     }
     return 1;
 }
@@ -491,8 +502,7 @@ pgp_keyid(uint8_t *keyid, const size_t idlen, const pgp_pubkey_t *key, pgp_hash_
 void 
 pgp_hash_add_int(pgp_hash_t *hash, unsigned n, unsigned length)
 {
-    uint8_t   c;
-
+    uint8_t c;
     while (length--) {
         c = n >> (length * 8);
         hash->add(hash, &c, 1);
@@ -596,11 +606,9 @@ pgp_str_to_hash_alg(const char *hash)
     if (netpgp_strcasecmp(hash, "SHA256") == 0) {
         return PGP_HASH_SHA256;
     }
-    /*
-        if (netpgp_strcasecmp(hash,"SHA224") == 0) {
+    if (netpgp_strcasecmp(hash,"SHA224") == 0) {
         return PGP_HASH_SHA224;
     }
-        */
     if (netpgp_strcasecmp(hash, "SHA512") == 0) {
         return PGP_HASH_SHA512;
     }
@@ -738,8 +746,10 @@ pgp_str_to_cipher(const char *cipher)
 void 
 pgp_random(void *dest, size_t length)
 {
-#ifdef HAVE_OPENSSL
+#if defined(HAVE_OPENSSL)
     RAND_bytes(dest, (int)length);
+#else
+    // TODO: Implement random bytes
 #endif
 }
 
@@ -813,7 +823,7 @@ void
 pgp_memory_add(pgp_memory_t *mem, const uint8_t *src, size_t length)
 {
     pgp_memory_pad(mem, length);
-    (void) memcpy(mem->buf + mem->length, src, length);
+    memcpy(mem->buf + mem->length, src, length);
     mem->length += length;
 }
 
@@ -919,9 +929,7 @@ void
 pgp_memory_free(pgp_memory_t *mem)
 {
     pgp_memory_release(mem);
-#ifdef HAVE_OPENSSL
     free(mem);
-#endif
 }
 
 /**
@@ -946,6 +954,12 @@ pgp_mem_data(pgp_memory_t *mem)
     return mem->buf;
 }
 
+#ifdef MAP_FILE
+#define NETPGP_MAP_FLAGS (MAP_PRIVATE | MAP_FILE)
+#else
+#define NETPGP_MAP_FLAGS (MAP_PRIVATE)
+#endif
+
 /* read a gile into an pgp_memory_t */
 int
 pgp_mem_readfile(pgp_memory_t *mem, const char *f)
@@ -961,13 +975,7 @@ pgp_mem_readfile(pgp_memory_t *mem, const char *f)
     }
     (void) fstat(fileno(fp), &st);
     mem->allocated = (size_t)st.st_size;
-    mem->buf = mmap(NULL, mem->allocated, PROT_READ,
-#ifdef MAP_FILE
-                MAP_PRIVATE | MAP_FILE,
-#else
-                MAP_PRIVATE,
-#endif
-		fileno(fp), 0);
+    mem->buf = mmap(NULL, mem->allocated, PROT_READ, NETPGP_MAP_FLAGS, fileno(fp), 0);
     if (mem->buf == MAP_FAILED) {
         /* mmap failed for some reason - try to allocate memory */
         if ((mem->buf = calloc(1, mem->allocated)) == NULL) {
@@ -1311,5 +1319,3 @@ __ops_set_debug_level(const char *f)
     return 1;
 }
 
-
-