[xmlsec] use 64 bits for serial numbers in xmlsec-nss

[Aleksey Sanin <aleksey src gnome org>]

commit a63f130510dca8a8ddfc9bbccfdf6e7197e838b0
Author: Aleksey Sanin <aleksey aleksey com>
Date:   Sun Apr 25 22:14:33 2010 -0700

    use 64 bits for serial numbers in xmlsec-nss

 ChangeLog         |    1 +
 TODO              |    4 +--
 src/nss/x509.c    |   24 ++++++++++++++-
 src/nss/x509vfy.c |   87 ++++++++++++++++++++++++++++++----------------------
 4 files changed, 75 insertions(+), 41 deletions(-)
---
diff --git a/ChangeLog b/ChangeLog
index a00456a..2739f89 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -4,6 +4,7 @@
 	* Added support for MD5 for digest and RSA in xmlsec-nss
 	* Minor fixes to xmlsec-nss (xmlsec bug #118630), bumping nss min 
 	version to 3.9 + nspr 4.4.1 or greater
+	Other minor fixes for xmlsec-nss
 
 2010-04-24  Aleksey Sanin  <aleksey aleksey com>
 	* Fix PK_CONFIG problem (reported by Roumen)
diff --git a/TODO b/TODO
index 1e9a307..c23de70 100644
--- a/TODO
+++ b/TODO
@@ -10,7 +10,7 @@ General
 Tests status
 -------------------------------------------------
 
-* xmlsec-nss (April 25, 2010)
+* xmlsec-nss (April 25, 2010 using NSS 3.12.6)
 
 ** Skipped tests (missing transforms: RIPEMD160, SHA224, RSA/OAEP, GOST)
 
@@ -30,9 +30,7 @@ merlin-xmlenc-five/encrypt-data-tripledes-cbc-rsa-oaep-mgf1p
 
 ** Failed tests
 
-aleksey-xmldsig-01/x509data-test
 aleksey-xmldsig-01/x509data-sn-test
-merlin-xmldsig-twenty-three/signature-x509-is
 merlin-xmldsig-twenty-three/signature-x509-ski
 phaos-xmldsig-three/signature-rsa-manifest-x509-data-ski
 merlin-xmldsig-twenty-three/signature-x509-crt-crl
diff --git a/src/nss/x509.c b/src/nss/x509.c
index e05edc0..2d7d881 100644
--- a/src/nss/x509.c
+++ b/src/nss/x509.c
@@ -1980,12 +1980,34 @@ xmlSecNssX509NameWrite(CERTName* nm) {
 static xmlChar*
 xmlSecNssASN1IntegerWrite(SECItem *num) {
     xmlChar *res = NULL;
+    int resLen = 64; /* not more than 64 chars */
+    PRUint64 val = 0;
+    unsigned int ii = 0;
+    int shift = 0;
 
     xmlSecAssert2(num != NULL, NULL);
+    xmlSecAssert2(num->type == siBuffer, NULL);
+    xmlSecAssert2(num->len <= 9, NULL);
+    xmlSecAssert2(num->data != NULL, NULL);
 
-    /* TODO : to be implemented after
+    /* HACK : to be fixed after
      * NSS bug http://bugzilla.mozilla.org/show_bug.cgi?id=212864 is fixed
      */
+    for(ii = num->len; ii > 0; --ii, shift += 8) {
+        val |= (num->data[ii - 1]) << shift;
+    }
+
+    res = (xmlChar*)xmlMalloc(resLen + 1);
+    if(res == NULL) {
+        xmlSecError(XMLSEC_ERRORS_HERE,
+                    NULL,
+                    "xmlStrdup",
+                    XMLSEC_ERRORS_R_MALLOC_FAILED,
+                    XMLSEC_ERRORS_NO_MESSAGE);
+        return (NULL);
+    }
+
+    PR_snprintf((char*)res, resLen, "%llu", val);
     return(res);
 }
 
diff --git a/src/nss/x509vfy.c b/src/nss/x509vfy.c
index 8bf4e27..8e3c2e1 100644
--- a/src/nss/x509vfy.c
+++ b/src/nss/x509vfy.c
@@ -70,7 +70,8 @@ static int              xmlSecNssX509NameStringRead     (xmlSecByte **str,
 static xmlSecByte *     xmlSecNssX509NameRead           (xmlSecByte *str,
                                                          int len);
 
-static void             xmlSecNssNumToItem(SECItem *it, unsigned long num);
+static int              xmlSecNssNumToItem              (SECItem *it, 
+                                                         PRUint64 num);
 
 
 static xmlSecKeyDataStoreKlass xmlSecNssX509StoreKlass = {
@@ -399,6 +400,7 @@ xmlSecNssX509FindCert(xmlChar *subjectName, xmlChar *issuerName,
     CERTName *name = NULL;
     SECItem *nameitem = NULL;
     PRArenaPool *arena = NULL;
+    int rv;
 
     if (subjectName != NULL) {
         name = xmlSecNssGetCertName(subjectName);
@@ -439,6 +441,7 @@ xmlSecNssX509FindCert(xmlChar *subjectName, xmlChar *issuerName,
 
     if((issuerName != NULL) && (issuerSerial != NULL)) {
         CERTIssuerAndSN issuerAndSN;
+        PRUint64 issuerSN = 0;
 
         name = xmlSecNssGetCertName(issuerName);
         if (name == NULL) {
@@ -467,8 +470,8 @@ xmlSecNssX509FindCert(xmlChar *subjectName, xmlChar *issuerName,
             xmlSecError(XMLSEC_ERRORS_HERE,
                         NULL,
                         "SEC_ASN1EncodeItem",
-                        XMLSEC_ERRORS_R_XMLSEC_FAILED,
-                                                "error code=%d", PORT_GetError());
+                        XMLSEC_ERRORS_R_CRYPTO_FAILED,
+                        "error code=%d", PORT_GetError());
             goto done;
         }
 
@@ -477,11 +480,28 @@ xmlSecNssX509FindCert(xmlChar *subjectName, xmlChar *issuerName,
         issuerAndSN.derIssuer.data = nameitem->data;
         issuerAndSN.derIssuer.len = nameitem->len;
 
+
         /* TBD: serial num can be arbitrarily long */
-        xmlSecNssNumToItem(&issuerAndSN.serialNumber, PORT_Atoi((char *)issuerSerial));
+        if(PR_sscanf((char *)issuerSerial, "%llu", &issuerSN) != 1) {
+            xmlSecError(XMLSEC_ERRORS_HERE,
+                        NULL,
+                        "PR_sscanf",
+                        XMLSEC_ERRORS_R_XMLSEC_FAILED,
+                        "error code=%d", PR_GetError());
+            goto done;
+        }
 
-        cert = CERT_FindCertByIssuerAndSN(CERT_GetDefaultCertDB(),
-                                          &issuerAndSN);
+        rv = xmlSecNssNumToItem(&issuerAndSN.serialNumber, issuerSN);
+        if(rv <= 0) {
+            xmlSecError(XMLSEC_ERRORS_HERE,
+                        NULL,
+                        "xmlSecNssNumToItem",
+                        XMLSEC_ERRORS_R_XMLSEC_FAILED,
+                        "error code=%d", PR_GetError());
+            goto done;
+        }
+
+        cert = CERT_FindCertByIssuerAndSN(CERT_GetDefaultCertDB(), &issuerAndSN);
         SECITEM_FreeItem(&issuerAndSN.serialNumber, PR_FALSE);
         goto done;
     }
@@ -690,47 +710,40 @@ xmlSecNssX509NameStringRead(xmlSecByte **str, int *strLen,
 }
 
 /* code lifted from NSS */
-static void
-xmlSecNssNumToItem(SECItem *it, unsigned long ui)
+static int
+xmlSecNssNumToItem(SECItem *it, PRUint64 ui)
 {
-    unsigned char bb[5];
-    int len;
+    unsigned char bb[9];
+    unsigned int zeros_len;
+
+    xmlSecAssert2(it != NULL, -1);
 
-    bb[0] = 0;
-    bb[1] = (unsigned char) (ui >> 24);
-    bb[2] = (unsigned char) (ui >> 16);
-    bb[3] = (unsigned char) (ui >> 8);
-    bb[4] = (unsigned char) (ui);
+    bb[0] = 0; /* important: we should have 0 at the beginning! */
+    bb[1] = (unsigned char) (ui >> 56);
+    bb[2] = (unsigned char) (ui >> 48);
+    bb[3] = (unsigned char) (ui >> 40);
+    bb[4] = (unsigned char) (ui >> 32);
+    bb[5] = (unsigned char) (ui >> 24);
+    bb[6] = (unsigned char) (ui >> 16);
+    bb[7] = (unsigned char) (ui >> 8);
+    bb[8] = (unsigned char) (ui);
 
     /*
     ** Small integers are encoded in a single byte. Larger integers
-    ** require progressively more space.
+    ** require progressively more space. Start from 1 because byte at 
+    ** position 0 is zero
     */
-    if (ui > 0x7f) {
-        if (ui > 0x7fff) {
-            if (ui > 0x7fffffL) {
-                if (ui >= 0x80000000L) {
-                    len = 5;
-                } else {
-                    len = 4;
-                }
-            } else {
-                len = 3;
-            }
-        } else {
-            len = 2;
-        }
-    } else {
-        len = 1;
-    }
+    for(zeros_len = 1; (zeros_len < sizeof(bb)) && (bb[zeros_len] == 0); ++zeros_len);
 
-    it->data = (unsigned char *)PORT_Alloc(len);
+    it->len = sizeof(bb) - (zeros_len - 1);
+    it->data = (unsigned char *)PORT_Alloc(it->len);
     if (it->data == NULL) {
-        return;
+        it->len = 0;
+        return (-1);
     }
 
-    it->len = len;
-    PORT_Memcpy(it->data, bb + (sizeof(bb) - len), len);
+    PORT_Memcpy(it->data, bb + (zeros_len - 1), it->len);
+    return(it->len);
 }
 #endif /* XMLSEC_NO_X509 */