[libxslt] Consolidate recursion checks

[Nick Wellnhofer <nwellnhof src gnome org>]

commit 1c8e0e556289582fece6f1a59113a7a5bef46ba4
Author: Nick Wellnhofer <wellnhofer aevum de>
Date:   Mon Jun 6 14:30:20 2016 +0200

    Consolidate recursion checks
    
    Move the check for potentially infinite recursion to
    xsltApplySequenceConstructor. In this function, both template and
    func:function calls can be handled. This also checks for the following
    case of infinite recursion in attribute sets found with afl-fuzz:
    
    <x:attribute-set name="set">
        <x:attribute name="attr">
            <elem x:use-attribute-sets="set"/>
        </x:attribute>
    </x:attribute-set>
    
    Rename funcLevel to depth and check against maxTemplateDepth. I hope it
    isn't a problem to rename an internal struct item.

 libexslt/functions.c    |   17 ++---------------
 libxslt/transform.c     |   38 ++++++++++++++++++++------------------
 libxslt/xsltInternals.h |    2 +-
 3 files changed, 23 insertions(+), 34 deletions(-)
---
diff --git a/libexslt/functions.c b/libexslt/functions.c
index b2c98d5..b49fbe6 100644
--- a/libexslt/functions.c
+++ b/libexslt/functions.c
@@ -56,8 +56,6 @@ static void exsltFuncFunctionFunction (xmlXPathParserContextPtr ctxt,
                                       int nargs);
 static exsltFuncFunctionData *exsltFuncNewFunctionData(void);
 
-#define MAX_FUNC_RECURSION 1000
-
 /*static const xmlChar *exsltResultDataID = (const xmlChar *) "EXSLT Result";*/
 
 /**
@@ -332,14 +330,6 @@ exsltFuncFunctionFunction (xmlXPathParserContextPtr ctxt, int nargs) {
                         "param == NULL\n");
        return;
     }
-    if (tctxt->funcLevel > MAX_FUNC_RECURSION) {
-       xsltGenericError(xsltGenericErrorContext,
-                        "{%s}%s: detected a recursion\n",
-                        ctxt->context->functionURI, ctxt->context->function);
-       ctxt->error = XPATH_MEMORY_ERROR;
-       return;
-    }
-    tctxt->funcLevel++;
 
     /*
      * We have a problem with the evaluation of function parameters.
@@ -423,7 +413,7 @@ exsltFuncFunctionFunction (xmlXPathParserContextPtr ctxt, int nargs) {
        xsltFreeStackElemList(params);
 
     if (data->error != 0)
-       goto error;
+       return;
 
     if (data->result != NULL) {
        ret = data->result;
@@ -451,13 +441,10 @@ exsltFuncFunctionFunction (xmlXPathParserContextPtr ctxt, int nargs) {
                         "executing a function\n",
                         ctxt->context->functionURI, ctxt->context->function);
        xmlFreeNode(fake);
-       goto error;
+       return;
     }
     xmlFreeNode(fake);
     valuePush(ctxt, ret);
-
-error:
-    tctxt->funcLevel--;
 }
 
 
diff --git a/libxslt/transform.c b/libxslt/transform.c
index 6b2000e..cd93bd0 100644
--- a/libxslt/transform.c
+++ b/libxslt/transform.c
@@ -2378,6 +2378,24 @@ xsltApplySequenceConstructor(xsltTransformContextPtr ctxt,
         return;
     CHECK_STOPPED;
 
+    /*
+    * Check for infinite recursion: stop if the maximum of nested templates
+    * is excceeded. Adjust xsltMaxDepth if you need more.
+    */
+    if (ctxt->depth >= ctxt->maxTemplateDepth) {
+        xsltTransformError(ctxt, NULL, list,
+           "xsltApplySequenceConstructor: A potential infinite template "
+            "recursion was detected.\n"
+           "You can adjust xsltMaxDepth (--maxdepth) in order to "
+           "raise the maximum number of nested template calls and "
+           "variables/params (currently set to %d).\n",
+           ctxt->maxTemplateDepth);
+        xsltDebug(ctxt, contextNode, list, NULL);
+       ctxt->state = XSLT_STATE_STOPPED;
+        return;
+    }
+    ctxt->depth++;
+
     oldLocalFragmentTop = ctxt->localRVT;
     oldInsert = insert = ctxt->insert;
     oldInst = oldCurInst = ctxt->inst;
@@ -3010,6 +3028,8 @@ error:
     ctxt->inst = oldInst;
     ctxt->insert = oldInsert;
 
+    ctxt->depth--;
+
 #ifdef WITH_DEBUGGER
     if ((ctxt->debugStatus != XSLT_DEBUG_NONE) && (addCallResult)) {
         xslDropCall();
@@ -3076,24 +3096,6 @@ xsltApplyXSLTTemplate(xsltTransformContextPtr ctxt,
         return;
     CHECK_STOPPED;
 
-    /*
-    * Check for infinite recursion: stop if the maximum of nested templates
-    * is excceeded. Adjust xsltMaxDepth if you need more.
-    */
-    if (ctxt->templNr >= ctxt->maxTemplateDepth)
-    {
-        xsltTransformError(ctxt, NULL, list,
-           "xsltApplyXSLTTemplate: A potential infinite template recursion "
-           "was detected.\n"
-           "You can adjust xsltMaxDepth (--maxdepth) in order to "
-           "raise the maximum number of nested template calls and "
-           "variables/params (currently set to %d).\n",
-           ctxt->maxTemplateDepth);
-        xsltDebug(ctxt, contextNode, list, NULL);
-       ctxt->state = XSLT_STATE_STOPPED;
-        return;
-    }
-
     if (ctxt->varsNr >= ctxt->maxTemplateVars)
        {
         xsltTransformError(ctxt, NULL, list,
diff --git a/libxslt/xsltInternals.h b/libxslt/xsltInternals.h
index b56cbdd..060b178 100644
--- a/libxslt/xsltInternals.h
+++ b/libxslt/xsltInternals.h
@@ -1785,7 +1785,7 @@ struct _xsltTransformContext {
                            exits */
     xmlDocPtr localRVTBase; /* Obsolete */
     int keyInitLevel;   /* Needed to catch recursive keys issues */
-    int funcLevel;      /* Needed to catch recursive functions issues */
+    int depth;          /* Needed to catch recursions */
     int maxTemplateDepth;
     int maxTemplateVars;
 };