GNOME.org
 

Threaded Transforms

I'm attaching a rough first pass at a threaded
wrapper for XSLT transformations.  I'm going to
be offline for a couple of days, and I know at
least Brent wanted to see this.

Basically, you make a YelpTransform struct with
some callback functions for chunks, errors, and
finalization.  The YelpTransform forks off a new
thread with the libxslt stuff.  Internally, it
manages a GAsyncQueue and adds idle handlers to
call your callbacks in the main thread (or, more
pedantically, in whatever thread your main loop
is running in).

Outside this module, you never see the thread or
the queue or any of that.  It encapsulates the
threading completely.

This implementation is not production-quality.
Notably, freeing a running YelpTransform will
result in Very Bad Things happening.  Ideally,
we'd find a way to set some sort of stop bit
in the xsltTransformContext on free and wait
for the thread to exit cleanly.  The thread
itself could then clean everything up in its
dying breath if the stop bit was set.  We'd
probably rename free to release, I guess.

It is not a GObject, because it's designed to
be managed from only one place.  In the brave
new YelpDocument world, a YelpDocument would
create a YelpTransform to do its grunt work
(if it uses XSLT, which a document doesn't
necessarily have to).  Thus, we don't really
need ref/unref, and we don't need to hook up
multiple callback functions to events.

We can probably save hideous small mallocs
with a memory pool.  GLib has routines for
this.  Same with thread pools.

The YelpTransformChunk bit is a stopgap so
that I could get something testable running.
The YelpDocument API calls for a universal
YelpPage object, and we'd use that.  (There
are some lifecylce issues we'll have to work
through, but they aren't insurmountable.)

You want me to write more, but I really need
to get going.  Compile test-transform and run
it with the full path of a DocBook file.  The
output isn't very exciting (wow, yay, it can
run XSLT, whoop-de-frikkin-doo), but what's
happening underneath is new and good.

Share and enjoy,
Shaun


Index: src/Makefile.am
===================================================================
RCS file: /cvs/gnome/yelp/src/Makefile.am,v
retrieving revision 1.95
diff -u -r1.95 Makefile.am
--- src/Makefile.am	12 Jun 2006 05:18:34 -0000	1.95
+++ src/Makefile.am	27 Jun 2006 13:17:42 -0000
@@ -101,7 +101,7 @@
 
 yelp_LDFLAGS = -R$(MOZILLA_HOME) $(AM_LDFLAGS)
 
-check_PROGRAMS = test-man-parser test-pager test-uri
+check_PROGRAMS = test-man-parser test-pager test-uri test-transform
 
 test_man_parser_SOURCES =					\
 	yelp-error.c		yelp-error.h			\
@@ -163,6 +163,16 @@
 test_uri_LDADD = $(YELP_LIBS)
 
 test_uri_LDFLAGS = $(AM_LDFLAGS)
+
+test_transform_SOURCES =				\
+	yelp-debug.c		yelp-debug.h		\
+	yelp-error.c		yelp-transform.h	\
+	yelp-transform.c	yelp-transform.h	\
+	test-transform.c
+test_transform_CFLAGS = $(YELP_CFLAGS) $(AM_CFLAGS) $(YELP_DEFINES)
+test_transform_LDADD = $(YELP_LIBS)
+test_transform_LDFLAGS = $(AM_LDFLAGS)
+
 
 @INTLTOOL_SERVER_RULE@
 
Index: src/yelp-error.c
===================================================================
RCS file: /cvs/gnome/yelp/src/yelp-error.c,v
retrieving revision 1.7
diff -u -r1.7 yelp-error.c
--- src/yelp-error.c	22 Feb 2005 23:44:27 -0000	1.7
+++ src/yelp-error.c	27 Jun 2006 13:17:42 -0000
@@ -67,3 +67,27 @@
     else
 	return error->message;
 }
+
+YelpError *
+yelp_error_new (gchar *title, gchar *format, ...)
+{
+    YelpError *error;
+    va_list args;
+
+    error = g_new0 (YelpError, 1);
+    error->title = g_strdup (title);
+
+    va_start (args, format);
+    error->text = g_strdup_vprintf (format, args);
+    va_end (args);
+
+    return error;
+}
+
+void
+yelp_error_free (YelpError *error)
+{
+    g_free (error->title);
+    g_free (error->text);
+    g_free (error);
+}
Index: src/yelp-error.h
===================================================================
RCS file: /cvs/gnome/yelp/src/yelp-error.h,v
retrieving revision 1.11
diff -u -r1.11 yelp-error.h
--- src/yelp-error.h	2 Jan 2005 06:06:26 -0000	1.11
+++ src/yelp-error.h	27 Jun 2006 13:17:42 -0000
@@ -1,4 +1,4 @@
-/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 4 -*- */
 /*
  * Copyright (C) 2002 Mikael Hallendal <micke imendio com>
  *
@@ -32,11 +32,22 @@
 	YELP_ERROR_FORMAT,        /* Format is not supported       */
 	YELP_ERROR_IO,            /* Error in IO                   */
 	YELP_ERROR_PROC           /* Error processing the document */
-} YelpError;
+} YelpErrorCode;
 
 GQuark           yelp_error_quark         (void) G_GNUC_CONST;
 
 const gchar *    yelp_error_get_primary   (GError      *error);
 const gchar *    yelp_error_get_secondary (GError      *error);
+
+typedef struct _YelpError YelpError;
+struct _YelpError {
+    gchar *title;
+    gchar *text;
+};
+
+YelpError *       yelp_error_new          (gchar       *title,
+					   gchar       *format,
+					   ...);
+void              yelp_error_free         (YelpError   *error);
 
 #endif /* __YELP_ERROR_H__ */

/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 4 -*- */
/*
 * Copyright (C) 2002 Shaun McCance
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc.,  59 Temple Place - Suite 330, Cambridge, MA 02139, USA.
 *
 * Author: Shaun McCance <shaunm gnome org>
 */

#include <glib.h>

#include "yelp-error.h"
#include "yelp-transform.h"

static void   chunk_func (YelpTransform      *transform,
			  YelpTransformChunk *chunk,
			  gpointer            user_data);
static void   error_func (YelpTransform      *transform,
			  YelpError          *error,
			  gpointer            user_data);
static void   final_func (YelpTransform      *transform,
			  gpointer            user_data);

GMainLoop *loop;

static void
chunk_func (YelpTransform      *transform,
	    YelpTransformChunk *chunk,
	    gpointer            user_data)
{
  printf ("CHUNK: %s\n", chunk->id);
  printf ("  %s\n", chunk->title);
}

static void
error_func (YelpTransform *transform,
	    YelpError     *error,
	    gpointer       user_data)
{
  printf ("ERROR: %s\n", error->title);
}

static void
final_func (YelpTransform *transform,
	    gpointer       user_data)
{
  printf ("FINAL\n");
  g_main_loop_quit (loop);
}


gint 
main (gint argc, gchar **argv) 
{
  xmlParserCtxtPtr parser;
  xmlDocPtr doc;
  YelpTransform *transform;
  gchar **params;

  g_thread_init (NULL);

  if (argc < 2) {
    g_error ("Usage: test-transform file\n");
    return 1;
  }

  params = g_new0 (gchar *, 7);
  params[0] = "db.chunk.extension";
  params[1] = "\"\"";
  params[2] = "db.chunk.info_basename";
  params[3] = "\"x-yelp-titlepage\"";
  params[4] = "db.chunk.max_depth";
  params[5] = "2";
  params[6] = NULL;

  transform = yelp_transform_new (DATADIR"/yelp/xslt/db2html.xsl",
				  chunk_func,
				  error_func,
				  final_func,
				  NULL);
  parser = xmlNewParserCtxt ();
  doc = xmlCtxtReadFile (parser,
			 argv[1],
			 NULL,
			 XML_PARSE_DTDLOAD | XML_PARSE_NOCDATA |
			 XML_PARSE_NOENT   | XML_PARSE_NONET   );
  xmlXIncludeProcessFlags (doc,
			   XML_PARSE_DTDLOAD | XML_PARSE_NOCDATA |
			   XML_PARSE_NOENT   | XML_PARSE_NONET   );
  yelp_transform_start (transform, doc, params);

  loop = g_main_loop_new(NULL, FALSE);
  g_main_loop_run (loop);

  return 0;
}

/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 4 -*- */
/*
 * Copyright (C) 2003-2006 Shaun McCance  <shaunm gnome org>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 02111-1307, USA.
 *
 * Author: Shaun McCance  <shaunm gnome org>
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <glib.h>
#include <glib/gi18n.h>
#include <libxml/parser.h>
#include <libxml/parserInternals.h>
#include <libxml/xinclude.h>
#include <libxslt/xslt.h>
#include <libxslt/templates.h>
#include <libxslt/transform.h>
#include <libxslt/extensions.h>
#include <libxslt/xsltInternals.h>
#include <libxslt/xsltutils.h>

#include "yelp-debug.h"
#include "yelp-error.h"
#include "yelp-transform.h"

#define YELP_NAMESPACE "http://www.gnome.org/yelp/ns";

static void      transform_run         (YelpTransform  *transform);
static void      transform_set_error   (YelpTransform  *transform,
					YelpError      *error);

static gboolean  transform_chunk       (YelpTransform  *transform);
static gboolean  transform_error       (YelpTransform  *transform);
static gboolean  transform_final       (YelpTransform  *transform);

static void      xslt_yelp_document    (xsltTransformContextPtr ctxt,
					xmlNodePtr              node,
					xmlNodePtr              inst,
					xsltStylePreCompPtr     comp);
static void      xslt_yelp_cache       (xsltTransformContextPtr ctxt,
					xmlNodePtr              node,
					xmlNodePtr              inst,
					xsltStylePreCompPtr     comp);

/******************************************************************************/

YelpTransform
*yelp_transform_new (gchar                   *stylesheet,
		     YelpTransformChunkFunc   chunk_func,
		     YelpTransformErrorFunc   error_func,
		     YelpTransformFinalFunc   final_func,
		     gpointer                 user_data)
{
    YelpTransform *transform;
    
    transform = g_new0 (YelpTransform, 1);

    transform->stylesheet = xsltParseStylesheetFile (BAD_CAST stylesheet);
    if (!transform->stylesheet) {
	transform->error =
	    yelp_error_new (_("Invalid Stylesheet"),
			    _("The XSLT stylesheet ‘%s’ is either missing, or it is "
			      "not valid."),
			    stylesheet);
	transform_error (transform);
	return NULL;
    }

    transform->chunk_func = chunk_func;
    transform->error_func = error_func;
    transform->final_func = final_func;

    transform->queue = g_async_queue_new ();

    transform->user_data = user_data;

    return transform;
}

void
yelp_transform_free (YelpTransform *transform)
{
    if (transform->outputDoc)
	xmlFreeDoc (transform->outputDoc);
    if (transform->stylesheet)
	xsltFreeStylesheet (transform->stylesheet);
    if (transform->context)
	xsltFreeTransformContext (transform->context);

    g_strfreev (transform->params);

    if (transform->error)
	yelp_error_free (transform->error);

    g_async_queue_unref (transform->queue);

    g_free (transform);
}

void
yelp_transform_start (YelpTransform *transform,
		      xmlDocPtr      document,
		      gchar        **params)
{
    transform->inputDoc = document;

    transform->context = xsltNewTransformContext (transform->stylesheet,
						  transform->inputDoc);
    if (!transform->context) {
	YelpError *error = 
	    yelp_error_new (_("Broken Transformation"),
			    _("An unknown error occurred while attempting to "
			      "transform the document."));
	transform_set_error (transform, error);
	return;
    }

    transform->params = g_strdupv (params);

    transform->context->_private = transform;
    xsltRegisterExtElement (transform->context,
			    BAD_CAST "document",
			    BAD_CAST YELP_NAMESPACE,
			    (xsltTransformFunction) xslt_yelp_document);
    xsltRegisterExtElement (transform->context,
			    BAD_CAST "cache",
			    BAD_CAST YELP_NAMESPACE,
			    (xsltTransformFunction) xslt_yelp_cache);

    transform->thread = g_thread_create ((GThreadFunc) transform_run,
					 transform, FALSE, NULL);
}

void
yelp_transform_chunk_free (YelpTransformChunk *chunk)
{
    g_free (chunk->id);
    g_free (chunk->title);
    g_free (chunk->contents);

    g_free (chunk);
}

/******************************************************************************/

static void
transform_run (YelpTransform *transform)
{
    transform->outputDoc = xsltApplyStylesheetUser (transform->stylesheet,
						    transform->inputDoc,
						    (const char **) transform->params,
						    NULL, NULL,
						    transform->context);
    // FIXME: do something with outputDoc?
    // FIXME: check for anything remaining on the queue
    g_idle_add ((GSourceFunc) transform_final, transform);
}

static void
transform_set_error (YelpTransform *transform,
		     YelpError     *error)
{
    if (transform->error)
	yelp_error_free (transform->error);
    transform->error = error;
    g_idle_add ((GSourceFunc) transform_error, transform);
}

static gboolean
transform_chunk (YelpTransform *transform)
{
    YelpTransformChunk *chunk;

    chunk = (YelpTransformChunk *) g_async_queue_try_pop (transform->queue);

    if (chunk) {
	if (transform->chunk_func)
	    transform->chunk_func (transform, chunk, transform->user_data);
	else
	    yelp_transform_chunk_free (chunk);
    }

    return FALSE;
}

static gboolean
transform_error (YelpTransform *transform)
{
    YelpError *error;

    error = transform->error;
    transform->error = NULL;

    if (transform->error_func)
	transform->error_func (transform, error, transform->user_data);
    else
	yelp_error_free (error);

    return FALSE;
}

static gboolean
transform_final (YelpTransform *transform)
{
    if (transform->final_func)
	transform->final_func (transform, transform->user_data);

    return FALSE;
}

/******************************************************************************/

static void
xslt_yelp_document (xsltTransformContextPtr ctxt,
		    xmlNodePtr              node,
		    xmlNodePtr              inst,
		    xsltStylePreCompPtr     comp)
{
    YelpTransform *transform;
    YelpTransformChunk *chunk;
    xmlChar *page_id = NULL;
    xmlChar *page_title = NULL;
    xmlChar *page_buf;
    gint     buf_size;
    xsltStylesheetPtr style = NULL;
    const char *old_outfile;
    xmlDocPtr   new_doc = NULL;
    xmlDocPtr   old_doc;
    xmlNodePtr  old_insert;
    xmlNodePtr  cur;

    if (!ctxt || !node || !inst || !comp)
	return;

    transform = (YelpTransform *) ctxt->_private;

    debug_print (DB_FUNCTION, "entering\n");

    page_id = xsltEvalAttrValueTemplate (ctxt, inst,
					 (const xmlChar *) "href",
					 NULL);
    if (page_id == NULL) {
	xsltTransformError (ctxt, NULL, inst,
			    _("No href attribute found on yelp:document"));
	/* FIXME: put a real error here */
	goto done;
    }
    debug_print (DB_FUNCTION, "  page_id = \"%s\"\n", page_id);

    old_outfile = ctxt->outputFile;
    old_doc     = ctxt->output;
    old_insert  = ctxt->insert;
    ctxt->outputFile = (const char *) page_id;

    style = xsltNewStylesheet ();
    if (style == NULL) {
	xsltTransformError (ctxt, NULL, inst,
			    _("Out of memory"));
	goto done;
    }

    style->omitXmlDeclaration = TRUE;

    new_doc = xmlNewDoc (BAD_CAST "1.0");
    new_doc->charset = XML_CHAR_ENCODING_UTF8;
    new_doc->dict = ctxt->dict;
    xmlDictReference (new_doc->dict);

    ctxt->output = new_doc;
    ctxt->insert = (xmlNodePtr) new_doc;

    xsltApplyOneTemplate (ctxt, node, inst->children, NULL, NULL);
    xsltSaveResultToString (&page_buf, &buf_size, new_doc, style);

    ctxt->outputFile = old_outfile;
    ctxt->output     = old_doc;
    ctxt->insert     = old_insert;

    if (!page_title)
	page_title = BAD_CAST g_strdup (_("Unknown Page"));

    chunk = g_new0 (YelpTransformChunk, 1);

    chunk->id = (gchar *) page_id;
    chunk->title = (gchar *) page_title;
    chunk->contents = (gchar *) page_buf;

    g_async_queue_push (transform->queue, chunk);
    g_idle_add ((GSourceFunc) transform_chunk, transform);

 done:
    if (new_doc)
	xmlFreeDoc (new_doc);
    if (style)
	xsltFreeStylesheet (style);
}

static void
xslt_yelp_cache (xsltTransformContextPtr ctxt,
		 xmlNodePtr              node,
		 xmlNodePtr              inst,
		 xsltStylePreCompPtr     comp)
{
}

/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 4 -*- */
/*
 * Copyright (C) 2003-2006 Shaun McCance  <shaunm gnome org>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 02111-1307, USA.
 *
 * Author: Shaun McCance  <shaunm gnome org>
 */

#ifndef __YELP_TRANSFORM_H__
#define __YELP_TRANSFORM_H__

#include <glib.h>
#include <libxml/tree.h>
#include <libxslt/xslt.h>
#include <libxslt/transform.h>

#include "yelp-error.h"

typedef struct _YelpTransform YelpTransform;
typedef struct _YelpTransformChunk YelpTransformChunk;

typedef void  (*YelpTransformChunkFunc)  (YelpTransform           *transform,
					  YelpTransformChunk      *chunk,
					  gpointer                 user_data);
typedef void  (*YelpTransformErrorFunc)  (YelpTransform           *transform,
					  YelpError               *error,
					  gpointer                 user_data);
typedef void  (*YelpTransformFinalFunc)  (YelpTransform           *transform,
					  gpointer                 user_data);

struct _YelpTransform {
    xmlDocPtr               inputDoc;
    xmlDocPtr               outputDoc;
    xsltStylesheetPtr       stylesheet;
    xsltTransformContextPtr context;

    YelpTransformChunkFunc  chunk_func;
    YelpTransformErrorFunc  error_func;
    YelpTransformFinalFunc  final_func;

    gchar                 **params;

    GThread                *thread;
    GAsyncQueue            *queue;

    gpointer                user_data;

    YelpError              *error;
};

struct _YelpTransformChunk {
    gchar *id;
    gchar *title;
    gchar *contents;
};

YelpTransform  *yelp_transform_new       (gchar                   *stylesheet,
					  YelpTransformChunkFunc   chunk_func,
					  YelpTransformErrorFunc   error_func,
					  YelpTransformFinalFunc   final_func,
					  gpointer                 user_data);
void            yelp_transform_free      (YelpTransform           *transform);
void            yelp_transform_start     (YelpTransform           *transform,
					  xmlDocPtr                document,
					  gchar                  **params);

void            yelp_transform_chunk_free  (YelpTransformChunk  *chunk);

#endif /* __YELP_TRANSFORM_H__ */
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]