[pan2/xzver] rough ...



commit 92e446d5a676dfac750e28ed9ae7a1407cea7a93
Author: Heinrich MÃller <sphemuel stud informatik uni-erlangen de>
Date:   Sat Jul 9 23:38:10 2011 +0200

    rough ...
    
    Conflicts:
    	pan.png
    	pan/general/debug.cc
    	pan/general/debug.h
    	pan/general/file-util.cc
    	pan/gui/pan.cc
    	pan/tasks/nntp.cc
    	pan/tasks/socket-impl-openssl.cc
    	pan/tasks/task-xover.cc
    	pan/tasks/task-xover.h

 configure.in                     |   11 ++-
 pan.cbp                          |    2 +
 pan.png                          |  Bin 4041 -> 8808 bytes
 pan/general/debug.cc             |    3 +
 pan/general/debug.h              |    3 +
 pan/gui/Makefile.am              |    4 +-
 pan/gui/pan.cc                   |    2 +
 pan/tasks/Makefile.am            |    4 +-
 pan/tasks/decoder.h              |    4 +-
 pan/tasks/nntp.cc                |   22 ++++-
 pan/tasks/nntp.h                 |   12 ++-
 pan/tasks/queue.cc               |   20 ++++-
 pan/tasks/queue.h                |    4 +
 pan/tasks/socket-impl-openssl.cc |    2 +-
 pan/tasks/task-xover.cc          |  209 ++++++++++++++++++++++++++++++++++---
 pan/tasks/task-xover.h           |   14 +++
 pan/tasks/task.h                 |    4 +
 17 files changed, 289 insertions(+), 31 deletions(-)
---
diff --git a/configure.in b/configure.in
index 827534e..5145556 100644
--- a/configure.in
+++ b/configure.in
@@ -87,12 +87,17 @@ AC_DEFINE_UNQUOTED([GETTEXT_PACKAGE],["$GETTEXT_PACKAGE"],[gettext package name]
 AM_GLIB_GNU_GETTEXT
 panlocaledir='${prefix}/${DATADIRNAME}/locale'
 
+dnl libz for xzver
+PKG_CHECK_MODULES([ZLIB], [zlib >= 1.2.7],AC_DEFINE(HAVE_ZLIB,[1],[Zlib for xzver support]),[])
+AC_SUBST(ZLIB_CFLAGS)
+AC_SUBST(ZLIB_LIBS)
+
 dnl D-Bus support
 AC_ARG_WITH(dbus, AS_HELP_STRING([--with-dbus],[enable D-Bus support (normally: no)]), [want_dbus=$withval], [want_dbus=no])
 if test "x$want_dbus" = "xyes" ; then
   AM_PATH_GLIB_2_0($GLIB_REQUIRED_FOR_DBUS,AC_DEFINE(HAVE_DBUS,[1],[D-Bus support for multiple Pan instances],exit 1,gobject gmodule gthread))
 else
-  AM_PATH_GLIB_2_0($GLIB_REQUIRED,,exit 1,gobject gmodule gthread)	
+  AM_PATH_GLIB_2_0($GLIB_REQUIRED,,exit 1,gobject gmodule gthread)
 fi
 
 
@@ -161,10 +166,10 @@ dnl GnuTLS support
 gnutls_msg=no
 AC_ARG_WITH(gnutls, AS_HELP_STRING([--with-gnutls],[enable GnuTLS support (normally: no)]), [want_gnutls=$withval], [want_gnutls=no])
 if test "x$want_gnutls" = "xyes" ; then
-  
+
   PKG_CHECK_MODULES([GNUTLS],[gnutls >= $GNUTLS_REQUIRED],
   [
-  GNUTLS_VER=`pkg-config --modversion gnutls`	
+  GNUTLS_VER=`pkg-config --modversion gnutls`
   gnutls_msg="yes = $GNUTLS_VER"
   AC_DEFINE(HAVE_GNUTLS,[1],[GnuTLS support])],
   [gnutls_msg=no
diff --git a/pan.cbp b/pan.cbp
index e9dc378..fe164b2 100644
--- a/pan.cbp
+++ b/pan.cbp
@@ -300,6 +300,8 @@
 		<Unit filename="pan/tasks/task.h" />
 		<Unit filename="pan/tasks/upload-queue.cc" />
 		<Unit filename="pan/tasks/upload-queue.h" />
+		<Unit filename="pan/tasks/xzver-decoder.cc" />
+		<Unit filename="pan/tasks/xzver-decoder.h" />
 		<Unit filename="pan/usenet-utils/MersenneTwister.h" />
 		<Unit filename="pan/usenet-utils/blowfish.cc" />
 		<Unit filename="pan/usenet-utils/blowfish.h" />
diff --git a/pan.png b/pan.png
index 7b01de9..9373470 100644
Binary files a/pan.png and b/pan.png differ
diff --git a/pan/general/debug.cc b/pan/general/debug.cc
index 9a8161c..76a3a22 100644
--- a/pan/general/debug.cc
+++ b/pan/general/debug.cc
@@ -8,6 +8,9 @@ namespace pan
   bool _verbose_flag = false;
   std::ofstream _dbg_file;
 
+
+  // imhotep
+  bool _xzver_support = false;
   bool _dbg_ssl = false;
 
 }
diff --git a/pan/general/debug.h b/pan/general/debug.h
index 23fada3..f257d5e 100644
--- a/pan/general/debug.h
+++ b/pan/general/debug.h
@@ -31,6 +31,9 @@ namespace pan
   extern bool _dbg_ssl;
   extern std::ofstream _dbg_file;
 
+  // imhotep
+  extern bool _xzver_support;
+
 }
 
 #define LINE_ID '(' << __FILE__ << ':' << __LINE__ << ':' << __func__ << ')'
diff --git a/pan/gui/Makefile.am b/pan/gui/Makefile.am
index 39cac93..c37cd73 100644
--- a/pan/gui/Makefile.am
+++ b/pan/gui/Makefile.am
@@ -1,5 +1,5 @@
 AM_CPPFLAGS = -I top_srcdir@ @GTKSPELL_CFLAGS@ @ENCHANT_CFLAGS@ @GTK_CFLAGS@ @GMIME_CFLAGS@ @GLIB_CFLAGS@ \
-							@GNUTLS_CFLAGS@ @LIBNOTIFY_CFLAGS@ @LIBGNOME_KEYRING_1_CFLAGS@ -DPANLOCALEDIR=\""$(panlocaledir)"\"
+							@GNUTLS_CFLAGS@ @LIBNOTIFY_CFLAGS@ @LIBGNOME_KEYRING_1_CFLAGS@ -DPANLOCALEDIR=\""$(panlocaledir)"\" @ZLIB_CFLAGS@
 
 noinst_LIBRARIES = libpangui.a
 
@@ -102,7 +102,7 @@ WINRCOBJ =
 endif
 
 pan_SOURCES = gui.cc pan.cc $(WINRC)
-pan_LDADD = ./libpangui.a $(WINRCOBJ) ../data-impl/libpandata.a ../tasks/libtasks.a ../data/libdata.a ../usenet-utils/libusenetutils.a ../general/libgeneralutils.a ../../uulib/libuu.a \
+pan_LDADD = @ZLIB_LIBS@ ./libpangui.a $(WINRCOBJ) ../data-impl/libpandata.a ../tasks/libtasks.a ../data/libdata.a ../usenet-utils/libusenetutils.a ../general/libgeneralutils.a ../../uulib/libuu.a \
 		@GTKSPELL_LIBS@ @ENCHANT_LIBS@ @GTK_LIBS@ @GMIME_LIBS@ @GLIB_LIBS@ @GNUTLS_LIBS@ @LIBNOTIFY_LIBS@ @LIBGNOME_KEYRING_1_LIBS@
 if HAVE_WIN32
 pan_LDFLAGS = -mwindows
diff --git a/pan/gui/pan.cc b/pan/gui/pan.cc
index 9f20a89..fafac0e 100644
--- a/pan/gui/pan.cc
+++ b/pan/gui/pan.cc
@@ -861,6 +861,8 @@ main (int argc, char *argv[])
       { usage (); return EXIT_SUCCESS; }
     else if (!strcmp(tok, "--verbose") )
       verbosed = true;
+    else if (!strcmp(tok,"--xzver"))
+      _xzver_support = true;
     else {
       nzb = true;
       nzb_files.push_back (tok);
diff --git a/pan/tasks/Makefile.am b/pan/tasks/Makefile.am
index df282b1..f7220b3 100644
--- a/pan/tasks/Makefile.am
+++ b/pan/tasks/Makefile.am
@@ -1,6 +1,6 @@
-AM_CPPFLAGS = -I top_srcdir@ @GMIME_CFLAGS@ @GLIB_CFLAGS@ @GNUTLS_CFLAGS@ @GTK_CFLAGS@
+AM_CPPFLAGS = -I top_srcdir@ @GMIME_CFLAGS@ @GLIB_CFLAGS@ @GNUTLS_CFLAGS@ @GTK_CFLAGS@ @ZLIB_CFLAGS@
 
-AM_LDFLAGS = ../../uulib/libuu.a @GNUTLS_LIBS@
+AM_LDFLAGS = ../../uulib/libuu.a @GNUTLS_LIBS@ @ZLIB_LIBS@
 
 noinst_LIBRARIES = libtasks.a
 
diff --git a/pan/tasks/decoder.h b/pan/tasks/decoder.h
index 0814694..b62fff7 100644
--- a/pan/tasks/decoder.h
+++ b/pan/tasks/decoder.h
@@ -72,7 +72,7 @@ namespace pan
 
     protected: // inherited from WorkerPool::Worker
 
-      void do_work();
+      virtual void do_work();
 
     private:
 
@@ -90,7 +90,7 @@ namespace pan
       volatile int num_scanned_files;
 
       static void uu_log(void *thiz, char *message, int severity);
-      double get_percentage (const uuprogress& p) const;
+      virtual double get_percentage (const uuprogress& p) const;
       static int uu_busy_poll(void * self, uuprogress *p);
       /** tell our task about the decode's progress */
       static gboolean progress_update_timer_func(gpointer decoder);
diff --git a/pan/tasks/nntp.cc b/pan/tasks/nntp.cc
index fc6b560..557fa20 100644
--- a/pan/tasks/nntp.cc
+++ b/pan/tasks/nntp.cc
@@ -63,6 +63,7 @@ NNTP :: fire_done_func (Health health, const StringView& response)
       debug ("I (" << (void*)this << ") am setting my _listener to 0");
       _listener = 0;
       l->on_nntp_done (this, health, response);
+      _xzver = false;
    }
 }
 
@@ -78,7 +79,7 @@ NNTP :: on_socket_response (Socket * sock UNUSED, const StringView& line_in)
    StringView line (line_in);
 
    // strip off trailing \r\n
-   if (line.len>=2 && line.str[line.len-2]=='\r' && line.str[line.len-1]=='\n')
+   if (line.len>=2 && line.str[line.len-2]=='\r' && line.str[line.len-1]=='\n' && !_xzver)
      line.truncate (line.len-2);
 
 //    std::cerr <<"_nntp_response_text: " << _nntp_response_text<<std::endl;
@@ -93,7 +94,7 @@ NNTP :: on_socket_response (Socket * sock UNUSED, const StringView& line_in)
       {
          state = CMD_MORE;
 
-         if (line.len>=2 && line.str[0]=='.' && line.str[1]=='.') // rfc 977: 2.4.1
+         if (line.len>=2 && line.str[0]=='.' && line.str[1]=='.' && !_xzver) // rfc 977: 2.4.1
             line.rtruncate (line.len-1);
 
          assert (_listener != 0);
@@ -325,6 +326,23 @@ NNTP :: xover (const Quark   & group,
 }
 
 void
+NNTP :: xzver (const Quark   & group,
+               uint64_t        low,
+               uint64_t        high,
+               Listener      * l)
+{
+   _listener = l;
+   _xzver = true;
+
+   if (group != _group)
+      _commands.push_back (build_command ("GROUP %s\r\n", group.c_str()));
+
+   _commands.push_back (build_command ("XZVER %"G_GUINT64_FORMAT"-%"G_GUINT64_FORMAT"\r\n", low, high));
+
+   write_next_command ();
+}
+
+void
 NNTP :: xover_count_only (const Quark   & group,
                           Listener      * l)
 {
diff --git a/pan/tasks/nntp.h b/pan/tasks/nntp.h
index c74ae06..fdfd0ca 100644
--- a/pan/tasks/nntp.h
+++ b/pan/tasks/nntp.h
@@ -153,7 +153,8 @@ namespace pan
           _listener(0),
           _username(username),
           _password(password),
-          _nntp_response_text(false)
+          _nntp_response_text(false),
+          _xzver(false)
        {}
 
        virtual ~NNTP ()
@@ -206,6 +207,13 @@ namespace pan
       void xover_count_only (const Quark        & group,
                              Listener           * l);
 
+
+      /** Experimental XZVER header compression support */
+      void  xzver           (const Quark   & group,
+                             uint64_t        low,
+                             uint64_t        high,
+                             Listener      * l) ;
+
       /**
        * Executes a LIST command: "LIST"
        *
@@ -319,6 +327,8 @@ namespace pan
       /** True if the server told us that we're getting a list back. */
       bool _nntp_response_text;
 
+      bool _xzver;
+
       typedef std::deque<std::string> strings_t;
       strings_t _commands;
       std::string _previous_command;
diff --git a/pan/tasks/queue.cc b/pan/tasks/queue.cc
index 133470a..dede377 100644
--- a/pan/tasks/queue.cc
+++ b/pan/tasks/queue.cc
@@ -329,10 +329,14 @@ Queue :: process_task (Task * task)
   }
   else if (state._work == Task::NEED_DECODER)
   {
-
     if (!_decoder_task)
       give_task_a_decoder (task);
   }
+  else if (state._work == Task::NEED_XZVER_DECODER)
+  {
+    if (!_decoder_task)
+      give_task_a_xzver_decoder (task);
+  }
   else if (state._work == Task::NEED_ENCODER)
   {
     if (!_encoder_task)
@@ -387,6 +391,20 @@ Queue :: find_first_task_needing_decoder ()
 }
 
 Task*
+Queue :: find_first_task_needing_xzver_decoder ()
+{
+  foreach (TaskSet, _tasks, it) {
+    const Task::State& state ((*it)->get_state ());
+    if  ((state._work == Task::NEED_XZVER_DECODER)
+      && (!_stopped.count (*it))
+      && (!_removing.count (*it)))
+      return *it;
+  }
+
+  return 0;
+}
+
+Task*
 Queue :: find_first_task_needing_encoder ()
 {
   foreach (TaskSet, _tasks, it) {
diff --git a/pan/tasks/queue.h b/pan/tasks/queue.h
index d2b2862..59479d6 100644
--- a/pan/tasks/queue.h
+++ b/pan/tasks/queue.h
@@ -26,6 +26,7 @@
 #include <pan/general/macros.h> // for UNUSED
 #include <pan/general/map-vector.h>
 #include <pan/tasks/decoder.h>
+#include <pan/tasks/xzver-decoder.h>
 #include <pan/tasks/encoder.h>
 #include <pan/general/quark.h>
 #include <pan/tasks/nntp-pool.h>
@@ -47,6 +48,7 @@ namespace pan
   class WorkerPool;
   struct Encoder;
   struct Decoder;
+  struct XZVERDecoder;
 
   /**
    * A Queue helper that saves tasks to disk and restores them from disk.
@@ -187,12 +189,14 @@ namespace pan
     protected:
       void process_task (Task *);
       void give_task_a_decoder (Task*);
+      void give_task_a_xzver_decoder (Task*);
       void give_task_a_encoder (Task*);
       void give_task_a_connection (Task*, NNTP*);
       ServerInfo& _server_info;
       bool _is_online;
       Task* find_first_task_needing_server (const Quark& server);
       Task* find_first_task_needing_decoder ();
+      Task* find_first_task_needing_xzver_decoder ();
       Task* find_first_task_needing_encoder ();
 
       void give_task_an_upload_slot (TaskUpload* task);
diff --git a/pan/tasks/socket-impl-openssl.cc b/pan/tasks/socket-impl-openssl.cc
index f7035c9..17175b9 100644
--- a/pan/tasks/socket-impl-openssl.cc
+++ b/pan/tasks/socket-impl-openssl.cc
@@ -4,7 +4,7 @@
  * Copyright (C) 2002-2006  Charles Kerr <charles rebelbase com>
  *
  * This file
- * Copyright (C) 2011 Heinrich Mü<henmull src gnome org>
+ * Copyright (C) 2011 Heinrich Mïller <henmull src gnome org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
diff --git a/pan/tasks/task-xover.cc b/pan/tasks/task-xover.cc
index 34a5a69..433846b 100644
--- a/pan/tasks/task-xover.cc
+++ b/pan/tasks/task-xover.cc
@@ -38,7 +38,7 @@ extern "C" {
 #include <pan/data/data.h>
 #include "nntp.h"
 #include "task-xover.h"
-
+#include "xzver-decoder.h"
 
 using namespace pan;
 
@@ -85,6 +85,16 @@ namespace
   }
 }
 
+namespace
+{
+  char* build_cachename (char* buf, size_t len, const char* name)
+  {
+    const char * home(file::get_pan_home().c_str());
+    g_snprintf(buf,len,"%s%c%s%c%s",home, G_DIR_SEPARATOR, "encode-cache", G_DIR_SEPARATOR, name);
+    return buf;
+  }
+}
+
 TaskXOver :: TaskXOver (Data         & data,
                         const Quark  & group,
                         Mode           mode,
@@ -100,9 +110,21 @@ TaskXOver :: TaskXOver (Data         & data,
   _bytes_so_far (0),
   _parts_so_far (0ul),
   _articles_so_far (0ul),
-  _total_minitasks (0)
+  _total_minitasks (0),
+  _running_minitasks (0),
+  _xzver (_xzver_support),
+  _decoder(0),
+  _decoder_has_run (false)
 {
 
+
+
+  if (_xzver)
+  {
+    char buf[4096];
+    _headers.open(build_cachename(buf,sizeof(buf), "xzver_test"), std::ios::out | std::ios::binary);
+  }
+
   debug ("ctor for " << group);
 
   // add a ``GROUP'' MiniTask for each server that has this group
@@ -131,7 +153,6 @@ TaskXOver :: ~TaskXOver ()
       _data.set_xover_high (_group, it->first, it->second);
     _data.xover_unref (_group);
   }
-  _data.fire_group_entered(_group, 1, 0);
 }
 
 void
@@ -166,7 +187,11 @@ TaskXOver :: use_nntp (NNTP* nntp)
       case MiniTask::XOVER:
         debug ("XOVER " << mt._low << '-' << mt._high << " to " << server);
         _last_xover_number[nntp] = mt._low;
-        nntp->xover (_group, mt._low, mt._high, this);
+        if (_xzver)
+          nntp->xzver (_group, mt._low, mt._high, this);
+        else
+          nntp->xover (_group, mt._low, mt._high, this);
+        --_running_minitasks;
         break;
       default:
         assert (0);
@@ -179,7 +204,6 @@ TaskXOver :: use_nntp (NNTP* nntp)
 ****
 ***/
 
-///TODO show low and high in UI (is this already there?)
 void
 TaskXOver :: on_nntp_group (NNTP          * nntp,
                             const Quark   & group,
@@ -220,15 +244,24 @@ TaskXOver :: on_nntp_group (NNTP          * nntp,
   if (l <= high)
   {
     //std::cerr << LINE_ID << " okay, I'll try to get articles in [" << l << "..." << h << ']' << std::endl;
-    add_steps (h-l);
-    const int INCREMENT(1000);
+    add_steps (_xzver ? 2 : 1*(h-l));
+    int INCREMENT = 1000;
     MiniTasks_t& minitasks (_server_to_minitasks[servername]);
-    for (uint64_t m=l; m<=h; m+=INCREMENT) {
-      MiniTask mt (MiniTask::XOVER, m, m+INCREMENT);
-      debug ("adding MiniTask for " << servername << ": xover [" << mt._low << '-' << mt._high << "]");
-      minitasks.push_front (mt);
+    if (_xzver)
+    {
+      MiniTask mt (MiniTask::XOVER, l, h);
       ++_total_minitasks;
     }
+    else
+    {
+        for (uint64_t m=l; m<=h; m+=INCREMENT) {
+        MiniTask mt (MiniTask::XOVER, m, m+INCREMENT);
+        debug ("adding MiniTask for " << servername << ": xover [" << mt._low << '-' << mt._high << "]");
+        minitasks.push_front (mt);
+        ++_total_minitasks;
+      }
+    }
+    _running_minitasks = _total_minitasks;
   }
   else
   {
@@ -279,6 +312,22 @@ TaskXOver :: on_nntp_line         (NNTP               * nntp,
                                    const StringView   & line)
 {
 
+    if (_xzver ) {
+        if (line.strstr("=ybegin line=128"))
+          _headers << line.str << " name=xzver_decoded\n";
+        else
+          _headers << line.str <<"\n";
+    }
+    else
+      on_nntp_line_process (nntp, line);
+    increment_step(1);
+}
+
+void
+TaskXOver :: on_nntp_line_process (NNTP               * nntp,
+                                   const StringView   & line)
+{
+
   pan_return_if_fail (nntp != 0);
   pan_return_if_fail (!nntp->_server.empty());
   pan_return_if_fail (!nntp->_group.empty());
@@ -325,10 +374,8 @@ TaskXOver :: on_nntp_line         (NNTP               * nntp,
       || author.empty() // missing author
       || date.empty() // missing date
       || mid.empty() // missing mid
-      || mid.front()!='<') // corrupt mid
-      /// Concerning bug : https://bugzilla.gnome.org/show_bug.cgi?id=650042
-      /// Even if we didn't get a proper reference here, continue.
-      //|| (!ref.empty() && ref.front()!='<'))
+      || mid.front()!='<' // corrupt mid
+      || (!ref.empty() && ref[0] != '<'))
     return;
 
   // if news server doesn't provide an xref, fake one
@@ -378,6 +425,8 @@ TaskXOver :: on_nntp_done (NNTP              * nntp,
                            Health              health,
                            const StringView  & response UNUSED)
 {
+  if (_running_minitasks == 0) process_headers(nntp);
+
   update_work (true);
   check_in (nntp, health);
 }
@@ -397,14 +446,140 @@ TaskXOver :: update_work (bool subtract_one_from_nntp_count)
 
   //std::cerr << LINE_ID << " servers: " << servers.size() << " nntp: " << nntp_count << std::endl;
 
+//  if (!servers.empty())
+//    _state.set_need_nntp (servers);
+//  else if (nntp_count)
+//    _state.set_working ();
+//  else {
+//    _state.set_completed();
+//    set_finished(OK);
+//  }
   if (!servers.empty())
     _state.set_need_nntp (servers);
   else if (nntp_count)
     _state.set_working ();
-  else {
+  else if (!_decoder && !_decoder_has_run) {
+    _state.set_need_xzverdecoder ();
+    set_step(0);
+  } else if (_decoder_has_run) {
     _state.set_completed();
-    set_finished(OK);
+    set_finished (OK);
+  } else assert(0 && "hm, missed a state.");
+}
+
+namespace
+{
+
+  #define CHUNK 16384
+
+  int inf(FILE *source, FILE *dest)
+  {
+    int ret;
+    unsigned have;
+    z_stream strm;
+    unsigned char in[CHUNK];
+    unsigned char out[CHUNK];
+
+    /* allocate inflate state */
+    strm.zalloc = Z_NULL;
+    strm.zfree = Z_NULL;
+    strm.opaque = Z_NULL;
+    strm.avail_in = 0;
+    strm.next_in = Z_NULL;
+    ret = inflateInit2(&strm,-15); //raw inflate
+    if (ret != Z_OK)
+        return ret;
+
+    /* decompress until deflate stream ends or end of file */
+    do {
+        strm.avail_in = fread(in, 1, CHUNK, source);
+        if (ferror(source)) {
+            (void)inflateEnd(&strm);
+            return Z_ERRNO;
+        }
+        if (strm.avail_in == 0)
+            break;
+        strm.next_in = in;
+
+        /* run inflate() on input until output buffer not full */
+        do {
+            strm.avail_out = CHUNK;
+            strm.next_out = out;
+            ret = inflate(&strm, Z_NO_FLUSH);
+            assert(ret != Z_STREAM_ERROR);  /* state not clobbered */
+            switch (ret) {
+            case Z_NEED_DICT:
+                ret = Z_DATA_ERROR;     /* and fall through */
+            case Z_DATA_ERROR:
+            case Z_MEM_ERROR:
+                (void)inflateEnd(&strm);
+                return ret;
+            }
+            have = CHUNK - strm.avail_out;
+            if (fwrite(out, 1, have, dest) != have || ferror(dest)) {
+                (void)inflateEnd(&strm);
+                return Z_ERRNO;
+            }
+        } while (strm.avail_out == 0);
+
+        /* done when inflate() says it's done */
+    } while (ret != Z_STREAM_END);
+
+    /* clean up and return */
+    (void)inflateEnd(&strm);
+    return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR;
+  }
+}
+
+void
+TaskXOver:: use_decoder (Decoder* decoder)
+{
+  if (_state._work != NEED_DECODER)
+    check_in (decoder);
+
+  _decoder = static_cast<XZVERDecoder*>(decoder);
+  init_steps(100);
+  _state.set_working();
+  _decoder->enqueue (this);
+  set_status_va (_("Decoding XZVER Headers"));
+  debug ("decoder thread was free, enqueued work");
+}
+
+void
+TaskXOver :: stop ()
+{
+  if (_decoder)
+      _decoder->cancel();
+}
+
+void
+ TaskXOver :: process_headers (NNTP* nntp)
+{
+  char buf[4096];
+
+  _headers.close();
+  /* yenc-decode */
+  UUInitialize ();
+  UULoadFile (build_cachename(buf,sizeof(buf), "xzver_test"), 0, 1);
+  std::cerr<<"uudecode "<<UUstrerror(UUDecodeFile (UUGetFileListItem (0), build_cachename(buf,sizeof(buf), "xzver_decoded")))<<"\n";
+  UUCleanUp ();
+
+  /* raw zlib inflate */
+  FILE * in = fopen (buf, "rb");
+  FILE * out = fopen (build_cachename(buf,sizeof(buf), "xzver_out"), "wb");
+  int res(Z_OK);
+  if (in && out) res = inf (in,out);
+
+  /* feed to on_nntp_line */
+  if (in) fclose(in);
+  if (out) fclose(out);
+  if (res==Z_OK)
+  {
+    std::ifstream f(buf, std::ios::in);
+    char buf[4096];
+    while (f.getline(buf,sizeof(buf))) on_nntp_line_process(nntp,StringView(buf));
   }
+
 }
 
 unsigned long
diff --git a/pan/tasks/task-xover.h b/pan/tasks/task-xover.h
index d1784de..c5ba3a8 100644
--- a/pan/tasks/task-xover.h
+++ b/pan/tasks/task-xover.h
@@ -32,6 +32,9 @@
 
 namespace pan
 {
+
+  struct XZVERDecoder;
+
   /**
    * Task for downloading a some or all of a newsgroups' headers
    * @ingroup tasks
@@ -45,11 +48,14 @@ namespace pan
 
     public: // task subclass
       virtual unsigned long get_bytes_remaining () const;
+      virtual void use_decoder (Decoder*);
+      void stop ();
 
     protected: // task subclass
       virtual void use_nntp (NNTP * nntp);
 
     private: // NNTP::Listener
+      void on_nntp_line_process (NNTP*, const StringView&);
       virtual void on_nntp_line (NNTP*, const StringView&);
       virtual void on_nntp_done (NNTP*, Health, const StringView&);
       virtual void on_nntp_group (NNTP*, const Quark&, unsigned long, uint64_t, uint64_t);
@@ -67,6 +73,7 @@ namespace pan
       server_to_minitasks_t _server_to_minitasks;
 
     private: // implementation
+      void process_headers (NNTP*);
       Data& _data;
       const Quark _group;
       std::string _short_group_name;
@@ -82,8 +89,15 @@ namespace pan
       unsigned long _bytes_so_far;
       unsigned long _parts_so_far;
       unsigned long _articles_so_far;
+      unsigned long _lines_so_far;
       unsigned long _total_minitasks;
+      int           _running_minitasks;
+      bool _xzver;
+      std::ofstream _headers;
 
+      friend class XZVERDecoder;
+      XZVERDecoder * _decoder;
+      bool _decoder_has_run;
   };
 }
 
diff --git a/pan/tasks/task.h b/pan/tasks/task.h
index b1a9455..aaabacf 100644
--- a/pan/tasks/task.h
+++ b/pan/tasks/task.h
@@ -56,6 +56,7 @@ namespace pan
             /** Task waiting for a decoder/encoder */
             NEED_DECODER,
             NEED_ENCODER,
+            NEED_XZVER_DECODER,
             /** Task is running */
             WORKING,
             /** Task is paused, woken up if 'current_connections < max_connections' */
@@ -99,6 +100,9 @@ namespace pan
                void set_need_decoder () {
                    _work = NEED_DECODER; _servers.clear(); }
 
+               void set_need_xzverdecoder () {
+                   _work = NEED_XZVER_DECODER; _servers.clear(); }
+
                void set_need_encoder () {
                    _work = NEED_ENCODER; _servers.clear(); }
 



[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]