[pan2/xzver] rough ...
- From: Heinrich MÃller <henmull src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [pan2/xzver] rough ...
- Date: Thu, 31 May 2012 18:16:45 +0000 (UTC)
commit 92e446d5a676dfac750e28ed9ae7a1407cea7a93
Author: Heinrich MÃller <sphemuel stud informatik uni-erlangen de>
Date: Sat Jul 9 23:38:10 2011 +0200
rough ...
Conflicts:
pan.png
pan/general/debug.cc
pan/general/debug.h
pan/general/file-util.cc
pan/gui/pan.cc
pan/tasks/nntp.cc
pan/tasks/socket-impl-openssl.cc
pan/tasks/task-xover.cc
pan/tasks/task-xover.h
configure.in | 11 ++-
pan.cbp | 2 +
pan.png | Bin 4041 -> 8808 bytes
pan/general/debug.cc | 3 +
pan/general/debug.h | 3 +
pan/gui/Makefile.am | 4 +-
pan/gui/pan.cc | 2 +
pan/tasks/Makefile.am | 4 +-
pan/tasks/decoder.h | 4 +-
pan/tasks/nntp.cc | 22 ++++-
pan/tasks/nntp.h | 12 ++-
pan/tasks/queue.cc | 20 ++++-
pan/tasks/queue.h | 4 +
pan/tasks/socket-impl-openssl.cc | 2 +-
pan/tasks/task-xover.cc | 209 ++++++++++++++++++++++++++++++++++---
pan/tasks/task-xover.h | 14 +++
pan/tasks/task.h | 4 +
17 files changed, 289 insertions(+), 31 deletions(-)
---
diff --git a/configure.in b/configure.in
index 827534e..5145556 100644
--- a/configure.in
+++ b/configure.in
@@ -87,12 +87,17 @@ AC_DEFINE_UNQUOTED([GETTEXT_PACKAGE],["$GETTEXT_PACKAGE"],[gettext package name]
AM_GLIB_GNU_GETTEXT
panlocaledir='${prefix}/${DATADIRNAME}/locale'
+dnl libz for xzver
+PKG_CHECK_MODULES([ZLIB], [zlib >= 1.2.7],AC_DEFINE(HAVE_ZLIB,[1],[Zlib for xzver support]),[])
+AC_SUBST(ZLIB_CFLAGS)
+AC_SUBST(ZLIB_LIBS)
+
dnl D-Bus support
AC_ARG_WITH(dbus, AS_HELP_STRING([--with-dbus],[enable D-Bus support (normally: no)]), [want_dbus=$withval], [want_dbus=no])
if test "x$want_dbus" = "xyes" ; then
AM_PATH_GLIB_2_0($GLIB_REQUIRED_FOR_DBUS,AC_DEFINE(HAVE_DBUS,[1],[D-Bus support for multiple Pan instances],exit 1,gobject gmodule gthread))
else
- AM_PATH_GLIB_2_0($GLIB_REQUIRED,,exit 1,gobject gmodule gthread)
+ AM_PATH_GLIB_2_0($GLIB_REQUIRED,,exit 1,gobject gmodule gthread)
fi
@@ -161,10 +166,10 @@ dnl GnuTLS support
gnutls_msg=no
AC_ARG_WITH(gnutls, AS_HELP_STRING([--with-gnutls],[enable GnuTLS support (normally: no)]), [want_gnutls=$withval], [want_gnutls=no])
if test "x$want_gnutls" = "xyes" ; then
-
+
PKG_CHECK_MODULES([GNUTLS],[gnutls >= $GNUTLS_REQUIRED],
[
- GNUTLS_VER=`pkg-config --modversion gnutls`
+ GNUTLS_VER=`pkg-config --modversion gnutls`
gnutls_msg="yes = $GNUTLS_VER"
AC_DEFINE(HAVE_GNUTLS,[1],[GnuTLS support])],
[gnutls_msg=no
diff --git a/pan.cbp b/pan.cbp
index e9dc378..fe164b2 100644
--- a/pan.cbp
+++ b/pan.cbp
@@ -300,6 +300,8 @@
<Unit filename="pan/tasks/task.h" />
<Unit filename="pan/tasks/upload-queue.cc" />
<Unit filename="pan/tasks/upload-queue.h" />
+ <Unit filename="pan/tasks/xzver-decoder.cc" />
+ <Unit filename="pan/tasks/xzver-decoder.h" />
<Unit filename="pan/usenet-utils/MersenneTwister.h" />
<Unit filename="pan/usenet-utils/blowfish.cc" />
<Unit filename="pan/usenet-utils/blowfish.h" />
diff --git a/pan.png b/pan.png
index 7b01de9..9373470 100644
Binary files a/pan.png and b/pan.png differ
diff --git a/pan/general/debug.cc b/pan/general/debug.cc
index 9a8161c..76a3a22 100644
--- a/pan/general/debug.cc
+++ b/pan/general/debug.cc
@@ -8,6 +8,9 @@ namespace pan
bool _verbose_flag = false;
std::ofstream _dbg_file;
+
+ // imhotep
+ bool _xzver_support = false;
bool _dbg_ssl = false;
}
diff --git a/pan/general/debug.h b/pan/general/debug.h
index 23fada3..f257d5e 100644
--- a/pan/general/debug.h
+++ b/pan/general/debug.h
@@ -31,6 +31,9 @@ namespace pan
extern bool _dbg_ssl;
extern std::ofstream _dbg_file;
+ // imhotep
+ extern bool _xzver_support;
+
}
#define LINE_ID '(' << __FILE__ << ':' << __LINE__ << ':' << __func__ << ')'
diff --git a/pan/gui/Makefile.am b/pan/gui/Makefile.am
index 39cac93..c37cd73 100644
--- a/pan/gui/Makefile.am
+++ b/pan/gui/Makefile.am
@@ -1,5 +1,5 @@
AM_CPPFLAGS = -I top_srcdir@ @GTKSPELL_CFLAGS@ @ENCHANT_CFLAGS@ @GTK_CFLAGS@ @GMIME_CFLAGS@ @GLIB_CFLAGS@ \
- @GNUTLS_CFLAGS@ @LIBNOTIFY_CFLAGS@ @LIBGNOME_KEYRING_1_CFLAGS@ -DPANLOCALEDIR=\""$(panlocaledir)"\"
+ @GNUTLS_CFLAGS@ @LIBNOTIFY_CFLAGS@ @LIBGNOME_KEYRING_1_CFLAGS@ -DPANLOCALEDIR=\""$(panlocaledir)"\" @ZLIB_CFLAGS@
noinst_LIBRARIES = libpangui.a
@@ -102,7 +102,7 @@ WINRCOBJ =
endif
pan_SOURCES = gui.cc pan.cc $(WINRC)
-pan_LDADD = ./libpangui.a $(WINRCOBJ) ../data-impl/libpandata.a ../tasks/libtasks.a ../data/libdata.a ../usenet-utils/libusenetutils.a ../general/libgeneralutils.a ../../uulib/libuu.a \
+pan_LDADD = @ZLIB_LIBS@ ./libpangui.a $(WINRCOBJ) ../data-impl/libpandata.a ../tasks/libtasks.a ../data/libdata.a ../usenet-utils/libusenetutils.a ../general/libgeneralutils.a ../../uulib/libuu.a \
@GTKSPELL_LIBS@ @ENCHANT_LIBS@ @GTK_LIBS@ @GMIME_LIBS@ @GLIB_LIBS@ @GNUTLS_LIBS@ @LIBNOTIFY_LIBS@ @LIBGNOME_KEYRING_1_LIBS@
if HAVE_WIN32
pan_LDFLAGS = -mwindows
diff --git a/pan/gui/pan.cc b/pan/gui/pan.cc
index 9f20a89..fafac0e 100644
--- a/pan/gui/pan.cc
+++ b/pan/gui/pan.cc
@@ -861,6 +861,8 @@ main (int argc, char *argv[])
{ usage (); return EXIT_SUCCESS; }
else if (!strcmp(tok, "--verbose") )
verbosed = true;
+ else if (!strcmp(tok,"--xzver"))
+ _xzver_support = true;
else {
nzb = true;
nzb_files.push_back (tok);
diff --git a/pan/tasks/Makefile.am b/pan/tasks/Makefile.am
index df282b1..f7220b3 100644
--- a/pan/tasks/Makefile.am
+++ b/pan/tasks/Makefile.am
@@ -1,6 +1,6 @@
-AM_CPPFLAGS = -I top_srcdir@ @GMIME_CFLAGS@ @GLIB_CFLAGS@ @GNUTLS_CFLAGS@ @GTK_CFLAGS@
+AM_CPPFLAGS = -I top_srcdir@ @GMIME_CFLAGS@ @GLIB_CFLAGS@ @GNUTLS_CFLAGS@ @GTK_CFLAGS@ @ZLIB_CFLAGS@
-AM_LDFLAGS = ../../uulib/libuu.a @GNUTLS_LIBS@
+AM_LDFLAGS = ../../uulib/libuu.a @GNUTLS_LIBS@ @ZLIB_LIBS@
noinst_LIBRARIES = libtasks.a
diff --git a/pan/tasks/decoder.h b/pan/tasks/decoder.h
index 0814694..b62fff7 100644
--- a/pan/tasks/decoder.h
+++ b/pan/tasks/decoder.h
@@ -72,7 +72,7 @@ namespace pan
protected: // inherited from WorkerPool::Worker
- void do_work();
+ virtual void do_work();
private:
@@ -90,7 +90,7 @@ namespace pan
volatile int num_scanned_files;
static void uu_log(void *thiz, char *message, int severity);
- double get_percentage (const uuprogress& p) const;
+ virtual double get_percentage (const uuprogress& p) const;
static int uu_busy_poll(void * self, uuprogress *p);
/** tell our task about the decode's progress */
static gboolean progress_update_timer_func(gpointer decoder);
diff --git a/pan/tasks/nntp.cc b/pan/tasks/nntp.cc
index fc6b560..557fa20 100644
--- a/pan/tasks/nntp.cc
+++ b/pan/tasks/nntp.cc
@@ -63,6 +63,7 @@ NNTP :: fire_done_func (Health health, const StringView& response)
debug ("I (" << (void*)this << ") am setting my _listener to 0");
_listener = 0;
l->on_nntp_done (this, health, response);
+ _xzver = false;
}
}
@@ -78,7 +79,7 @@ NNTP :: on_socket_response (Socket * sock UNUSED, const StringView& line_in)
StringView line (line_in);
// strip off trailing \r\n
- if (line.len>=2 && line.str[line.len-2]=='\r' && line.str[line.len-1]=='\n')
+ if (line.len>=2 && line.str[line.len-2]=='\r' && line.str[line.len-1]=='\n' && !_xzver)
line.truncate (line.len-2);
// std::cerr <<"_nntp_response_text: " << _nntp_response_text<<std::endl;
@@ -93,7 +94,7 @@ NNTP :: on_socket_response (Socket * sock UNUSED, const StringView& line_in)
{
state = CMD_MORE;
- if (line.len>=2 && line.str[0]=='.' && line.str[1]=='.') // rfc 977: 2.4.1
+ if (line.len>=2 && line.str[0]=='.' && line.str[1]=='.' && !_xzver) // rfc 977: 2.4.1
line.rtruncate (line.len-1);
assert (_listener != 0);
@@ -325,6 +326,23 @@ NNTP :: xover (const Quark & group,
}
void
+NNTP :: xzver (const Quark & group,
+ uint64_t low,
+ uint64_t high,
+ Listener * l)
+{
+ _listener = l;
+ _xzver = true;
+
+ if (group != _group)
+ _commands.push_back (build_command ("GROUP %s\r\n", group.c_str()));
+
+ _commands.push_back (build_command ("XZVER %"G_GUINT64_FORMAT"-%"G_GUINT64_FORMAT"\r\n", low, high));
+
+ write_next_command ();
+}
+
+void
NNTP :: xover_count_only (const Quark & group,
Listener * l)
{
diff --git a/pan/tasks/nntp.h b/pan/tasks/nntp.h
index c74ae06..fdfd0ca 100644
--- a/pan/tasks/nntp.h
+++ b/pan/tasks/nntp.h
@@ -153,7 +153,8 @@ namespace pan
_listener(0),
_username(username),
_password(password),
- _nntp_response_text(false)
+ _nntp_response_text(false),
+ _xzver(false)
{}
virtual ~NNTP ()
@@ -206,6 +207,13 @@ namespace pan
void xover_count_only (const Quark & group,
Listener * l);
+
+ /** Experimental XZVER header compression support */
+ void xzver (const Quark & group,
+ uint64_t low,
+ uint64_t high,
+ Listener * l) ;
+
/**
* Executes a LIST command: "LIST"
*
@@ -319,6 +327,8 @@ namespace pan
/** True if the server told us that we're getting a list back. */
bool _nntp_response_text;
+ bool _xzver;
+
typedef std::deque<std::string> strings_t;
strings_t _commands;
std::string _previous_command;
diff --git a/pan/tasks/queue.cc b/pan/tasks/queue.cc
index 133470a..dede377 100644
--- a/pan/tasks/queue.cc
+++ b/pan/tasks/queue.cc
@@ -329,10 +329,14 @@ Queue :: process_task (Task * task)
}
else if (state._work == Task::NEED_DECODER)
{
-
if (!_decoder_task)
give_task_a_decoder (task);
}
+ else if (state._work == Task::NEED_XZVER_DECODER)
+ {
+ if (!_decoder_task)
+ give_task_a_xzver_decoder (task);
+ }
else if (state._work == Task::NEED_ENCODER)
{
if (!_encoder_task)
@@ -387,6 +391,20 @@ Queue :: find_first_task_needing_decoder ()
}
Task*
+Queue :: find_first_task_needing_xzver_decoder ()
+{
+ foreach (TaskSet, _tasks, it) {
+ const Task::State& state ((*it)->get_state ());
+ if ((state._work == Task::NEED_XZVER_DECODER)
+ && (!_stopped.count (*it))
+ && (!_removing.count (*it)))
+ return *it;
+ }
+
+ return 0;
+}
+
+Task*
Queue :: find_first_task_needing_encoder ()
{
foreach (TaskSet, _tasks, it) {
diff --git a/pan/tasks/queue.h b/pan/tasks/queue.h
index d2b2862..59479d6 100644
--- a/pan/tasks/queue.h
+++ b/pan/tasks/queue.h
@@ -26,6 +26,7 @@
#include <pan/general/macros.h> // for UNUSED
#include <pan/general/map-vector.h>
#include <pan/tasks/decoder.h>
+#include <pan/tasks/xzver-decoder.h>
#include <pan/tasks/encoder.h>
#include <pan/general/quark.h>
#include <pan/tasks/nntp-pool.h>
@@ -47,6 +48,7 @@ namespace pan
class WorkerPool;
struct Encoder;
struct Decoder;
+ struct XZVERDecoder;
/**
* A Queue helper that saves tasks to disk and restores them from disk.
@@ -187,12 +189,14 @@ namespace pan
protected:
void process_task (Task *);
void give_task_a_decoder (Task*);
+ void give_task_a_xzver_decoder (Task*);
void give_task_a_encoder (Task*);
void give_task_a_connection (Task*, NNTP*);
ServerInfo& _server_info;
bool _is_online;
Task* find_first_task_needing_server (const Quark& server);
Task* find_first_task_needing_decoder ();
+ Task* find_first_task_needing_xzver_decoder ();
Task* find_first_task_needing_encoder ();
void give_task_an_upload_slot (TaskUpload* task);
diff --git a/pan/tasks/socket-impl-openssl.cc b/pan/tasks/socket-impl-openssl.cc
index f7035c9..17175b9 100644
--- a/pan/tasks/socket-impl-openssl.cc
+++ b/pan/tasks/socket-impl-openssl.cc
@@ -4,7 +4,7 @@
* Copyright (C) 2002-2006 Charles Kerr <charles rebelbase com>
*
* This file
- * Copyright (C) 2011 Heinrich Mü<henmull src gnome org>
+ * Copyright (C) 2011 Heinrich Mïller <henmull src gnome org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
diff --git a/pan/tasks/task-xover.cc b/pan/tasks/task-xover.cc
index 34a5a69..433846b 100644
--- a/pan/tasks/task-xover.cc
+++ b/pan/tasks/task-xover.cc
@@ -38,7 +38,7 @@ extern "C" {
#include <pan/data/data.h>
#include "nntp.h"
#include "task-xover.h"
-
+#include "xzver-decoder.h"
using namespace pan;
@@ -85,6 +85,16 @@ namespace
}
}
+namespace
+{
+ char* build_cachename (char* buf, size_t len, const char* name)
+ {
+ const char * home(file::get_pan_home().c_str());
+ g_snprintf(buf,len,"%s%c%s%c%s",home, G_DIR_SEPARATOR, "encode-cache", G_DIR_SEPARATOR, name);
+ return buf;
+ }
+}
+
TaskXOver :: TaskXOver (Data & data,
const Quark & group,
Mode mode,
@@ -100,9 +110,21 @@ TaskXOver :: TaskXOver (Data & data,
_bytes_so_far (0),
_parts_so_far (0ul),
_articles_so_far (0ul),
- _total_minitasks (0)
+ _total_minitasks (0),
+ _running_minitasks (0),
+ _xzver (_xzver_support),
+ _decoder(0),
+ _decoder_has_run (false)
{
+
+
+ if (_xzver)
+ {
+ char buf[4096];
+ _headers.open(build_cachename(buf,sizeof(buf), "xzver_test"), std::ios::out | std::ios::binary);
+ }
+
debug ("ctor for " << group);
// add a ``GROUP'' MiniTask for each server that has this group
@@ -131,7 +153,6 @@ TaskXOver :: ~TaskXOver ()
_data.set_xover_high (_group, it->first, it->second);
_data.xover_unref (_group);
}
- _data.fire_group_entered(_group, 1, 0);
}
void
@@ -166,7 +187,11 @@ TaskXOver :: use_nntp (NNTP* nntp)
case MiniTask::XOVER:
debug ("XOVER " << mt._low << '-' << mt._high << " to " << server);
_last_xover_number[nntp] = mt._low;
- nntp->xover (_group, mt._low, mt._high, this);
+ if (_xzver)
+ nntp->xzver (_group, mt._low, mt._high, this);
+ else
+ nntp->xover (_group, mt._low, mt._high, this);
+ --_running_minitasks;
break;
default:
assert (0);
@@ -179,7 +204,6 @@ TaskXOver :: use_nntp (NNTP* nntp)
****
***/
-///TODO show low and high in UI (is this already there?)
void
TaskXOver :: on_nntp_group (NNTP * nntp,
const Quark & group,
@@ -220,15 +244,24 @@ TaskXOver :: on_nntp_group (NNTP * nntp,
if (l <= high)
{
//std::cerr << LINE_ID << " okay, I'll try to get articles in [" << l << "..." << h << ']' << std::endl;
- add_steps (h-l);
- const int INCREMENT(1000);
+ add_steps (_xzver ? 2 : 1*(h-l));
+ int INCREMENT = 1000;
MiniTasks_t& minitasks (_server_to_minitasks[servername]);
- for (uint64_t m=l; m<=h; m+=INCREMENT) {
- MiniTask mt (MiniTask::XOVER, m, m+INCREMENT);
- debug ("adding MiniTask for " << servername << ": xover [" << mt._low << '-' << mt._high << "]");
- minitasks.push_front (mt);
+ if (_xzver)
+ {
+ MiniTask mt (MiniTask::XOVER, l, h);
++_total_minitasks;
}
+ else
+ {
+ for (uint64_t m=l; m<=h; m+=INCREMENT) {
+ MiniTask mt (MiniTask::XOVER, m, m+INCREMENT);
+ debug ("adding MiniTask for " << servername << ": xover [" << mt._low << '-' << mt._high << "]");
+ minitasks.push_front (mt);
+ ++_total_minitasks;
+ }
+ }
+ _running_minitasks = _total_minitasks;
}
else
{
@@ -279,6 +312,22 @@ TaskXOver :: on_nntp_line (NNTP * nntp,
const StringView & line)
{
+ if (_xzver ) {
+ if (line.strstr("=ybegin line=128"))
+ _headers << line.str << " name=xzver_decoded\n";
+ else
+ _headers << line.str <<"\n";
+ }
+ else
+ on_nntp_line_process (nntp, line);
+ increment_step(1);
+}
+
+void
+TaskXOver :: on_nntp_line_process (NNTP * nntp,
+ const StringView & line)
+{
+
pan_return_if_fail (nntp != 0);
pan_return_if_fail (!nntp->_server.empty());
pan_return_if_fail (!nntp->_group.empty());
@@ -325,10 +374,8 @@ TaskXOver :: on_nntp_line (NNTP * nntp,
|| author.empty() // missing author
|| date.empty() // missing date
|| mid.empty() // missing mid
- || mid.front()!='<') // corrupt mid
- /// Concerning bug : https://bugzilla.gnome.org/show_bug.cgi?id=650042
- /// Even if we didn't get a proper reference here, continue.
- //|| (!ref.empty() && ref.front()!='<'))
+ || mid.front()!='<' // corrupt mid
+ || (!ref.empty() && ref[0] != '<'))
return;
// if news server doesn't provide an xref, fake one
@@ -378,6 +425,8 @@ TaskXOver :: on_nntp_done (NNTP * nntp,
Health health,
const StringView & response UNUSED)
{
+ if (_running_minitasks == 0) process_headers(nntp);
+
update_work (true);
check_in (nntp, health);
}
@@ -397,14 +446,140 @@ TaskXOver :: update_work (bool subtract_one_from_nntp_count)
//std::cerr << LINE_ID << " servers: " << servers.size() << " nntp: " << nntp_count << std::endl;
+// if (!servers.empty())
+// _state.set_need_nntp (servers);
+// else if (nntp_count)
+// _state.set_working ();
+// else {
+// _state.set_completed();
+// set_finished(OK);
+// }
if (!servers.empty())
_state.set_need_nntp (servers);
else if (nntp_count)
_state.set_working ();
- else {
+ else if (!_decoder && !_decoder_has_run) {
+ _state.set_need_xzverdecoder ();
+ set_step(0);
+ } else if (_decoder_has_run) {
_state.set_completed();
- set_finished(OK);
+ set_finished (OK);
+ } else assert(0 && "hm, missed a state.");
+}
+
+namespace
+{
+
+ #define CHUNK 16384
+
+ int inf(FILE *source, FILE *dest)
+ {
+ int ret;
+ unsigned have;
+ z_stream strm;
+ unsigned char in[CHUNK];
+ unsigned char out[CHUNK];
+
+ /* allocate inflate state */
+ strm.zalloc = Z_NULL;
+ strm.zfree = Z_NULL;
+ strm.opaque = Z_NULL;
+ strm.avail_in = 0;
+ strm.next_in = Z_NULL;
+ ret = inflateInit2(&strm,-15); //raw inflate
+ if (ret != Z_OK)
+ return ret;
+
+ /* decompress until deflate stream ends or end of file */
+ do {
+ strm.avail_in = fread(in, 1, CHUNK, source);
+ if (ferror(source)) {
+ (void)inflateEnd(&strm);
+ return Z_ERRNO;
+ }
+ if (strm.avail_in == 0)
+ break;
+ strm.next_in = in;
+
+ /* run inflate() on input until output buffer not full */
+ do {
+ strm.avail_out = CHUNK;
+ strm.next_out = out;
+ ret = inflate(&strm, Z_NO_FLUSH);
+ assert(ret != Z_STREAM_ERROR); /* state not clobbered */
+ switch (ret) {
+ case Z_NEED_DICT:
+ ret = Z_DATA_ERROR; /* and fall through */
+ case Z_DATA_ERROR:
+ case Z_MEM_ERROR:
+ (void)inflateEnd(&strm);
+ return ret;
+ }
+ have = CHUNK - strm.avail_out;
+ if (fwrite(out, 1, have, dest) != have || ferror(dest)) {
+ (void)inflateEnd(&strm);
+ return Z_ERRNO;
+ }
+ } while (strm.avail_out == 0);
+
+ /* done when inflate() says it's done */
+ } while (ret != Z_STREAM_END);
+
+ /* clean up and return */
+ (void)inflateEnd(&strm);
+ return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR;
+ }
+}
+
+void
+TaskXOver:: use_decoder (Decoder* decoder)
+{
+ if (_state._work != NEED_DECODER)
+ check_in (decoder);
+
+ _decoder = static_cast<XZVERDecoder*>(decoder);
+ init_steps(100);
+ _state.set_working();
+ _decoder->enqueue (this);
+ set_status_va (_("Decoding XZVER Headers"));
+ debug ("decoder thread was free, enqueued work");
+}
+
+void
+TaskXOver :: stop ()
+{
+ if (_decoder)
+ _decoder->cancel();
+}
+
+void
+ TaskXOver :: process_headers (NNTP* nntp)
+{
+ char buf[4096];
+
+ _headers.close();
+ /* yenc-decode */
+ UUInitialize ();
+ UULoadFile (build_cachename(buf,sizeof(buf), "xzver_test"), 0, 1);
+ std::cerr<<"uudecode "<<UUstrerror(UUDecodeFile (UUGetFileListItem (0), build_cachename(buf,sizeof(buf), "xzver_decoded")))<<"\n";
+ UUCleanUp ();
+
+ /* raw zlib inflate */
+ FILE * in = fopen (buf, "rb");
+ FILE * out = fopen (build_cachename(buf,sizeof(buf), "xzver_out"), "wb");
+ int res(Z_OK);
+ if (in && out) res = inf (in,out);
+
+ /* feed to on_nntp_line */
+ if (in) fclose(in);
+ if (out) fclose(out);
+ if (res==Z_OK)
+ {
+ std::ifstream f(buf, std::ios::in);
+ char buf[4096];
+ while (f.getline(buf,sizeof(buf))) on_nntp_line_process(nntp,StringView(buf));
}
+
}
unsigned long
diff --git a/pan/tasks/task-xover.h b/pan/tasks/task-xover.h
index d1784de..c5ba3a8 100644
--- a/pan/tasks/task-xover.h
+++ b/pan/tasks/task-xover.h
@@ -32,6 +32,9 @@
namespace pan
{
+
+ struct XZVERDecoder;
+
/**
* Task for downloading a some or all of a newsgroups' headers
* @ingroup tasks
@@ -45,11 +48,14 @@ namespace pan
public: // task subclass
virtual unsigned long get_bytes_remaining () const;
+ virtual void use_decoder (Decoder*);
+ void stop ();
protected: // task subclass
virtual void use_nntp (NNTP * nntp);
private: // NNTP::Listener
+ void on_nntp_line_process (NNTP*, const StringView&);
virtual void on_nntp_line (NNTP*, const StringView&);
virtual void on_nntp_done (NNTP*, Health, const StringView&);
virtual void on_nntp_group (NNTP*, const Quark&, unsigned long, uint64_t, uint64_t);
@@ -67,6 +73,7 @@ namespace pan
server_to_minitasks_t _server_to_minitasks;
private: // implementation
+ void process_headers (NNTP*);
Data& _data;
const Quark _group;
std::string _short_group_name;
@@ -82,8 +89,15 @@ namespace pan
unsigned long _bytes_so_far;
unsigned long _parts_so_far;
unsigned long _articles_so_far;
+ unsigned long _lines_so_far;
unsigned long _total_minitasks;
+ int _running_minitasks;
+ bool _xzver;
+ std::ofstream _headers;
+ friend class XZVERDecoder;
+ XZVERDecoder * _decoder;
+ bool _decoder_has_run;
};
}
diff --git a/pan/tasks/task.h b/pan/tasks/task.h
index b1a9455..aaabacf 100644
--- a/pan/tasks/task.h
+++ b/pan/tasks/task.h
@@ -56,6 +56,7 @@ namespace pan
/** Task waiting for a decoder/encoder */
NEED_DECODER,
NEED_ENCODER,
+ NEED_XZVER_DECODER,
/** Task is running */
WORKING,
/** Task is paused, woken up if 'current_connections < max_connections' */
@@ -99,6 +100,9 @@ namespace pan
void set_need_decoder () {
_work = NEED_DECODER; _servers.clear(); }
+ void set_need_xzverdecoder () {
+ _work = NEED_XZVER_DECODER; _servers.clear(); }
+
void set_need_encoder () {
_work = NEED_ENCODER; _servers.clear(); }
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
