[pan2: 127/268] added xzver support, enable with ./pan --xzver on the command line. EXPERIMENTAL, see the source!!!
- From: Heinrich MÃller <henmull src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [pan2: 127/268] added xzver support, enable with ./pan --xzver on the command line. EXPERIMENTAL, see the source!!!
- Date: Mon, 2 Jan 2012 15:48:40 +0000 (UTC)
commit ef96111cf71212a56af5b33ebd1219bc5203caa8
Author: Heinrich MÃller <sphemuel stud informatik uni-erlangen de>
Date: Sat Jul 9 23:38:10 2011 +0200
added xzver support, enable with ./pan --xzver on the command line.
EXPERIMENTAL, see the source!!!
pan.png | Bin 4909 -> 3897 bytes
pan/general/debug.cc | 3 +
pan/general/debug.h | 3 +
pan/general/file-util.cc | 35 ---------
pan/gui/pan.cc | 2 +
pan/icons/icon_pan.png | Bin 3862 -> 3897 bytes
pan/icons/icon_pan_about_logo.png | Bin 9389 -> 9327 bytes
pan/tasks/nntp.cc | 24 +++++-
pan/tasks/nntp.h | 18 ++++-
pan/tasks/socket-impl-openssl.cc | 1 -
pan/tasks/task-xover.cc | 148 +++++++++++++++++++++++++++++++++++-
pan/tasks/task-xover.h | 5 +
12 files changed, 193 insertions(+), 46 deletions(-)
---
diff --git a/pan.png b/pan.png
index b7cc52a..1d6ccad 100644
Binary files a/pan.png and b/pan.png differ
diff --git a/pan/general/debug.cc b/pan/general/debug.cc
index 2e6cfb4..749f27f 100644
--- a/pan/general/debug.cc
+++ b/pan/general/debug.cc
@@ -5,4 +5,7 @@ namespace pan
{
bool _debug_flag = false;
bool _debug_verbose_flag = false;
+
+ // imhotep
+ bool _xzver_support = false;
}
diff --git a/pan/general/debug.h b/pan/general/debug.h
index f005916..1d06c0f 100644
--- a/pan/general/debug.h
+++ b/pan/general/debug.h
@@ -26,6 +26,9 @@ namespace pan
{
extern bool _debug_flag;
extern bool _debug_verbose_flag;
+
+ // imhotep
+ extern bool _xzver_support;
}
#define LINE_ID '(' << __FILE__ << ':' << __LINE__ << ':' << __func__ << ')'
diff --git a/pan/general/file-util.cc b/pan/general/file-util.cc
index bfe70a8..2e08bc2 100644
--- a/pan/general/file-util.cc
+++ b/pan/general/file-util.cc
@@ -51,41 +51,6 @@ using namespace pan;
***/
std::string
-file :: get_uulib_path()
-{
- char * pch (g_build_filename (file::get_pan_home().c_str(), "uulib-encode-cache", NULL));
- file :: ensure_dir_exists (pch);
- std::string path (pch);
- if (pch) g_free (pch);
- return path;
-}
-
-// delete cached files to avoid "disk full" problems
-void
-file :: uulib_cache_clear ()
-{
-// DIR *dp;
-// struct dirent *ep;
-// struct stat st;
-// const char * dir = get_uulib_path().c_str();
-// std::cerr<<"opening dir : "<<dir<<std::endl;
-// dp = opendir (dir);
-// if (dp != NULL)
-// {
-// while (ep = readdir (dp))
-// {
-// stat(ep->d_name, &st);
-// if (S_ISDIR(st.st_mode) == 0) {
-// std::cerr<<"deleting file :"<<ep->d_name<<"\n";
-// unlink(ep->d_name);
-// }
-// }
-// (void) closedir (dp);
-// } else
-// std::cerr<<"error clearing uulib cache!\n";
-}
-
-std::string
file :: get_pan_home ()
{
static std::string pan_home;
diff --git a/pan/gui/pan.cc b/pan/gui/pan.cc
index 5d8a8e4..c3afc2f 100644
--- a/pan/gui/pan.cc
+++ b/pan/gui/pan.cc
@@ -286,6 +286,8 @@ main (int argc, char *argv[])
nzb_output_path = tok+9;
else if (!strcmp(tok,"-h") || !strcmp(tok,"--help"))
{ usage (); return 0; }
+ else if (!strcmp(tok,"--xzver"))
+ _xzver_support = true;
else {
nzb = true;
nzb_files.push_back (tok);
diff --git a/pan/icons/icon_pan.png b/pan/icons/icon_pan.png
index 16632ce..1d6ccad 100644
Binary files a/pan/icons/icon_pan.png and b/pan/icons/icon_pan.png differ
diff --git a/pan/icons/icon_pan_about_logo.png b/pan/icons/icon_pan_about_logo.png
index 8e6b816..ac0ec9b 100644
Binary files a/pan/icons/icon_pan_about_logo.png and b/pan/icons/icon_pan_about_logo.png differ
diff --git a/pan/tasks/nntp.cc b/pan/tasks/nntp.cc
index 2b6da4c..f4cb2c7 100644
--- a/pan/tasks/nntp.cc
+++ b/pan/tasks/nntp.cc
@@ -63,6 +63,7 @@ NNTP :: fire_done_func (Health health, const StringView& response)
debug ("I (" << (void*)this << ") am setting my _listener to 0");
_listener = 0;
l->on_nntp_done (this, health, response);
+ _xzver = false;
}
}
@@ -78,7 +79,7 @@ NNTP :: on_socket_response (Socket * sock UNUSED, const StringView& line_in)
StringView line (line_in);
// strip off trailing \r\n
- if (line.len>=2 && line.str[line.len-2]=='\r' && line.str[line.len-1]=='\n')
+ if (line.len>=2 && line.str[line.len-2]=='\r' && line.str[line.len-1]=='\n' && !_xzver)
line.truncate (line.len-2);
// std::cerr <<"_nntp_response_text: " << _nntp_response_text<<std::endl;
@@ -93,7 +94,7 @@ NNTP :: on_socket_response (Socket * sock UNUSED, const StringView& line_in)
{
state = CMD_MORE;
- if (line.len>=2 && line.str[0]=='.' && line.str[1]=='.') // rfc 977: 2.4.1
+ if (line.len>=2 && line.str[0]=='.' && line.str[1]=='.' && !_xzver) // rfc 977: 2.4.1
line.rtruncate (line.len-1);
assert (_listener != 0);
@@ -244,7 +245,7 @@ NNTP :: on_socket_response (Socket * sock UNUSED, const StringView& line_in)
switch (state) {
case CMD_FAIL: fire_done_func (ERR_COMMAND, line); more = false; break;
case CMD_DONE: if (_commands.empty()) fire_done_func (OK, line); more = false; break;
- case CMD_MORE: more = true; break; // keep listining for more on this command
+ case CMD_MORE: more = true; break; // keep listening for more on this command
case CMD_NEXT: more = false; break; // no more responses on this command; wait for next...
case CMD_RETRY: fire_done_func (ERR_NETWORK, line); more = false; break;
default: abort(); break;
@@ -317,6 +318,23 @@ NNTP :: xover (const Quark & group,
write_next_command ();
}
+void
+NNTP :: xzver (const Quark & group,
+ uint64_t low,
+ uint64_t high,
+ Listener * l)
+{
+ _listener = l;
+ _xzver = true;
+
+ if (group != _group)
+ _commands.push_back (build_command ("GROUP %s\r\n", group.c_str()));
+
+ _commands.push_back (build_command ("XZVER %"G_GUINT64_FORMAT"-%"G_GUINT64_FORMAT"\r\n", low, high));
+
+ write_next_command ();
+}
+
void
NNTP :: list_newsgroups (Listener * l)
diff --git a/pan/tasks/nntp.h b/pan/tasks/nntp.h
index c913600..c83de4b 100644
--- a/pan/tasks/nntp.h
+++ b/pan/tasks/nntp.h
@@ -109,7 +109,7 @@ namespace pan
* lines for an ARTICLE command.
*/
virtual void on_nntp_line (NNTP * nntp UNUSED,
- const StringView & line UNUSED) {}
+ const StringView & line UNUSED) {}
/**
* Called at the end of an NNTP command. If the command was
@@ -126,6 +126,10 @@ namespace pan
Health health UNUSED,
const StringView & response UNUSED) {}
+ virtual void on_xover_done (NNTP * nntp UNUSED,
+ Health health UNUSED,
+ const StringView & response UNUSED) {}
+
/**
* Called whenever an NNTP object sets the current group.
*/
@@ -149,7 +153,8 @@ namespace pan
_listener(0),
_username(username),
_password(password),
- _nntp_response_text(false)
+ _nntp_response_text(false),
+ _xzver(false)
{}
virtual ~NNTP ()
@@ -185,6 +190,13 @@ namespace pan
uint64_t high,
Listener * l);
+
+ /** Experimental XZVER header compression support */
+ void xzver (const Quark & group,
+ uint64_t low,
+ uint64_t high,
+ Listener * l) ;
+
/**
* Executes a LIST command: "LIST"
*
@@ -298,6 +310,8 @@ namespace pan
/** True if the server told us that we're getting a list back. */
bool _nntp_response_text;
+ bool _xzver;
+
typedef std::deque<std::string> strings_t;
strings_t _commands;
std::string _previous_command;
diff --git a/pan/tasks/socket-impl-openssl.cc b/pan/tasks/socket-impl-openssl.cc
index c2e651e..f6acdc1 100644
--- a/pan/tasks/socket-impl-openssl.cc
+++ b/pan/tasks/socket-impl-openssl.cc
@@ -464,7 +464,6 @@ namespace
{
if (ret) *ret = 0;
if(SSL_get_error(chan->ssl, err) == SSL_ERROR_WANT_READ) {
- std::cerr<<"again\n";
return G_IO_STATUS_AGAIN;
}
return ssl_errno(errno);
diff --git a/pan/tasks/task-xover.cc b/pan/tasks/task-xover.cc
index 85ac342..77fd955 100644
--- a/pan/tasks/task-xover.cc
+++ b/pan/tasks/task-xover.cc
@@ -26,11 +26,12 @@ extern "C" {
#include <uulib/uudeview.h>
#include <glib/gi18n.h>
#include <gmime/gmime-utils.h>
-
+ #include <zlib.h>
}
#include <fstream>
#include <iostream>
#include <pan/general/debug.h>
+#include <pan/general/file-util.h>
#include <pan/general/macros.h>
#include <pan/general/messages.h>
#include <pan/general/utf8-utils.h>
@@ -38,6 +39,7 @@ extern "C" {
#include "nntp.h"
#include "task-xover.h"
+
using namespace pan;
namespace
@@ -83,6 +85,16 @@ namespace
}
}
+namespace
+{
+ char* build_cachename (char* buf, size_t len, const char* name)
+ {
+ const char * home(file::get_pan_home().c_str());
+ g_snprintf(buf,len,"%s%c%s%c%s",home, G_DIR_SEPARATOR, "encode-cache", G_DIR_SEPARATOR, name);
+ return buf;
+ }
+}
+
TaskXOver :: TaskXOver (Data & data,
const Quark & group,
Mode mode,
@@ -98,9 +110,19 @@ TaskXOver :: TaskXOver (Data & data,
_bytes_so_far (0),
_parts_so_far (0ul),
_articles_so_far (0ul),
- _total_minitasks (0)
+ _total_minitasks (0),
+ _running_minitasks (0),
+ _xzver (_xzver_support)
{
+
+
+ if (_xzver)
+ {
+ char buf[4096];
+ _headers.open(build_cachename(buf,sizeof(buf), "xzver_test"), std::ios::out | std::ios::binary);
+ }
+
debug ("ctor for " << group);
// add a ``GROUP'' MiniTask for each server that has this group
@@ -163,7 +185,11 @@ TaskXOver :: use_nntp (NNTP* nntp)
case MiniTask::XOVER:
debug ("XOVER " << mt._low << '-' << mt._high << " to " << server);
_last_xover_number[nntp] = mt._low;
- nntp->xover (_group, mt._low, mt._high, this);
+ if (_xzver)
+ nntp->xzver (_group, mt._low, mt._high, this);
+ else
+ nntp->xover (_group, mt._low, mt._high, this);
+ --_running_minitasks;
break;
default:
assert (0);
@@ -217,14 +243,15 @@ TaskXOver :: on_nntp_group (NNTP * nntp,
{
//std::cerr << LINE_ID << " okay, I'll try to get articles in [" << l << "..." << h << ']' << std::endl;
add_steps (h-l);
- const int INCREMENT (1000);
+ int INCREMENT = _xzver ? 100000 : 1000;
MiniTasks_t& minitasks (_server_to_minitasks[servername]);
for (uint64_t m=l; m<=h; m+=INCREMENT) {
MiniTask mt (MiniTask::XOVER, m, m+INCREMENT);
- debug ("adding MiniTask for " << servername << ": xover [" << mt._low << '-' << mt._high << ']');
+ debug ("adding MiniTask for " << servername << ": xover [" << mt._low << '-' << mt._high << "]");
minitasks.push_front (mt);
++_total_minitasks;
}
+ _running_minitasks = _total_minitasks;
}
else
{
@@ -275,6 +302,21 @@ TaskXOver :: on_nntp_line (NNTP * nntp,
const StringView & line)
{
+ if (_xzver ) {
+ if (line.strstr("=ybegin line=128"))
+ _headers << line.str << " name=xzver_decoded\n";
+ else
+ _headers << line.str <<"\n";
+ }
+ else
+ on_nntp_line_process (nntp, line);
+}
+
+void
+TaskXOver :: on_nntp_line_process (NNTP * nntp,
+ const StringView & line)
+{
+
pan_return_if_fail (nntp != 0);
pan_return_if_fail (!nntp->_server.empty());
pan_return_if_fail (!nntp->_group.empty());
@@ -372,6 +414,8 @@ TaskXOver :: on_nntp_done (NNTP * nntp,
Health health,
const StringView & response UNUSED)
{
+ if (_running_minitasks == 0) process_headers(nntp);
+
update_work (true);
check_in (nntp, health);
}
@@ -401,6 +445,100 @@ TaskXOver :: update_work (bool subtract_one_from_nntp_count)
}
}
+namespace
+{
+
+ #define CHUNK 16384
+
+ int inf(FILE *source, FILE *dest)
+ {
+ int ret;
+ unsigned have;
+ z_stream strm;
+ unsigned char in[CHUNK];
+ unsigned char out[CHUNK];
+
+ /* allocate inflate state */
+ strm.zalloc = Z_NULL;
+ strm.zfree = Z_NULL;
+ strm.opaque = Z_NULL;
+ strm.avail_in = 0;
+ strm.next_in = Z_NULL;
+ ret = inflateInit2(&strm,-15); //raw inflate
+ if (ret != Z_OK)
+ return ret;
+
+ /* decompress until deflate stream ends or end of file */
+ do {
+ strm.avail_in = fread(in, 1, CHUNK, source);
+ if (ferror(source)) {
+ (void)inflateEnd(&strm);
+ return Z_ERRNO;
+ }
+ if (strm.avail_in == 0)
+ break;
+ strm.next_in = in;
+
+ /* run inflate() on input until output buffer not full */
+ do {
+ strm.avail_out = CHUNK;
+ strm.next_out = out;
+ ret = inflate(&strm, Z_NO_FLUSH);
+ assert(ret != Z_STREAM_ERROR); /* state not clobbered */
+ switch (ret) {
+ case Z_NEED_DICT:
+ ret = Z_DATA_ERROR; /* and fall through */
+ case Z_DATA_ERROR:
+ case Z_MEM_ERROR:
+ (void)inflateEnd(&strm);
+ return ret;
+ }
+ have = CHUNK - strm.avail_out;
+ if (fwrite(out, 1, have, dest) != have || ferror(dest)) {
+ (void)inflateEnd(&strm);
+ return Z_ERRNO;
+ }
+ } while (strm.avail_out == 0);
+
+ /* done when inflate() says it's done */
+ } while (ret != Z_STREAM_END);
+
+ /* clean up and return */
+ (void)inflateEnd(&strm);
+ return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR;
+ }
+}
+
+void
+ TaskXOver :: process_headers (NNTP* nntp)
+{
+ char buf[4096];
+
+ _headers.close();
+ /* yenc-decode */
+ UUInitialize ();
+ UULoadFile (build_cachename(buf,sizeof(buf), "xzver_test"), 0, 1);
+ UUDecodeFile (UUGetFileListItem (0), build_cachename(buf,sizeof(buf), "xzver_decoded"));
+ UUCleanUp ();
+
+ /* raw zlib inflate */
+ FILE * in = fopen (buf, "rb");
+ FILE * out = fopen (build_cachename(buf,sizeof(buf), "xzver_out"), "wb");
+ int res(Z_OK);
+ if (in && out) res = inf (in,out);
+
+ /* feed to on_nntp_line */
+ if (in) fclose(in);
+ if (out) fclose(out);
+ if (res==Z_OK)
+ {
+ std::ifstream f(buf, std::ios::in);
+ char buf[4096];
+ while (f.getline(buf,sizeof(buf))) on_nntp_line_process(nntp,StringView(buf));
+ }
+
+}
+
unsigned long
TaskXOver :: get_bytes_remaining () const
{
diff --git a/pan/tasks/task-xover.h b/pan/tasks/task-xover.h
index 26633a6..ba146e1 100644
--- a/pan/tasks/task-xover.h
+++ b/pan/tasks/task-xover.h
@@ -68,6 +68,7 @@ namespace pan
server_to_minitasks_t _server_to_minitasks;
private: // implementation
+ void process_headers (NNTP*);
Data& _data;
const Quark _group;
std::string _short_group_name;
@@ -85,6 +86,10 @@ namespace pan
unsigned long _articles_so_far;
unsigned long _lines_so_far;
unsigned long _total_minitasks;
+ int _running_minitasks;
+ bool _xzver;
+// FILE * _headers;
+ std::ofstream _headers;
};
}
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
