[pan2/xzver_support: 44/47] dsfsdf
- From: Heinrich MÃller <henmull src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [pan2/xzver_support: 44/47] dsfsdf
- Date: Sun, 10 Jun 2012 19:20:12 +0000 (UTC)
commit 1681de597b994b900cbe35400a92529a2a477d22
Author: Heinrich MÃller <henmull src gnome org>
Date: Sun Jun 10 20:39:51 2012 +0200
dsfsdf
pan/gui/gui.cc | 2 +-
pan/gui/pan.cc | 4 +-
pan/gui/server-ui.cc | 76 ++++++++++++-
pan/tasks/Makefile.am | 2 +-
pan/tasks/decoder.cc | 19 ++-
pan/tasks/encoder.cc | 2 +-
pan/tasks/encoder.h | 2 +-
pan/tasks/nntp.cc | 23 +++-
pan/tasks/nntp.h | 10 +-
pan/tasks/task-xover.cc | 185 +++++++++--------------------
pan/tasks/task-xover.h | 13 ++-
pan/tasks/xzver-decoder.cc | 278 +++++++++++++++++++++++++++++++++++---------
pan/tasks/xzver-decoder.h | 67 +++++------
13 files changed, 435 insertions(+), 248 deletions(-)
---
diff --git a/pan/gui/gui.cc b/pan/gui/gui.cc
index 041397f..c7b430f 100644
--- a/pan/gui/gui.cc
+++ b/pan/gui/gui.cc
@@ -1505,7 +1505,7 @@ void GUI :: do_delete_article ()
void GUI :: do_clear_article_cache ()
{
_cache.clear ();
- _encode_cache.clear();
+// _encode_cache.clear();
}
void GUI :: do_mark_article_read ()
diff --git a/pan/gui/pan.cc b/pan/gui/pan.cc
index 513aca6..ac74b4e 100644
--- a/pan/gui/pan.cc
+++ b/pan/gui/pan.cc
@@ -861,8 +861,6 @@ main (int argc, char *argv[])
{ usage (); return EXIT_SUCCESS; }
else if (!strcmp(tok, "--verbose") )
verbosed = true;
- else if (!strcmp(tok,"--xzver"))
- _xzver_support = true;
else {
nzb = true;
nzb_files.push_back (tok);
@@ -1045,7 +1043,7 @@ main (int argc, char *argv[])
if (prefs.get_flag("clear-article-cache-on-shutdown", false)) {
cache.clear ();
- encode_cache.clear();
+// encode_cache.clear();
}
delete queue_and_gui;
diff --git a/pan/gui/server-ui.cc b/pan/gui/server-ui.cc
index e0d9c8b..b2c8ded 100644
--- a/pan/gui/server-ui.cc
+++ b/pan/gui/server-ui.cc
@@ -52,6 +52,7 @@ using namespace pan;
namespace
{
+
struct ServerEditDialog
{
Data& data;
@@ -69,7 +70,10 @@ namespace
GtkWidget * rank_combo;
GtkWidget * ssl_combo;
GtkWidget * always_trust_checkbox;
+ GtkWidget * compression_checkbox;
ServerEditDialog (Data& d, Queue& q, Prefs& p): data(d), queue(q), prefs(p) {}
+ CompressionType compressiontype;
+
};
void pan_entry_set_text (GtkWidget * w, const StringView& v)
@@ -98,15 +102,64 @@ namespace
void ssl_changed_cb(GtkComboBox* w, ServerEditDialog* d)
{
+ if (!gtk_widget_get_realized(d->dialog)) return;
int ssl(0);
#ifdef HAVE_GNUTLS
GtkTreeIter iter;
if (gtk_combo_box_get_active_iter (w, &iter))
gtk_tree_model_get (gtk_combo_box_get_model(w), &iter, 1, &ssl, -1);
+ if (ssl==0)
+ {
+ gtk_widget_hide(d->always_trust_checkbox);
+ }
+ else
+ {
+ gtk_widget_show(d->always_trust_checkbox);
+ }
pan_spin_button_set (d->port_spin, ssl == 0 ? STD_NNTP_PORT : STD_SSL_PORT);
#endif
}
+ void address_entry_changed_cb (GtkEditable *editable,
+ gpointer gp)
+ {
+ ServerEditDialog* d (static_cast<ServerEditDialog*>(gp));
+ if (!d) return;
+
+ gchar* text = gtk_editable_get_chars (editable, 0, -1);
+ StringView t(text);
+
+ CompressionType type(NONE);
+
+ // 0 == NONE
+
+ if (t.strstr("astraweb")) // 1
+ {
+ type = XZVER;
+ }
+ if (t.strstr("giganews")) // 2
+ {
+ type = XFEATURE;
+ }
+ char* others[] = {"newshosting", "easynews","usenetserver" };
+ for (int i= 0; i < G_N_ELEMENTS(others); i++)
+ {
+ if (t.strstr(others[i]))
+ {
+ type = DIABLO; // 3
+ break;
+ }
+ }
+ d->compressiontype = type;
+
+ if (type != NONE)
+ gtk_widget_show(d->compression_checkbox);
+ else
+ gtk_widget_hide(d->compression_checkbox);
+
+ gtk_toggle_button_set_active (GTK_TOGGLE_BUTTON(d->compression_checkbox), type != NONE);
+ }
+
void
edit_dialog_populate (Data&, Prefs& prefs, const Quark& server, ServerEditDialog * d)
{
@@ -117,6 +170,7 @@ namespace
d->server = server;
int port(STD_NNTP_PORT), max_conn(4), age(31*3), rank(1), ssl(0), trust(0);
+ CompressionType compression(NONE);
std::string addr, user, cert;
gchar* pass(NULL);
if (!server.empty()) {
@@ -128,6 +182,7 @@ namespace
ssl = d->data.get_server_ssl_support(server);
cert = d->data.get_server_cert(server);
d->data.get_server_trust (server, trust);
+ d->data.get_server_compression_type (server, compression);
}
pan_entry_set_text (d->address_entry, addr);
@@ -136,6 +191,11 @@ namespace
pan_entry_set_text (d->auth_username_entry, user);
pan_entry_set_text (d->auth_password_entry, pass);
d->cert = cert;
+ d->compressiontype = compression;
+ std::cerr<<compression<<"\n";
+
+ gtk_toggle_button_set_active (GTK_TOGGLE_BUTTON(d->compression_checkbox), d->compressiontype != NONE);
+
// set the age combobox
GtkComboBox * combo (GTK_COMBO_BOX (d->expiration_age_combo));
@@ -221,6 +281,8 @@ namespace
trust = gtk_toggle_button_get_active(GTK_TOGGLE_BUTTON(d->always_trust_checkbox)) ? 1 : 0;
#endif
+ int header_comp = gtk_toggle_button_get_active(GTK_TOGGLE_BUTTON(d->compression_checkbox)) ? 1 : 0;
+
const char * err_msg (0);
if (addr.empty())
err_msg = _("Please specify the server's address.");
@@ -245,7 +307,7 @@ namespace
d->data.set_server_ssl_support(d->server, ssl);
d->data.set_server_cert(d->server,cert);
d->data.set_server_trust(d->server,trust);
-
+ d->data.set_server_compression_type(d->server, header_comp);
d->data.save_server_info(d->server);
d->queue.upkeep ();
@@ -321,6 +383,7 @@ pan :: server_edit_dialog_new (Data& data, Queue& queue, Prefs& prefs, GtkWindow
GtkWidget * w = d->address_entry = gtk_entry_new ();
gtk_widget_set_tooltip_text( w, _("The news server's actual address, e.g. \"news.mynewsserver.com\""));
HIG::workarea_add_row (t, &row, _("_Address:"), w, NULL);
+ g_signal_connect (w, "changed", G_CALLBACK(address_entry_changed_cb), d);
GtkAdjustment * a = GTK_ADJUSTMENT (gtk_adjustment_new (1.0, 1.0, ULONG_MAX, 1.0, 1.0, 0.0));
w = d->port_spin = gtk_spin_button_new (GTK_ADJUSTMENT(a), 1.0, 0u);
@@ -397,6 +460,10 @@ pan :: server_edit_dialog_new (Data& data, Queue& queue, Prefs& prefs, GtkWindow
gtk_widget_set_tooltip_text( e, _("Fallback servers are used for articles that can't be found on the primaries. One common approach is to use free servers as primaries and subscription servers as fallbacks."));
HIG::workarea_add_row (t, &row, e, w);
+ // header compression options
+ d->compression_checkbox = w = gtk_check_button_new_with_label (_("Enable header compression for speedup"));
+ HIG::workarea_add_row (t, &row, e, w);
+
// ssl 3.0 option
#ifdef HAVE_GNUTLS
// select ssl/plaintext
@@ -441,6 +508,13 @@ pan :: server_edit_dialog_new (Data& data, Queue& queue, Prefs& prefs, GtkWindow
d->server = server;
edit_dialog_populate (data, prefs, server, d);
gtk_widget_show_all (d->dialog);
+
+ // perhaps hide compression checkbox
+ if (d->compressiontype==NONE)
+ {
+ gtk_widget_hide(d->compression_checkbox);
+ }
+
return d->dialog;
}
diff --git a/pan/tasks/Makefile.am b/pan/tasks/Makefile.am
index 9841de8..edf1e1b 100644
--- a/pan/tasks/Makefile.am
+++ b/pan/tasks/Makefile.am
@@ -1,6 +1,6 @@
AM_CPPFLAGS = -I top_srcdir@ @GMIME_CFLAGS@ @GLIB_CFLAGS@ @GNUTLS_CFLAGS@ @GTK_CFLAGS@ @ZLIB_CFLAGS@
-AM_LDFLAGS = ../../uulib/libuu.a @GNUTLS_LIBS@ @ZLIB_LIBS@ @GTK_LIBS@
+AM_LDFLAGS = ../../uulib/libuu.a @GNUTLS_LIBS@ @ZLIB_LIBS@
noinst_LIBRARIES = libtasks.a
diff --git a/pan/tasks/decoder.cc b/pan/tasks/decoder.cc
index 6b35f8a..b3c0864 100644
--- a/pan/tasks/decoder.cc
+++ b/pan/tasks/decoder.cc
@@ -290,9 +290,10 @@ int
Decoder :: uu_busy_poll (void * d, uuprogress *p)
{
Decoder * self (static_cast<Decoder*>(d));
+ if (!self) return true;
self->mut.lock();
- self->percent = self->get_percentage(*p);
- self->current_file = p->curfile;
+ self->percent = self->get_percentage(*p);
+ self->current_file = p->curfile;
self->mut.unlock();
return self->was_cancelled(); // returning true tells uulib to abort
@@ -303,17 +304,21 @@ gboolean
Decoder :: progress_update_timer_func (gpointer decoder)
{
Decoder *self = static_cast<Decoder *>(decoder);
+ if (!self) return false;
Task *task = self->task;
if (!task || self->was_cancelled()) return false;
self->mut.lock();
- const double percent (self->percent);
- const std::string f (content_to_utf8 (self->current_file));
+ const double percent (self->percent);
+ if (!self->current_file.empty())
+ {
+ const std::string f (content_to_utf8 (self->current_file));
+ if (!f.empty())
+ task->set_status_va (_("Decoding %s"), f.c_str());
+ }
+ task->set_step(int(percent));
self->mut.unlock();
- task->set_step(int(percent));
- task->set_status_va (_("Decoding %s"), f.c_str());
-
return true; // keep timer func running
}
diff --git a/pan/tasks/encoder.cc b/pan/tasks/encoder.cc
index 887ef76..21900c3 100644
--- a/pan/tasks/encoder.cc
+++ b/pan/tasks/encoder.cc
@@ -262,9 +262,9 @@ Encoder :: progress_update_timer_func (gpointer decoder)
self->mut.lock();
const double percent (self->percent);
const std::string f (content_to_utf8 (self->current_file));
+ task->set_step(int(percent));
self->mut.unlock();
- task->set_step(int(percent));
task->set_status_va (_("Encoding %s"), f.c_str());
return true; // keep timer func running
diff --git a/pan/tasks/encoder.h b/pan/tasks/encoder.h
index 89eb821..3ed5ac2 100644
--- a/pan/tasks/encoder.h
+++ b/pan/tasks/encoder.h
@@ -48,7 +48,7 @@ namespace pan
* @author Charles Kerr <charles rebelbase com>
* @ingroup tasks
* @see Queue
- * @see TaskArticle
+ * @see TaskUpload
*/
class Encoder: public WorkerPool::Worker
{
diff --git a/pan/tasks/nntp.cc b/pan/tasks/nntp.cc
index 557fa20..822bdb5 100644
--- a/pan/tasks/nntp.cc
+++ b/pan/tasks/nntp.cc
@@ -63,7 +63,6 @@ NNTP :: fire_done_func (Health health, const StringView& response)
debug ("I (" << (void*)this << ") am setting my _listener to 0");
_listener = 0;
l->on_nntp_done (this, health, response);
- _xzver = false;
}
}
@@ -79,7 +78,7 @@ NNTP :: on_socket_response (Socket * sock UNUSED, const StringView& line_in)
StringView line (line_in);
// strip off trailing \r\n
- if (line.len>=2 && line.str[line.len-2]=='\r' && line.str[line.len-1]=='\n' && !_xzver)
+ if (line.len>=2 && line.str[line.len-2]=='\r' && line.str[line.len-1]=='\n')
line.truncate (line.len-2);
// std::cerr <<"_nntp_response_text: " << _nntp_response_text<<std::endl;
@@ -94,7 +93,7 @@ NNTP :: on_socket_response (Socket * sock UNUSED, const StringView& line_in)
{
state = CMD_MORE;
- if (line.len>=2 && line.str[0]=='.' && line.str[1]=='.' && !_xzver) // rfc 977: 2.4.1
+ if (line.len>=2 && line.str[0]=='.' && line.str[1]=='.') // rfc 977: 2.4.1
line.rtruncate (line.len-1);
assert (_listener != 0);
@@ -332,7 +331,6 @@ NNTP :: xzver (const Quark & group,
Listener * l)
{
_listener = l;
- _xzver = true;
if (group != _group)
_commands.push_back (build_command ("GROUP %s\r\n", group.c_str()));
@@ -343,6 +341,23 @@ NNTP :: xzver (const Quark & group,
}
void
+NNTP :: xfeat (const Quark & group,
+ uint64_t low,
+ uint64_t high,
+ Listener * l)
+{
+ _listener = l;
+
+ if (group != _group)
+ _commands.push_back (build_command ("GROUP %s\r\n", group.c_str()));
+
+ _commands.push_back ("XFEATURE COMPRESS GZIP");
+ _commands.push_back (build_command ("XOVER %"G_GUINT64_FORMAT"-%"G_GUINT64_FORMAT"\r\n", low, high));
+
+ write_next_command ();
+}
+
+void
NNTP :: xover_count_only (const Quark & group,
Listener * l)
{
diff --git a/pan/tasks/nntp.h b/pan/tasks/nntp.h
index 178d376..73793af 100644
--- a/pan/tasks/nntp.h
+++ b/pan/tasks/nntp.h
@@ -153,8 +153,7 @@ namespace pan
_listener(0),
_username(username),
_password(password),
- _nntp_response_text(false),
- _xzver(false)
+ _nntp_response_text(false)
{}
virtual ~NNTP ()
@@ -200,6 +199,11 @@ namespace pan
uint64_t high,
Listener * l);
+ void xfeat (const Quark & group,
+ uint64_t low,
+ uint64_t high,
+ Listener * l);
+
/**
* Executes an XOVER command: "XOVER" to count
* the xover numbers internally
@@ -330,8 +334,6 @@ namespace pan
/** True if the server told us that we're getting a list back. */
bool _nntp_response_text;
- bool _xzver;
-
typedef std::deque<std::string> strings_t;
strings_t _commands;
std::string _previous_command;
diff --git a/pan/tasks/task-xover.cc b/pan/tasks/task-xover.cc
index 18403ed..26e7bc2 100644
--- a/pan/tasks/task-xover.cc
+++ b/pan/tasks/task-xover.cc
@@ -12,7 +12,7 @@
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
- * You should have received a copy of the GNU General Public License
+ * You should _have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
@@ -45,6 +45,7 @@ using namespace pan;
namespace
{
+
std::string
get_short_name (const StringView& in)
{
@@ -113,17 +114,11 @@ TaskXOver :: TaskXOver (Data & data,
_articles_so_far (0ul),
_total_minitasks (0),
_running_minitasks (0),
- _xzver (false),//_xzver_support),
_decoder(0),
- _decoder_has_run (false),
- _extract_done(false)
+ _decoder_has_run (false)
{
- if (_xzver)
- {
- char buf[4096];
- _headers.open(build_cachename(buf,sizeof(buf), "xzver_test"), std::ios::out | std::ios::binary);
- }
+
debug ("ctor for " << group);
@@ -146,6 +141,12 @@ TaskXOver :: TaskXOver (Data & data,
update_work ();
}
+void
+TaskXOver :: setHigh(const Quark& server, uint64_t& h)
+{
+ _high[server] = h;
+}
+
TaskXOver :: ~TaskXOver ()
{
if (_group_xover_is_reffed) {
@@ -155,6 +156,8 @@ TaskXOver :: ~TaskXOver ()
}
if (_decoder)
_decoder->cancel_silently();
+
+ _data.fire_group_entered(_group, 1, 0);
}
void
@@ -162,6 +165,10 @@ TaskXOver :: use_nntp (NNTP* nntp)
{
const Quark& server (nntp->_server);
debug ("got an nntp from " << nntp->_server);
+ CompressionType type;
+ _data.get_server_compression_type(nntp->_server, type);
+
+ _compression_enabled = type != NONE;
// if this is the first nntp we've gotten, ref the xover data
if (!_group_xover_is_reffed) {
@@ -172,7 +179,7 @@ TaskXOver :: use_nntp (NNTP* nntp)
MiniTasks_t& minitasks (_server_to_minitasks[server]);
if (minitasks.empty())
{
- debug ("That's interesting, I got a socket for " << server << " but have no use for it!");
+ debug ("That's interesting, I got a socket for " << server << " but _have no use for it!");
_state._servers.erase (server);
check_in (nntp, OK);
}
@@ -189,8 +196,16 @@ TaskXOver :: use_nntp (NNTP* nntp)
case MiniTask::XOVER:
debug ("XOVER " << mt._low << '-' << mt._high << " to " << server);
_last_xover_number[nntp] = mt._low;
- if (_xzver)
- nntp->xzver (_group, mt._low, mt._high, this);
+ if (_compression_enabled)
+ {
+ char buf[4096];
+ _headers.open(build_cachename(buf,sizeof(buf), "xzver_test"), std::ios::out | std::ios::binary);
+// if (type == XZVER)
+ nntp->xzver (_group, mt._low, mt._high, this);
+// if (type == XFEATURE)
+// nntp->xfeat (_group, mt._low, mt._high, this);
+ // TODO diablo
+ }
else
nntp->xover (_group, mt._low, mt._high, this);
--_running_minitasks;
@@ -245,12 +260,12 @@ TaskXOver :: on_nntp_group (NNTP * nntp,
if (l <= high)
{
- std::cerr << LINE_ID << " okay, I'll try to get articles in [" << l << "..." << h << ']' << std::endl;
+// std::cerr << LINE_ID << " okay, I'll try to get articles in [" << l << "..." << h << ']' << std::endl;
add_steps (h-l);
MiniTasks_t& minitasks (_server_to_minitasks[servername]);
- if (_xzver)
+ if (_compression_enabled)
{
MiniTask mt (MiniTask::XOVER, l, h);
minitasks.push_front (mt);
@@ -318,12 +333,13 @@ void
TaskXOver :: on_nntp_line (NNTP * nntp,
const StringView & line)
{
- if (_xzver ) {
+ if (_compression_enabled) {
if (line.strstr("=ybegin line=128"))
+ {
_headers << line.str << " name=xzver_decoded\n";
+ }
else
_headers << line.str <<"\n";
- increment_step(1);
}
else
{
@@ -336,6 +352,8 @@ TaskXOver :: on_nntp_line_process (NNTP * nntp,
const StringView & line)
{
+ _dbg<<line.str<<"\n";
+
pan_return_if_fail (nntp != 0);
pan_return_if_fail (!nntp->_server.empty());
pan_return_if_fail (!nntp->_group.empty());
@@ -356,6 +374,7 @@ TaskXOver :: on_nntp_line_process (NNTP * nntp,
//handle multiple "References:"-message-ids correctly.
ok = ok && l.pop_token (tmp, '\t');
+ ref += tmp;
do
{
if (tmp.empty()) continue;
@@ -382,8 +401,10 @@ TaskXOver :: on_nntp_line_process (NNTP * nntp,
|| author.empty() // missing author
|| date.empty() // missing date
|| mid.empty() // missing mid
- || mid.front()!='<' // corrupt mid
- || (!ref.empty() && ref[0] != '<'))
+ || mid.front()!='<') // corrupt mid
+ /// Concerning bug : https://bugzilla.gnome.org/show_bug.cgi?id=650042
+ /// Even if we didn't get a proper reference here, continue.
+ //|| (!ref.empty() && ref.front()!='<'))
return;
// if news server doesn't provide an xref, fake one
@@ -394,11 +415,6 @@ TaskXOver :: on_nntp_line_process (NNTP * nntp,
nntp->_group.c_str(),
number);
- uint64_t& h (_high[nntp->_server]);
- h = std::max (h, number);
-
-// std::cerr<<h<<" "<<number<<"\n";
-
const char * fallback_charset = NULL; // FIXME
// are we done?
@@ -419,11 +435,12 @@ TaskXOver :: on_nntp_line_process (NNTP * nntp,
if (article)
++_articles_so_far;
- // emit a status update
+// emit a status update
uint64_t& prev = _last_xover_number[nntp];
+ uint64_t& h (_high[nntp->_server]);
+ h = std::max (h, number);
increment_step (number - prev);
-// std::cerr<<"stepping "<<number<<" "<<prev<<" "<<get_steps()<<" "<<get_step()<<" "<<get_progress_of_100()<<"\n";
prev = number;
if (!(_parts_so_far % 500))
set_status_va (_("%s (%lu parts, %lu articles)"), _short_group_name.c_str(), _parts_so_far, _articles_so_far);
@@ -437,8 +454,12 @@ TaskXOver :: on_nntp_done (NNTP * nntp,
Health health,
const StringView & response UNUSED)
{
- if (_running_minitasks == 0) //process_headers(nntp);
+ if (_running_minitasks == 0 && _compression_enabled)
+ {
_backup_nntp = new NNTP(nntp->_server, nntp->get_username(), "", 0);
+ _backup_nntp->_group = nntp->_group;
+ }
+
update_work (true);
check_in (nntp, health);
}
@@ -460,20 +481,20 @@ TaskXOver :: update_work (bool subtract_one_from_nntp_count)
_state.set_need_nntp (servers);
else if (nntp_count)
_state.set_working ();
- else if (!_decoder && !_decoder_has_run && _xzver) {
+ else if (!_decoder && !_decoder_has_run && _compression_enabled) {
+ std::cerr<<"decode\n";
_headers.close();
_state.set_need_xzverdecoder ();
- set_step(0);
- } else if (_decoder_has_run && !_extract_done && _xzver) {
- _state.set_working();
- process_headers(_backup_nntp);
- } else if (_decoder_has_run && _extract_done && _xzver) {
+ } else if (_decoder_has_run) {
+ std::cerr<<"finished\n";
_state.set_completed();
set_finished (OK);
- } else if (!_xzver)
+ _dbg.close();
+ } else if (!_compression_enabled)
{
_state.set_completed();
set_finished (OK);
+ _dbg.close();
} else assert(0 && "hm, missed a state.");
}
@@ -484,10 +505,9 @@ TaskXOver:: use_decoder (Decoder* decoder)
check_in (decoder);
_decoder = static_cast<XZVERDecoder*>(decoder);
- init_steps(100);
_state.set_working();
- _decoder->enqueue (this);
- set_status_va (_("Decoding XZVER Headers"));
+
+ _decoder->enqueue (this, &_high[_backup_nntp->_server], _backup_nntp, &_data);
debug ("decoder thread was free, enqueued work");
}
@@ -498,96 +518,6 @@ TaskXOver :: stop ()
_decoder->cancel();
}
-namespace
-{
- #define CHUNK 16384
-
- int inflate_xzver(FILE *source, FILE *dest)
- {
- int ret;
- unsigned have;
- z_stream strm;
- unsigned char in[CHUNK];
- unsigned char out[CHUNK];
-
- /* allocate inflate state */
- strm.zalloc = Z_NULL;
- strm.zfree = Z_NULL;
- strm.opaque = Z_NULL;
- strm.avail_in = 0;
- strm.next_in = Z_NULL;
- ret = inflateInit2(&strm,-15); //raw inflate
- if (ret != Z_OK)
- return ret;
-
- /* decompress until deflate stream ends or end of file */
- do {
- strm.avail_in = fread(in, 1, CHUNK, source);
- if (ferror(source)) {
- (void)inflateEnd(&strm);
- return Z_ERRNO;
- }
- if (strm.avail_in == 0)
- break;
- strm.next_in = in;
-
- /* run inflate() on input until output buffer not full */
- do {
- strm.avail_out = CHUNK;
- strm.next_out = out;
- ret = inflate(&strm, Z_NO_FLUSH);
- assert(ret != Z_STREAM_ERROR); /* state not clobbered */
- switch (ret) {
- case Z_NEED_DICT:
- ret = Z_DATA_ERROR; /* and fall through */
- case Z_DATA_ERROR:
- case Z_MEM_ERROR:
- (void)inflateEnd(&strm);
- return ret;
- }
- have = CHUNK - strm.avail_out;
- if (fwrite(out, 1, have, dest) != have || ferror(dest)) {
- (void)inflateEnd(&strm);
- return Z_ERRNO;
- }
- } while (strm.avail_out == 0);
-
- /* done when inflate() says it's done */
- } while (ret != Z_STREAM_END);
-
- /* clean up and return */
- (void)inflateEnd(&strm);
- return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR;
- }
-}
-
-void
-TaskXOver :: process_headers (NNTP* nntp)
-{
- char buf[4096];
-
- build_cachename(buf,sizeof(buf), "xzver_decoded");
-
- /* raw zlib inflate */
- FILE * in = fopen (buf, "rb");
- FILE * out = fopen (build_cachename(buf,sizeof(buf), "xzver_out"), "wb");
- int res(Z_OK);
- if (in && out) res = inflate_xzver (in,out);
-
- /* feed to on_nntp_line */
- if (in) fclose(in);
- if (out) fclose(out);
- if (res==Z_OK)
- {
- std::ifstream f(buf, std::ios::in);
- char buf[4096];
- while (f.getline(buf,sizeof(buf))) on_nntp_line_process(nntp,StringView(buf));
- }
- _extract_done = true;
- update_work();
-}
-
-
unsigned long
TaskXOver :: get_bytes_remaining () const
{
@@ -609,8 +539,6 @@ TaskXOver :: on_worker_done (bool cancelled)
assert(_decoder);
if (!_decoder) return;
- std::cerr<<"worker done\n";
-
if (!cancelled)
{
// the decoder is done... catch up on all housekeeping
@@ -641,7 +569,6 @@ TaskXOver :: on_worker_done (bool cancelled)
_state.set_health (ERR_LOCAL);
else {
_state.set_completed();
- set_step (100);
_decoder_has_run = true;
}
}
diff --git a/pan/tasks/task-xover.h b/pan/tasks/task-xover.h
index 8f8051e..8f1d7f2 100644
--- a/pan/tasks/task-xover.h
+++ b/pan/tasks/task-xover.h
@@ -23,6 +23,7 @@
#include <map>
#include <vector>
#include <sstream>
+#include <zlib.h>
#include <pan/data/data.h>
#include <pan/tasks/task.h>
@@ -98,15 +99,21 @@ namespace pan
unsigned long _articles_so_far;
unsigned long _lines_so_far;
unsigned long _total_minitasks;
- int _running_minitasks;
- bool _xzver;
+ unsigned long _running_minitasks;
+ bool _compression_enabled;
std::ofstream _headers;
NNTP * _backup_nntp;
+ CompressionType _compressiontype;
+
friend class XZVERDecoder;
XZVERDecoder * _decoder;
bool _decoder_has_run;
- bool _extract_done;
+
+ std::ofstream _dbg;
+
+ public:
+ void setHigh(const Quark& server, uint64_t& h);
};
}
diff --git a/pan/tasks/xzver-decoder.cc b/pan/tasks/xzver-decoder.cc
index d2be871..87eb7ff 100644
--- a/pan/tasks/xzver-decoder.cc
+++ b/pan/tasks/xzver-decoder.cc
@@ -48,6 +48,8 @@ using namespace pan;
XZVERDecoder :: XZVERDecoder (WorkerPool& pool) :
Decoder(pool)
{
+// out = g_string_sized_new (CHUNK*1024);
+ _cnt = 0;
}
XZVERDecoder :: ~XZVERDecoder()
@@ -56,16 +58,17 @@ XZVERDecoder :: ~XZVERDecoder()
}
void
-XZVERDecoder :: enqueue (TaskXOver * task)
+XZVERDecoder :: enqueue (TaskXOver * task, uint64_t* h, NNTP * nntp, Data* data)
{
+ _cnt = 0;
+
disable_progress_update ();
this->xtask = task;
+ this->nntp = nntp;
+ this->data = data;
+ this->high = *h;
- mark_read = false;
- percent = 0;
- num_scanned_files = 0;
- current_file.clear ();
log_infos.clear();
log_errors.clear();
@@ -83,6 +86,145 @@ namespace
}
}
+namespace
+{
+ unsigned long view_to_ul (const StringView& view)
+ {
+ unsigned long ul = 0ul;
+
+ if (!view.empty()) {
+ errno = 0;
+ ul = strtoul (view.str, 0, 10);
+ if (errno)
+ ul = 0ul;
+ }
+
+ return ul;
+ }
+ uint64_t view_to_ull (const StringView& view)
+ {
+ uint64_t ul = 0ul;
+
+ if (!view.empty()) {
+ errno = 0;
+ ul = g_ascii_strtoull (view.str, 0, 10);
+ if (errno)
+ ul = 0ul;
+ }
+
+ return ul;
+ }
+
+ bool header_is_nonencoded_utf8 (const StringView& in)
+ {
+ const bool is_nonencoded (!in.strstr("=?"));
+ const bool is_utf8 (g_utf8_validate (in.str, in.len, 0));
+ return is_nonencoded && is_utf8;
+ }
+}
+
+void
+XZVERDecoder :: on_nntp_batch_process (StringView & line)
+{
+ unsigned int lines=0u;
+ unsigned long bytes=0ul;
+ uint64_t number=0;
+
+ StringView subj, author, date, mid, tmp, xref;
+ StringView& l = line;
+ std::string ref;
+
+ bool ok = !l.empty();
+ ok = ok && l.pop_token (tmp, '\t'); if (ok) number = view_to_ull (tmp); tmp.clear();
+ ok = ok && l.pop_token (subj, '\t'); if (ok) subj.trim ();
+ ok = ok && l.pop_token (author, '\t'); if (ok) author.trim ();
+ ok = ok && l.pop_token (date, '\t'); if (ok) date.trim ();
+ ok = ok && l.pop_token (mid, '\t'); if (ok) mid.trim ();
+
+ //handle multiple "References:"-message-ids correctly.
+ ok = ok && l.pop_token (tmp, '\t');
+ do
+ {
+ if (tmp.empty()) continue;
+ if (tmp.front() == '<')
+ {
+ tmp.trim();
+ ref += tmp;
+ tmp.clear();
+ } else break;
+ } while ((ok = ok && l.pop_token (tmp, '\t'))) ;
+
+ if (ok) bytes = view_to_ul (tmp); tmp.clear();
+ ok = ok && l.pop_token (tmp, '\t'); if (ok) lines = view_to_ul (tmp);
+ ok = ok && l.pop_token (xref, '\t'); if (ok) xref.trim ();
+
+ if (xref.len>6 && !strncmp(xref.str,"Xref: ", 6)) {
+ xref = xref.substr (xref.str+6, 0);
+ xref.trim ();
+ }
+
+ // is this header corrupt?
+ if (!number // missing number
+ || subj.empty() // missing subject
+ || author.empty() // missing author
+ || date.empty() // missing date
+ || mid.empty() // missing mid
+ || mid.front()!='<') // corrupt mid
+ /// Concerning bug : https://bugzilla.gnome.org/show_bug.cgi?id=650042
+ /// Even if we didn't get a proper reference here, continue.
+ //|| (!ref.empty() && ref.front()!='<'))
+ return;
+
+ // if news server doesn't provide an xref, fake one
+ char * buf (0);
+ if (xref.empty())
+ xref = buf = g_strdup_printf ("%s %s:%"G_GUINT64_FORMAT,
+ nntp->_server.c_str(),
+ nntp->_group.c_str(),
+ number);
+
+ const char * fallback_charset = NULL; // FIXME
+ const time_t time_posted = g_mime_utils_header_decode_date (date.str, NULL);
+
+ data->xover_add (
+ nntp->_server, nntp->_group,
+ (header_is_nonencoded_utf8(subj) ? subj : header_to_utf8(subj,fallback_charset).c_str()),
+ (header_is_nonencoded_utf8(author) ? author : header_to_utf8(author,fallback_charset).c_str()),
+ time_posted, mid, StringView(ref), bytes, lines, xref);
+
+ high = std::max (high, number);
+
+ g_free (buf);
+}
+
+void
+XZVERDecoder :: find_line()
+{
+ StringView ret;
+// char * buf = out->str;
+// int pos(0), _apos(0);
+// while (*buf) {
+// if (*buf == '\n')
+// {
+// int _pos(std::min(pos+1,(int)out->len));
+// ret.assign(out->str, _pos);
+// g_string_erase (out, 0, _pos);
+// xtask->on_nntp_line_process(nntp, ret);
+// _cnt++;
+// }
+// ++pos; ++buf;
+// }
+ while(!stream.eof())
+ {
+ std::string out;
+ getline(stream, out);
+ ret.assign(out);
+ xtask->on_nntp_line_process(nntp, ret);
+ _cnt++;
+ }
+}
+
+
// save article IN A WORKER THREAD to avoid network stalls
void
XZVERDecoder :: do_work()
@@ -90,81 +232,103 @@ XZVERDecoder :: do_work()
const int bufsz = 4096;
char buf[bufsz];
- enable_progress_update();
+ disable_progress_update();
int res;
if (((res = UUInitialize())) != UURET_OK)
log_errors.push_back(_("Error initializing uulib")); // log error
else
{
- UUSetMsgCallback (this, uu_log);
- UUSetOption (UUOPT_DESPERATE, 0, NULL); // keep incompletes; they're useful to par2
+// UUSetMsgCallback (this, uu_log);
+ UUSetOption (UUOPT_DESPERATE, 0, NULL);
UUSetOption (UUOPT_IGNMODE, 1, NULL); // don't save file as executable
- UUSetBusyCallback (this, uu_busy_poll, 500); // .5 secs busy poll?
+// UUSetBusyCallback (this, uu_busy_poll, 500); // .5 secs busy poll?
UULoadFile (build_cachename(buf,sizeof(buf), "xzver_test"), 0, 1);
UUDecodeFile (UUGetFileListItem (0), build_cachename(buf,sizeof(buf), "xzver_decoded"));
UUCleanUp ();
}
- disable_progress_update();
-}
+// disable_progress_update();
-/***
-****
-***/
+ _strm.zalloc = Z_NULL;
+ _strm.zfree = Z_NULL;
+ _strm.opaque = Z_NULL;
+ _strm.avail_in = 0;
+ _strm.next_in = Z_NULL;
-void
-XZVERDecoder :: uu_log (void* data, char* message, int severity)
-{
- XZVERDecoder *self = static_cast<XZVERDecoder *>(data);
- char * pch (g_locale_to_utf8 (message, -1, 0, 0, 0));
+ _zret = inflateInit2(&_strm,-MAX_WBITS); //raw inflate
+ if (_zret != Z_OK)
+ {
+ log_errors.push_back(_("Error initializing zlib deflate"));
+ return;
+ }
- if (severity >= UUMSG_WARNING)
- self->file_errors.push_back (pch ? pch : message);
+ std::ifstream headers;
+ FILE * in = fopen(build_cachename(buf,sizeof(buf), "xzver_decoded"), "rb");
- if (severity >= UUMSG_ERROR)
- self->log_errors.push_back (pch ? pch : message);
- else if (severity >= UUMSG_NOTE)
- self->log_infos.push_back (pch ? pch : message);
+ if (!in)
+ {
+ char tmpbuf[2048];
+ g_snprintf(tmpbuf, sizeof(tmpbuf), _("Error opening header file %s"), buf);
+ log_errors.push_back(tmpbuf);
+ return;
+ }
+
+ int ret;
+ unsigned char fbuf[CHUNK];
+ StringView line;
+
+ do
+ {
+ size_t len = fread(fbuf, sizeof(char), CHUNK, in);
+ if (len==0 || ferror(in)) break;
+ ret = inflate_xzver (len, fbuf);
+// find_line();
+ } while (!feof(in) && _zret == Z_OK);
+
+ find_line();
+
+// std::cerr<<"final "<<out->len<<"\n";
+
+ if (in) fclose(in);
+ (void)inflateEnd(&_strm);
+
+ std::cerr<<"count "<<_cnt<<"\n";
+
+ xtask->setHigh(nntp->_server, high);
- g_free (pch);
}
-double
-XZVERDecoder :: get_percentage (const uuprogress& p) const
+int
+XZVERDecoder :: inflate_xzver (size_t len, unsigned char* buf)
{
- // These should add up to 100.
- // We can tweak these as needed. Calin sees more time spent
- // in COPYING, but I'm seeing it in DECODING, so I've split
- // the difference here and given them the same weight.
- static const double WEIGHT_SCANNING = 10;
- static const double WEIGHT_DECODING = 45;
- static const double WEIGHT_COPYING = 45;
-
- double base = 0;
-
- if (p.action != UUACT_SCANNING)
- base += WEIGHT_SCANNING;
- else {
- const double percent = (100.0 + p.percent);
- return base + (percent / (100.0/WEIGHT_SCANNING));
- }
- if (p.action != UUACT_DECODING)
- base += WEIGHT_DECODING;
- else {
- const double percent = ((100.0) + p.percent);
- return base + (percent / (100.0/WEIGHT_DECODING));
- }
+ InflateChunk ret;
- if (p.action != UUACT_COPYING)
- base += WEIGHT_COPYING;
- else {
- const double percent = p.percent;
- return base + (percent / (100.0/WEIGHT_COPYING));
- }
+ _strm.avail_in = len;
+ _strm.next_in = buf;
+
+ /* run inflate() on input until output buffer not full */
+ do {
+ _strm.avail_out = CHUNK;
+ _strm.next_out = ret.tmpbuf;
+ _zret = inflate(&_strm, Z_NO_FLUSH);
+ assert(_zret != Z_STREAM_ERROR); /* state not clobbered */
+ switch (_zret) {
+ case Z_NEED_DICT:
+ _zret = Z_DATA_ERROR; /* and fall through */
+ case Z_DATA_ERROR:
+ case Z_MEM_ERROR:
+ (void)inflateEnd(&_strm);
+ return _zret;
+ }
+// g_string_append_len(out, (char*)ret.tmpbuf, CHUNK - _strm.avail_out);
+ stream<<ret.tmpbuf;
+
+ } while (_strm.avail_out == 0);
+
+ return _zret == Z_STREAM_END ? Z_OK : Z_STREAM_ERROR;
- return 0;
}
diff --git a/pan/tasks/xzver-decoder.h b/pan/tasks/xzver-decoder.h
index 838d9cb..00ea32f 100644
--- a/pan/tasks/xzver-decoder.h
+++ b/pan/tasks/xzver-decoder.h
@@ -27,7 +27,11 @@
#include <list>
#include <string>
+#include <sstream>
#include <vector>
+
+#include <fstream>
+
#include <pan/general/locking.h>
#include <pan/general/worker-pool.h>
#include <pan/tasks/task-xover.h>
@@ -39,67 +43,58 @@ extern "C" {
namespace pan
{
+ #define CHUNK 16384
class Decoder;
/**
- * Decodes attachments in a worker thread.
- *
+ * Decodes XZVER yenc-encoded and zlib-deflated
+ * headers to process with TaskXOver
+ * @author Heinrich Mueller <heinrich mueller82 gmail com>
* @author Calin Culianu <calin ajvar org>
* @author Charles Kerr <charles rebelbase com>
* @ingroup tasks
* @see Queue
- * @see TaskArticle
+ * @see TaskXOver
*/
class XZVERDecoder: public Decoder
{
public:
+ struct InflateChunk
+ {
+ int ret;
+ unsigned char tmpbuf[CHUNK]; // dbg
+ };
+
XZVERDecoder (WorkerPool&);
~XZVERDecoder ();
typedef std::vector<std::string> strings_t;
- void enqueue (TaskXOver * task);
-
-// public:
-//
-// typedef std::list<std::string> log_t;
-// log_t log_severe, log_errors, log_infos, file_errors;
-// bool mark_read;
-// Health health;
+ void enqueue (TaskXOver * task, uint64_t*, NNTP*, Data*);
protected: // inherited from WorkerPool::Worker
void do_work();
TaskXOver * xtask;
-// private:
-//
-// TaskArticle * task;
-// std::string save_path;
-// strings_t input_files;
-// TaskArticle::SaveMode save_mode;
-// TaskArticle::SaveOptions options;
-// StringView attachment_filename;
-//
-// // These are set in the worker thread and polled in the main thread.
-// Mutex mut;
-// volatile double percent;
-// std::string current_file; // the current file we are decoding, with path
-// volatile int num_scanned_files;
-//
- static void uu_log(void *thiz, char *message, int severity);
- double get_percentage (const uuprogress& p) const;
-// static int uu_busy_poll(void * self, uuprogress *p);
-// /** tell our task about the decode's progress */
-// static gboolean progress_update_timer_func(gpointer decoder);
-//
-// WorkerPool& _worker_pool;
-// int _gsourceid;
-// void disable_progress_update();
-// void enable_progress_update();
+ z_stream _strm;
+ int _zret;
+ NNTP* nntp;
+ int _cnt;
+ Data* data;
+ uint64_t high;
+ GString * out;
+ unsigned char outbuf[4096];
+
+ std::stringstream stream;
+
+ int inflate_xzver (size_t len, unsigned char* buf);
+ void on_nntp_batch_process (StringView&);
+ void find_line();
+
};
}
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
