[pan2: 51/268] encodecache done. debugging ...
- From: Heinrich MÃller <henmull src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [pan2: 51/268] encodecache done. debugging ...
- Date: Mon, 2 Jan 2012 15:42:17 +0000 (UTC)
commit 9cb5c680d6e812169c14b8180f277ea072a281d1
Author: Heinrich Mueller <sphemuel stud informatik uni-erlangen de>
Date: Sat Jun 4 16:58:05 2011 +0200
encodecache done. debugging ...
pan/data-impl/data-impl.cc | 20 ++-
pan/data-impl/data-impl.h | 7 +-
pan/data-impl/task-archive.cc | 2 +-
pan/data/data.h | 1 +
pan/data/encode-cache.cc | 340 +++++++++++++++++++++++++++++++++++++++++
pan/data/encode-cache.h | 123 +++++++++++++++
pan/data/encode-cache.o | Bin 0 -> 495716 bytes
pan/gui/gui.cc | 30 ++--
pan/gui/gui.h | 5 +-
pan/gui/log-ui.cc | 1 -
pan/gui/pan.cc | 16 +-
pan/gui/post-ui.cc | 33 ++--
pan/gui/post-ui.h | 17 +--
pan/tasks/encoder.cc | 49 ++++---
pan/tasks/encoder.h | 22 ++-
pan/tasks/nzb-test.cc | 4 +-
pan/tasks/nzb.cc | 48 ++++---
pan/tasks/nzb.h | 2 +
pan/tasks/task-upload.cc | 127 +++++++--------
pan/tasks/task-upload.h | 55 ++------
20 files changed, 675 insertions(+), 227 deletions(-)
---
diff --git a/pan/data-impl/data-impl.cc b/pan/data-impl/data-impl.cc
index 8cc8acc..f270995 100644
--- a/pan/data-impl/data-impl.cc
+++ b/pan/data-impl/data-impl.cc
@@ -50,20 +50,22 @@ namespace
return path;
}
-// std::string get_encode_cache_path ()
-// {
-// char * pch (g_build_filename (file::get_pan_home().c_str(), "encode-cache", NULL));
-// file :: ensure_dir_exists (pch);
-// std::string path (pch);
-// g_free (pch);
-// return path;
-// }
+ std::string get_encode_cache_path ()
+ {
+ char * pch (g_build_filename (file::get_pan_home().c_str(), "encode-cache", NULL));
+ file :: ensure_dir_exists (pch);
+ std::string path (pch);
+ g_free (pch);
+ return path;
+ }
+
}
+///SEE
DataImpl :: DataImpl (bool unit_test, int cache_megs, DataIO * io):
ProfilesImpl (*io),
_cache (get_cache_path(), cache_megs),
-// _encode_cache (get_encode_cache_path(), cache_megs),
+ _encode_cache (get_encode_cache_path(), cache_megs),
_unit_test (unit_test),
_data_io (io),
_descriptions_loaded (false),
diff --git a/pan/data-impl/data-impl.h b/pan/data-impl/data-impl.h
index cdb9b51..669fafe 100644
--- a/pan/data-impl/data-impl.h
+++ b/pan/data-impl/data-impl.h
@@ -35,6 +35,7 @@
#include <pan/usenet-utils/scorefile.h>
#include <pan/data/article.h>
#include <pan/data/article-cache.h>
+#include <pan/data/encode-cache.h>
#include <pan/data/data.h>
#include <pan/tasks/queue.h>
#include <pan/data-impl/data-io.h>
@@ -72,10 +73,10 @@ namespace pan
virtual ArticleCache& get_cache () { return _cache; }
virtual const ArticleCache& get_cache () const { return _cache; }
-// virtual EncodeCache& get_encode_cache () { return _encode_cache; }
-// virtual const EncodeCache& get_encode_cache () const { return _encode_cache; }
+ virtual EncodeCache& get_encode_cache () { return _encode_cache; }
+ virtual const EncodeCache& get_encode_cache () const { return _encode_cache; }
private:
-// EncodeCache _encode_cache;
+ EncodeCache _encode_cache;
ArticleCache _cache;
private:
diff --git a/pan/data-impl/task-archive.cc b/pan/data-impl/task-archive.cc
index 5c94e4b..1e91dad 100644
--- a/pan/data-impl/task-archive.cc
+++ b/pan/data-impl/task-archive.cc
@@ -53,6 +53,6 @@ DataImpl :: load_tasks (std::vector<Task*>& setme)
full += *it;
char * dir (g_get_current_dir ()); // hmm, maybe we could add a tag to nzb for this?
- NZB :: tasks_from_nzb_string (StringView(full), dir, get_cache(), *this, *this, *this, setme);
+ NZB :: tasks_from_nzb_string (StringView(full), dir, get_cache(), get_encode_cache(), *this, *this, *this, setme);
g_free (dir);
}
diff --git a/pan/data/data.h b/pan/data/data.h
index 4e03785..cea4604 100644
--- a/pan/data/data.h
+++ b/pan/data/data.h
@@ -31,6 +31,7 @@
#include <pan/usenet-utils/scorefile.h>
#include <pan/data/article.h>
#include <pan/data/article-cache.h>
+#include <pan/data/encode-cache.h>
#include <pan/data/server-info.h>
namespace pan
diff --git a/pan/data/encode-cache.cc b/pan/data/encode-cache.cc
new file mode 100644
index 0000000..d9bd86e
--- /dev/null
+++ b/pan/data/encode-cache.cc
@@ -0,0 +1,340 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/*
+ * Pan - A Newsreader for Gtk+
+ * Copyright (C) 2002-2006 Charles Kerr <charles rebelbase com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <config.h>
+#include <fstream>
+#include <sstream>
+
+extern "C"
+{
+ #include <errno.h>
+ #include <sys/types.h>
+ #include <sys/stat.h>
+ #include <unistd.h>
+ #include <dirent.h>
+
+ #include <glib.h>
+ #include <glib/gi18n.h>
+ #include <gmime/gmime.h>
+}
+
+#include <pan/general/debug.h>
+#include <pan/general/file-util.h>
+#include <pan/general/macros.h>
+#include <pan/general/messages.h>
+#include <pan/general/log.h>
+#include <pan/general/string-view.h>
+#include <pan/usenet-utils/mime-utils.h>
+#include "article.h"
+#include "encode-cache.h"
+
+using namespace pan;
+
+/*****
+******
+*****/
+namespace
+{
+ char*
+ message_id_to_filename (char * buf, const Quark& mid)
+ {
+ int partno();
+ g_snprintf (buf, sizeof(buf), "%s.%d", mid.c_str(), partno);
+ std::cerr<<"ec mid to fn "<<buf<<"\n";
+ return buf;
+ }
+
+ int
+ filename_to_message_id (char * buf, int len, const char * basename)
+ {
+ const char * in;
+ char * out;
+ char * pch;
+ char tmp_basename[PATH_MAX];
+
+ // sanity clause
+ pan_return_val_if_fail (basename && *basename, 0);
+ pan_return_val_if_fail (buf!=NULL, 0);
+ pan_return_val_if_fail (len>0, 0);
+
+ // remove the trailing ".msg"
+ g_strlcpy (tmp_basename, basename, sizeof(tmp_basename));
+ if ((pch = g_strrstr (tmp_basename, ".msg")))
+ *pch = '\0';
+ g_strstrip (tmp_basename);
+
+ // transform
+ out = buf;
+ *out++ = '<';
+ for (in=tmp_basename; *in; ++in) {
+ if (in[0]!='%' || !g_ascii_isxdigit(in[1]) || !g_ascii_isxdigit(in[2]))
+ *out++ = *in;
+ else {
+ char buf[3];
+ buf[0] = *++in;
+ buf[1] = *++in;
+ buf[2] = '\0';
+ *out++ = (char) strtoul (buf, NULL, 16);
+ }
+ }
+ *out++ = '>';
+ *out = '\0';
+
+ return out - buf;
+ std::cerr<<"ec fn to mid "<<(out-buf)<<"\n";
+ }
+};
+
+/*****
+******
+*****/
+
+EncodeCache :: EncodeCache (const StringView& path, size_t max_megs):
+ _path (path.str, path.len),
+ _max_megs (max_megs),
+ _current_bytes (0ul)
+{
+ GError * err = NULL;
+ GDir * dir = g_dir_open (_path.c_str(), 0, &err);
+ if (err != NULL)
+ {
+ Log::add_err_va (_("Error opening directory: \"%s\": %s"), _path.c_str(), err->message);
+ g_clear_error (&err);
+ }
+ else
+ {
+
+
+ ///TODO import articles, see nzb.ccc
+// char filename[PATH_MAX];
+// const char * fname;
+// while ((fname = g_dir_read_name (dir)))
+// {
+// struct stat stat_p;
+// g_snprintf (filename, sizeof(filename), "%s%c%s", _path.c_str(), G_DIR_SEPARATOR, fname);
+// if (!stat (filename, &stat_p))
+// {
+// char str[2048];
+// const int len (filename_to_message_id (str, sizeof(str), fname));
+// if (len != 0)
+// {
+// MsgInfo info;
+// info._message_id = StringView (str, len);
+// info._size = stat_p.st_size;
+// info._date = stat_p.st_mtime;
+//// _current_bytes += info._size;
+// _mid_to_info.insert (mid_to_info_t::value_type (info._message_id, info));
+// }
+// }
+// }
+// g_dir_close (dir);
+// debug ("loaded " << _mid_to_info.size() << " articles into cache from " << _path);
+ }
+}
+
+EncodeCache :: ~EncodeCache ()
+{
+}
+
+/*****
+******
+*****/
+
+void
+EncodeCache :: fire_added (const Quark& mid)
+{
+ for (listeners_t::iterator it(_listeners.begin()), end(_listeners.end()); it!=end; )
+ (*it++)->on_cache_added (mid);
+}
+
+void
+EncodeCache :: fire_removed (const quarks_t& mids)
+{
+ for (listeners_t::iterator it(_listeners.begin()), end(_listeners.end()); it!=end; )
+ (*it++)->on_cache_removed (mids);
+}
+
+/*****
+******
+*****/
+
+bool
+EncodeCache :: contains (const Quark& mid) const
+{
+ return _mid_to_info.find (mid) != _mid_to_info.end();
+}
+
+void
+EncodeCache :: get_filename (char* buf, const Quark& mid) const
+{
+ const char* base = g_path_get_basename(mid.c_str());
+ g_snprintf (buf, PATH_MAX, "%s%c%s", _path.c_str(), G_DIR_SEPARATOR, base);
+ g_free((gpointer)base);
+ std::cerr<<"ec get_filename "<<buf<<"\n";
+}
+
+///TODO give function a reference to vector!!
+FILE*
+EncodeCache :: get_fp_from_mid(const Quark& mid)
+{
+ std::cerr<<"ec get_fp from mid "<<mid.c_str()<<"\n";
+ std::cerr<<"mid to info : "<<_mid_to_info[mid]._message_id<<std::endl;
+ return _mid_to_info[mid]._fp;
+}
+
+FILE*
+EncodeCache :: add (const Quark& message_id)
+{
+
+ std::cerr<<"ec add "<<message_id<<"\n";
+
+ pan_return_val_if_fail (!message_id.empty(), false);
+
+ FILE * fp = 0;
+ char filename[PATH_MAX];
+ struct stat sb;
+ get_filename (filename, message_id);
+ fp = fopen (filename, "wb+");
+
+ if (!fp)
+ {
+ Log::add_err_va (_("Unable to save \"%s\" %s"),
+ filename, file::pan_strerror(errno));
+ }
+ else
+ {
+ std::cerr<<"building msginfo\n";
+ MsgInfo info;
+ info._fp = fp;
+ info._message_id = message_id;
+ stat (message_id, &sb);
+ info._size = sb.st_size;
+ info._date = time(0);
+ _current_bytes += info._size;
+ _mid_to_info.insert (mid_to_info_t::value_type (info._message_id, info));
+ fire_added (message_id);
+ resize ();
+ }
+ return fp;
+}
+
+/***
+****
+***/
+
+void
+EncodeCache :: reserve (const mid_sequence_t& mids)
+{
+ std::cerr<<"ec reserve\n";
+ foreach_const (mid_sequence_t, mids, it)
+ ++_locks[*it];
+}
+
+void
+EncodeCache :: release (const mid_sequence_t& mids)
+{
+ std::cerr<<"ec release\n";
+ foreach_const (mid_sequence_t, mids, it)
+ if (!--_locks[*it])
+ _locks.erase (*it);
+}
+
+/***
+****
+***/
+
+void
+EncodeCache :: resize ()
+{
+ // let's shrink it to 80% of the maximum size
+ const double buffer_zone (0.8);
+ guint64 max_bytes (_max_megs * 1024 * 1024);
+ max_bytes = (guint64) ((double)max_bytes * buffer_zone);
+ resize (max_bytes);
+}
+
+void
+EncodeCache :: clear ()
+{
+ resize (0);
+}
+
+void
+EncodeCache :: get_data(std::string& data, const Quark& where)
+{
+ char buf[4096];
+ get_filename(buf, where);
+ std::ifstream in(buf, std::ifstream::in);
+ std::stringstream out;
+ if (in.bad())
+ std::cerr<<"error getting file "<<where.c_str()<<std::endl;
+ while (in.good())
+ out << (char) in.get();
+
+ data = out.str();
+}
+
+void
+EncodeCache :: resize (guint64 max_bytes)
+{
+ quarks_t removed;
+ if (_current_bytes > max_bytes)
+ {
+ // sort from oldest to youngest
+ typedef std::set<MsgInfo, MsgInfoCompare> sorted_info_t;
+ sorted_info_t si;
+ for (mid_to_info_t::const_iterator it=_mid_to_info.begin(),
+ end=_mid_to_info.end(); it!=end; ++it)
+ si.insert (it->second);
+
+ // start blowing away files
+ for (sorted_info_t::const_iterator it=si.begin(), end=si.end();
+ _current_bytes>max_bytes && it!=end; ++it) {
+ const Quark& mid (it->_message_id);
+ if (_locks.find(mid) == _locks.end()) {
+ char buf[PATH_MAX];
+ get_filename (buf, mid);
+ unlink (buf);
+ _current_bytes -= it->_size;
+ removed.insert (mid);
+ if (_mid_to_info[mid]._fp) fclose(_mid_to_info[mid]._fp);
+ _mid_to_info.erase (mid);
+ }
+ }
+ }
+
+ std::cerr<< "cache expired " << removed.size() << " articles, "
+ "has " << _mid_to_info.size() << " active "
+ "and " << _locks.size() << " locked.\n";
+
+ if (!removed.empty())
+ fire_removed (removed);
+}
+
+EncodeCache :: strings_t
+EncodeCache :: get_filenames (const mid_sequence_t& mids)
+{
+ strings_t ret;
+ char filename[PATH_MAX];
+ foreach_const (mid_sequence_t, mids, it) {
+ get_filename (filename, *it);
+ ret.push_back (filename);
+ }
+ return ret;
+}
diff --git a/pan/data/encode-cache.h b/pan/data/encode-cache.h
new file mode 100644
index 0000000..de39479
--- /dev/null
+++ b/pan/data/encode-cache.h
@@ -0,0 +1,123 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/*
+ * Pan - A Newsreader for Gtk+
+ * Copyright (C) 2002-2006 Charles Kerr <charles rebelbase com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#ifndef _EncodeCache_h_
+#define _EncodeCache_h_
+
+#include <map>
+#include <vector>
+extern "C" {
+ #include <glib/gtypes.h> // for guint64
+}
+#include <pan/general/string-view.h>
+#include <pan/general/quark.h>
+
+namespace pan
+{
+ class Article;
+ class StringView;
+
+ /**
+ * A disk cache for binary attachments.
+ *
+ * This allows a cache to be set to a certain maximum size, where
+ * the oldest articles will be aged out when the cache is full.
+ *
+ * It also has a lock/unlock mechanism to allow the cache to grow
+ * past its limit briefly to allow large multipart articles' pieces
+ * to all be held at once (for decoding).
+ *
+ * FIXME: This should probably be an interface class implemented in
+ * data-impl in the same way profiles was.
+ *
+ * @ingroup data
+ */
+ class EncodeCache
+ {
+ public:
+
+ EncodeCache (const StringView& path, size_t max_megs=10);
+ ~EncodeCache ();
+
+ FILE* get_fp_from_mid(const Quark& mid);
+ void get_filename (char* buf, const Quark& mid) const;
+
+ typedef std::vector<Quark> mid_sequence_t;
+
+ bool contains (const Quark& message_id) const;
+ FILE* add (const Quark& message_id);
+ void get_data(std::string& data, const Quark& where);
+ void reserve (const mid_sequence_t& mids);
+ void release (const mid_sequence_t& mids);
+ void resize ();
+ void clear ();
+
+ typedef std::vector<std::string> strings_t;
+ strings_t get_filenames (const mid_sequence_t&);
+
+ public:
+
+ /** Interface class for objects that listen to an EncodeCache's events. */
+ struct Listener {
+ virtual ~Listener () {}
+ virtual void on_cache_added (const Quark& mid) = 0;
+ virtual void on_cache_removed (const quarks_t& mid) = 0;
+ };
+ void add_listener (Listener * l) { _listeners.insert(l); }
+ void remove_listener (Listener * l) { _listeners.erase(l); }
+
+ private:
+
+ std::map<Quark,int> _locks;
+
+ struct MsgInfo {
+ FILE * _fp;
+ Quark _message_id;
+ size_t _size;
+ time_t _date;
+ MsgInfo(): _size(0), _date(0) {}
+ };
+
+ typedef std::map<Quark,MsgInfo> mid_to_info_t;
+ mid_to_info_t _mid_to_info;
+
+ struct MsgInfoCompare {
+ bool operator()(const MsgInfo& a, const MsgInfo& b) const {
+ if (a._date != b._date)
+ return a._date < b._date;
+ return a._message_id < b._message_id;
+ }
+ };
+
+ std::string _path;
+ const size_t _max_megs;
+ guint64 _current_bytes;
+
+ typedef std::set<Listener*> listeners_t;
+ listeners_t _listeners;
+
+ void fire_added (const Quark& mid);
+ void fire_removed (const quarks_t& mid);
+
+ void resize (guint64 max_bytes);
+ };
+}
+
+
+#endif // __EncodeCache_h__
diff --git a/pan/data/encode-cache.o b/pan/data/encode-cache.o
new file mode 100644
index 0000000..d0b69c0
Binary files /dev/null and b/pan/data/encode-cache.o differ
diff --git a/pan/gui/gui.cc b/pan/gui/gui.cc
index 4e0922b..f8e60f1 100644
--- a/pan/gui/gui.cc
+++ b/pan/gui/gui.cc
@@ -161,11 +161,11 @@ namespace
}
-GUI :: GUI (Data& data, Queue& queue, ArticleCache& cache, /*EncodeCache& encode_cache, */Prefs& prefs, GroupPrefs& group_prefs):
+GUI :: GUI (Data& data, Queue& queue, ArticleCache& cache, EncodeCache& encode_cache, Prefs& prefs, GroupPrefs& group_prefs):
_data (data),
_queue (queue),
_cache (cache),
-// _encode_cache (encode_cache),
+ _encode_cache (encode_cache),
_prefs (prefs),
_group_prefs (group_prefs),
_root (gtk_vbox_new (FALSE, 0)),
@@ -600,7 +600,7 @@ void GUI :: do_save_articles_to_nzb ()
// write them to a file
std::ofstream tmp(file.c_str());
- if (tmp.good())
+ if (tmp.good())
NZB :: nzb_to_xml_file (tmp, tasks);
tmp.close();
}
@@ -615,12 +615,13 @@ namespace
GtkWidget * _root;
Prefs& _prefs;
ArticleCache& _cache;
+ EncodeCache & _encode_cache;
const Article _article;
const std::string _path;
SaveArticlesFromNZB (Data& d, Queue& q, GtkWidget *r, Prefs& p,
- ArticleCache& c, const Article& a, const std::string& path):
- _data(d), _queue(q), _root(r), _prefs(p), _cache(c), _article(a), _path(path) {}
+ ArticleCache& c, EncodeCache& ec, const Article& a, const std::string& path):
+ _data(d), _queue(q), _root(r), _prefs(p), _cache(c), _encode_cache(ec), _article(a), _path(path) {}
static void foreach_part_cb (GMimeObject */*parent*/, GMimeObject *o, gpointer self)
{
@@ -642,7 +643,7 @@ namespace
const GByteArray * buffer (GMIME_STREAM_MEM(mem_stream)->buffer);
const StringView nzb ((const char*)buffer->data, buffer->len);
Queue::tasks_t tasks;
- NZB :: tasks_from_nzb_string (nzb, _path, _cache, _data, _data, _data, tasks);
+ NZB :: tasks_from_nzb_string (nzb, _path, _cache, _encode_cache, _data, _data, _data, tasks);
if (!tasks.empty())
_queue.add_tasks (tasks, Queue::BOTTOM);
g_object_unref (mem_stream);
@@ -671,7 +672,8 @@ void GUI :: do_save_articles_from_nzb ()
const std::string path (GUI :: prompt_user_for_save_path (get_window(_root), _prefs));
if (!path.empty())
{
- SaveArticlesFromNZB * listener = new SaveArticlesFromNZB (_data, _queue, _root, _prefs, _cache, *article, path);
+ SaveArticlesFromNZB * listener = new SaveArticlesFromNZB (_data, _queue, _root,
+ _prefs, _cache, _encode_cache, *article, path);
Task * t = new TaskArticle (_data, _data, *article, _cache, _data, listener);
_queue.add_task (t, Queue::TOP);
}
@@ -725,7 +727,7 @@ void GUI :: do_import_tasks ()
const std::string path (prompt_user_for_save_path (get_window(_root), _prefs));
if (!path.empty())
foreach_const (strings_t, filenames, it)
- NZB :: tasks_from_nzb_file (*it, path, _cache, _data, _data, _data, tasks);
+ NZB :: tasks_from_nzb_file (*it, path, _cache, _encode_cache, _data, _data, _data, tasks);
}
if (!tasks.empty())
@@ -1119,7 +1121,7 @@ void GUI :: do_supersede_article ()
g_object_unref (content_object);
g_object_unref (stream);
- PostUI * post = PostUI :: create_window (0, _data, _queue, _data, _data, new_message, _prefs, _group_prefs);
+ PostUI * post = PostUI :: create_window (0, _data, _queue, _data, _data, new_message, _prefs, _group_prefs, _encode_cache);
if (post)
{
gtk_widget_show_all (post->root());
@@ -1182,7 +1184,7 @@ void GUI :: do_cancel_article ()
g_object_unref (stream);
g_free (cancel_message);
- PostUI * post = PostUI :: create_window (0, _data, _queue, _data, _data, cancel, _prefs, _group_prefs);
+ PostUI * post = PostUI :: create_window (0, _data, _queue, _data, _data, cancel, _prefs, _group_prefs, _encode_cache);
if (post)
{
gtk_widget_show_all (post->root());
@@ -1216,7 +1218,7 @@ void GUI :: do_delete_article ()
void GUI :: do_clear_article_cache ()
{
_cache.clear ();
-// _encode_cache.clear();
+ _encode_cache.clear();
}
void GUI :: do_mark_article_read ()
@@ -1264,7 +1266,7 @@ GUI :: do_post ()
g_mime_message_set_mime_part (message, GMIME_OBJECT(part));
g_object_unref (part);
- PostUI * post = PostUI :: create_window (0, _data, _queue, _data, _data, message, _prefs, _group_prefs);
+ PostUI * post = PostUI :: create_window (0, _data, _queue, _data, _data, message, _prefs, _group_prefs, _encode_cache);
if (post)
gtk_widget_show_all (post->root());
g_object_unref (message);
@@ -1274,7 +1276,7 @@ void GUI :: do_followup_to ()
{
GMimeMessage * message = _body_pane->create_followup_or_reply (false);
if (message) {
- PostUI * post = PostUI :: create_window(0, _data, _queue, _data, _data, message, _prefs, _group_prefs);
+ PostUI * post = PostUI :: create_window(0, _data, _queue, _data, _data, message, _prefs, _group_prefs, _encode_cache);
if (post)
gtk_widget_show_all (post->root());
g_object_unref (message);
@@ -1284,7 +1286,7 @@ void GUI :: do_reply_to ()
{
GMimeMessage * message = _body_pane->create_followup_or_reply (true);
if (message) {
- PostUI * post = PostUI :: create_window (0, _data, _queue, _data, _data, message, _prefs, _group_prefs);
+ PostUI * post = PostUI :: create_window (0, _data, _queue, _data, _data, message, _prefs, _group_prefs, _encode_cache);
if (post)
gtk_widget_show_all (post->root());
g_object_unref (message);
diff --git a/pan/gui/gui.h b/pan/gui/gui.h
index b3ab564..72a6e43 100644
--- a/pan/gui/gui.h
+++ b/pan/gui/gui.h
@@ -22,6 +22,7 @@
#include <pan/general/log.h>
#include <pan/general/progress.h>
#include <pan/data/article-cache.h>
+#include <pan/data/encode-cache.h>
#include <pan/tasks/queue.h>
#include <pan/gui/action-manager.h>
@@ -53,7 +54,7 @@ namespace pan
private Prefs::Listener
{
public:
- GUI (Data& data, Queue&, ArticleCache&, /*EncodeCache&,*/ Prefs&, GroupPrefs&);
+ GUI (Data& data, Queue&, ArticleCache&, EncodeCache&, Prefs&, GroupPrefs&);
virtual ~GUI ();
GtkWidget* root () { return _root; }
@@ -188,7 +189,7 @@ namespace pan
Data& _data;
Queue& _queue;
ArticleCache& _cache;
-// EncodeCache& _encode_cache;
+ EncodeCache& _encode_cache;
Prefs& _prefs;
GroupPrefs& _group_prefs;
diff --git a/pan/gui/log-ui.cc b/pan/gui/log-ui.cc
index ccee55e..5102312 100644
--- a/pan/gui/log-ui.cc
+++ b/pan/gui/log-ui.cc
@@ -143,7 +143,6 @@ namespace
COL_MESSAGE, &*it, -1);
if (!it->messages.empty())
{
- std::cerr<<"multi\n";
foreach_const (Log::entries_t, it->messages, lit)
{
gtk_tree_store_prepend (store, &child, &top );
diff --git a/pan/gui/pan.cc b/pan/gui/pan.cc
index 09545b0..da07709 100644
--- a/pan/gui/pan.cc
+++ b/pan/gui/pan.cc
@@ -128,7 +128,7 @@ namespace
}
void run_pan_in_window (ArticleCache & cache,
-// EncodeCache & encode_cache,
+ EncodeCache & encode_cache,
Data & data,
Queue & queue,
Prefs & prefs,
@@ -138,7 +138,7 @@ namespace
{
const gulong delete_cb_id = g_signal_connect (window, "delete-event", G_CALLBACK(delete_event_cb), 0);
- GUI gui (data, queue, cache, /*encode_cache,*/ prefs, group_prefs);
+ GUI gui (data, queue, cache, encode_cache, prefs, group_prefs);
gtk_container_add (GTK_CONTAINER(window), gui.root());
gtk_widget_show (GTK_WIDGET(window));
@@ -179,7 +179,7 @@ namespace
~PanKiller() { q.remove_listener(this); }
/** Method from Queue::Listener interface: quits program on zero sized Q*/
- void on_queue_size_changed (Queue&, int active, int total)
+ void on_queue_size_changed (Queue&, int active, int total)
{ if (!active && !total) mainloop_quit(); }
// all below methods from Queue::Listener interface are noops
@@ -282,7 +282,7 @@ main (int argc, char *argv[])
nzb = true;
else if (!strcmp (tok, "--version"))
{ std::cerr << "Pan " << VERSION << '\n'; return 0; }
- else if (!strcmp (tok, "-o") && i<argc-1)
+ else if (!strcmp (tok, "-o") && i<argc-1)
nzb_output_path = argv[++i];
else if (!memcmp (tok, "--output=", 9))
nzb_output_path = tok+9;
@@ -317,7 +317,7 @@ main (int argc, char *argv[])
const int cache_megs = prefs.get_int ("cache-size-megs", 10);
DataImpl data (false, cache_megs);
ArticleCache& cache (data.get_cache ());
-// EncodeCache& encode_cache (data.get_encode_cache());
+ EncodeCache& encode_cache (data.get_encode_cache());
if (nzb && data.get_servers().empty()) {
std::cerr << _("Please configure Pan's news servers before using it as an nzb client.") << std::endl;
@@ -351,7 +351,7 @@ main (int argc, char *argv[])
// load the nzb files...
std::vector<Task*> tasks;
foreach_const (strings_t, nzb_files, it)
- NZB :: tasks_from_nzb_file (*it, nzb_output_path, cache, data, data, data, tasks);
+ NZB :: tasks_from_nzb_file (*it, nzb_output_path, cache, encode_cache, data, data, data, tasks);
queue.add_tasks (tasks, Queue::BOTTOM);
}
@@ -390,14 +390,14 @@ main (int argc, char *argv[])
gtk_window_set_resizable (GTK_WINDOW(window), true);
gtk_window_set_default_icon (pixbuf);
g_object_unref (pixbuf);
- run_pan_in_window (cache, /*encode_cache,*/ data, queue, prefs, group_prefs, GTK_WINDOW(window));
+ run_pan_in_window (cache, encode_cache, data, queue, prefs, group_prefs, GTK_WINDOW(window));
}
worker_pool.cancel_all_silently ();
if (prefs.get_flag("clear-article-cache-on-shutdown", false)) {
cache.clear ();
-// encode_cache.clear();
+ encode_cache.clear();
}
}
diff --git a/pan/gui/post-ui.cc b/pan/gui/post-ui.cc
index af45011..c0da668 100644
--- a/pan/gui/post-ui.cc
+++ b/pan/gui/post-ui.cc
@@ -2191,15 +2191,16 @@ PostUI :: create_parts_tab ()
gtk_tree_view_set_rules_hint(GTK_TREE_VIEW(w),TRUE);
gtk_tree_view_columns_autosize(GTK_TREE_VIEW(w));
+ ++row;
+ gtk_table_attach (GTK_TABLE(t), w, 1, 2, row, row+1, fe, fill, 0, 0);
+
//append scroll window
w = gtk_scrolled_window_new (NULL, NULL);
gtk_scrolled_window_set_policy (GTK_SCROLLED_WINDOW(w), GTK_POLICY_AUTOMATIC, GTK_POLICY_AUTOMATIC);
gtk_scrolled_window_set_shadow_type (GTK_SCROLLED_WINDOW(w), GTK_SHADOW_IN);
- gtk_container_add (GTK_CONTAINER(w), _parts_store);
- ++row;
- gtk_table_attach (GTK_TABLE(t), w, 0, 2, row, row+1, fe, fill, 0, 0);
+ gtk_container_add (GTK_CONTAINER(w), t);
- return t;
+ return w;
}
GtkWidget*
@@ -2491,17 +2492,15 @@ PostUI :: PostUI (GtkWindow * parent,
Profiles & profiles,
GMimeMessage * message,
Prefs & prefs,
- GroupPrefs & group_prefs
-// ,
-// ArticleCache & cache):
- ):
+ GroupPrefs & group_prefs,
+ EncodeCache & cache):
_data (data),
_queue (queue),
_gs (gs),
_profiles (profiles),
_prefs (prefs),
_group_prefs (group_prefs),
-// _cache (cache),
+ _cache (cache),
_root (0),
_from_combo (0),
_subject_entry (0),
@@ -2517,8 +2516,7 @@ PostUI :: PostUI (GtkWindow * parent,
_group_entry_changed_idle_tag (0),
_file_queue_empty(true),
_upload_ptr(0),
- _total_parts(0),
- _upload_cursor(0)
+ _total_parts(0)
{
g_assert (profiles.has_profiles());
g_return_if_fail (message != 0);
@@ -2571,9 +2569,8 @@ PostUI :: create_window (GtkWindow * parent,
Profiles & profiles,
GMimeMessage * message,
Prefs & prefs,
- GroupPrefs & group_prefs)
-// ,
-// ArticleCache & cache)
+ GroupPrefs & group_prefs,
+ EncodeCache & cache)
{
// can't post without a profile...
if (!profiles.has_profiles())
@@ -2594,7 +2591,7 @@ PostUI :: create_window (GtkWindow * parent,
return 0;
}
- return new PostUI (0, data, queue, gs, profiles, message, prefs, group_prefs);//, cache);
+ return new PostUI (0, data, queue, gs, profiles, message, prefs, group_prefs, cache);
}
void
@@ -2648,9 +2645,11 @@ PostUI :: prompt_user_for_queueable_files (tasks_v& queue, GtkWindow * parent, c
int i(0);
for (; cur; cur = cur->next, ++i)
{
+ _uploaded.push_back(new Article());
+ Article& a(*_uploaded.back());
TaskUpload* tmp = new TaskUpload(std::string((const char*)cur->data),
- profile.posting_server, //_cache,
- groups, subject, author, 0, 0,
+ profile.posting_server, _cache,
+ groups, subject, author, a, 0, 0,
TaskUpload::YENC);
_file_queue_tasks.push_back(tmp);
}
diff --git a/pan/gui/post-ui.h b/pan/gui/post-ui.h
index 1df55d7..e85f2af 100644
--- a/pan/gui/post-ui.h
+++ b/pan/gui/post-ui.h
@@ -25,7 +25,7 @@
#include <pan/general/progress.h>
#include <pan/tasks/queue.h>
#include <pan/usenet-utils/text-massager.h>
-#include <pan/data/article-cache.h>
+#include <pan/data/encode-cache.h>
#include "group-prefs.h"
namespace pan
@@ -46,13 +46,13 @@ namespace pan
{
public:
static PostUI* create_window (GtkWindow*, Data&, Queue&, GroupServer&, Profiles&,
- GMimeMessage*, Prefs&, GroupPrefs&);//, ArticleCache&);
+ GMimeMessage*, Prefs&, GroupPrefs&, EncodeCache&);
void prompt_user_for_queueable_files (tasks_v& queue, GtkWindow * parent, const Prefs& prefs);
protected:
PostUI (GtkWindow*, Data&, Queue&, GroupServer&, Profiles&,
- GMimeMessage*, Prefs&, GroupPrefs&);//, ArticleCache&);
+ GMimeMessage*, Prefs&, GroupPrefs&, EncodeCache&);
public:
~PostUI ();
@@ -139,6 +139,7 @@ namespace pan
GtkWidget * _post_dialog;
TaskPost * _post_task;
TaskUpload* _upload_task;
+ std::vector<Article*> _uploaded;
typedef std::map<std::string, std::string> str2str_t;
str2str_t _hidden_headers;
str2str_t _profile_headers;
@@ -167,11 +168,6 @@ namespace pan
GtkWidget* create_log_tab ();
TaskUpload* _upload_ptr;
int _total_parts;
- typedef std::map<int, TaskUpload::Needed> needed_t;
- typedef std::pair<int, TaskUpload::Needed> needed_p;
- typedef std::vector<needed_t> needed_v;
- needed_v _needed_v;
- int _upload_cursor;
private:
std::string utf8ize (const StringView&) const;
@@ -187,8 +183,8 @@ namespace pan
static gboolean group_entry_changed_idle (gpointer);
static void group_entry_changed_cb (GtkEditable*, gpointer);
-// protected:
-// ArticleCache& _cache;
+ protected:
+ EncodeCache& _cache;
public:
void set_spellcheck_enabled (bool);
@@ -213,7 +209,6 @@ namespace pan
public:
TaskUpload* upload_ptr() { return _upload_ptr; }
- int cursor() { return _upload_cursor; }
};
}
diff --git a/pan/tasks/encoder.cc b/pan/tasks/encoder.cc
index 2333b63..3351701 100644
--- a/pan/tasks/encoder.cc
+++ b/pan/tasks/encoder.cc
@@ -55,24 +55,28 @@ Encoder :: ~Encoder()
***/
void
-Encoder :: enqueue (TaskUpload * task,
- std::string & filename,
- std::string & basename,
- std::string & groups,
- std::string & subject,
- std::string & author,
+Encoder :: enqueue (TaskUpload * task,
+ const Article::mid_sequence_t & mids,
+ EncodeCache * cache,
+ std::string & filename,
+ std::string & basename,
+ std::string & groups,
+ std::string & subject,
+ std::string & author,
const TaskUpload::EncodeMode & enc)
{
disable_progress_update ();
this->task = task;
- this->filename = filename;
this->basename = basename;
+ this->filename = filename;
this->encode_mode = encode_mode;
this->groups = groups;
this->subject = subject;
this->author = author;
+ this-> mids = mids;
+ this->cache = cache;
percent = 0;
current_file.clear ();
@@ -83,17 +87,15 @@ Encoder :: enqueue (TaskUpload * task,
_worker_pool.push_work (this, task, false);
}
-// save article IN A WORKER THREAD to avoid network stalls
void
Encoder :: do_work()
{
const int bufsz = 4096;
char buf[bufsz], buf2[bufsz];
- unsigned long cnt(1);
+ int cnt(1);
crc32_t crcptr;
FILE* outfile, * infile ;
- std::string uulib(file :: get_uulib_path());
enable_progress_update();
int res;
@@ -104,21 +106,27 @@ Encoder :: do_work()
UUSetMsgCallback (this, uu_log);
UUSetBusyCallback (this, uu_busy_poll, 200);
- g_snprintf(buf,bufsz,"%s/%s.%d", uulib.c_str(), basename.c_str(), cnt);
- outfile = fopen(buf,"wb");
- while (1) {
-
+ foreach_const(Article::mid_sequence_t, mids, it) {
+
+ FILE * fp = cache->get_fp_from_mid(*it);
+ if (!(*it))
+ {
+ g_snprintf(buf, bufsz, _("Error loading %s from cache."), it->c_str());
+ log_errors.push_back(buf); // log error
+ ++cnt;
+ continue;
+ }
+ cache->get_filename(buf, *it);
+ std::cerr<<"do work "<<buf<<std::endl;
// 4000 lines SHOULD be OK for ANY nntp server ...
- res = UUE_PrepPartial (outfile, NULL, (char*)filename.c_str(),YENC_ENCODED,
- (char*)basename.c_str(),0644, cnt, 4000,
+ res = UUE_PrepPartial (fp, NULL, (char*)filename.c_str(),YENC_ENCODED,
+ (char*)basename.c_str(),0644, cnt++, 4000,
0, (char*)groups.c_str(),
(char*)author.c_str(), (char*)subject.c_str(),
0);
-
- if (outfile) fclose(outfile);
+ if (fp) fclose(fp);
+ std::cerr<<"encoder said : "<<UUstrerror(res)<<std::endl;
if (res != UURET_CONT) break;
- g_snprintf(buf,bufsz,"%s/%s.%d", uulib.c_str(), basename.c_str(), ++cnt);
- outfile = fopen(buf,"wb");
}
if (res != UURET_OK)
@@ -195,6 +203,7 @@ Encoder :: uu_busy_poll (void * d, uuprogress *p)
self->mut.lock();
self->percent = self->get_percentage(*p);
self->current_file = p->curfile;
+ self->parts = p->numparts;
self->mut.unlock();
return self->was_cancelled(); // returning true tells uulib to abort
diff --git a/pan/tasks/encoder.h b/pan/tasks/encoder.h
index 142c46c..83f3ad3 100644
--- a/pan/tasks/encoder.h
+++ b/pan/tasks/encoder.h
@@ -27,6 +27,7 @@
#include <list>
#include <string>
#include <vector>
+#include <sstream>
#include <pan/general/locking.h>
#include <pan/general/worker-pool.h>
#include <pan/tasks/task-upload.h>
@@ -58,12 +59,14 @@ namespace pan
typedef std::vector<std::string> strings_t;
- void enqueue (TaskUpload * task,
- std::string & filename,
- std::string & basename,
- std::string & groups,
- std::string & subject,
- std::string & author,
+ void enqueue (TaskUpload * task,
+ const Article::mid_sequence_t & mids,
+ EncodeCache * cache,
+ std::string & filename,
+ std::string & basename,
+ std::string & groups,
+ std::string & subject,
+ std::string & author,
const TaskUpload::EncodeMode & enc = TaskUpload::YENC);
public:
@@ -77,14 +80,15 @@ namespace pan
private:
- std::set<int>* parts;
friend class TaskUpload;
friend class PostUI;
+ int parts;
TaskUpload * task;
TaskUpload::EncodeMode encode_mode;
- std::string filename;
- std::string basename;
+ std::string basename, filename;
std::string subject, author, groups;
+ EncodeCache * cache;
+ Article::mid_sequence_t mids;
// These are set in the worker thread and polled in the main thread.
Mutex mut;
diff --git a/pan/tasks/nzb-test.cc b/pan/tasks/nzb-test.cc
index f066c54..46f6361 100644
--- a/pan/tasks/nzb-test.cc
+++ b/pan/tasks/nzb-test.cc
@@ -4,6 +4,7 @@
#include <pan/general/string-view.h>
#include <pan/general/test.h>
#include <pan/data/article-cache.h>
+#include <pan/data/encode-cache.h>
#include "nzb.h"
#include "task-article.h"
@@ -62,10 +63,11 @@ int main ()
MyGroupServer gs (gmap);
ArticleCache cache ("/tmp");
+ EncodeCache e_cache ("/tmp");
MyArticleRead read;
StringView v (test_1);
std::vector<Task*> tasks;
- NZB :: tasks_from_nzb_string (v, StringView("/tmp"), cache, read, ranks, gs, tasks);
+ NZB :: tasks_from_nzb_string (v, StringView("/tmp"), cache, e_cache, read, ranks, gs, tasks);
check (tasks.size() == 1)
const Article a (dynamic_cast<TaskArticle*>(tasks[0])->get_article());
check (a.author == "Joe Bloggs <bloggs nowhere example>")
diff --git a/pan/tasks/nzb.cc b/pan/tasks/nzb.cc
index d4ac6dd..0a897fe 100644
--- a/pan/tasks/nzb.cc
+++ b/pan/tasks/nzb.cc
@@ -50,11 +50,10 @@ namespace
std::vector<std::string> groups_str; // TaskUpload
TaskUpload::needed_t needed_parts; // TaskUpload
Article a;
- Quark posting_server;
- bool encoded; // encoder was done already
PartBatch parts;
tasks_t tasks;
ArticleCache& cache;
+ EncodeCache& encode_cache;
ArticleRead& read;
const ServerRank& ranks;
const GroupServer& gs;
@@ -62,9 +61,9 @@ namespace
size_t bytes;
size_t number;
- MyContext (ArticleCache& ac, ArticleRead& r,
+ MyContext (ArticleCache& ac, EncodeCache& ec, ArticleRead& r,
const ServerRank& rank, const GroupServer& g, const StringView& p):
- cache(ac), read(r), ranks(rank), gs(g), fallback_path(p) {}
+ cache(ac), encode_cache(ec), read(r), ranks(rank), gs(g), fallback_path(p) {}
void file_clear () {
groups.clear ();
@@ -102,7 +101,6 @@ namespace
for (const char **k(attribute_names), **v(attribute_vals); *k; ++k, ++v) {
if (!strcmp (*k,"author")) mc.a.author = *v;
else if (!strcmp (*k,"subject")) mc.a.subject = *v;
- else if (!strcmp (*k,"server")) mc.posting_server = Quark(*v);
}
}
@@ -149,13 +147,18 @@ namespace
}
else if (!strcmp(element_name, "part") && mc.number && !mc.text.empty()) {
- TaskUpload::Needed n;
- n.bytes = mc.bytes;
- n.filename = mc.text;
- n.partno = mc.number;
- n.partial = false;
- std::pair<int,TaskUpload::Needed> tmp(mc.number,n);
- mc.needed_parts.insert(tmp);
+// TaskUpload::Needed n;
+// n.bytes = mc.bytes;
+// n.filename = mc.text;
+// n.partno = mc.number;
+// n.partial = false;
+// std::pair<int,TaskUpload::Needed> tmp(mc.number,n);
+// mc.needed_parts.insert(tmp);
+ if (mc.a.message_id.empty()) {
+ mc.a.message_id = mc.text;
+ mc.parts.init (mc.text);
+ }
+ mc.parts.add_part (mc.number, mc.text, mc.bytes);
}
else if (!strcmp(element_name,"path"))
@@ -180,11 +183,16 @@ namespace
else if (!strcmp (element_name, "upload"))
{
debug("adding taskupload from nzb.\n");
- TaskUpload::needed_t tmp2;
- foreach (TaskUpload::needed_t, mc.needed_parts, it)
- tmp2.insert(*it);
- TaskUpload* tmp = new TaskUpload (mc.path, const_cast<const Quark&>(mc.posting_server), //mc.cache,
- mc.groups, mc.a.subject.to_string(), mc.a.author.to_string(), &tmp2, 0, TaskUpload::YENC);
+// TaskUpload::needed_t tmp2;
+// foreach (TaskUpload::needed_t, mc.needed_parts, it)
+// tmp2.insert(*it);
+ mc.parts.sort ();
+ mc.a.set_parts (mc.parts);
+ foreach_const (quarks_t, mc.groups, git)
+ mc.a.xref.insert (Quark("dummy"), *git,0);
+
+ TaskUpload* tmp = new TaskUpload (mc.path, Quark("dummy"), mc.encode_cache,
+ mc.groups, mc.a.subject.to_string(), mc.a.author.to_string(), mc.a, 0, 0, TaskUpload::YENC);
mc.tasks.push_back (tmp);
}
}
@@ -203,13 +211,14 @@ void
NZB :: tasks_from_nzb_string (const StringView & nzb_in,
const StringView & save_path,
ArticleCache & cache,
+ EncodeCache & encode_cache,
ArticleRead & read,
const ServerRank & ranks,
const GroupServer & gs,
std::vector<Task*> & appendme)
{
const std::string nzb (clean_utf8 (nzb_in));
- MyContext mc (cache, read, ranks, gs, save_path);
+ MyContext mc (cache, encode_cache, read, ranks, gs, save_path);
GMarkupParser p;
p.start_element = start_element;
p.end_element = end_element;
@@ -232,6 +241,7 @@ void
NZB :: tasks_from_nzb_file (const StringView & filename,
const StringView & save_path,
ArticleCache & c,
+ EncodeCache & ec,
ArticleRead & r,
const ServerRank & ranks,
const GroupServer & gs,
@@ -239,7 +249,7 @@ NZB :: tasks_from_nzb_file (const StringView & filename,
{
std::string nzb;
if (file :: get_text_file_contents (filename, nzb))
- tasks_from_nzb_string (nzb, save_path, c, r, ranks, gs, appendme);
+ tasks_from_nzb_string (nzb, save_path, c, ec, r, ranks, gs, appendme);
}
namespace
diff --git a/pan/tasks/nzb.h b/pan/tasks/nzb.h
index fe49dfa..d8b4152 100644
--- a/pan/tasks/nzb.h
+++ b/pan/tasks/nzb.h
@@ -38,6 +38,7 @@ namespace pan
static void tasks_from_nzb_string (const StringView & nzb,
const StringView & save_path,
ArticleCache & cache,
+ EncodeCache & encode_cache,
ArticleRead & read,
const ServerRank & ranks,
const GroupServer & gs,
@@ -46,6 +47,7 @@ namespace pan
static void tasks_from_nzb_file (const StringView & filename,
const StringView & save_path,
ArticleCache & cache,
+ EncodeCache & encode_cache,
ArticleRead & read,
const ServerRank & ranks,
const GroupServer & gs,
diff --git a/pan/tasks/task-upload.cc b/pan/tasks/task-upload.cc
index 3c9db68..1733790 100644
--- a/pan/tasks/task-upload.cc
+++ b/pan/tasks/task-upload.cc
@@ -36,7 +36,7 @@ extern "C" {
#include <pan/general/log.h>
#include <pan/general/macros.h>
#include <pan/usenet-utils/mime-utils.h>
-#include <pan/data/article-cache.h>
+#include <pan/data/encode-cache.h>
#include "encoder.h"
#include "task-upload.h"
@@ -53,35 +53,28 @@ namespace
return buf;
}
- std::string get_basename(const char* f)
- {
- char buf[1024];
- char * freeme = g_path_get_basename(f);
- snprintf (buf, sizeof(buf), _("%s"), freeme);
- g_free(freeme);
- return buf;
- }
+// std::string get_basename(const char* f)
+// {
+// char buf[4096];
+// char * freeme = g_path_get_basename(f);
+// snprintf (buf, sizeof(buf), _("%s"), freeme);
+// g_free(freeme);
+// return buf;
+// }
}
-namespace
-{
- void delete_cache(const TaskUpload::Needed& n)
- {
- unlink(n.filename.c_str());
- }
-}
-
/***
****
***/
TaskUpload :: TaskUpload ( const std::string & filename,
const Quark & server,
-// ArticleCache & cache,
+ EncodeCache & cache,
quarks_t & groups,
std::string subject,
std::string author,
+ Article & article,
needed_t * imported,
Progress::Listener * listener,
const TaskUpload::EncodeMode enc):
@@ -89,7 +82,7 @@ TaskUpload :: TaskUpload ( const std::string & filename,
_filename(filename),
_basename (g_path_get_basename(filename.c_str())),
_server(server),
-// _cache(cache),
+ _cache(cache),
_groups(groups),
_subject (subject),
_author(author),
@@ -111,6 +104,7 @@ TaskUpload :: TaskUpload ( const std::string & filename,
_bytes = sb.st_size;
build_needed_tasks(imported);
+ _cache.reserve(_article.get_part_mids());
update_work ();
}
@@ -118,36 +112,46 @@ void
TaskUpload :: build_needed_tasks(bool imported)
{
+ char buf[4096];
+ char buf2[4096];
+
_total_parts = (int) (((long)get_byte_count() + (4000*128-1)) / (4000*128));
- TaskUpload::Needed tmp;
- char buf[2048];
- struct stat sb;
- const char* uulib = file::get_uulib_path().c_str();
- mid_sequence_t names;
- int cnt(0);
+ int cnt(1);
+
+ quarks_t groups;
+ foreach_const (Xref, _article.xref, it)
+ groups.insert (it->group);
- for (int i=1;i<=_total_parts; ++i)
+ for (int i=1; i<=_total_parts; ++i)
{
if (imported)
{
- needed_t::iterator it = _needed.find(i);
- if (it == _needed.end())
- continue;
+ needed_t::iterator it = _needed.find(cnt);
+ if (it == _needed.end())
+ continue;
}
- tmp.partno = i;
- g_snprintf(buf,sizeof(buf),"%s/%s.%d", uulib, _basename.c_str(), i);
- tmp.filename = buf;
- tmp.partial = false;
- tmp.nntp = 0;
- stat(buf, &sb);
- tmp.bytes = (unsigned long)sb.st_size;
- _needed.insert(std::pair<int,Needed>(i,tmp));
- ++cnt;
+
+ g_snprintf(buf,sizeof(buf),"%s.%d", _filename.c_str(), i);
+ _article.add_part(i, StringView(buf), 0);
}
- _needed_parts = cnt;
- // reserve cache
-// _cache.reserve (names);
+ for (Article::part_iterator i(_article.pbegin()), e(_article.pend()); i!=e; ++i, ++cnt)
+ {
+ const std::string mid (i.mid ());
+
+ TaskUpload::Needed n;
+ n.message_id = mid;
+ n.bytes = i.bytes();
+ n.partno = cnt;
+
+ foreach_const (quarks_t, groups, git)
+ n.xref.insert (_server, *git, mid==_article.message_id.to_string()
+ ? _article.xref.find_number(_server,*git) : 0);
+
+ std::cerr<<"needed insert "<<cnt<<std::endl;
+ _needed.insert(std::pair<int,Needed>(cnt,n));
+ }
+ _needed_parts = cnt;
}
void
@@ -209,14 +213,9 @@ TaskUpload :: use_nntp (NNTP * nntp)
set_status_va (_("Uploading %s - Part %d of %d"), _basename.c_str(), needed->partno, _total_parts);
- std::stringstream tmp;
- std::ifstream in(needed->filename.c_str(), std::ifstream::in);
- if (in.bad())
- debug("error with stream!!!\n");
- while (in.good())
- tmp << (char) in.get();
- in.close();
- nntp->post(StringView(tmp.str()), this);
+ std::string data;
+ _cache.get_data(data,needed->message_id.c_str());
+ nntp->post(StringView(data), this);
update_work ();
}
}
@@ -248,14 +247,16 @@ TaskUpload :: on_nntp_done (NNTP * nntp,
switch (health)
{
case OK:
+ // save to cache
_needed.erase (it);
post_ok = true;
increment_step(1);
break;
case ERR_NETWORK:
+ //reset
+ it->second.nntp = 0;
goto _end;
case ERR_COMMAND:
- delete_cache(it->second);
_needed.erase (it);
break;
}
@@ -263,22 +264,13 @@ TaskUpload :: on_nntp_done (NNTP * nntp,
switch (atoi(response.str))
{
case NO_POSTING:
- g_snprintf(buf,sizeof(buf), _("Posting of File %s (Part %d of %d) failed: No Posts allowed by server."),
+ Log :: add_err_va (_("Posting of File %s (Part %d of %d) failed: No Posts allowed by server."),
_basename.c_str(), it->second.partno, _total_parts);
- tmp.message = buf;
- tmp.severity = Log :: PAN_SEVERITY_ERROR;
- _logfile.push_back(tmp);
- Log::add_entry_list (tmp, _logfile);
- std::cerr<<LINE_ID<<" "<<_logfile.size()<<std::endl;
this->stop();
break;
case POSTING_FAILED:
- g_snprintf(buf,sizeof(buf), _("Posting of File %s (Part %d of %d) failed: %s"),
+ Log :: add_err_va ( _("Posting of File %s (Part %d of %d) failed: %s"),
_basename.c_str(), it->second.partno, _total_parts, response.str);
- tmp.severity = Log :: PAN_SEVERITY_ERROR;
- tmp.message = buf;
- _logfile.push_back(tmp);
- std::cerr<<LINE_ID<<" "<<_logfile.size()<<std::endl;
break;
case ARTICLE_POSTED_OK:
tmp.severity = Log :: PAN_SEVERITY_INFO;
@@ -303,12 +295,8 @@ TaskUpload :: on_nntp_done (NNTP * nntp,
std::cerr<<LINE_ID<<" "<<_logfile.size()<<std::endl;
} else
{
- g_snprintf(buf,sizeof(buf), _("Posting of file %s not successful: Check the popup log!"),
+ Log :: add_err_va (_("Posting of file %s not successful: Check the popup log!"),
_basename.c_str(), response.str);
- tmp.message = buf;
- tmp.severity = Log :: PAN_SEVERITY_ERROR;
- _logfile.push_back(tmp);
- Log::add_entry_list (tmp, _logfile);
std::cerr<<LINE_ID<<" "<<_logfile.size()<<std::endl;
}
break;
@@ -356,7 +344,11 @@ TaskUpload :: use_encoder (Encoder* encoder)
if (i<_groups.size()&& i>0 && _groups.size()>1) groups += ",";
groups += (*it).to_string();
}
- _encoder->enqueue (this, _filename, _basename, groups, _subject, _author, YENC);
+ const Article::mid_sequence_t mids (_article.get_part_mids());
+ foreach_const (Article::mid_sequence_t, mids, it)
+ _cache.add(*it) ? std::cerr<<"fp valid!\n" :std::cerr<<"fp invalid!\n";
+ _encoder->enqueue (this, mids, &_cache, _filename, _basename,
+ groups, _subject, _author, YENC);
debug ("encoder thread was free, enqueued work");
}
@@ -390,6 +382,7 @@ TaskUpload :: on_worker_done (bool cancelled)
set_error (_encoder->log_errors.front());
{
+ std::cerr<<_encoder->parts<<" "<<_total_parts<<std::endl;
set_step (0);
init_steps(_needed_parts);
_encoder_has_run = true;
diff --git a/pan/tasks/task-upload.h b/pan/tasks/task-upload.h
index 8b557dc..df81964 100644
--- a/pan/tasks/task-upload.h
+++ b/pan/tasks/task-upload.h
@@ -27,7 +27,7 @@
#include <pan/general/worker-pool.h>
#include <pan/general/locking.h>
#include <pan/data/article.h>
-//#include <pan/data/article-cache.h>
+#include <pan/data/encode-cache.h>
#include <pan/data/data.h>
#include <pan/general/log.h>
#include <pan/data/xref.h>
@@ -59,6 +59,9 @@ namespace pan
int partno;
NNTP* nntp;
bool partial;
+ std::string message_id;
+ Xref xref;
+
Needed (): nntp(0), bytes(0) , partial(false) {}
void reset() { nntp = 0; }
};
@@ -75,13 +78,15 @@ namespace pan
// life cycle
TaskUpload ( const std::string & filename,
const Quark & server,
-// ArticleCache & cache,
+ EncodeCache & cache,
quarks_t & groups,
std::string subject,
std::string author,
+ Article & article,
needed_t * imported=0,
Progress::Listener * listener= 0,
TaskUpload::EncodeMode enc= YENC);
+
virtual ~TaskUpload ();
public: // Task subclass
@@ -129,56 +134,16 @@ namespace pan
int _total_parts, _needed_parts;
unsigned long _bytes;
Mutex mut;
-// ArticleCache& _cache;
+ EncodeCache& _cache;
std::deque<Log::Entry> _logfile; // for intermediate updates
- Log :: Severity _severity_final;
-
- void build_needed_tasks(bool);
+ Article _article;
private:
needed_t _needed;
void update_work (NNTP * checkin_pending = 0);
+ void build_needed_tasks(bool);
};
-
-// from mime-utils.cc
-namespace
-{
-
- const char*
- __yenc_extract_tag_val_char (const char * line, const char *tag)
- {
- const char * retval = NULL;
-
- const char * tmp = strstr (line, tag);
- if (tmp != NULL) {
- tmp += strlen (tag);
- if (*tmp != '\0')
- retval = tmp;
- }
-
- return retval;
- }
-
- guint
- __yenc_extract_tag_val_int_base (const char * line,
- const char * tag,
- int base)
- {
- guint retval = 0;
-
- const char * tmp = __yenc_extract_tag_val_char (line, tag);
- if (tmp != NULL) {
- char * tail = NULL;
- retval = strtoul (tmp, &tail, base);
- if (tmp == tail)
- retval = 0;
- }
-
- return retval;
- }
-}
-
}
#endif
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
