[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
Re: [Pan-devel] Memory woes ... no more! (download next N headers)
|
From: |
Rico |
|
Subject: |
Re: [Pan-devel] Memory woes ... no more! (download next N headers) |
|
Date: |
Wed, 26 Feb 2003 13:00:02 -0800 (PST) |
Yeah babes! yah! :D
Now it looks more like decently functional. For those of you who are keen to
give this a spin, I'd appreciate if you'd comment about how you like or dislike
the feature. One argument for it is that presently, downloading N new headers
might be misleading. If there are 2N new headers, the N oldest of those can
only
be obtained by downloading _all_ headers in the group, which might include
1000's of older articles below the oldest local one. The user might not be
interested in those.
Charles, it seems to me that each time download_headers_response_cb() is
called,
the callback to switch_to_group() leads us to add another task_headers_new() to
the queue. Isn't that a redundant request to the server? The ..._response_cb()
does the job already, right? Unless I've missed something, the fix is part of
the patch which follows.
The keg's back there... Enjoy!
Rico.
================================================================================
diff -u -3 -r pan-0.13.3.93/pan/articlelist.c
pan-0.13.3.93_src/pan/articlelist.c
--- pan-0.13.3.93/pan/articlelist.c Thu Feb 13 20:06:39 2003
+++ pan-0.13.3.93_src/pan/articlelist.c Thu Feb 27 03:57:28 2003
@@ -3000,12 +3000,15 @@
return 0;
}
+
static int
set_group_mainthread_end (gpointer p)
{
ArgSet * argset;
Group * group;
Group * old_group;
+ gboolean fetch_headers; /* when getting next sample, and coming
from
switch_to_group(),
+ we must not download
SAMPLE_SIZE_DEFAULT headers */
StatusItem * status;
GPtrArray * articles;
GPtrArray * tmp;
@@ -3015,9 +3018,10 @@
/* pump out the arguments */
argset = (ArgSet*) p;
group = (Group*) argset_get (argset, 0);
- status = (StatusItem*) argset_get (argset, 1);
- articles = (GPtrArray*) argset_get (argset, 2);
- sort = GPOINTER_TO_INT (argset_get (argset, 3));
+ fetch_headers = !((gboolean) argset_get (argset, 1));
+ status = (StatusItem*) argset_get (argset, 2);
+ articles = (GPtrArray*) argset_get (argset, 3);
+ sort = GPOINTER_TO_INT (argset_get (argset, 4));
/* update the UI */
pan_lock ();
@@ -3051,8 +3055,8 @@
/* Now that the articlelist is fully loaded,
get new headers, if required. */
- if (fetch_new_on_group_load && !group_is_folder(group))
- queue_insert_tasks
(g_slist_append(NULL,task_headers_new(group,HEADERS_NEW)), 0);
+ if (fetch_headers && fetch_new_on_group_load && !group_is_folder(group))
+ queue_insert_tasks (g_slist_append(NULL,task_headers_new_sample
(group,
SAMPLE_SIZE_DEFAULT, FALSE)), 0);
debug_exit ("set_group_mainthread_end");
return 0;
@@ -3072,7 +3076,7 @@
/* pump out the arguments */
argset = (ArgSet*) p;
group = (Group*) argset_get (argset, 0);
- status = (StatusItem*) argset_get (argset, 1);
+ status = (StatusItem*) argset_get (argset, 2);
/* we're switching. all other switch requests: please hold */
g_static_mutex_lock (&switch_mutex);
@@ -3170,6 +3174,8 @@
}
else
{
- gui_queue_add (set_group_mainthread_begin, argset_new1(group));
+ /* second argument in argset used to avoid race with switch_to_group()
*/
+ gui_queue_add (set_group_mainthread_begin,
+ argset_new2(group, (gpointer)
group->_no_fetch_on_load));
}
}
diff -u -3 -r pan-0.13.3.93/pan/base/file-grouplist.c
pan-0.13.3.93_src/pan/base/file-grouplist.c
--- pan-0.13.3.93/pan/base/file-grouplist.c Fri Jan 24 01:47:18 2003
+++ pan-0.13.3.93_src/pan/base/file-grouplist.c Thu Feb 27 03:47:34 2003
@@ -131,7 +131,7 @@
groups = g_ptr_array_new ();
if (version==1 || version==2 || version==3 || version==4 ||
version==5 || version==6 || version==7 || version==8 ||
- version==9 || version==10 || version==11 || version==12)
+ version==9 || version==10 || version==11 || version==12 ||
version==13)
{
long i;
long qty = get_next_token_long (march_idx, '\n', &march_idx);
@@ -220,13 +220,13 @@
if (abs(g->old_sort_style) ==
abs(g->new_sort_style))
g->old_sort_style = 0;
- g->article_low = (gulong) get_next_token_ulong
(march_idx, '\n',
&march_idx);
-
if (version>=9)
g->loaded_since_last_fetch = get_next_token_int
(march_idx, '\n',
&march_idx) != 0;
else
g->loaded_since_last_fetch = FALSE;
+ g->article_low = (gulong) get_next_token_ulong
(march_idx, '\n',
&march_idx);
+
g->article_high = (gulong) get_next_token_ulong
(march_idx, '\n',
&march_idx);
/* filter_bits got widened from guint16 to guint32 in
version 7 */
@@ -238,6 +238,10 @@
g->article_high_old = (gulong)
get_next_token_ulong (march_idx, '\n',
&march_idx);
else
g->article_high_old = g->article_high;
+
+ /* article_last_low_fetched added in version 13 */
+ if (version>=13)
+ g->article_last_low_fetched = (gulong)
get_next_token_ulong (march_idx,
'\n', &march_idx);
g->article_qty = (gint32) get_next_token_int
(march_idx, '\n',
&march_idx);
g->article_read_qty = (gint32) get_next_token_int
(march_idx, '\n',
&march_idx);
@@ -257,7 +261,7 @@
g->permission = ch;
skip_next_token (march_idx, '\n', &march_idx);
}
-
+
if (is_nonempty_string(newsrc))
newsrc_init (group_get_newsrc(g), newsrc,
g->article_low,
g->article_high);
@@ -412,7 +416,7 @@
/* Write the group information... */
pos = 0;
- fprintf (idx_fp, "12\n"); /* file format version number */
+ fprintf (idx_fp, "13\n"); /* file format version number */
fprintf (idx_fp, "%u\n", groups->len); /* number of groups */
for (i=0; i!=groups->len; ++i)
{
@@ -439,7 +443,7 @@
/* write the non-string fields. */
fprintf (idx_fp,
"%ld\n" "%ld\n" "%ld\n" "%ld\n" "%ld\n" "%ld\n"
"%ld\n" "%ld\n"
- "%u\n" "%lu\n" "%lu\n" "%d\n" "%d\n" "%d\n" "%lu\n"
"%lu\n" "%lu\n"
"%d\n" "%d\n" "%c\n",
+ "%u\n" "%lu\n" "%lu\n" "%d\n" "%d\n" "%d\n" "%lu\n"
"%lu\n" "%lu\n"
"%lu\n" "%d\n" "%d\n" "%c\n",
name_idx, desc_idx, path_idx, char_idx, news_idx,
dead_idx,
filt_idx, iden_idx,
(unsigned)(group->flags & ~GROUP_NEW),
group->filter_show,
@@ -450,6 +454,7 @@
(unsigned long)group->article_low,
(unsigned long)group->article_high,
(unsigned long)group->article_high_old,
+ (unsigned long)group->article_last_low_fetched,
(int)group->article_qty,
(int)group->article_read_qty,
(char)(group->permission!='\0' ? group->permission :
'?'));
diff -u -3 -r pan-0.13.3.93/pan/base/group.c pan-0.13.3.93_src/pan/base/group.c
--- pan-0.13.3.93/pan/base/group.c Mon Feb 10 04:29:48 2003
+++ pan-0.13.3.93_src/pan/base/group.c Tue Feb 25 06:00:02 2003
@@ -78,6 +78,7 @@
g->article_qty = (gint32)0;
g->article_read_qty = (gint32)0;
g->article_low = (gulong)0;
+ g->article_last_low_fetched = (gulong)0;
g->article_high_old = (gulong)0;
g->permission = '?';
g->article_high = (gulong)0;
@@ -96,6 +97,7 @@
g->_articles = NULL;
g->_deleted_articles = NULL;
g->_one_big_chunk = NULL;
+ g->_no_fetch_on_load = FALSE;
}
static GMutex*
@@ -335,13 +337,21 @@
void
group_set_article_range (Group * g,
- gulong article_low,
- gulong article_high)
+ gulong remote_article_low,
+ gulong remote_article_high,
+ gulong lowest_from_last_fetch)
{
g_return_if_fail (group_is_valid(g));
- g->article_low = article_low;
- g->article_high = article_high;
+ if(g->article_low == 0 || g->article_low > lowest_from_last_fetch)
+ g->article_low = lowest_from_last_fetch;
+
+ g->article_high = remote_article_high;
+
+ g->article_last_low_fetched = lowest_from_last_fetch;
+
newsrc_set_group_range (group_get_newsrc(g), g->article_low,
g->article_high);
newsrc_set_group_range (group_get_purged(g), g->article_low,
g->article_high);
@@ -577,7 +587,7 @@
/* if user wants to clear the range, do that too */
if (clear_counts)
- group_set_article_range (group, 0, 0);
+ group_set_article_range (group, 0, 0, 0);
fire_groups_changed (&group, 1);
}
@@ -777,7 +787,10 @@
{
/* maybe update the article range */
if (high!=group->article_high || low!=group->article_low)
- group_set_article_range (group, low, high);
+ group_set_article_range (group,
+ low,
+ high,
+ low!=group->article_low ? low
: group->article_last_low_fetched);
/* update the group article stats */
group_set_article_qty_impl (group, g_hash_table_size(ours));
diff -u -3 -r pan-0.13.3.93/pan/base/group.h pan-0.13.3.93_src/pan/base/group.h
--- pan-0.13.3.93/pan/base/group.h Tue Dec 10 19:22:34 2002
+++ pan-0.13.3.93_src/pan/base/group.h Wed Feb 26 13:05:21 2003
@@ -92,6 +92,7 @@
gint32 article_qty; /* (read only) # of
articles
in group */
gint32 article_read_qty; /* (read only) # of
read
articles */
gulong article_low; /* (read only) low
article
number */
+ gulong article_last_low_fetched;
gulong article_high; /* (read only) high
article
number */
gulong article_high_old; /* (read only)
previous high
article */
@@ -101,7 +102,8 @@
char * identity_name; /* last identity used
for
this group */
char * description; /* maybe null */
char * download_dir; /* where to save
files, or
NULL */
- char * default_charset; /* default charset for
articles that don't specify, or NULL */
+ char * default_charset; /* default charset for
articles that don't specify,
+
or NULL */
GStringChunk * chunk; /* string chunk used by
articles */
/* PRIVATE */
@@ -112,6 +114,9 @@
GHashTable * _articles; /* articles */
GPtrArray * _deleted_articles; /* deleted articles */
char * _one_big_chunk; /* used by loading
local
headers */
+ gboolean _no_fetch_on_load; /* for
switch_to_group() to
disable the download
+
in set_group_mainthread_end() */
+
}
Group;
@@ -134,7 +139,8 @@
void group_set_article_range (Group * group,
gulong article_low,
- gulong article_high);
+ gulong article_high,
+ gulong
low_to_fetch);
void group_get_article_range (const Group * group,
gulong * article_low,
diff -u -3 -r pan-0.13.3.93/pan/base/newsrc-port.c
pan-0.13.3.93_src/pan/base/newsrc-port.c
--- pan-0.13.3.93/pan/base/newsrc-port.c Mon Dec 30 08:03:48 2002
+++ pan-0.13.3.93_src/pan/base/newsrc-port.c Sun Feb 23 21:59:24 2003
@@ -147,7 +147,9 @@
subscribed = FALSE;
newsrc_import_line (n, &subscribed, line->str);
g_ptr_array_add (subscribed?sub_y:sub_n, group);
- group_set_article_range (group, n->group_low,
n->group_high);
+ group_set_article_range (group,
+ n->group_low,
n->group_high,
+
group->article_last_low_fetched);
/* track how many groups imported */
++groups_imported_qty;
diff -u -3 -r pan-0.13.3.93/pan/dialogs/dialog-headers.c
pan-0.13.3.93_src/pan/dialogs/dialog-headers.c
--- pan-0.13.3.93/pan/dialogs/dialog-headers.c Mon Nov 25 22:08:10 2002
+++ pan-0.13.3.93_src/pan/dialogs/dialog-headers.c Tue Feb 25 04:48:20 2003
@@ -66,9 +66,14 @@
switch_to_group (gpointer callobj, gpointer callarg, gpointer client)
{
Group * group = GROUP (client);
-
+ gboolean tmp = group->_no_fetch_on_load;
+
+ group->_no_fetch_on_load = TRUE;
if (group->article_qty > 0)
articlelist_set_group (group);
+
+ group->_no_fetch_on_load = tmp;
+
}
static void
@@ -205,10 +210,11 @@
/* some new headers */
hbox = gtk_hbox_new (FALSE, 0);
w = gtk_radio_button_new_with_mnemonic_from_widget (GTK_RADIO_BUTTON(w),
_("Download _recent headers: "));
+ gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(w), TRUE);
gtk_box_pack_start (GTK_BOX(hbox), w, FALSE, FALSE, 0);
dialog->latest_n_headers_rb = w;
adj = GTK_ADJUSTMENT(gtk_adjustment_new (
- 150, 0, INT_MAX, 50, 50, 1));
+ 600, 0, INT_MAX, 50, 50, 1));
w = gtk_spin_button_new (adj, 1, 0);
gtk_box_pack_start (GTK_BOX(hbox), w, FALSE, FALSE, 0);
dialog->n_headers_spinbutton = w;
diff -u -3 -r pan-0.13.3.93/pan/nntp.c pan-0.13.3.93_src/pan/nntp.c
--- pan-0.13.3.93/pan/nntp.c Fri Jan 24 19:26:28 2003
+++ pan-0.13.3.93_src/pan/nntp.c Mon Feb 24 01:11:18 2003
@@ -744,6 +744,7 @@
gulong total;
gulong first;
gulong last;
+ gulong low_no_overlap = low;
gboolean read_status = 0;
gint result;
gint val;
@@ -787,8 +788,11 @@
/**
*** Tell the server that we want a series of article headers...
**/
+
+ if (low > group->article_low && low < group->article_high_old)
+ low_no_overlap = group->article_high_old + 1;
- val = *abort ? -1 : nntp_command_va (status, sock, &response, "XOVER
%lu-%lu", low, high);
+ val = *abort ? -1 : nntp_command_va (status, sock, &response, "XOVER
%lu-%lu", low_no_overlap, high);
if (val != 224) {
status_item_emit_status_va (status, _("Getting header list
failed: %s"),
response);
return val==-1 ? TASK_FAIL : TASK_FAIL_HOPELESS;
diff -u -3 -r pan-0.13.3.93/pan/task-headers.c
pan-0.13.3.93_src/pan/task-headers.c
--- pan-0.13.3.93/pan/task-headers.c Fri Jan 24 19:26:28 2003
+++ pan-0.13.3.93_src/pan/task-headers.c Tue Feb 25 17:07:00 2003
@@ -46,8 +46,6 @@
********************** Defines / Enumerated types
*********************/
-#define SAMPLE_SIZE_DEFAULT 150
-
/*********************
********************** Macros
*********************/
@@ -255,6 +253,25 @@
&task->hint_abort,
progress_fmt,
articles);
+
+ /* if there are fewer _new_ headers than the user requested,
include the
difference
+ as next most recent articles lying below the lowest article
we have
locally */
+ if (low_to_fetch > task_h->group->article_low &&
+ low_to_fetch < task_h->group->article_high) {
+ if (task_h->group->article_low > low_in_group) {
+ high_to_fetch = task_h->group->article_low - 1;
+ low_to_fetch = high_to_fetch -
(task_h->group->article_high -
low_to_fetch);
+ result = nntp_download_headers (status,
+ sock,
+ task_h->group,
+ low_to_fetch,
+ high_to_fetch,
+ &task->hint_abort,
+ progress_fmt,
+ articles);
+ }
+ }
+
if (result != TASK_SUCCESS) {
state_set = TRUE;
state = result;
@@ -301,7 +318,7 @@
group_mark_new_article_number (task_h->group,
task_h->group->article_high);
group_add_articles_remove_unused (task_h->group,
articles, status);
group_expire_articles_not_in_range (task_h->group,
low_in_group,
high_in_group);
- group_set_article_range (task_h->group, low_in_group,
high_in_group);
+ group_set_article_range (task_h->group, low_in_group,
high_in_group,
low_to_fetch);
/* apply the rules to the new headers */
rule_manager_process_incoming_articles (articles);
@@ -378,12 +395,12 @@
static int
task_get_header_range (TaskHeaders * task,
PanSocket * sock,
- gulong * setme_lo_article_in_group,
- gulong * setme_hi_article_in_group,
+ gulong * setme_lo_article_in_group,
+ gulong * setme_hi_article_in_group,
gulong * setme_lo_article_to_fetch,
- gulong * setme_hi_article_to_fetch,
- gulong * setme_total_in_group,
- const char ** setme_progress_fmt)
+ gulong * setme_hi_article_to_fetch,
+ gulong * setme_total_in_group,
+ const char ** setme_progress_fmt)
{
const gboolean * const abort = &TASK(task)->hint_abort;
Group *group = TASK_HEADERS(task)->group;
@@ -432,14 +449,32 @@
hi_article_to_fetch = hi_article_in_group;
progress_fmt = _("New %lu of %lu");
}
-
else if (dl_type==HEADERS_SAMPLE)
{
+/* Two cases to consider when sampling:-
+ 1) We already have the very latest article(s) on the server
+ i) Are there articles sandwich'ed between the last article
from the
previous
+ download and the highest local article _before_ the
download ?
+ ii) Are there articles older than the very lowest local one ?
+ 2) When we don't have the very latest articles, then we get them
+*/
const int qty = MIN (total_in_group, task->sample_size);
status_item_emit_status_va (
STATUS_ITEM(task), _("Sampling %d articles"), qty);
- lo_article_to_fetch = hi_article_in_group - qty;
- hi_article_to_fetch = hi_article_in_group;
+ if(hi_article_in_group == group->article_high) {
+ if(group->article_last_low_fetched -
group->article_high_old > 1) {
+ hi_article_to_fetch =
group->article_last_low_fetched - 1;
+ lo_article_to_fetch = hi_article_to_fetch - qty;
+ }
+ else if(group->article_low > lo_article_in_group) {
+ hi_article_to_fetch = group->article_low - 1;
+ lo_article_to_fetch = hi_article_to_fetch - qty;
+ }
+ }
+ else {
+ lo_article_to_fetch = hi_article_in_group - qty;
+ hi_article_to_fetch = hi_article_in_group;
+ }
progress_fmt = _("Sampling %lu of %lu");
}
else if (dl_type==HEADERS_ALL)
diff -u -3 -r pan-0.13.3.93/pan/task-headers.h
pan-0.13.3.93_src/pan/task-headers.h
--- pan-0.13.3.93/pan/task-headers.h Sun Aug 4 23:50:53 2002
+++ pan-0.13.3.93_src/pan/task-headers.h Sat Feb 22 14:32:51 2003
@@ -29,6 +29,7 @@
**/
#define TASK_HEADERS(a) ((TaskHeaders *)a)
+#define SAMPLE_SIZE_DEFAULT 600
typedef struct _TaskHeaders TaskHeaders;
__________________________________________________
Do you Yahoo!?
Yahoo! Tax Center - forms, calculators, tips, more
http://taxes.yahoo.com/