* gnus-agent.el (gnus-agentize): Updated documentation to match
[gnus] / lisp / gnus-agent.el
index 527cb69..bdda87e 100644 (file)
 
 (require 'gnus)
 (require 'gnus-cache)
+(require 'nnmail)
 (require 'nnvirtual)
 (require 'gnus-sum)
 (require 'gnus-score)
 (require 'gnus-srvr)
+(require 'gnus-util)
 (eval-when-compile
   (if (featurep 'xemacs)
       (require 'itimer)
   :group 'gnus-agent
   :type 'hook)
 
+(defcustom gnus-agent-fetched-hook nil
+  "Hook run when finished fetching articles."
+  :group 'gnus-agent
+  :type 'hook)
+
 (defcustom gnus-agent-handle-level gnus-level-subscribed
   "Groups on levels higher than this variable will be ignored by the Agent."
   :group 'gnus-agent
@@ -153,21 +160,26 @@ this limit."
 
 (defcustom gnus-agent-enable-expiration 'ENABLE
   "The default expiration state for each group.
-When set to ENABLE, the default, gnus-agent-expire will expire old
+When set to ENABLE, the default, `gnus-agent-expire' will expire old
 contents from a group's local storage.  This value may be overridden
 to disable expiration in specific categories, topics, and groups.  Of
-course, you could change gnus-agent-enable-expiration to nil then
+course, you could change gnus-agent-enable-expiration to DISABLE then
 enable expiration per categories, topics, and groups."
   :group 'gnus-agent
   :type '(radio (const :format "Enable " ENABLE)
                 (const :format "Disable " DISABLE)))
 
+(defcustom gnus-agent-expire-unagentized-dirs t
+"Have gnus-agent-expire scan the directories under
+\(gnus-agent-directory) for groups that are no longer agentized.  When
+found, offer to remove them.")
+
 ;;; Internal variables
 
 (defvar gnus-agent-history-buffers nil)
 (defvar gnus-agent-buffer-alist nil)
 (defvar gnus-agent-article-alist nil
-"An assoc list identifying the articles whose headers have been fetched.  
+  "An assoc list identifying the articles whose headers have been fetched.  
 If successfully fetched, these headers will be stored in the group's overview
 file.  The key of each assoc pair is the article ID, the value of each assoc
 pair is a flag indicating whether the identified article has been downloaded
@@ -176,8 +188,7 @@ NOTES:
 1) The last element of this list can not be expired as some 
    routines (for example, get-agent-fetch-headers) use the last
    value to track which articles have had their headers retrieved.
-2) The gnus-agent-regenerate may destructively modify the value.
-")
+2) The function `gnus-agent-regenerate' may destructively modify the value.")
 (defvar gnus-agent-group-alist nil)
 (defvar gnus-category-alist nil)
 (defvar gnus-agent-current-history nil)
@@ -273,56 +284,61 @@ node `(gnus)Server Buffer'.")
                     (setq category (cdr category)))))))
   category)
 
-(defmacro gnus-agent-cat-defaccessor (name prop-name)
-  "Define accessor and setter methods for manipulating a list of the form
-(NAME (PROPERTY1 VALUE1) ... (PROPERTY_N VALUE_N)).
+(eval-when-compile
+  (defmacro gnus-agent-cat-defaccessor (name prop-name)
+    "Define accessor and setter methods for manipulating a list of the form
+\(NAME (PROPERTY1 VALUE1) ... (PROPERTY_N VALUE_N)).
 Given the call (gnus-agent-cat-defaccessor func PROPERTY1), the list may be
 manipulated as follows:
   (func LIST): Returns VALUE1
   (setf (func LIST) NEW_VALUE1): Replaces VALUE1 with NEW_VALUE1."
-  `(progn (defmacro ,name (category)
-            (list (quote cdr) (list (quote assq)
-                                    (quote (quote ,prop-name)) category)))
-
-          (define-setf-method ,name (category)
-            (let* ((--category--temp-- (gensym "--category--"))
-                   (--value--temp-- (gensym "--value--")))
-              (list (list --category--temp--) ; temporary-variables
-                    (list category)     ; value-forms
-                    (list --value--temp--) ; store-variables
-                    (let* ((category --category--temp--) ; store-form
-                           (value --value--temp--))
-                      (list (quote gnus-agent-cat-set-property)
-                            category
-                            (quote (quote ,prop-name))
-                            value))
-                    (list (quote ,name) --category--temp--) ; access-form
-                    )))))
+    `(progn (defmacro ,name (category)
+              (list (quote cdr) (list (quote assq)
+                                      (quote (quote ,prop-name)) category)))
+
+            (define-setf-method ,name (category)
+              (let* ((--category--temp-- (make-symbol "--category--"))
+                     (--value--temp-- (make-symbol "--value--")))
+                (list (list --category--temp--) ; temporary-variables
+                      (list category)   ; value-forms
+                      (list --value--temp--) ; store-variables
+                      (let* ((category --category--temp--) ; store-form
+                             (value --value--temp--))
+                        (list (quote gnus-agent-cat-set-property)
+                              category
+                              (quote (quote ,prop-name))
+                              value))
+                      (list (quote ,name) --category--temp--) ; access-form
+                      )))))
+  )
 
 (defmacro gnus-agent-cat-name (category)
   `(car ,category))
 
 (gnus-agent-cat-defaccessor
- gnus-agent-cat-days-until-old    agent-days-until-old)
+ gnus-agent-cat-days-until-old             agent-days-until-old)
 (gnus-agent-cat-defaccessor
- gnus-agent-cat-enable-expiration agent-enable-expiration)
+ gnus-agent-cat-enable-expiration          agent-enable-expiration)
 (gnus-agent-cat-defaccessor
- gnus-agent-cat-groups            agent-groups)
+ gnus-agent-cat-groups                     agent-groups)
 (gnus-agent-cat-defaccessor
- gnus-agent-cat-high-score        agent-high-score)
+ gnus-agent-cat-high-score                 agent-high-score)
 (gnus-agent-cat-defaccessor
- gnus-agent-cat-length-when-long  agent-length-when-long)
+ gnus-agent-cat-length-when-long           agent-length-when-long)
 (gnus-agent-cat-defaccessor
- gnus-agent-cat-length-when-short agent-length-when-short)
+ gnus-agent-cat-length-when-short          agent-length-when-short)
 (gnus-agent-cat-defaccessor
- gnus-agent-cat-low-score         agent-low-score)
+ gnus-agent-cat-low-score                  agent-low-score)
 (gnus-agent-cat-defaccessor
- gnus-agent-cat-predicate         agent-predicate)
+ gnus-agent-cat-predicate                  agent-predicate)
 (gnus-agent-cat-defaccessor
- gnus-agent-cat-score-file        agent-score-file)
+ gnus-agent-cat-score-file                 agent-score-file)
+(gnus-agent-cat-defaccessor
+ gnus-agent-cat-disable-undownloaded-faces agent-disable-undownloaded-faces)
 
-(defsetf gnus-agent-cat-groups (category) (groups)
-  (list 'gnus-agent-set-cat-groups category groups))
+(eval-when-compile
+  (defsetf gnus-agent-cat-groups (category) (groups)
+    (list 'gnus-agent-set-cat-groups category groups)))
 
 (defun gnus-agent-set-cat-groups (category groups)
   (unless (eq groups 'ignore)
@@ -357,8 +373,8 @@ manipulated as follows:
                            (setcdr category (cons cell (cdr category)))
                            cell)) groups))))))
 
-(defsubst gnus-agent-cat-make (name)
-  (list name '(agent-predicate . false)))
+(defsubst gnus-agent-cat-make (name &optional default-agent-predicate)
+  (list name `(agent-predicate . ,(or default-agent-predicate 'false))))
 
 ;;; Fetching setup functions.
 
@@ -387,6 +403,10 @@ manipulated as follows:
 (defmacro gnus-agent-append-to-list (tail value)
   `(setq ,tail (setcdr ,tail (cons ,value nil))))
 
+(defmacro gnus-agent-message (level &rest args)
+  `(if (<= ,level gnus-verbose)
+       (message ,@args)))
+
 ;;;
 ;;; Mode infestation
 ;;;
@@ -416,7 +436,13 @@ manipulated as follows:
                                                     buffer))))
            minor-mode-map-alist))
     (when (eq major-mode 'gnus-group-mode)
-      (gnus-agent-toggle-plugged gnus-plugged))
+      (let ((init-plugged gnus-plugged)
+            (gnus-agent-go-online nil))
+        ;; g-a-t-p does nothing when gnus-plugged isn't changed.
+        ;; Therefore, make certain that the current value does not
+        ;; match the desired initial value.
+        (setq gnus-plugged :unknown)
+        (gnus-agent-toggle-plugged init-plugged)))
     (gnus-run-hooks 'gnus-agent-mode-hook
                    (intern (format "gnus-agent-%s-mode-hook" buffer)))))
 
@@ -558,10 +584,10 @@ manipulated as follows:
 ;;;###autoload
 (defun gnus-agentize ()
   "Allow Gnus to be an offline newsreader.
-The normal usage of this command is to put the following as the
-last form in your `.gnus.el' file:
 
-\(gnus-agentize)
+The gnus-agentize function is now called internally by gnus when
+gnus-agent is set.  If you wish to avoid calling gnus-agentize,
+customize gnus-agent to nil.
 
 This will modify the `gnus-setup-news-hook', and
 `message-send-mail-real-function' variables, and install the Gnus agent
@@ -585,14 +611,18 @@ minor mode in all Gnus buffers."
                       gnus-agent-covered-methods ))))
      (append (list gnus-select-method) gnus-secondary-select-methods))))
 
-(defun gnus-agent-queue-setup ()
-  "Make sure the queue group exists."
-  (unless (gnus-gethash "nndraft:queue" gnus-newsrc-hashtb)
-    (gnus-request-create-group "queue" '(nndraft ""))
+(defun gnus-agent-queue-setup (&optional group-name)
+  "Make sure the queue group exists.
+Optional arg GROUP-NAME allows to specify another group."
+  (unless (gnus-gethash (format "nndraft:%s" (or group-name "queue"))
+                       gnus-newsrc-hashtb)
+    (gnus-request-create-group (or group-name "queue") '(nndraft ""))
     (let ((gnus-level-default-subscribed 1))
-      (gnus-subscribe-group "nndraft:queue" nil '(nndraft "")))
+      (gnus-subscribe-group (format "nndraft:%s" (or group-name "queue"))
+                           nil '(nndraft "")))
     (gnus-group-set-parameter
-     "nndraft:queue" 'gnus-dummy '((gnus-draft-mode)))))
+     (format "nndraft:%s" (or group-name "queue"))
+     'gnus-dummy '((gnus-draft-mode)))))
 
 (defun gnus-agent-send-mail ()
   (if gnus-plugged
@@ -740,8 +770,7 @@ be a select method."
          (gnus-message 1 "Couldn't open server %s" (nth 1 gnus-command-method))
        (while (not (eobp))
          (if (null (eval (read (current-buffer))))
-             (progn (forward-line)
-                    (kill-line -1))
+             (gnus-delete-line)
            (write-file (gnus-agent-lib-file "flags"))
            (error "Couldn't set flags from file %s"
                   (gnus-agent-lib-file "flags"))))
@@ -883,6 +912,7 @@ article's mark is toggled."
              (headers (sort (mapcar (lambda (h)
                                       (mail-header-number h))
                                     gnus-newsgroup-headers) '<))
+             (cached (and gnus-use-cache gnus-newsgroup-cached))
              (undownloaded (list nil))
              (tail-undownloaded undownloaded)
              (unfetched (list nil))
@@ -893,23 +923,30 @@ article's mark is toggled."
            (cond ((< a h)
                   ;; Ignore IDs in the alist that are not being
                   ;; displayed in the summary.
-                  (pop alist))
+                  (setq alist (cdr alist)))
                  ((> a h)
                    ;; Headers that are not in the alist should be
                    ;; fictious (see nnagent-retrieve-headers); they
                    ;; imply that this article isn't in the agent.
                   (gnus-agent-append-to-list tail-undownloaded h)
                   (gnus-agent-append-to-list tail-unfetched    h)
-                   (pop headers)) 
+                   (setq headers (cdr headers))) 
                  ((cdar alist)
-                  (pop alist)
-                  (pop headers)
+                  (setq alist (cdr alist))
+                  (setq headers (cdr headers))
                   nil                  ; ignore already downloaded
                   )
                  (t
-                  (pop alist)
-                  (pop headers)
-                  (gnus-agent-append-to-list tail-undownloaded a)))))
+                  (setq alist (cdr alist))
+                  (setq headers (cdr headers))
+                   
+                   ;; This article isn't in the agent.  Check to see
+                   ;; if it is in the cache.  If it is, it's been
+                   ;; downloaded.
+                   (while (and cached (< (car cached) a))
+                     (setq cached (cdr cached)))
+                   (unless (equal a (car cached))
+                     (gnus-agent-append-to-list tail-undownloaded a))))))
 
        (while headers
           (let ((num (pop headers)))
@@ -1008,14 +1045,21 @@ This can be added to `gnus-select-article-hook' or
             (list gnus-current-article))
        (setq gnus-newsgroup-undownloaded
              (delq gnus-current-article gnus-newsgroup-undownloaded))
-       (gnus-summary-update-article-line
-        gnus-current-article
-        (gnus-summary-article-header gnus-current-article))))))
+        (gnus-summary-update-download-mark gnus-current-article)))))
 
 ;;;
 ;;; Internal functions
 ;;;
 
+;;; NOTES:
+;;; The agent's active range is defined as follows:
+;;;  If the agent has no record of the group, use the actual active
+;;;    range.
+;;;  If the agent has a record, set the agent's active range to
+;;;    include the max limit of the actual active range.
+;;;  When expiring, update the min limit to match the smallest of the
+;;;    min article not expired or the min actual active range.
+
 (defun gnus-agent-save-active (method)
   (gnus-agent-save-active-1 method 'gnus-active-to-gnus-format))
 
@@ -1029,32 +1073,41 @@ This can be added to `gnus-select-article-hook' or
       (erase-buffer)
       (nnheader-insert-file-contents file))))
 
-(defun gnus-agent-write-active (file new)
-  (let ((orig (gnus-make-hashtable (count-lines (point-min) (point-max))))
-       (file (gnus-agent-lib-file "active"))
-       elem osym)
-    (when (file-exists-p file)
+(defun gnus-agent-write-active (file new &optional literal-replacement)
+  (let ((old new))
+    (when (and (not literal-replacement)
+               (file-exists-p file))
+      (setq old (gnus-make-hashtable (count-lines (point-min) (point-max))))
       (with-temp-buffer
-       (nnheader-insert-file-contents file)
-       (gnus-active-to-gnus-format nil orig))
+        (nnheader-insert-file-contents file)
+        (gnus-active-to-gnus-format nil old))
+      ;; Iterate over the current active groups, the current active
+      ;; range may expand, but NOT CONTRACT, the agent's active range.
       (mapatoms
-       (lambda (sym)
-        (when (and sym (boundp sym))
-          (if (and (boundp (setq osym (intern (symbol-name sym) orig)))
-                   (setq elem (symbol-value osym)))
-              (progn
-                (if (and (integerp (car (symbol-value sym)))
-                         (> (car elem) (car (symbol-value sym))))
-                    (setcar elem (car (symbol-value sym))))
-                (if (integerp (cdr (symbol-value sym)))
-                    (setcdr elem (cdr (symbol-value sym)))))
-            (set (intern (symbol-name sym) orig) (symbol-value sym)))))
+       (lambda (nsym)
+         (let ((new-active (and nsym (boundp nsym) (symbol-value nsym))))
+           (when new-active
+             (let* ((osym       (intern (symbol-name nsym) old))
+                    (old-active (and (boundp osym) (symbol-value osym))))
+               (if old-active
+                   (let ((new-min (car new-active))
+                         (old-min (car old-active))
+                         (new-max (cdr new-active))
+                         (old-max (cdr old-active)))
+                     (if (and (integerp new-min)
+                              (< new-min old-min))
+                         (setcar old-active new-min))
+                     (if (and (integerp new-max)
+                              (> new-max old-max))
+                         (setcdr old-active new-max)))
+                 (set osym new-active))))))
        new))
     (gnus-make-directory (file-name-directory file))
     (let ((nnmail-active-file-coding-system gnus-agent-file-coding-system))
-      ;; The hashtable contains real names of groups,  no more prefix
-      ;; removing, so set `full' to `t'.
-      (gnus-write-active-file file orig t))))
+      ;; The hashtable contains real names of groups.  However, do NOT
+      ;; add the foreign server prefix as gnus-active-to-gnus-format
+      ;; will add it while reading the file.
+      (gnus-write-active-file file old nil))))
 
 (defun gnus-agent-save-groups (method)
   (gnus-agent-save-active-1 method 'gnus-groups-to-gnus-format))
@@ -1065,39 +1118,57 @@ This can be added to `gnus-select-article-hook' or
           (coding-system-for-write nnheader-file-coding-system)
           (file-name-coding-system nnmail-pathname-coding-system)
           (file (gnus-agent-lib-file "active"))
-          oactive-min)
+          oactive-min oactive-max)
       (gnus-make-directory (file-name-directory file))
       (with-temp-file file
        ;; Emacs got problem to match non-ASCII group in multibyte buffer.
        (mm-disable-multibyte)
        (when (file-exists-p file)
-         (nnheader-insert-file-contents file))
-       (goto-char (point-min))
-       (when (re-search-forward
-              (concat "^" (regexp-quote group) " ") nil t)
-         (save-excursion
-           (read (current-buffer))                      ;; max
-           (setq oactive-min (read (current-buffer))))  ;; min
-         (gnus-delete-line))
+         (nnheader-insert-file-contents file)
+
+          (goto-char (point-min))
+          (when (re-search-forward
+                 (concat "^" (regexp-quote group) " ") nil t)
+            (save-excursion
+              (setq oactive-max (read (current-buffer)) ;; max
+                    oactive-min (read (current-buffer)))) ;; min
+            (gnus-delete-line)))
        (insert (format "%S %d %d y\n" (intern group)
-                       (cdr active)
-                       (or oactive-min (car active))))
+                       (max (or oactive-max (cdr active)) (cdr active))
+                        (min (or oactive-min (car active)) (car active))))
        (goto-char (point-max))
        (while (search-backward "\\." nil t)
          (delete-char 1))))))
 
 (defun gnus-agent-group-path (group)
   "Translate GROUP into a file name."
-  (if nnmail-use-long-file-names
-      (gnus-group-real-name group)
-    (nnheader-translate-file-chars
-     (nnheader-replace-chars-in-string
-      (nnheader-replace-duplicate-chars-in-string
-       (nnheader-replace-chars-in-string
-       (gnus-group-real-name group)
-       ?/ ?_)
-       ?. ?_)
-      ?. ?/))))
+
+  ;; NOTE: This is what nnmail-group-pathname does as of Apr 2003.
+  ;; The two methods must be kept synchronized, which is why
+  ;; gnus-agent-group-pathname was added.
+
+  (setq group
+        (nnheader-translate-file-chars
+         (nnheader-replace-duplicate-chars-in-string
+          (nnheader-replace-chars-in-string 
+           (gnus-group-real-name group)
+           ?/ ?_)
+          ?. ?_)))
+  (if (or nnmail-use-long-file-names
+          (file-directory-p (expand-file-name group (gnus-agent-directory))))
+      group
+    (mm-encode-coding-string
+     (nnheader-replace-chars-in-string group ?. ?/)
+     nnmail-pathname-coding-system)))
+
+(defun gnus-agent-group-pathname (group)
+  "Translate GROUP into a file name."
+  ;; nnagent uses nnmail-group-pathname to read articles while
+  ;; unplugged.  The agent must, therefore, use the same directory
+  ;; while plugged.
+  (let ((gnus-command-method (or gnus-command-method
+                                 (gnus-find-method-for-group group))))
+    (nnmail-group-pathname (gnus-group-real-name group) (gnus-agent-directory))))
 
 (defun gnus-agent-get-function (method)
   (if (gnus-online method)
@@ -1194,9 +1265,7 @@ This can be added to `gnus-select-article-hook' or
       (when (or (cdr selected-sets) (car selected-sets))
         (let* ((fetched-articles (list nil))
                (tail-fetched-articles fetched-articles)
-               (dir (concat
-                     (gnus-agent-directory)
-                     (gnus-agent-group-path group) "/"))
+               (dir (gnus-agent-group-pathname group))
                (date (time-to-days (current-time)))
                (case-fold-search t)
                pos crosses id)
@@ -1265,7 +1334,7 @@ This can be added to `gnus-select-article-hook' or
                       (gnus-agent-append-to-list
                       tail-fetched-articles (caar pos)))
                     (widen)
-                    (pop pos))))
+                    (setq pos (cdr pos)))))
 
             (gnus-agent-save-alist group (cdr fetched-articles) date)
             (gnus-message 7 ""))
@@ -1299,7 +1368,7 @@ This can be added to `gnus-select-article-hook' or
        (insert (string-to-number (cdar crosses)))
        (insert-buffer-substring gnus-agent-overview-buffer beg end)
         (gnus-agent-check-overview-buffer))
-      (pop crosses))))
+      (setq crosses (cdr crosses)))))
 
 (defun gnus-agent-backup-overview-buffer ()
   (when gnus-newsgroup-name
@@ -1367,7 +1436,7 @@ and that there are no duplicates."
                      (gnus-agent-article-name ".overview"
                                               (caar gnus-agent-buffer-alist))
                      nil 'silent))
-      (pop gnus-agent-buffer-alist))
+      (setq gnus-agent-buffer-alist (cdr gnus-agent-buffer-alist)))
     (while gnus-agent-group-alist
       (with-temp-file (gnus-agent-article-name
                       ".agentview" (caar gnus-agent-group-alist))
@@ -1375,7 +1444,7 @@ and that there are no duplicates."
        (insert "\n")
         (princ 1 (current-buffer))
        (insert "\n"))
-      (pop gnus-agent-group-alist))))
+      (setq gnus-agent-group-alist (cdr gnus-agent-group-alist)))))
 
 (defun gnus-agent-find-parameter (group symbol)
   "Search for GROUPs SYMBOL in the group's parameters, the group's
@@ -1386,13 +1455,14 @@ variables.  Returns the first non-nil value found."
       (symbol-value
        (cdr
         (assq symbol
-         '((agent-short-article . gnus-agent-short-article)
-           (agent-long-article . gnus-agent-long-article)
-           (agent-low-score . gnus-agent-low-score)
-           (agent-high-score . gnus-agent-high-score)
-           (agent-days-until-old . gnus-agent-expire-days)
-           (agent-enable-expiration
-            . gnus-agent-enable-expiration)))))))
+              '((agent-short-article . gnus-agent-short-article)
+                (agent-long-article . gnus-agent-long-article)
+                (agent-low-score . gnus-agent-low-score)
+                (agent-high-score . gnus-agent-high-score)
+                (agent-days-until-old . gnus-agent-expire-days)
+                (agent-enable-expiration
+                 . gnus-agent-enable-expiration)
+                (agent-predicate . gnus-agent-predicate)))))))
 
 (defun gnus-agent-fetch-headers (group &optional force)
   "Fetch interesting headers into the agent.  The group's overview
@@ -1691,8 +1761,7 @@ FILE and places the combined headers into `nntp-server-buffer'."
 (defun gnus-agent-article-name (article group)
   (expand-file-name article
                    (file-name-as-directory
-                    (expand-file-name (gnus-agent-group-path group)
-                                      (gnus-agent-directory)))))
+                     (gnus-agent-group-pathname group))))
 
 (defun gnus-agent-batch-confirmation (msg)
   "Show error message and return t."
@@ -1746,8 +1815,8 @@ FILE and places the combined headers into `nntp-server-buffer'."
                                       (error-message-string err)))
                       (signal 'quit
                               "Cannot fetch articles into the Gnus agent")))))))))
-       (pop methods))
-      (run-hooks 'gnus-agent-fetch-hook)
+       (setq methods (cdr methods)))
+      (gnus-run-hooks 'gnus-agent-fetched-hook)
       (gnus-message 6 "Finished fetching articles into the Gnus agent"))))
 
 (defun gnus-agent-fetch-group-1 (group method)
@@ -1820,7 +1889,7 @@ FILE and places the combined headers into `nntp-server-buffer'."
           ;; If the selection predicate requires scoring, score each header
           (unless (memq predicate '(gnus-agent-true gnus-agent-false))
             (let ((score-param
-                   (gnus-agent-find-parameter group 'agent-score)))
+                   (gnus-agent-find-parameter group 'agent-score-file)))
               ;; Translate score-param into real one
               (cond
                ((not score-param))
@@ -1865,6 +1934,9 @@ FILE and places the combined headers into `nntp-server-buffer'."
                                       (gnus-agent-long-article
                                        (gnus-agent-find-parameter
                                         group 'agent-long-article))
+                                      (gnus-agent-short-article
+                                       (gnus-agent-find-parameter
+                                        group 'agent-short-article))
                                       (gnus-agent-low-score
                                        (gnus-agent-find-parameter
                                         group 'agent-low-score))
@@ -1953,6 +2025,9 @@ General format specifiers can also be used.  See Info node
 (defvar gnus-category-mode-line-format "Gnus: %%b"
   "The format specification for the category mode line.")
 
+(defvar gnus-agent-predicate 'false
+  "The selection predicate used when no other source is available.")
+
 (defvar gnus-agent-short-article 100
   "Articles that have fewer lines than this are short.")
 
@@ -2110,14 +2185,16 @@ The following commands are available:
                 (mapcar
                  (lambda (c)
                    (setcdr c
-                           (mapcar*
-                            (lambda (valu symb)
-                              (cons symb valu))
-                            (cdr c)
-                            '(agent-predicate agent-score-file agent-groups)))
+                           (delq nil
+                                 (gnus-mapcar
+                                  (lambda (valu symb)
+                                    (if valu
+                                        (cons symb valu)))
+                                  (cdr c)
+                                  '(agent-predicate agent-score-file agent-groups))))
                    c)
                  old-list)))))
-         (list (gnus-agent-cat-make 'default)))))
+         (list (gnus-agent-cat-make 'default 'short)))))
 
 (defun gnus-category-write ()
   "Write the category alist."
@@ -2300,7 +2377,7 @@ The following commands are available:
   (cond
    ;; Functions are just returned as is.
    ((or (symbolp predicate)
-       (gnus-functionp predicate))
+       (functionp predicate))
     `(,(or (cdr (assq predicate gnus-category-predicate-alist))
           predicate)))
    ;; More complex predicate.
@@ -2337,7 +2414,7 @@ return only unread articles."
          nil)
         ((not function)
          nil)
-        ((symbolp function)
+        ((functionp function)
          'ignore)
         ((memq (car function) '(or and not))
          (apply (car function)
@@ -2390,18 +2467,19 @@ FORCE is equivalent to setting the expiration predicates to true."
               (overview (gnus-get-buffer-create " *expire overview*"))
               orig)
           (unwind-protect
-              (when (file-exists-p (gnus-agent-lib-file "active"))
-                (with-temp-buffer
-                  (nnheader-insert-file-contents
-                   (gnus-agent-lib-file "active"))
-                  (gnus-active-to-gnus-format
-                   gnus-command-method
-                   (setq orig (gnus-make-hashtable
-                               (count-lines (point-min) (point-max))))))
-                (save-excursion
-                  (gnus-agent-expire-group-1
-                   group overview (gnus-gethash-safe group orig)
-                   articles force)))
+              (let ((active-file (gnus-agent-lib-file "active")))
+                (when (file-exists-p active-file)
+                  (with-temp-buffer
+                    (nnheader-insert-file-contents active-file)
+                    (gnus-active-to-gnus-format
+                     gnus-command-method
+                     (setq orig (gnus-make-hashtable
+                                 (count-lines (point-min) (point-max))))))
+                  (save-excursion
+                    (gnus-agent-expire-group-1
+                     group overview (gnus-gethash-safe group orig)
+                     articles force))
+                  (gnus-agent-write-active active-file orig t)))
             (kill-buffer overview))))
     (gnus-message 4 "Expiry...done")))
 
@@ -2409,310 +2487,321 @@ FORCE is equivalent to setting the expiration predicates to true."
   ;; Internal function - requires caller to have set
   ;; gnus-command-method, initialized overview buffer, and to have
   ;; provided a non-nil active
-  (interactive)
 
-  (if (eq 'DISABLE (gnus-agent-find-parameter group 'agent-enable-expiration))
-      (gnus-message 5 "Expiry skipping over %s" group)
-    (gnus-message 5 "Expiring articles in %s" group)
-    (gnus-agent-load-alist group)
-    (let* ((info (gnus-get-info group))
-           (alist gnus-agent-article-alist)
-           (dir (concat
-                 (gnus-agent-directory)
-                 (gnus-agent-group-path group)
-                 "/"))
-           (day (- (time-to-days (current-time))
-                   (gnus-agent-find-parameter group 'agent-days-until-old)))
-           (specials (if (and alist
-                              (not force))
-                         ;; This could be a bit of a problem.  I need to
-                         ;; keep the last article to avoid refetching
-                         ;; headers when using nntp in the backend.  At
-                         ;; the same time, if someone uses a backend
-                         ;; that supports article moving then I may have
-                         ;; to remove the last article to complete the
-                         ;; move.  Right now, I'm going to assume that
-                         ;; FORCE overrides specials.
-                         (list (caar (last alist)))))
-           (unreads ;; Articles that are excluded from the
-            ;; expiration process
-            (cond (gnus-agent-expire-all
-                   ;; All articles are marked read by global decree
-                   nil)
-                  ((eq articles t)
-                   ;; All articles are marked read by function
-                   ;; parameter
-                   nil)
-                  ((not articles)
-                   ;; Unread articles are marked protected from
-                   ;; expiration Don't call
-                   ;; gnus-list-of-unread-articles as it returns
-                   ;; articles that have not been fetched into the
-                   ;; agent.
-                   (ignore-errors
-                    (gnus-agent-unread-articles group)))
-                  (t
-                   ;; All articles EXCEPT those named by the caller
-                   ;; are protected from expiration
-                   (gnus-sorted-difference
-                    (gnus-uncompress-range
-                     (cons (caar alist)
-                           (caar (last alist))))
-                    (sort articles '<)))))
-           (marked ;; More articles that are exluded from the
-            ;; expiration process
-            (cond (gnus-agent-expire-all
-                   ;; All articles are unmarked by global decree
-                   nil)
-                  ((eq articles t)
-                   ;; All articles are unmarked by function
-                   ;; parameter
-                   nil)
-                  (articles
-                   ;; All articles may as well be unmarked as the
-                   ;; unreads list already names the articles we are
-                   ;; going to keep
-                   nil)
-                  (t
-                   ;; Ticked and/or dormant articles are excluded
-                   ;; from expiration
-                   (nconc
-                    (gnus-uncompress-range
-                     (cdr (assq 'tick (gnus-info-marks info))))
-                    (gnus-uncompress-range
-                     (cdr (assq 'dormant
-                                (gnus-info-marks info))))))))
-           (nov-file (concat dir ".overview"))
-           (cnt 0)
-           (completed -1)
-           dlist
-           type)
-
-      ;; The normal article alist contains elements that look like
-      ;; (article# .  fetch_date) I need to combine other
-      ;; information with this list.  For example, a flag indicating
-      ;; that a particular article MUST BE KEPT.  To do this, I'm
-      ;; going to transform the elements to look like (article#
-      ;; fetch_date keep_flag NOV_entry_marker) Later, I'll reverse
-      ;; the process to generate the expired article alist.
-
-      ;; Convert the alist elements to (article# fetch_date nil
-      ;; nil).
-      (setq dlist (mapcar (lambda (e)
-                            (list (car e) (cdr e) nil nil)) alist))
-
-      ;; Convert the keep lists to elements that look like (article#
-      ;; nil keep_flag nil) then append it to the expanded dlist
-      ;; These statements are sorted by ascending precidence of the
-      ;; keep_flag.
-      (setq dlist (nconc dlist
-                         (mapcar (lambda (e)
-                                   (list e nil 'unread  nil))
-                                 unreads)))
-      (setq dlist (nconc dlist
-                         (mapcar (lambda (e)
-                                   (list e nil 'marked  nil))
-                                 marked)))
-      (setq dlist (nconc dlist
-                         (mapcar (lambda (e)
-                                   (list e nil 'special nil))
-                                 specials)))
-
-      (set-buffer overview)
-      (erase-buffer)
-      (when (file-exists-p nov-file)
-        (gnus-message 7 "gnus-agent-expire: Loading overview...")
-        (nnheader-insert-file-contents nov-file)
-        (goto-char (point-min))
-
-        (let (p)
-          (while (< (setq p (point)) (point-max))
-            (condition-case nil
-                ;; If I successfully read an integer (the plus zero
-                ;; ensures a numeric type), prepend a marker entry
-                ;; to the list
-                (push (list (+ 0 (read (current-buffer))) nil nil
-                            (set-marker (make-marker) p))
-                      dlist)
-              (error
-               (gnus-message 1 "gnus-agent-expire: read error \
+  (let ((dir (gnus-agent-group-pathname group)))
+    (when (boundp 'gnus-agent-expire-current-dirs)
+      (set 'gnus-agent-expire-current-dirs 
+           (cons dir 
+                 (symbol-value 'gnus-agent-expire-current-dirs))))
+
+    (if (and (not force)
+             (eq 'DISABLE (gnus-agent-find-parameter group 
+                                                     'agent-enable-expiration)))
+        (gnus-message 5 "Expiry skipping over %s" group)
+      (gnus-message 5 "Expiring articles in %s" group)
+      (gnus-agent-load-alist group)
+      (let* ((info (gnus-get-info group))
+             (alist gnus-agent-article-alist)
+             (day (- (time-to-days (current-time))
+                     (gnus-agent-find-parameter group 'agent-days-until-old)))
+             (specials (if (and alist
+                                (not force))
+                           ;; This could be a bit of a problem.  I need to
+                           ;; keep the last article to avoid refetching
+                           ;; headers when using nntp in the backend.  At
+                           ;; the same time, if someone uses a backend
+                           ;; that supports article moving then I may have
+                           ;; to remove the last article to complete the
+                           ;; move.  Right now, I'm going to assume that
+                           ;; FORCE overrides specials.
+                           (list (caar (last alist)))))
+             (unreads ;; Articles that are excluded from the
+              ;; expiration process
+              (cond (gnus-agent-expire-all
+                     ;; All articles are marked read by global decree
+                     nil)
+                    ((eq articles t)
+                     ;; All articles are marked read by function
+                     ;; parameter
+                     nil)
+                    ((not articles)
+                     ;; Unread articles are marked protected from
+                     ;; expiration Don't call
+                     ;; gnus-list-of-unread-articles as it returns
+                     ;; articles that have not been fetched into the
+                     ;; agent.
+                     (ignore-errors
+                       (gnus-agent-unread-articles group)))
+                    (t
+                     ;; All articles EXCEPT those named by the caller
+                     ;; are protected from expiration
+                     (gnus-sorted-difference
+                      (gnus-uncompress-range
+                       (cons (caar alist)
+                             (caar (last alist))))
+                      (sort articles '<)))))
+             (marked ;; More articles that are exluded from the
+              ;; expiration process
+              (cond (gnus-agent-expire-all
+                     ;; All articles are unmarked by global decree
+                     nil)
+                    ((eq articles t)
+                     ;; All articles are unmarked by function
+                     ;; parameter
+                     nil)
+                    (articles
+                     ;; All articles may as well be unmarked as the
+                     ;; unreads list already names the articles we are
+                     ;; going to keep
+                     nil)
+                    (t
+                     ;; Ticked and/or dormant articles are excluded
+                     ;; from expiration
+                     (nconc
+                      (gnus-uncompress-range
+                       (cdr (assq 'tick (gnus-info-marks info))))
+                      (gnus-uncompress-range
+                       (cdr (assq 'dormant
+                                  (gnus-info-marks info))))))))
+             (nov-file (concat dir ".overview"))
+             (cnt 0)
+             (completed -1)
+             dlist
+             type)
+
+        ;; The normal article alist contains elements that look like
+        ;; (article# .  fetch_date) I need to combine other
+        ;; information with this list.  For example, a flag indicating
+        ;; that a particular article MUST BE KEPT.  To do this, I'm
+        ;; going to transform the elements to look like (article#
+        ;; fetch_date keep_flag NOV_entry_marker) Later, I'll reverse
+        ;; the process to generate the expired article alist.
+
+        ;; Convert the alist elements to (article# fetch_date nil
+        ;; nil).
+        (setq dlist (mapcar (lambda (e)
+                              (list (car e) (cdr e) nil nil)) alist))
+
+        ;; Convert the keep lists to elements that look like (article#
+        ;; nil keep_flag nil) then append it to the expanded dlist
+        ;; These statements are sorted by ascending precidence of the
+        ;; keep_flag.
+        (setq dlist (nconc dlist
+                           (mapcar (lambda (e)
+                                     (list e nil 'unread  nil))
+                                   unreads)))
+        (setq dlist (nconc dlist
+                           (mapcar (lambda (e)
+                                     (list e nil 'marked  nil))
+                                   marked)))
+        (setq dlist (nconc dlist
+                           (mapcar (lambda (e)
+                                     (list e nil 'special nil))
+                                   specials)))
+
+        (set-buffer overview)
+        (erase-buffer)
+        (buffer-disable-undo)
+        (when (file-exists-p nov-file)
+          (gnus-message 7 "gnus-agent-expire: Loading overview...")
+          (nnheader-insert-file-contents nov-file)
+          (goto-char (point-min))
+
+          (let (p)
+            (while (< (setq p (point)) (point-max))
+              (condition-case nil
+                  ;; If I successfully read an integer (the plus zero
+                  ;; ensures a numeric type), prepend a marker entry
+                  ;; to the list
+                  (push (list (+ 0 (read (current-buffer))) nil nil
+                              (set-marker (make-marker) p))
+                        dlist)
+                (error
+                 (gnus-message 1 "gnus-agent-expire: read error \
 occurred when reading expression at %s in %s.  Skipping to next \
 line." (point) nov-file)))
-            ;; Whether I succeeded, or failed, it doesn't matter.
-            ;; Move to the next line then try again.
-            (forward-line 1)))
-        (gnus-message
-         7 "gnus-agent-expire: Loading overview... Done"))
-      (set-buffer-modified-p nil)
-
-      ;; At this point, all of the information is in dlist.  The
-      ;; only problem is that much of it is spread across multiple
-      ;; entries.  Sort then MERGE!!
-      (gnus-message 7 "gnus-agent-expire: Sorting entries... ")
-      ;; If two entries have the same article-number then sort by
-      ;; ascending keep_flag.
-      (let ((special 0)
-            (marked 1)
-            (unread 2))
-        (setq dlist
-              (sort dlist
-                    (lambda (a b)
-                      (cond ((< (nth 0 a) (nth 0 b))
-                             t)
-                            ((> (nth 0 a) (nth 0 b))
-                             nil)
-                            (t
-                             (let ((a (or (symbol-value (nth 2 a))
-                                          3))
-                                   (b (or (symbol-value (nth 2 b))
-                                          3)))
-                               (<= a b))))))))
-      (gnus-message 7 "gnus-agent-expire: Sorting entries... Done")
-      (gnus-message 7 "gnus-agent-expire: Merging entries... ")
-      (let ((dlist dlist))
-        (while (cdr dlist)              ; I'm not at the end-of-list
-          (if (eq (caar dlist) (caadr dlist))
-              (let ((first (cdr (car dlist)))
-                    (secnd (cdr (cadr dlist))))
-                (setcar first (or (car first)
-                                  (car secnd))) ; fetch_date
-                (setq first (cdr first)
-                      secnd (cdr secnd))
-                (setcar first (or (car first)
-                                  (car secnd))) ; Keep_flag
-                (setq first (cdr first)
-                      secnd (cdr secnd))
-                (setcar first (or (car first)
-                                  (car secnd))) ; NOV_entry_marker
-
-                (setcdr dlist (cddr dlist)))
-            (setq dlist (cdr dlist)))))
-      (gnus-message 7 "gnus-agent-expire: Merging entries... Done")
-
-      (let* ((len (float (length dlist)))
-             (alist (list nil))
-             (tail-alist alist))
-        (while dlist
-          (let ((new-completed (truncate (* 100.0
-                                            (/ (setq cnt (1+ cnt))
-                                               len)))))
-            (when (> new-completed completed)
-              (setq completed new-completed)
-              (gnus-message 9 "%3d%% completed..."  completed)))
-          (let* ((entry          (car dlist))
-                 (article-number (nth 0 entry))
-                 (fetch-date     (nth 1 entry))
-                 (keep           (nth 2 entry))
-                 (marker         (nth 3 entry)))
-
-            (cond
-             ;; Kept articles are unread, marked, or special.
-             (keep
-              (gnus-message 10
-                            "gnus-agent-expire: Article %d: Kept %s article."
-                            article-number keep)
-              (when fetch-date
-                (unless (file-exists-p
-                         (concat dir (number-to-string
-                                      article-number)))
-                  (setf (nth 1 entry) nil)
-                  (gnus-message 3 "gnus-agent-expire cleared \
+              ;; Whether I succeeded, or failed, it doesn't matter.
+              ;; Move to the next line then try again.
+              (forward-line 1)))
+
+          (gnus-message
+           7 "gnus-agent-expire: Loading overview... Done"))
+        (set-buffer-modified-p nil)
+
+        ;; At this point, all of the information is in dlist.  The
+        ;; only problem is that much of it is spread across multiple
+        ;; entries.  Sort then MERGE!!
+        (gnus-message 7 "gnus-agent-expire: Sorting entries... ")
+        ;; If two entries have the same article-number then sort by
+        ;; ascending keep_flag.
+        (let ((special 0)
+              (marked 1)
+              (unread 2))
+          (setq dlist
+                (sort dlist
+                      (lambda (a b)
+                        (cond ((< (nth 0 a) (nth 0 b))
+                               t)
+                              ((> (nth 0 a) (nth 0 b))
+                               nil)
+                              (t
+                               (let ((a (or (symbol-value (nth 2 a))
+                                            3))
+                                     (b (or (symbol-value (nth 2 b))
+                                            3)))
+                                 (<= a b))))))))
+        (gnus-message 7 "gnus-agent-expire: Sorting entries... Done")
+        (gnus-message 7 "gnus-agent-expire: Merging entries... ")
+        (let ((dlist dlist))
+          (while (cdr dlist)            ; I'm not at the end-of-list
+            (if (eq (caar dlist) (caadr dlist))
+                (let ((first (cdr (car dlist)))
+                      (secnd (cdr (cadr dlist))))
+                  (setcar first (or (car first)
+                                    (car secnd))) ; fetch_date
+                  (setq first (cdr first)
+                        secnd (cdr secnd))
+                  (setcar first (or (car first)
+                                    (car secnd))) ; Keep_flag
+                  (setq first (cdr first)
+                        secnd (cdr secnd))
+                  (setcar first (or (car first)
+                                    (car secnd))) ; NOV_entry_marker
+
+                  (setcdr dlist (cddr dlist)))
+              (setq dlist (cdr dlist)))))
+        (gnus-message 7 "gnus-agent-expire: Merging entries... Done")
+
+        (let* ((len (float (length dlist)))
+               (alist (list nil))
+               (tail-alist alist))
+          (while dlist
+            (let ((new-completed (truncate (* 100.0
+                                              (/ (setq cnt (1+ cnt))
+                                                 len)))))
+              (when (> new-completed completed)
+                (setq completed new-completed)
+                (gnus-message 7 "%3d%% completed..."  completed)))
+            (let* ((entry          (car dlist))
+                   (article-number (nth 0 entry))
+                   (fetch-date     (nth 1 entry))
+                   (keep           (nth 2 entry))
+                   (marker         (nth 3 entry)))
+
+              (cond
+               ;; Kept articles are unread, marked, or special.
+               (keep
+                (gnus-agent-message 10
+                                    "gnus-agent-expire: Article %d: Kept %s article."
+                                    article-number keep)
+                (when fetch-date
+                  (unless (file-exists-p
+                           (concat dir (number-to-string
+                                        article-number)))
+                    (setf (nth 1 entry) nil)
+                    (gnus-agent-message 3 "gnus-agent-expire cleared \
 download flag on article %d as the cached article file is missing."
-                                (caar dlist)))
-                (unless marker
-                  (gnus-message 1 "gnus-agent-expire detected a \
+                                        (caar dlist)))
+                  (unless marker
+                    (gnus-message 1 "gnus-agent-expire detected a \
 missing NOV entry.  Run gnus-agent-regenerate-group to restore it.")))
-              (gnus-agent-append-to-list
-               tail-alist
-               (cons article-number fetch-date)))
-
-             ;; The following articles are READ, UNMARKED, and
-             ;; ORDINARY.  See if they can be EXPIRED!!!
-             ((setq type
-                    (cond
-                     ((not (integerp fetch-date))
-                      'read) ;; never fetched article (may expire
-                     ;; right now)
-                     ((not (file-exists-p
-                            (concat dir (number-to-string
-                                         article-number))))
-                      (setf (nth 1 entry) nil)
-                      'externally-expired) ;; Can't find the cached
-                     ;; article.  Handle case
-                     ;; as though this article
-                     ;; was never fetched.
-
-                     ;; We now have the arrival day, so we see
-                     ;; whether it's old enough to be expired.
-                     ((< fetch-date day)
-                      'expired)
-                     (force
-                      'forced)))
-
-              ;; I found some reason to expire this entry.
-
-              (let ((actions nil))
-                (when (memq type '(forced expired))
-                  (ignore-errors        ; Just being paranoid.
-                   (delete-file (concat dir (number-to-string
-                                             article-number)))
-                   (push "expired cached article" actions))
-                  (setf (nth 1 entry) nil)
-                  )
-
-                (when marker
-                  (push "NOV entry removed" actions)
-                  (goto-char marker)
-                  (gnus-delete-line))
-
-                ;; If considering all articles is set, I can only
-                ;; expire article IDs that are no longer in the
-                ;; active range.
-                (if (and gnus-agent-consider-all-articles
-                         (>= article-number (car active)))
-                    ;; I have to keep this ID in the alist
-                    (gnus-agent-append-to-list
-                     tail-alist (cons article-number fetch-date))
-                  (push (format "Removed %s article number from \
+                (gnus-agent-append-to-list
+                 tail-alist
+                 (cons article-number fetch-date)))
+
+               ;; The following articles are READ, UNMARKED, and
+               ;; ORDINARY.  See if they can be EXPIRED!!!
+               ((setq type
+                      (cond
+                       ((not (integerp fetch-date))
+                        'read) ;; never fetched article (may expire
+                       ;; right now)
+                       ((not (file-exists-p
+                              (concat dir (number-to-string
+                                           article-number))))
+                        (setf (nth 1 entry) nil)
+                        'externally-expired) ;; Can't find the cached
+                       ;; article.  Handle case
+                       ;; as though this article
+                       ;; was never fetched.
+
+                       ;; We now have the arrival day, so we see
+                       ;; whether it's old enough to be expired.
+                       ((< fetch-date day)
+                        'expired)
+                       (force
+                        'forced)))
+
+                ;; I found some reason to expire this entry.
+
+                (let ((actions nil))
+                  (when (memq type '(forced expired))
+                    (ignore-errors      ; Just being paranoid.
+                      (delete-file (concat dir (number-to-string
+                                                article-number)))
+                      (push "expired cached article" actions))
+                    (setf (nth 1 entry) nil)
+                    )
+
+                  (when marker
+                    (push "NOV entry removed" actions)
+                    (goto-char marker)
+                    (gnus-delete-line))
+
+                  ;; If considering all articles is set, I can only
+                  ;; expire article IDs that are no longer in the
+                  ;; active range (That is, articles that preceed the
+                  ;; first article in the new alist).
+                  (if (and gnus-agent-consider-all-articles
+                           (>= article-number (car active)))
+                      ;; I have to keep this ID in the alist
+                      (gnus-agent-append-to-list
+                       tail-alist (cons article-number fetch-date))
+                    (push (format "Removed %s article number from \
 article alist" type) actions))
 
-                (gnus-message 7 "gnus-agent-expire: Article %d: %s"
-                              article-number
-                              (mapconcat 'identity actions ", "))))
-             (t
-              (gnus-message
-               10 "gnus-agent-expire: Article %d: Article kept as \
+                  (gnus-agent-message 8 "gnus-agent-expire: Article %d: %s"
+                                      article-number
+                                      (mapconcat 'identity actions ", "))))
+               (t
+                (gnus-agent-message
+                 10 "gnus-agent-expire: Article %d: Article kept as \
 expiration tests failed." article-number)
-              (gnus-agent-append-to-list
-               tail-alist (cons article-number fetch-date)))
-             )
+                (gnus-agent-append-to-list
+                 tail-alist (cons article-number fetch-date)))
+               )
 
-            ;; Clean up markers as I want to recycle this buffer
-            ;; over several groups.
-            (when marker
-              (set-marker marker nil))
+              ;; Clean up markers as I want to recycle this buffer
+              ;; over several groups.
+              (when marker
+                (set-marker marker nil))
 
-            (setq dlist (cdr dlist))))
+              (setq dlist (cdr dlist))))
 
-        (setq alist (cdr alist))
+          (setq alist (cdr alist))
 
-        (let ((inhibit-quit t))
-          (unless (equal alist gnus-agent-article-alist)
-            (setq gnus-agent-article-alist alist)
-            (gnus-agent-save-alist group))
+          (let ((inhibit-quit t))
+            (unless (equal alist gnus-agent-article-alist)
+              (setq gnus-agent-article-alist alist)
+              (gnus-agent-save-alist group)
 
-          (when (buffer-modified-p)
-            (let ((coding-system-for-write
-                   gnus-agent-file-coding-system))
-              (gnus-make-directory dir)
-              (write-region (point-min) (point-max) nov-file nil
-                            'silent)
-              ;; clear the modified flag as that I'm not confused by
-              ;; its status on the next pass through this routine.
-              (set-buffer-modified-p nil)))
+              ;; The active list changed, set the agent's active range
+              ;; to match the beginning of the list.
+              (if alist
+                  (setcar active (caar alist))))
 
-          (when (eq articles t)
-            (gnus-summary-update-info)))))))
+            (when (buffer-modified-p)
+              (let ((coding-system-for-write
+                     gnus-agent-file-coding-system))
+                (gnus-make-directory dir)
+                (write-region (point-min) (point-max) nov-file nil
+                              'silent)
+                ;; clear the modified flag as that I'm not confused by
+                ;; its status on the next pass through this routine.
+                (set-buffer-modified-p nil)))
+
+            (when (eq articles t)
+              (gnus-summary-update-info))))))))
 
 (defun gnus-agent-expire (&optional articles group force)
   "Expire all old articles.
@@ -2733,30 +2822,118 @@ FORCE is equivalent to setting the expiration predicates to true."
             (yes-or-no-p "Are you sure that you want to expire all \
 articles in every agentized group."))
         (let ((methods gnus-agent-covered-methods)
+              ;; Bind gnus-agent-expire-current-dirs to enable tracking
+              ;; of agent directories.
+              (gnus-agent-expire-current-dirs nil)
               gnus-command-method overview orig)
           (setq overview (gnus-get-buffer-create " *expire overview*"))
           (unwind-protect
               (while (setq gnus-command-method (pop methods))
-                (when (file-exists-p (gnus-agent-lib-file "active"))
-                  (with-temp-buffer
-                    (nnheader-insert-file-contents
-                     (gnus-agent-lib-file "active"))
-                    (gnus-active-to-gnus-format
-                     gnus-command-method
-                     (setq orig (gnus-make-hashtable
-                                 (count-lines (point-min) (point-max))))))
-                  (dolist (expiring-group (gnus-groups-from-server
-                                           gnus-command-method))
-                    (let* ((active
-                            (gnus-gethash-safe expiring-group orig)))
+                (let ((active-file (gnus-agent-lib-file "active")))
+                  (when (file-exists-p active-file)
+                    (with-temp-buffer
+                      (nnheader-insert-file-contents active-file)
+                      (gnus-active-to-gnus-format
+                       gnus-command-method
+                       (setq orig (gnus-make-hashtable
+                                   (count-lines (point-min) (point-max))))))
+                    (dolist (expiring-group (gnus-groups-from-server
+                                             gnus-command-method))
+                      (let* ((active
+                              (gnus-gethash-safe expiring-group orig)))
                                         
-                      (when active
-                        (save-excursion
-                          (gnus-agent-expire-group-1
-                           expiring-group overview active articles force)))))))
+                        (when active
+                          (save-excursion
+                            (gnus-agent-expire-group-1
+                             expiring-group overview active articles force)))))
+                    (gnus-agent-write-active active-file orig t))))
             (kill-buffer overview))
+          (gnus-agent-expire-unagentized-dirs)
           (gnus-message 4 "Expiry...done")))))
 
+(defun gnus-agent-expire-unagentized-dirs ()
+  (when (and gnus-agent-expire-unagentized-dirs
+             (boundp 'gnus-agent-expire-current-dirs))
+    (let* ((keep (gnus-make-hashtable))
+          ;; Formally bind gnus-agent-expire-current-dirs so that the
+          ;; compiler will not complain about free references.
+          (gnus-agent-expire-current-dirs
+           (symbol-value 'gnus-agent-expire-current-dirs))
+           dir)
+
+      (gnus-sethash gnus-agent-directory t keep)
+      (while gnus-agent-expire-current-dirs
+       (setq dir (pop gnus-agent-expire-current-dirs))
+       (when (and (stringp dir)
+                  (file-directory-p dir))
+         (while (not (gnus-gethash dir keep))
+           (gnus-sethash dir t keep)
+           (setq dir (file-name-directory (directory-file-name dir))))))
+
+      (let* (to-remove
+             checker
+             (checker
+              (function
+               (lambda (d)
+                 "Given a directory, check it and its subdirectories for 
+              membership in the keep hash.  If it isn't found, add 
+              it to to-remove." 
+                 (let ((files (directory-files d))
+                       file)
+                   (while (setq file (pop files))
+                     (cond ((equal file ".") ; Ignore self
+                            nil)
+                           ((equal file "..") ; Ignore parent
+                            nil)
+                           ((equal file ".overview") 
+                            ;; Directory must contain .overview to be
+                            ;; agent's cache of a group.
+                            (let ((d (file-name-as-directory d))
+                                  r)
+                              ;; Search ancestor's for last directory NOT
+                              ;; found in keep hash.
+                              (while (not (gnus-gethash
+                                           (setq d (file-name-directory d)) keep))
+                                (setq r d
+                                      d (directory-file-name d)))
+                              ;; if ANY ancestor was NOT in keep hash and
+                              ;; it it's already in to-remove, add it to
+                              ;; to-remove.                          
+                              (if (and r
+                                       (not (member r to-remove)))
+                                  (push r to-remove))))
+                           ((file-directory-p (setq file (nnheader-concat d file)))
+                            (funcall checker file)))))))))
+        (funcall checker (expand-file-name gnus-agent-directory))
+
+        (when (and to-remove
+                   (or gnus-expert-user
+                       (gnus-y-or-n-p
+                        "gnus-agent-expire has identified local directories that are\
+ not currently required by any agentized group.         Do you wish to consider\
+ deleting them?")))
+          (while to-remove
+            (let ((dir (pop to-remove)))
+              (if (gnus-y-or-n-p (format "Delete %s? " dir))
+                  (let* (delete-recursive
+                         (delete-recursive
+                          (function
+                           (lambda (f-or-d)
+                             (ignore-errors
+                               (if (file-directory-p f-or-d)
+                                   (condition-case nil
+                                       (delete-directory f-or-d)
+                                     (file-error
+                                      (mapcar (lambda (f)
+                                                (or (member f '("." ".."))
+                                                    (funcall delete-recursive
+                                                             (nnheader-concat
+                                                              f-or-d f))))
+                                              (directory-files f-or-d))
+                                      (delete-directory f-or-d)))
+                                 (delete-file f-or-d)))))))
+                    (funcall delete-recursive dir))))))))))
+
 ;;;###autoload
 (defun gnus-agent-batch ()
   "Start Gnus, send queue and fetch session."
@@ -2783,7 +2960,7 @@ articles in every agentized group."))
                        (gnus-agent-append-to-list tail-unread candidate)
                        nil)
                       ((> candidate max)
-                       (pop read)))))))
+                       (setq read (cdr read))))))))
     (while known
       (gnus-agent-append-to-list tail-unread (car (pop known))))
     (cdr unread)))
@@ -2811,14 +2988,14 @@ has been fetched."
               (v2 (caar ref)))
           (cond ((< v1 v2) ; v1 does not appear in the reference list
                 (gnus-agent-append-to-list tail-uncached v1)
-                 (pop arts))
+                 (setq arts (cdr arts)))
                 ((= v1 v2)
                  (unless (or cached-header (cdar ref)) ; v1 is already cached
                   (gnus-agent-append-to-list tail-uncached v1))
-                 (pop arts)
-                 (pop ref))
+                 (setq arts (cdr arts))
+                 (setq ref (cdr ref)))
                 (t ; reference article (v2) preceeds the list being filtered
-                 (pop ref)))))
+                 (setq ref (cdr ref))))))
       (while arts
        (gnus-agent-append-to-list tail-uncached (pop arts)))
       (cdr uncached))
@@ -2963,10 +3140,7 @@ has been fetched."
                  (not gnus-plugged))
              (numberp article))
     (let* ((gnus-command-method (gnus-find-method-for-group group))
-           (file (concat
-                 (gnus-agent-directory)
-                 (gnus-agent-group-path group) "/"
-                 (number-to-string article)))
+           (file (gnus-agent-article-name (number-to-string article) group))
            (buffer-read-only nil))
       (when (and (file-exists-p file)
                  (> (nth 7 (file-attributes file)) 0))
@@ -3042,7 +3216,7 @@ If REREAD is not nil, downloaded articles are marked as unread."
                           (gnus-message 4 "gnus-agent-regenerate-group: NOV\
  entries contained duplicate of article %s.     Duplicate deleted." l1)
                            (gnus-delete-line)
-                           (pop nov-arts)))))
+                           (setq nov-arts (cdr nov-arts))))))
                  (t
                  (gnus-message 1 "gnus-agent-regenerate-group: NOV\
  entries contained line that did not begin with an article number.  Deleted\
@@ -3088,12 +3262,12 @@ If REREAD is not nil, downloaded articles are marked as unread."
                            (nth 5 (file-attributes
                                    (concat dir (number-to-string
                                                 (car downloaded))))))) alist)
-              (pop downloaded)
-              (pop nov-arts))
+              (setq downloaded (cdr downloaded))
+              (setq nov-arts (cdr nov-arts)))
              (t
               ;; This entry in the overview has not been downloaded
               (push (cons (car nov-arts) nil) alist)
-              (pop nov-arts))))
+              (setq nov-arts (cdr nov-arts)))))
 
      ;; When gnus-agent-consider-all-articles is set,
      ;; gnus-agent-regenerate-group should NOT remove article IDs from
@@ -3117,15 +3291,15 @@ If REREAD is not nil, downloaded articles are marked as unread."
                    (oID (caar o)))
                (cond ((not nID)
                       (setq n (setcdr n (list (list oID))))
-                      (pop o))
+                      (setq o (cdr o)))
                      ((< oID nID)
                       (setcdr n (cons (list oID) (cdr n)))
-                      (pop o))
+                      (setq o (cdr o)))
                      ((= oID nID)
-                      (pop o)
-                      (pop n))
+                      (setq o (cdr o))
+                      (setq n (cdr n)))
                      (t
-                      (pop n)))))
+                      (setq n (cdr n))))))
            (setq alist (cdr merged)))
        ;; Restore the last article ID if it is not already in the new alist
        (let ((n (last alist))
@@ -3277,7 +3451,7 @@ If CLEAN, don't read existing active files."
                                                  (caar days)
                                                  group))
                                       (throw 'found (cadar days)))
-                                    (pop days))
+                                    (setq days (cdr days)))
                                   nil)))
                       (when day
                         (gnus-group-set-parameter group 'agent-days-until-old