(spam-bogofilter-score): check bogofilter headers before
[gnus] / lisp / spam.el
index ed6b9d9..e39524b 100644 (file)
@@ -1,5 +1,5 @@
 ;;; spam.el --- Identifying spam
-;; Copyright (C) 2002 Free Software Foundation, Inc.
+;; Copyright (C) 2002, 2003 Free Software Foundation, Inc.
 
 ;; Author: Lars Magne Ingebrigtsen <larsi@gnus.org>
 ;; Keywords: network
@@ -40,6 +40,9 @@
 (require 'gnus)        ; for the definitions of group content classification and spam processors
 (require 'message)                     ;for the message-fetch-field functions
 
+;; for nnimap-split-download-body-default
+(eval-when-compile (require 'nnimap))
+
 ;; autoload executable-find
 (eval-and-compile
   ;; executable-find is not autoloaded in Emacs 20
@@ -69,7 +72,15 @@ When nil, only ham and unclassified groups will have their spam moved
 to the spam-process-destination.  When t, spam will also be moved from
 spam groups."
   :type 'boolean
-  :group 'spam-ifile)
+  :group 'spam)
+
+(defcustom spam-mark-ham-unread-before-move-from-spam-group nil
+  "Whether ham should be marked unread before it's moved out of a spam
+group according to ham-process-destination.  This variable is an
+official entry in the international Longest Variable Name
+Competition."
+  :type 'boolean
+  :group 'spam)
 
 (defcustom spam-whitelist (expand-file-name "whitelist" spam-directory)
   "The location of the whitelist.
@@ -100,11 +111,23 @@ The regular expression is matched against the address."
   :type 'boolean
   :group 'spam)
 
+(defcustom spam-use-whitelist-exclusive nil
+  "Whether whitelist-exclusive should be used by spam-split.
+Exclusive whitelisting means that all messages from senders not in the whitelist
+are considered spam."
+  :type 'boolean
+  :group 'spam)
+
 (defcustom spam-use-blackholes nil
   "Whether blackholes should be used by spam-split."
   :type 'boolean
   :group 'spam)
 
+(defcustom spam-use-hashcash nil
+  "Whether hashcash payments should be detected by spam-split."
+  :type 'boolean
+  :group 'spam)
+
 (defcustom spam-use-regex-headers nil
   "Whether a header regular expression match should be used by spam-split.
 Also see the variable `spam-spam-regex-headers' and `spam-ham-regex-headers'."
@@ -128,6 +151,13 @@ Enable this if you want Gnus to invoke Bogofilter on new messages."
   :type 'boolean
   :group 'spam)
 
+(defcustom spam-use-BBDB-exclusive nil
+  "Whether BBDB-exclusive should be used by spam-split.
+Exclusive BBDB means that all messages from senders not in the BBDB are 
+considered spam."
+  :type 'boolean
+  :group 'spam)
+
 (defcustom spam-use-ifile nil
   "Whether ifile should be used by spam-split."
   :type 'boolean
@@ -155,27 +185,9 @@ All unmarked article in such group receive the spam mark on group entry."
   :type '(repeat (string :tag "Server"))
   :group 'spam)
 
-(defcustom spam-ham-marks (list 'gnus-del-mark 'gnus-read-mark 
-                               'gnus-killed-mark 'gnus-kill-file-mark 
-                               'gnus-low-score-mark)
-  "Marks considered as being ham (positively not spam).
-Such articles will be processed as ham (non-spam) on group exit."
-  :type '(set
-         (variable-item gnus-del-mark)
-         (variable-item gnus-read-mark)
-         (variable-item gnus-killed-mark)
-         (variable-item gnus-kill-file-mark)
-         (variable-item gnus-low-score-mark))
-  :group 'spam)
-
-(defcustom spam-spam-marks (list 'gnus-spam-mark)
-  "Marks considered as being spam (positively spam).
-Such articles will be transmitted to `bogofilter -s' on group exit."
-  :type '(set 
-         (variable-item gnus-spam-mark)
-         (variable-item gnus-killed-mark)
-         (variable-item gnus-kill-file-mark)
-         (variable-item gnus-low-score-mark))
+(defcustom spam-blackhole-good-server-regex nil
+  "String matching IP addresses that should not be checked in the blackholes"
+  :type 'regexp
   :group 'spam)
 
 (defcustom spam-face 'gnus-splash-face
@@ -243,6 +255,21 @@ your main source of newsgroup names."
   :type 'string
   :group 'spam-bogofilter)
 
+(defcustom spam-bogofilter-spam-switch "-s"
+  "The switch that Bogofilter uses to register spam messages."
+  :type 'string
+  :group 'spam-bogofilter)
+
+(defcustom spam-bogofilter-ham-switch "-n"
+  "The switch that Bogofilter uses to register ham messages."
+  :type 'string
+  :group 'spam-bogofilter)
+
+(defcustom spam-bogofilter-bogosity-positive-spam-header "^\\(Yes\\|Spam\\)"
+  "The regex on `spam-bogofilter-header' for positive spam identification."
+  :type 'regexp
+  :group 'spam-bogofilter)
+
 (defcustom spam-bogofilter-database-directory nil
   "Directory path of the Bogofilter databases."
   :type '(choice (directory :tag "Location of the Bogofilter database directory")
@@ -266,6 +293,29 @@ your main source of newsgroup names."
       gnus-summary-highlight)
 
 ;; convenience functions
+(defun spam-group-ham-mark-p (group mark &optional spam)
+  (when (stringp group)
+    (let* ((marks (spam-group-ham-marks group spam))
+          (marks (if (symbolp mark) 
+                     marks 
+                   (mapcar 'symbol-value marks))))
+      (memq mark marks))))
+
+(defun spam-group-spam-mark-p (group mark)
+  (spam-group-ham-mark-p group mark t))
+
+(defun spam-group-ham-marks (group &optional spam)
+  (when (stringp group)
+    (let* ((marks (if spam
+                    (gnus-parameter-spam-marks group)
+                  (gnus-parameter-ham-marks group)))
+          (marks (car marks))
+          (marks (if (listp (car marks)) (car marks) marks)))
+      marks)))
+
+(defun spam-group-spam-marks (group)
+  (spam-group-ham-marks group t))
+
 (defun spam-group-spam-contents-p (group)
   (if (stringp group)
       (or (member group spam-junk-mailgroups)
@@ -312,6 +362,9 @@ your main source of newsgroup names."
 (defun spam-group-ham-processor-BBDB-p (group)
   (spam-group-processor-p group 'gnus-group-ham-exit-processor-BBDB))
 
+(defun spam-group-ham-processor-copy-p (group)
+  (spam-group-processor-p group 'gnus-group-ham-exit-processor-copy))
+
 ;;; Summary entry and exit processing.
 
 (defun spam-summary-prepare ()
@@ -319,61 +372,68 @@ your main source of newsgroup names."
 
 (add-hook 'gnus-summary-prepare-hook 'spam-summary-prepare)
 
+;; The spam processors are invoked for any group, spam or ham or neither
 (defun spam-summary-prepare-exit ()
-  ;; The spam processors are invoked for any group, spam or ham or neither
-  (gnus-message 6 "Exiting summary buffer and applying spam rules")
-  (when (and spam-bogofilter-path
-            (spam-group-spam-processor-bogofilter-p gnus-newsgroup-name))
-    (gnus-message 5 "Registering spam with bogofilter")
-    (spam-bogofilter-register-spam-routine))
+  (unless gnus-group-is-exiting-without-update-p
+    (gnus-message 6 "Exiting summary buffer and applying spam rules")
+    (when (and spam-bogofilter-path
+              (spam-group-spam-processor-bogofilter-p gnus-newsgroup-name))
+      (gnus-message 5 "Registering spam with bogofilter")
+      (spam-bogofilter-register-spam-routine))
   
-  (when (and spam-ifile-path
-            (spam-group-spam-processor-ifile-p gnus-newsgroup-name))
-    (gnus-message 5 "Registering spam with ifile")
-    (spam-ifile-register-spam-routine))
+    (when (and spam-ifile-path
+              (spam-group-spam-processor-ifile-p gnus-newsgroup-name))
+      (gnus-message 5 "Registering spam with ifile")
+      (spam-ifile-register-spam-routine))
   
-  (when (spam-group-spam-processor-stat-p gnus-newsgroup-name)
-    (gnus-message 5 "Registering spam with spam-stat")
-    (spam-stat-register-spam-routine))
-
-  (when (spam-group-spam-processor-blacklist-p gnus-newsgroup-name)
-    (gnus-message 5 "Registering spam with the blacklist")
-    (spam-blacklist-register-routine))
-
-  (if spam-move-spam-nonspam-groups-only      
-      (when (not (spam-group-spam-contents-p gnus-newsgroup-name))
-       (spam-mark-spam-as-expired-and-move-routine
-        (gnus-parameter-spam-process-destination gnus-newsgroup-name)))
-    (gnus-message 5 "Marking spam as expired and moving it")
-    (spam-mark-spam-as-expired-and-move-routine 
-     (gnus-parameter-spam-process-destination gnus-newsgroup-name)))
-
-  ;; now we redo spam-mark-spam-as-expired-and-move-routine to only
-  ;; expire spam, in case the above did not expire them
-  (spam-mark-spam-as-expired-and-move-routine nil)
-
-  (when (spam-group-ham-contents-p gnus-newsgroup-name)
-    (when (spam-group-ham-processor-whitelist-p gnus-newsgroup-name)
-      (gnus-message 5 "Registering ham with the whitelist")
-      (spam-whitelist-register-routine))
-    (when (spam-group-ham-processor-ifile-p gnus-newsgroup-name)
-      (gnus-message 5 "Registering ham with ifile")
-      (spam-ifile-register-ham-routine))
-    (when (spam-group-ham-processor-bogofilter-p gnus-newsgroup-name)
-      (gnus-message 5 "Registering ham with Bogofilter")
-      (spam-bogofilter-register-ham-routine))
-    (when (spam-group-ham-processor-stat-p gnus-newsgroup-name)
-      (gnus-message 5 "Registering ham with spam-stat")
-      (spam-stat-register-ham-routine))
-    (when (spam-group-ham-processor-BBDB-p gnus-newsgroup-name)
-      (gnus-message 5 "Registering ham with the BBDB")
-      (spam-BBDB-register-routine)))
-
-  ;; now move all ham articles out of spam groups
-  (when (spam-group-spam-contents-p gnus-newsgroup-name)
-    (gnus-message 5 "Moving ham messages from spam group")
-    (spam-ham-move-routine
-     (gnus-parameter-ham-process-destination gnus-newsgroup-name))))
+    (when (spam-group-spam-processor-stat-p gnus-newsgroup-name)
+      (gnus-message 5 "Registering spam with spam-stat")
+      (spam-stat-register-spam-routine))
+
+    (when (spam-group-spam-processor-blacklist-p gnus-newsgroup-name)
+      (gnus-message 5 "Registering spam with the blacklist")
+      (spam-blacklist-register-routine))
+
+    (if spam-move-spam-nonspam-groups-only      
+       (when (not (spam-group-spam-contents-p gnus-newsgroup-name))
+         (spam-mark-spam-as-expired-and-move-routine
+          (gnus-parameter-spam-process-destination gnus-newsgroup-name)))
+      (gnus-message 5 "Marking spam as expired and moving it to %s" gnus-newsgroup-name)
+      (spam-mark-spam-as-expired-and-move-routine 
+       (gnus-parameter-spam-process-destination gnus-newsgroup-name)))
+
+    ;; now we redo spam-mark-spam-as-expired-and-move-routine to only
+    ;; expire spam, in case the above did not expire them
+    (gnus-message 5 "Marking spam as expired without moving it")
+    (spam-mark-spam-as-expired-and-move-routine nil)
+
+    (when (spam-group-ham-contents-p gnus-newsgroup-name)
+      (when (spam-group-ham-processor-whitelist-p gnus-newsgroup-name)
+       (gnus-message 5 "Registering ham with the whitelist")
+       (spam-whitelist-register-routine))
+      (when (spam-group-ham-processor-ifile-p gnus-newsgroup-name)
+       (gnus-message 5 "Registering ham with ifile")
+       (spam-ifile-register-ham-routine))
+      (when (spam-group-ham-processor-bogofilter-p gnus-newsgroup-name)
+       (gnus-message 5 "Registering ham with Bogofilter")
+       (spam-bogofilter-register-ham-routine))
+      (when (spam-group-ham-processor-stat-p gnus-newsgroup-name)
+       (gnus-message 5 "Registering ham with spam-stat")
+       (spam-stat-register-ham-routine))
+      (when (spam-group-ham-processor-BBDB-p gnus-newsgroup-name)
+       (gnus-message 5 "Registering ham with the BBDB")
+       (spam-BBDB-register-routine)))
+
+    (when (spam-group-ham-processor-copy-p gnus-newsgroup-name)
+      (gnus-message 5 "Copying ham")
+      (spam-ham-move-routine
+       (gnus-parameter-ham-process-destination gnus-newsgroup-name) t))
+
+    ;; now move all ham articles out of spam groups
+    (when (spam-group-spam-contents-p gnus-newsgroup-name)
+      (gnus-message 5 "Moving ham messages from spam group")
+      (spam-ham-move-routine
+       (gnus-parameter-ham-process-destination gnus-newsgroup-name)))))
 
 (add-hook 'gnus-summary-prepare-exit-hook 'spam-summary-prepare-exit)
 
@@ -390,49 +450,57 @@ your main source of newsgroup names."
          (gnus-summary-mark-article article gnus-spam-mark))))))
 
 (defun spam-mark-spam-as-expired-and-move-routine (&optional group)
+  (gnus-summary-kill-process-mark)
   (let ((articles gnus-newsgroup-articles)
-       article)
-    (while articles
-      (setq article (pop articles))
+       article tomove)
+    (dolist (article articles)
       (when (eq (gnus-summary-article-mark article) gnus-spam-mark)
        (gnus-summary-mark-article article gnus-expirable-mark)
-       (when (stringp group)
-         (let ((gnus-current-article article))
-           (gnus-summary-move-article nil group)))))))
+       (push article tomove)))
+
+    ;; now do the actual move
+    (when (and tomove
+              (stringp group))
+      (dolist (article tomove)
+       (gnus-summary-set-process-mark article))
+      (when tomove (gnus-summary-move-article nil group))))
+  (gnus-summary-yank-process-mark))
  
-(defun spam-ham-move-routine (&optional group)
+(defun spam-ham-move-routine (&optional group copy)
+  (gnus-summary-kill-process-mark)
   (let ((articles gnus-newsgroup-articles)
-       article ham-mark-values mark)
-    (dolist (mark spam-ham-marks)
-      (push (symbol-value mark) ham-mark-values))
-
-    (while articles
-      (setq article (pop articles))
-      (when (and (memq mark ham-mark-values)
-                (stringp group))
-         (let ((gnus-current-article article))
-           (gnus-summary-move-article nil group))))))
+       article mark tomove)
+    (when (stringp group)              ; this routine will do nothing
+                                       ; without a valid group
+      (dolist (article articles)
+       (when (spam-group-ham-mark-p gnus-newsgroup-name
+                                    (gnus-summary-article-mark article))
+         (push article tomove)))
+
+      ;; now do the actual move
+      (when tomove
+       (dolist (article tomove)
+         (when spam-mark-ham-unread-before-move-from-spam-group
+           (gnus-summary-mark-article article gnus-unread-mark))           
+         (gnus-summary-set-process-mark article))
+       (if copy
+           (gnus-summary-copy-article nil group)
+         (gnus-summary-move-article nil group)))))
+  (gnus-summary-yank-process-mark))
  
 (defun spam-generic-register-routine (spam-func ham-func)
   (let ((articles gnus-newsgroup-articles)
-       article mark ham-articles spam-articles spam-mark-values 
-       ham-mark-values)
-
-    ;; marks are stored as symbolic values, so we have to dereference
-    ;; them for memq to work.  we wouldn't have to do this if
-    ;; gnus-summary-article-mark returned a symbol.
-    (dolist (mark spam-ham-marks)
-      (push (symbol-value mark) ham-mark-values))
-
-    (dolist (mark spam-spam-marks)
-      (push (symbol-value mark) spam-mark-values))
+       article mark ham-articles spam-articles)
 
     (while articles
       (setq article (pop articles)
            mark (gnus-summary-article-mark article))
-      (cond ((memq mark spam-mark-values) (push article spam-articles))
+      (cond ((spam-group-spam-mark-p gnus-newsgroup-name mark) 
+            (push article spam-articles))
            ((memq article gnus-newsgroup-saved))
-           ((memq mark ham-mark-values) (push article ham-articles))))
+           ((spam-group-ham-mark-p gnus-newsgroup-name mark)
+            (push article ham-articles))))
+
     (when (and ham-articles ham-func)
       (mapc ham-func ham-articles))    ; we use mapc because unlike
                                        ; mapcar it discards the
@@ -465,14 +533,15 @@ your main source of newsgroup names."
        (setq article-buffer (get-buffer gnus-article-buffer))))
     article-buffer))
 
-(defun spam-get-article-as-filename (article)
-  (let ((article-filename))
-    (when (numberp article)
-      (nnml-possibly-change-directory (gnus-group-real-name gnus-newsgroup-name))
-      (setq article-filename (expand-file-name (int-to-string article) nnml-current-directory)))
-    (if (file-exists-p article-filename)
-       article-filename
-      nil)))
+;; disabled for now
+;; (defun spam-get-article-as-filename (article)
+;;   (let ((article-filename))
+;;     (when (numberp article)
+;;       (nnml-possibly-change-directory (gnus-group-real-name gnus-newsgroup-name))
+;;       (setq article-filename (expand-file-name (int-to-string article) nnml-current-directory)))
+;;     (if (file-exists-p article-filename)
+;;     article-filename
+;;       nil)))
 
 (defun spam-fetch-field-from-fast (article)
   "Fetch the `from' field quickly, using the internal gnus-data-list function"
@@ -499,20 +568,26 @@ your main source of newsgroup names."
     (spam-use-ifile                    .       spam-check-ifile)
     (spam-use-stat                     .       spam-check-stat)
     (spam-use-blackholes               .       spam-check-blackholes)
+    (spam-use-hashcash                 .       spam-check-hashcash)
     (spam-use-bogofilter-headers       .       spam-check-bogofilter-headers)
     (spam-use-bogofilter               .       spam-check-bogofilter))
 "The spam-list-of-checks list contains pairs associating a parameter
 variable with a spam checking function.  If the parameter variable is
 true, then the checking function is called, and its value decides what
-happens.  Each individual check may return `nil', `t', or a mailgroup
-name.  The value `nil' means that the check does not yield a decision,
-and so, that further checks are needed.  The value `t' means that the
+happens.  Each individual check may return nil, t, or a mailgroup
+name.  The value nil means that the check does not yield a decision,
+and so, that further checks are needed.  The value t means that the
 message is definitely not spam, and that further spam checks should be
 inhibited.  Otherwise, a mailgroup name is returned where the mail
 should go, and further checks are also inhibited.  The usual mailgroup
 name is the value of `spam-split-group', meaning that the message is
 definitely a spam.")
 
+(defvar spam-list-of-statistical-checks
+  '(spam-use-ifile spam-use-stat spam-use-bogofilter)
+"The spam-list-of-statistical-checks list contains all the mail
+splitters that need to have the full message body available.")
+
 (defun spam-split ()
   "Split this message into the `spam' group if it is spam.
 This function can be used as an entry in `nnmail-split-fancy', for
@@ -520,20 +595,33 @@ example like this: (: spam-split)
 
 See the Info node `(gnus)Fancy Mail Splitting' for more details."
   (interactive)
+  (save-excursion
+    (save-restriction
+      (dolist (check spam-list-of-statistical-checks)
+       (when (symbol-value check)
+         (widen)
+         (gnus-message 8 "spam-split: widening the buffer (%s requires it)"
+                       (symbol-name check))
+         (return)))
+      ;;   (progn (widen) (debug (buffer-string)))
+      (let ((list-of-checks spam-list-of-checks)
+           decision)
+       (while (and list-of-checks (not decision))
+         (let ((pair (pop list-of-checks)))
+           (when (symbol-value (car pair))
+             (gnus-message 5 "spam-split: calling the %s function" (symbol-name (cdr pair)))
+             (setq decision (funcall (cdr pair))))))
+       (if (eq decision t)
+           nil
+         decision)))))
   
-  ;; load the spam-stat tables if needed
-  (when spam-use-stat (spam-stat-load))
-
-  (let ((list-of-checks spam-list-of-checks)
-       decision)
-    (while (and list-of-checks (not decision))
-      (let ((pair (pop list-of-checks)))
-       (when (symbol-value (car pair))
-         (gnus-message 5 "spam-split: calling the %s function" (symbol-name (cdr pair)))
-         (setq decision (funcall (cdr pair))))))
-    (if (eq decision t)
-       nil
-      decision)))
+(defun spam-setup-widening ()
+  (dolist (check spam-list-of-statistical-checks)
+    (when (symbol-value check)
+      (setq nnimap-split-download-body-default t))))
+
+(add-hook 'gnus-get-new-news-hook 'spam-setup-widening)
+
 \f
 ;;;; Regex headers
 
@@ -575,21 +663,38 @@ See the Info node `(gnus)Fancy Mail Splitting' for more details."
                ips)))
       (dolist (server spam-blackhole-servers)
        (dolist (ip ips)
-         (let ((query-string (concat ip "." server)))
-           (if spam-use-dig
-               (let ((query-result (query-dig query-string)))
-                 (when query-result
-                   (gnus-message 5 "spam (DIG): positive blackhole check '%s'" query-result)
-                   (push (list ip server query-result)
-                         matches)))
-             ;; else, if not using dig.el
-             (when (query-dns query-string)
-               (gnus-message 5 "spam: positive blackhole check '%s'" query-result)
-               (push (list ip server (query-dns query-string 'TXT))
-                     matches)))))))
+         (unless (and spam-blackhole-good-server-regex
+                      (string-match spam-blackhole-good-server-regex ip))
+           (let ((query-string (concat ip "." server)))
+             (if spam-use-dig
+                 (let ((query-result (query-dig query-string)))
+                   (when query-result
+                     (gnus-message 5 "(DIG): positive blackhole check '%s'" 
+                                   query-result)
+                     (push (list ip server query-result)
+                           matches)))
+               ;; else, if not using dig.el
+               (when (query-dns query-string)
+                 (gnus-message 5 "positive blackhole check")
+                 (push (list ip server (query-dns query-string 'TXT))
+                       matches))))))))
     (when matches
       spam-split-group)))
 \f
+;;;; Hashcash.
+
+(condition-case nil
+    (progn
+      (require 'hashcash)
+      
+      (defun spam-check-hashcash ()
+       "Check the headers for hashcash payments."
+       (mail-check-payment)))          ;mail-check-payment returns a boolean
+
+  (file-error (progn
+               (defalias 'mail-check-payment 'ignore)
+               (defalias 'spam-check-hashcash 'ignore))))
+\f
 ;;;; BBDB 
 
 ;;; original idea for spam-check-BBDB from Alexander Kotelnikov
@@ -623,13 +728,15 @@ See the Info node `(gnus)Fancy Mail Splitting' for more details."
        (spam-enter-ham-BBDB (spam-fetch-field-from-fast article)))))
 
   (defun spam-check-BBDB ()
-    "Mail from people in the BBDB is never considered spam"
+    "Mail from people in the BBDB is classified as ham or non-spam"
     (let ((who (message-fetch-field "from")))
       (when who
-       (setq who (regexp-quote (cadr
-                                (gnus-extract-address-components who))))
+       (setq who (cadr (gnus-extract-address-components who)))
        (if (bbdb-search-simple nil who)
-           nil spam-split-group)))))
+           t 
+         (if spam-use-BBDB-exclusive
+             spam-split-group
+           nil))))))
 
   (file-error (progn
                (defalias 'bbdb-search-simple 'ignore)
@@ -683,7 +790,7 @@ Uses `gnus-newsgroup-name' if category is nil (for ham registration)."
     (let ((category (or category gnus-newsgroup-name))
           (db-param (spam-get-ifile-database-parameter)))
       (with-temp-buffer
-       (insert-string article-string)
+       (insert article-string)
        (if db-param
             (call-process-region (point-min) (point-max) spam-ifile-path 
                                  nil nil nil 
@@ -726,10 +833,9 @@ Uses `gnus-newsgroup-name' if category is nil (for ham registration)."
         (lambda (article)
           (let ((article-string (spam-get-article-as-string article)))
             (with-temp-buffer
-              (insert-string article-string)
+              (insert article-string)
               (spam-stat-buffer-is-spam))))
-        nil)
-       (spam-stat-save))
+        nil))
 
       (defun spam-stat-register-ham-routine ()
        (spam-generic-register-routine 
@@ -737,9 +843,19 @@ Uses `gnus-newsgroup-name' if category is nil (for ham registration)."
         (lambda (article)
           (let ((article-string (spam-get-article-as-string article)))
             (with-temp-buffer
-              (insert-string article-string)
-              (spam-stat-buffer-is-non-spam)))))
-       (spam-stat-save)))
+              (insert article-string)
+              (spam-stat-buffer-is-non-spam))))))
+
+      (defun spam-maybe-spam-stat-load ()
+       (when spam-use-stat (spam-stat-load)))
+      
+      (defun spam-maybe-spam-stat-save ()
+       (when spam-use-stat (spam-stat-save)))
+
+      ;; Add hooks for loading and saving the spam stats
+      (add-hook 'gnus-save-newsrc-hook 'spam-maybe-spam-stat-save)
+      (add-hook 'gnus-get-top-new-news-hook 'spam-maybe-spam-stat-load)
+      (add-hook 'gnus-startup-hook 'spam-maybe-spam-stat-load))
 
   (file-error (progn
                (defalias 'spam-stat-register-ham-routine 'ignore)
@@ -783,12 +899,16 @@ Uses `gnus-newsgroup-name' if category is nil (for ham registration)."
     (insert address "\n")
     (save-buffer)))
 
-;;; returns nil if the sender is in the whitelist, spam-split-group otherwise
+;;; returns t if the sender is in the whitelist, nil or spam-split-group otherwise
 (defun spam-check-whitelist ()
   ;; FIXME!  Should it detect when file timestamps change?
   (unless spam-whitelist-cache
     (setq spam-whitelist-cache (spam-parse-list spam-whitelist)))
-  (if (spam-from-listed-p spam-whitelist-cache) nil spam-split-group))
+  (if (spam-from-listed-p spam-whitelist-cache) 
+      t
+    (if spam-use-whitelist-exclusive
+       spam-split-group
+      nil)))
 
 (defun spam-check-blacklist ()
   ;; FIXME!  Should it detect when file timestamps change?
@@ -846,12 +966,12 @@ Uses `gnus-newsgroup-name' if category is nil (for ham registration)."
 (defun spam-check-bogofilter-headers (&optional score)
   (let ((header (message-fetch-field spam-bogofilter-header)))
       (when (and header
-              (string-match "^Yes" header))
+                (string-match spam-bogofilter-bogosity-positive-spam-header
+                              header))
          (if score
              (when (string-match "spamicity=\\([0-9.]+\\)" header)
                (match-string 1 header))
            spam-split-group))))
-         
 
 ;; return something sensible if the score can't be determined
 (defun spam-bogofilter-score ()
@@ -860,7 +980,8 @@ Uses `gnus-newsgroup-name' if category is nil (for ham registration)."
   (save-window-excursion
     (gnus-summary-show-article t)
     (set-buffer gnus-article-buffer)
-    (let ((score (spam-check-bogofilter t)))
+    (let ((score (or (spam-check-bogofilter-headers t)
+                    (spam-check-bogofilter t))))
       (message "Spamicity score %s" score)
       (or score "0"))))
 
@@ -885,9 +1006,10 @@ Uses `gnus-newsgroup-name' if category is nil (for ham registration)."
 (defun spam-bogofilter-register-with-bogofilter (article-string spam)
   "Register an article, given as a string, as spam or non-spam."
   (when (stringp article-string)
-    (let ((switch (if spam "-s" "-n")))
+    (let ((switch (if spam spam-bogofilter-spam-switch 
+                   spam-bogofilter-ham-switch)))
       (with-temp-buffer
-       (insert-string article-string)
+       (insert article-string)
        (if spam-bogofilter-database-directory
            (call-process-region (point-min) (point-max) 
                                 spam-bogofilter-path