93613a84ad8158ad8be22afb71a54f479498e207
[gnus] / lisp / spam.el
1 ;;; spam.el --- Identifying spam
2
3 ;; Copyright (C) 2002-2015 Free Software Foundation, Inc.
4
5 ;; Author: Lars Magne Ingebrigtsen <larsi@gnus.org>
6 ;; Maintainer: Ted Zlatanov <tzz@lifelogs.com>
7 ;; Keywords: network, spam, mail, bogofilter, BBDB, dspam, dig, whitelist, blacklist, gmane, hashcash, spamassassin, bsfilter, ifile, stat, crm114, spamoracle
8
9 ;; This file is part of GNU Emacs.
10
11 ;; GNU Emacs is free software: you can redistribute it and/or modify
12 ;; it under the terms of the GNU General Public License as published by
13 ;; the Free Software Foundation, either version 3 of the License, or
14 ;; (at your option) any later version.
15
16 ;; GNU Emacs is distributed in the hope that it will be useful,
17 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
18 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 ;; GNU General Public License for more details.
20
21 ;; You should have received a copy of the GNU General Public License
22 ;; along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.
23
24 ;;; Commentary:
25
26 ;;; This module addresses a few aspects of spam control under Gnus.  Page
27 ;;; breaks are used for grouping declarations and documentation relating to
28 ;;; each particular aspect.
29
30 ;;; The integration with Gnus is not yet complete.  See various `FIXME'
31 ;;; comments, below, for supplementary explanations or discussions.
32
33 ;;; Several TODO items are marked as such
34
35 ;; TODO: cross-server splitting, remote processing, training through files
36
37 ;;; Code:
38
39 ;;{{{ compilation directives and autoloads/requires
40
41 (eval-when-compile (require 'cl))
42
43 (require 'message)              ;for the message-fetch-field functions
44 (require 'gnus-sum)
45 (require 'gnus-uu)                      ; because of key prefix issues
46 ;;; for the definitions of group content classification and spam processors
47 (require 'gnus)
48
49 (eval-when-compile (require 'hashcash))
50
51 ;; for nnimap-split-download-body-default
52 (eval-when-compile (require 'nnimap))
53
54 ;; autoload query-dig
55 (autoload 'query-dig "dig")
56
57 ;; autoload spam-report
58 (autoload 'spam-report-gmane "spam-report")
59 (autoload 'spam-report-gmane-spam "spam-report")
60 (autoload 'spam-report-gmane-ham "spam-report")
61 (autoload 'spam-report-resend "spam-report")
62
63 ;; autoload gnus-registry
64 (autoload 'gnus-registry-group-count "gnus-registry")
65 (autoload 'gnus-registry-get-id-key "gnus-registry")
66 (autoload 'gnus-registry-set-id-key "gnus-registry")
67 (autoload 'gnus-registry-handle-action "gnus-registry")
68
69 ;; autoload dns-query
70 (autoload 'dns-query "dns")
71
72 ;;}}}
73
74 ;;{{{ Main parameters.
75 (defvar spam-backends nil
76   "List of spam.el backends with all the pertinent data.
77 Populated by `spam-install-backend-super'.")
78
79 (defgroup spam nil
80   "Spam configuration."
81   :version "22.1"
82   :group 'mail
83   :group 'news)
84
85 (defcustom spam-summary-exit-behavior 'default
86   "Exit behavior at the time of summary exit.
87 Note that setting the `spam-use-move' or `spam-use-copy' backends on
88 a group through group/topic parameters overrides this mechanism."
89   :type '(choice
90           (const
91            :tag "Move spam out of all groups and ham out of spam groups"
92            default)
93           (const
94            :tag "Move spam out of all groups and ham out of all groups"
95            move-all)
96           (const
97            :tag "Never move spam or ham out of any groups"
98            move-none))
99   :group 'spam)
100
101 (defcustom spam-directory (nnheader-concat gnus-directory "spam/")
102   "Directory for spam whitelists and blacklists."
103   :type 'directory
104   :group 'spam)
105
106 (defcustom spam-mark-new-messages-in-spam-group-as-spam t
107   "Whether new messages in a spam group should get the spam-mark."
108   :type 'boolean
109   ;; :version "22.1" ;; Gnus 5.10.8 / No Gnus 0.3
110   :group 'spam)
111
112 (defcustom spam-log-to-registry nil
113   "Whether spam/ham processing should be logged in the registry."
114   :type 'boolean
115   :group 'spam)
116
117 (defcustom spam-split-symbolic-return nil
118   "Whether `spam-split' should work with symbols or group names."
119   :type 'boolean
120   :group 'spam)
121
122 (defcustom spam-split-symbolic-return-positive nil
123   "Whether `spam-split' should ALWAYS work with symbols or group names.
124 Do not set this if you use `spam-split' in a fancy split method."
125   :type 'boolean
126   :group 'spam)
127
128 (defcustom spam-mark-only-unseen-as-spam t
129   "Whether only unseen articles should be marked as spam in spam groups.
130 When nil, all unread articles in a spam group are marked as
131 spam.  Set this if you want to leave an article unread in a spam group
132 without losing it to the automatic spam-marking process."
133   :type 'boolean
134   :group 'spam)
135
136 (defcustom spam-mark-ham-unread-before-move-from-spam-group nil
137   "Whether ham should be marked unread before it's moved.
138 The article is moved out of a spam group according to `ham-process-destination'.
139 This variable is an official entry in the international Longest Variable Name
140 Competition."
141   :type 'boolean
142   :group 'spam)
143
144 (defcustom spam-disable-spam-split-during-ham-respool nil
145   "Whether `spam-split' should be ignored while resplitting ham.
146 This is useful to prevent ham from ending up in the same spam
147 group after the resplit.  Don't set this to t if you have `spam-split' as the
148 last rule in your split configuration."
149   :type 'boolean
150   :group 'spam)
151
152 (defcustom spam-autodetect-recheck-messages nil
153   "Should spam.el recheck all messages when autodetecting?
154 Normally this is nil, so only unseen messages will be checked."
155   :type 'boolean
156   :group 'spam)
157
158 (defcustom spam-whitelist (expand-file-name "whitelist" spam-directory)
159   "The location of the whitelist.
160 The file format is one regular expression per line.
161 The regular expression is matched against the address."
162   :type 'file
163   :group 'spam)
164
165 (defcustom spam-blacklist (expand-file-name "blacklist" spam-directory)
166   "The location of the blacklist.
167 The file format is one regular expression per line.
168 The regular expression is matched against the address."
169   :type 'file
170   :group 'spam)
171
172 (defcustom spam-use-dig t
173   "Whether `query-dig' should be used instead of `dns-query'."
174   :type 'boolean
175   :group 'spam)
176
177 (defcustom spam-use-gmane-xref nil
178   "Whether the Gmane spam xref should be used by `spam-split'."
179   :type 'boolean
180   :group 'spam)
181
182 (defcustom spam-use-blacklist nil
183   "Whether the blacklist should be used by `spam-split'."
184   :type 'boolean
185   :group 'spam)
186
187 (defcustom spam-blacklist-ignored-regexes nil
188   "Regular expressions that the blacklist should ignore."
189   :type '(repeat (regexp :tag "Regular expression to ignore when blacklisting"))
190   :group 'spam)
191
192 (defcustom spam-use-whitelist nil
193   "Whether the whitelist should be used by `spam-split'."
194   :type 'boolean
195   :group 'spam)
196
197 (defcustom spam-use-whitelist-exclusive nil
198   "Whether whitelist-exclusive should be used by `spam-split'.
199 Exclusive whitelisting means that all messages from senders not in the whitelist
200 are considered spam."
201   :type 'boolean
202   :group 'spam)
203
204 (defcustom spam-use-blackholes nil
205   "Whether blackholes should be used by `spam-split'."
206   :type 'boolean
207   :group 'spam)
208
209 (defcustom spam-use-hashcash nil
210   "Whether hashcash payments should be detected by `spam-split'."
211   :type 'boolean
212   :group 'spam)
213
214 (defcustom spam-use-regex-headers nil
215   "Whether a header regular expression match should be used by `spam-split'.
216 Also see the variables `spam-regex-headers-spam' and `spam-regex-headers-ham'."
217   :type 'boolean
218   :group 'spam)
219
220 (defcustom spam-use-regex-body nil
221   "Whether a body regular expression match should be used by `spam-split'.
222 Also see the variables `spam-regex-body-spam' and `spam-regex-body-ham'."
223   :type 'boolean
224   :group 'spam)
225
226 (defcustom spam-use-bogofilter-headers nil
227   "Whether bogofilter headers should be used by `spam-split'.
228 Enable this if you pre-process messages with Bogofilter BEFORE Gnus sees them."
229   :type 'boolean
230   :group 'spam)
231
232 (defcustom spam-use-bogofilter nil
233   "Whether bogofilter should be invoked by `spam-split'.
234 Enable this if you want Gnus to invoke Bogofilter on new messages."
235   :type 'boolean
236   :group 'spam)
237
238 (defcustom spam-use-bsfilter-headers nil
239   "Whether bsfilter headers should be used by `spam-split'.
240 Enable this if you pre-process messages with Bsfilter BEFORE Gnus sees them."
241   :type 'boolean
242   :group 'spam)
243
244 (defcustom spam-use-bsfilter nil
245   "Whether bsfilter should be invoked by `spam-split'.
246 Enable this if you want Gnus to invoke Bsfilter on new messages."
247   :type 'boolean
248   :group 'spam)
249
250 (defcustom spam-use-BBDB nil
251   "Whether BBDB should be used by `spam-split'."
252   :type 'boolean
253   :group 'spam)
254
255 (defcustom spam-use-BBDB-exclusive nil
256   "Whether BBDB-exclusive should be used by `spam-split'.
257 Exclusive BBDB means that all messages from senders not in the BBDB are
258 considered spam."
259   :type 'boolean
260   :group 'spam)
261
262 (defcustom spam-use-ifile nil
263   "Whether ifile should be used by `spam-split'."
264   :type 'boolean
265   :group 'spam)
266
267 (defcustom spam-use-stat nil
268   "Whether `spam-stat' should be used by `spam-split'."
269   :type 'boolean
270   :group 'spam)
271
272 (defcustom spam-use-spamoracle nil
273   "Whether spamoracle should be used by `spam-split'."
274   :type 'boolean
275   :group 'spam)
276
277 (defcustom spam-use-spamassassin nil
278   "Whether spamassassin should be invoked by `spam-split'.
279 Enable this if you want Gnus to invoke SpamAssassin on new messages."
280   :type 'boolean
281   :group 'spam)
282
283 (defcustom spam-use-spamassassin-headers nil
284   "Whether spamassassin headers should be checked by `spam-split'.
285 Enable this if you pre-process messages with SpamAssassin BEFORE Gnus sees
286 them."
287   :type 'boolean
288   :group 'spam)
289
290 (defcustom spam-use-crm114 nil
291   "Whether the CRM114 Mailfilter should be used by `spam-split'."
292   :type 'boolean
293   :group 'spam)
294
295 (defcustom spam-install-hooks (or
296                                spam-use-dig
297                                spam-use-gmane-xref
298                                spam-use-blacklist
299                                spam-use-whitelist
300                                spam-use-whitelist-exclusive
301                                spam-use-blackholes
302                                spam-use-hashcash
303                                spam-use-regex-headers
304                                spam-use-regex-body
305                                spam-use-bogofilter
306                                spam-use-bogofilter-headers
307                                spam-use-spamassassin
308                                spam-use-spamassassin-headers
309                                spam-use-bsfilter
310                                spam-use-bsfilter-headers
311                                spam-use-BBDB
312                                spam-use-BBDB-exclusive
313                                spam-use-ifile
314                                spam-use-stat
315                                spam-use-spamoracle
316                                spam-use-crm114)
317   "Whether the spam hooks should be installed.
318 Default to t if one of the spam-use-* variables is set."
319   :group 'spam
320   :type 'boolean)
321
322 (defcustom spam-split-group "spam"
323   "Group name where incoming spam should be put by `spam-split'."
324   :type 'string
325   :group 'spam)
326
327 ;;; TODO: deprecate this variable, it's confusing since it's a list of strings,
328 ;;; not regular expressions
329 (defcustom spam-junk-mailgroups (cons
330                                  spam-split-group
331                                  '("mail.junk" "poste.pourriel"))
332   "Mailgroups with spam contents.
333 All unmarked article in such group receive the spam mark on group entry."
334   :type '(repeat (string :tag "Group"))
335   :group 'spam)
336
337
338 (defcustom spam-gmane-xref-spam-group "gmane.spam.detected"
339   "The group where spam xrefs can be found on Gmane.
340 Only meaningful if you enable `spam-use-gmane-xref'."
341   :type 'string
342   :group 'spam)
343
344 (defcustom spam-blackhole-servers '("bl.spamcop.net" "relays.ordb.org"
345                                     "dev.null.dk" "relays.visi.com")
346   "List of blackhole servers.
347 Only meaningful if you enable `spam-use-blackholes'."
348   :type '(repeat (string :tag "Server"))
349   :group 'spam)
350
351 (defcustom spam-blackhole-good-server-regex nil
352   "String matching IP addresses that should not be checked in the blackholes.
353 Only meaningful if you enable `spam-use-blackholes'."
354   :type '(radio (const nil) regexp)
355   :group 'spam)
356
357 (defface spam
358   '((((class color) (type tty) (background dark))
359      (:foreground "gray80" :background "gray50"))
360     (((class color) (type tty) (background light))
361      (:foreground "gray50" :background "gray80"))
362     (((class color) (background dark))
363      (:foreground "ivory2"))
364     (((class color) (background light))
365      (:foreground "ivory4"))
366     (t :inverse-video t))
367   "Face for spam-marked articles."
368   :group 'spam)
369 ;; backward-compatibility alias
370 (put 'spam-face 'face-alias 'spam)
371 (put 'spam-face 'obsolete-face "22.1")
372
373 (defcustom spam-face 'spam
374   "Face for spam-marked articles."
375   :type 'face
376   :group 'spam)
377
378 (defcustom spam-regex-headers-spam '("^X-Spam-Flag: YES")
379   "Regular expression for positive header spam matches.
380 Only meaningful if you enable `spam-use-regex-headers'."
381   :type '(repeat (regexp :tag "Regular expression to match spam header"))
382   :group 'spam)
383
384 (defcustom spam-regex-headers-ham '("^X-Spam-Flag: NO")
385   "Regular expression for positive header ham matches.
386 Only meaningful if you enable `spam-use-regex-headers'."
387   :type '(repeat (regexp :tag "Regular expression to match ham header"))
388   :group 'spam)
389
390 (defcustom spam-regex-body-spam '()
391   "Regular expression for positive body spam matches.
392 Only meaningful if you enable `spam-use-regex-body'."
393   :type '(repeat (regexp :tag "Regular expression to match spam body"))
394   :group 'spam)
395
396 (defcustom spam-regex-body-ham '()
397   "Regular expression for positive body ham matches.
398 Only meaningful if you enable `spam-use-regex-body'."
399   :type '(repeat (regexp :tag "Regular expression to match ham body"))
400   :group 'spam)
401
402 (defcustom spam-summary-score-preferred-header nil
403   "Preferred header to use for `spam-summary-score'."
404   :type '(choice :tag "Header name"
405           (symbol :tag "SpamAssassin etc" X-Spam-Status)
406           (symbol :tag "Bogofilter"       X-Bogosity)
407           (const  :tag "No preference, take best guess." nil))
408   :group 'spam)
409
410 (defgroup spam-ifile nil
411   "Spam ifile configuration."
412   :group 'spam)
413
414 (make-obsolete-variable 'spam-ifile-path 'spam-ifile-program
415                         "Gnus 5.10.9 (Emacs 22.1)")
416 (defcustom spam-ifile-program (executable-find "ifile")
417   "Name of the ifile program."
418   :type '(choice (file :tag "Location of ifile")
419                  (const :tag "ifile is not installed"))
420   :group 'spam-ifile)
421
422 (make-obsolete-variable 'spam-ifile-database-path 'spam-ifile-database
423                         "Gnus 5.10.9 (Emacs 22.1)")
424 (defcustom spam-ifile-database nil
425   "File name of the ifile database."
426   :type '(choice (file :tag "Location of the ifile database")
427                  (const :tag "Use the default"))
428   :group 'spam-ifile)
429
430 (defcustom spam-ifile-spam-category "spam"
431   "Name of the spam ifile category."
432   :type 'string
433   :group 'spam-ifile)
434
435 (defcustom spam-ifile-ham-category nil
436   "Name of the ham ifile category.
437 If nil, the current group name will be used."
438   :type '(choice (string :tag "Use a fixed category")
439                  (const :tag "Use the current group name"))
440   :group 'spam-ifile)
441
442 (defcustom spam-ifile-all-categories nil
443   "Whether the ifile check will return all categories, or just spam.
444 Set this to t if you want to use the `spam-split' invocation of ifile as
445 your main source of newsgroup names."
446   :type 'boolean
447   :group 'spam-ifile)
448
449 (defgroup spam-bogofilter nil
450   "Spam bogofilter configuration."
451   :group 'spam)
452
453 (make-obsolete-variable 'spam-bogofilter-path 'spam-bogofilter-program
454                         "Gnus 5.10.9 (Emacs 22.1)")
455 (defcustom spam-bogofilter-program (executable-find "bogofilter")
456   "Name of the Bogofilter program."
457   :type '(choice (file :tag "Location of bogofilter")
458                  (const :tag "Bogofilter is not installed"))
459   :group 'spam-bogofilter)
460
461 (defvar spam-bogofilter-valid 'unknown "Is the bogofilter version valid?")
462
463 (defcustom spam-bogofilter-header "X-Bogosity"
464   "The header that Bogofilter inserts in messages."
465   :type 'string
466   :group 'spam-bogofilter)
467
468 (defcustom spam-bogofilter-spam-switch "-s"
469   "The switch that Bogofilter uses to register spam messages."
470   :type 'string
471   :group 'spam-bogofilter)
472
473 (defcustom spam-bogofilter-ham-switch "-n"
474   "The switch that Bogofilter uses to register ham messages."
475   :type 'string
476   :group 'spam-bogofilter)
477
478 (defcustom spam-bogofilter-spam-strong-switch "-S"
479   "The switch that Bogofilter uses to unregister ham messages."
480   :type 'string
481   :group 'spam-bogofilter)
482
483 (defcustom spam-bogofilter-ham-strong-switch "-N"
484   "The switch that Bogofilter uses to unregister spam messages."
485   :type 'string
486   :group 'spam-bogofilter)
487
488 (defcustom spam-bogofilter-bogosity-positive-spam-header "^\\(Yes\\|Spam\\)"
489   "The regex on `spam-bogofilter-header' for positive spam identification."
490   :type 'regexp
491   :group 'spam-bogofilter)
492
493 (defcustom spam-bogofilter-database-directory nil
494   "Location of the Bogofilter database.
495 When nil, use the default location."
496   :type '(choice (directory
497                   :tag "Location of the Bogofilter database directory")
498                  (const :tag "Use the default"))
499   :group 'spam-bogofilter)
500
501 (defgroup spam-bsfilter nil
502   "Spam bsfilter configuration."
503   :group 'spam)
504
505 (make-obsolete-variable 'spam-bsfilter-path 'spam-bsfilter-program
506                         "Gnus 5.10.9 (Emacs 22.1)")
507 (defcustom spam-bsfilter-program (executable-find "bsfilter")
508   "Name of the Bsfilter program."
509   :type '(choice (file :tag "Location of bsfilter")
510                  (const :tag "Bsfilter is not installed"))
511   :group 'spam-bsfilter)
512
513 (defcustom spam-bsfilter-header "X-Spam-Flag"
514   "The header inserted by Bsfilter to flag spam."
515   :type 'string
516   :group 'spam-bsfilter)
517
518 (defcustom spam-bsfilter-probability-header "X-Spam-Probability"
519   "The header that Bsfilter inserts in messages."
520   :type 'string
521   :group 'spam-bsfilter)
522
523 (defcustom spam-bsfilter-spam-switch "--add-spam"
524   "The switch that Bsfilter uses to register spam messages."
525   :type 'string
526   :group 'spam-bsfilter)
527
528 (defcustom spam-bsfilter-ham-switch "--add-clean"
529   "The switch that Bsfilter uses to register ham messages."
530   :type 'string
531   :group 'spam-bsfilter)
532
533 (defcustom spam-bsfilter-spam-strong-switch "--sub-spam"
534   "The switch that Bsfilter uses to unregister ham messages."
535   :type 'string
536   :group 'spam-bsfilter)
537
538 (defcustom spam-bsfilter-ham-strong-switch "--sub-clean"
539   "The switch that Bsfilter uses to unregister spam messages."
540   :type 'string
541   :group 'spam-bsfilter)
542
543 (defcustom spam-bsfilter-database-directory nil
544   "Directory path of the Bsfilter databases."
545   :type '(choice (directory
546                   :tag "Location of the Bsfilter database directory")
547                  (const :tag "Use the default"))
548   :group 'spam-bsfilter)
549
550 (defgroup spam-spamoracle nil
551   "Spam spamoracle configuration."
552   :group 'spam)
553
554 (defcustom spam-spamoracle-database nil
555   "Location of spamoracle database file.
556 When nil, use the default spamoracle database."
557   :type '(choice (directory :tag "Location of spamoracle database file.")
558                  (const :tag "Use the default"))
559   :group 'spam-spamoracle)
560
561 (defcustom spam-spamoracle-binary (executable-find "spamoracle")
562   "Location of the spamoracle binary."
563   :type '(choice (directory :tag "Location of the spamoracle binary")
564                  (const :tag "Use the default"))
565   :group 'spam-spamoracle)
566
567 (defgroup spam-spamassassin nil
568   "Spam SpamAssassin configuration."
569   :group 'spam)
570
571 (make-obsolete-variable 'spam-spamassassin-path
572   'spam-spamassassin-program "Gnus 5.10.9 (Emacs 22.1)")
573 (defcustom spam-assassin-program (executable-find "spamassassin")
574   "Name of the spamassassin program.
575 Hint: set this to \"spamc\" if you have spamd running.  See the spamc and
576 spamd man pages for more information on these programs."
577   :type '(choice (file :tag "Location of spamc")
578                  (const :tag "spamassassin is not installed"))
579   :group 'spam-spamassassin)
580
581 (defcustom spam-spamassassin-arguments ()
582   "Arguments to pass to the spamassassin executable.
583 This must be a list.  For example, `(\"-C\" \"configfile\")'."
584   :type '(restricted-sexp :match-alternatives (listp))
585   :group 'spam-spamassassin)
586
587 (defcustom spam-spamassassin-spam-flag-header "X-Spam-Flag"
588   "The header inserted by SpamAssassin to flag spam."
589   :type 'string
590   :group 'spam-spamassassin)
591
592 (defcustom spam-spamassassin-positive-spam-flag-header "YES"
593   "The regex on `spam-spamassassin-spam-flag-header' for positive spam
594 identification"
595   :type 'string
596   :group 'spam-spamassassin)
597
598 (defcustom spam-spamassassin-spam-status-header "X-Spam-Status"
599   "The header inserted by SpamAssassin, giving extended scoring information"
600   :type 'string
601   :group 'spam-spamassassin)
602
603 (make-obsolete-variable 'spam-sa-learn-path 'spam-sa-learn-program
604                         "Gnus 5.10.9 (Emacs 22.1)")
605 (defcustom spam-sa-learn-program (executable-find "sa-learn")
606   "Name of the sa-learn program."
607   :type '(choice (file :tag "Location of spamassassin")
608                  (const :tag "spamassassin is not installed"))
609   :group 'spam-spamassassin)
610
611 (defcustom spam-sa-learn-rebuild t
612   "Whether sa-learn should rebuild the database every time it is called
613 Enable this if you want sa-learn to rebuild the database automatically.  Doing
614 this will slightly increase the running time of the spam registration process.
615 If you choose not to do this, you will have to run \"sa-learn --rebuild\" in
616 order for SpamAssassin to recognize the new registered spam."
617   :type 'boolean
618   :group 'spam-spamassassin)
619
620 (defcustom spam-sa-learn-spam-switch "--spam"
621   "The switch that sa-learn uses to register spam messages."
622   :type 'string
623   :group 'spam-spamassassin)
624
625 (defcustom spam-sa-learn-ham-switch "--ham"
626   "The switch that sa-learn uses to register ham messages."
627   :type 'string
628   :group 'spam-spamassassin)
629
630 (defcustom spam-sa-learn-unregister-switch "--forget"
631   "The switch that sa-learn uses to unregister messages messages."
632   :type 'string
633   :group 'spam-spamassassin)
634
635 (defgroup spam-crm114 nil
636   "Spam CRM114 Mailfilter configuration."
637   :group 'spam)
638
639 (defcustom spam-crm114-program (executable-find "mailfilter.crm")
640   "File path of the CRM114 Mailfilter executable program."
641   :type '(choice (file :tag "Location of CRM114 Mailfilter")
642          (const :tag "CRM114 Mailfilter is not installed"))
643   :group 'spam-crm114)
644
645 (defcustom spam-crm114-header "X-CRM114-Status"
646   "The header that CRM114 Mailfilter inserts in messages."
647   :type 'string
648   :group 'spam-crm114)
649
650 (defcustom spam-crm114-spam-switch "--learnspam"
651   "The switch that CRM114 Mailfilter uses to register spam messages."
652   :type 'string
653   :group 'spam-crm114)
654
655 (defcustom spam-crm114-ham-switch "--learnnonspam"
656   "The switch that CRM114 Mailfilter uses to register ham messages."
657   :type 'string
658   :group 'spam-crm114)
659
660 (defcustom spam-crm114-spam-strong-switch "--unlearn"
661   "The switch that CRM114 Mailfilter uses to unregister ham messages."
662   :type 'string
663   :group 'spam-crm114)
664
665 (defcustom spam-crm114-ham-strong-switch "--unlearn"
666   "The switch that CRM114 Mailfilter uses to unregister spam messages."
667   :type 'string
668   :group 'spam-crm114)
669
670 (defcustom spam-crm114-positive-spam-header "^SPAM"
671   "The regex on `spam-crm114-header' for positive spam identification."
672   :type 'regexp
673   :group 'spam-crm114)
674
675 (defcustom spam-crm114-database-directory nil
676   "Directory path of the CRM114 Mailfilter databases."
677   :type '(choice (directory
678           :tag "Location of the CRM114 Mailfilter database directory")
679          (const :tag "Use the default"))
680   :group 'spam-crm114)
681
682 ;;; Key bindings for spam control.
683
684 (gnus-define-keys gnus-summary-mode-map
685   "St" spam-generic-score
686   "Sx" gnus-summary-mark-as-spam
687   "Mst" spam-generic-score
688   "Msx" gnus-summary-mark-as-spam
689   "\M-d" gnus-summary-mark-as-spam
690   "$" gnus-summary-mark-as-spam)
691
692 (defvar spam-cache-lookups t
693   "Whether spam.el will try to cache lookups using `spam-caches'.")
694
695 (defvar spam-caches (make-hash-table
696                      :size 10
697                      :test 'equal)
698   "Cache of spam detection entries.")
699
700 (defvar spam-old-articles nil
701   "List of old ham and spam articles, generated when a group is entered.")
702
703 (defvar spam-split-disabled nil
704   "If non-nil, `spam-split' is disabled, and always returns nil.")
705
706 (defvar spam-split-last-successful-check nil
707   "Internal variable.
708 `spam-split' will set this to nil or a spam-use-XYZ check if it
709 finds ham or spam.")
710
711 ;; internal variables for backends
712 ;; TODO: find a way to create these on the fly in spam-install-backend-super
713 (defvar spam-use-copy nil)
714 (defvar spam-use-move nil)
715 (defvar spam-use-gmane nil)
716 (defvar spam-use-resend nil)
717
718 ;;}}}
719
720 ;;{{{ convenience functions
721
722 (defun spam-clear-cache (symbol)
723   "Clear the `spam-caches' entry for a check."
724   (remhash symbol spam-caches))
725
726 (defun spam-xor (a b)
727   "Logical A xor B."
728   (and (or a b) (not (and a b))))
729
730 (defun spam-set-difference (list1 list2)
731   "Return a set difference of LIST1 and LIST2.
732 When either list is nil, the other is returned."
733   (if (and list1 list2)
734       ;; we have two non-nil lists
735       (progn
736         (dolist (item (append list1 list2))
737           (when (and (memq item list1) (memq item list2))
738             (setq list1 (delq item list1))
739             (setq list2 (delq item list2))))
740         (append list1 list2))
741     ;; if either of the lists was nil, return the other one
742     (if list1 list1 list2)))
743
744 (defun spam-group-ham-mark-p (group mark &optional spam)
745   "Checks if MARK is considered a ham mark in GROUP."
746   (when (stringp group)
747     (let* ((marks (spam-group-ham-marks group spam))
748            (marks (if (symbolp mark)
749                       marks
750                     (mapcar 'symbol-value marks))))
751       (memq mark marks))))
752
753 (defun spam-group-spam-mark-p (group mark)
754   "Checks if MARK is considered a spam mark in GROUP."
755   (spam-group-ham-mark-p group mark t))
756
757 (defun spam-group-ham-marks (group &optional spam)
758   "In GROUP, get all the ham marks."
759   (when (stringp group)
760     (let* ((marks (if spam
761                       (gnus-parameter-spam-marks group)
762                     (gnus-parameter-ham-marks group)))
763            (marks (car marks))
764            (marks (if (listp (car marks)) (car marks) marks)))
765       marks)))
766
767 (defun spam-group-spam-marks (group)
768   "In GROUP, get all the spam marks."
769   (spam-group-ham-marks group t))
770
771 (defun spam-group-spam-contents-p (group)
772   "Is GROUP a spam group?"
773   (if (and (stringp group) (< 0 (length group)))
774       (or (member group spam-junk-mailgroups)
775           (memq 'gnus-group-spam-classification-spam
776                 (gnus-parameter-spam-contents group)))
777     nil))
778
779 (defun spam-group-ham-contents-p (group)
780   "Is GROUP a ham group?"
781   (if (stringp group)
782       (memq 'gnus-group-spam-classification-ham
783             (gnus-parameter-spam-contents group))
784     nil))
785
786 (defun spam-classifications ()
787   "Return list of valid classifications"
788   '(spam ham))
789
790 (defun spam-classification-valid-p (classification)
791   "Is CLASSIFICATION a valid spam/ham classification?"
792   (memq classification (spam-classifications)))
793
794 (defun spam-backend-properties ()
795   "Return list of valid classifications."
796   '(statistical mover check hrf srf huf suf))
797
798 (defun spam-backend-property-valid-p (property)
799   "Is PROPERTY a valid backend property?"
800   (memq property (spam-backend-properties)))
801
802 (defun spam-backend-function-type-valid-p (type)
803   (or (eq type 'registration)
804       (eq type 'unregistration)))
805
806 (defun spam-process-type-valid-p (process-type)
807   (or (eq process-type 'incoming)
808       (eq process-type 'process)))
809
810 (defun spam-list-articles (articles classification)
811   (let ((mark-check (if (eq classification 'spam)
812                         'spam-group-spam-mark-p
813                       'spam-group-ham-mark-p))
814         alist mark-cache-yes mark-cache-no)
815     (dolist (article articles)
816       (let ((mark (gnus-summary-article-mark article)))
817         (unless (or (memq mark mark-cache-yes)
818                     (memq mark mark-cache-no))
819           (if (funcall mark-check
820                        gnus-newsgroup-name
821                        mark)
822               (push mark mark-cache-yes)
823             (push mark mark-cache-no)))
824         (when (memq mark mark-cache-yes)
825           (push article alist))))
826     alist))
827
828 ;;}}}
829
830 ;;{{{ backend installation functions and procedures
831
832 (defun spam-install-backend-super (backend &rest properties)
833   "Install BACKEND for spam.el.
834 Accepts incoming CHECK, ham registration function HRF, spam
835 registration function SRF, ham unregistration function HUF, spam
836 unregistration function SUF, and an indication whether the
837 backend is STATISTICAL."
838   (setq spam-backends (add-to-list 'spam-backends backend))
839   (while properties
840     (let ((property (pop properties))
841           (value (pop properties)))
842       (if (spam-backend-property-valid-p property)
843           (put backend property value)
844         (gnus-error
845          5
846          "spam-install-backend-super got an invalid property %s"
847          property)))))
848
849 (defun spam-backend-list (&optional type)
850   "Return a list of all the backend symbols, constrained by TYPE.
851 When TYPE is 'non-mover, only non-mover backends are returned.
852 When TYPE is 'mover, only mover backends are returned."
853   (let (list)
854     (dolist (backend spam-backends)
855       (when (or
856              (null type)                ;either no type was requested
857              ;; or the type is 'mover and the backend is a mover
858              (and
859               (eq type 'mover)
860               (spam-backend-mover-p backend))
861              ;; or the type is 'non-mover and the backend is not a mover
862              (and
863               (eq type 'non-mover)
864               (not (spam-backend-mover-p backend))))
865         (push backend list)))
866       list))
867
868 (defun spam-backend-check (backend)
869   "Get the check function for BACKEND.
870 Each individual check may return nil, t, or a mailgroup name.
871 The value nil means that the check does not yield a decision, and
872 so, that further checks are needed.  The value t means that the
873 message is definitely not spam, and that further spam checks
874 should be inhibited.  Otherwise, a mailgroup name or the symbol
875 'spam (depending on `spam-split-symbolic-return') is returned where
876 the mail should go, and further checks are also inhibited.  The
877 usual mailgroup name is the value of `spam-split-group', meaning
878 that the message is definitely a spam."
879   (get backend 'check))
880
881 (defun spam-backend-valid-p (backend)
882   "Is BACKEND valid?"
883   (member backend (spam-backend-list)))
884
885 (defun spam-backend-info (backend)
886   "Return information about BACKEND."
887   (if (spam-backend-valid-p backend)
888       (let (info)
889         (setq info (format "Backend %s has the following properties:\n"
890                            backend))
891         (dolist (property (spam-backend-properties))
892           (setq info (format "%s%s=%s\n"
893                              info
894                              property
895                              (get backend property))))
896         info)
897     (gnus-error 5 "spam-backend-info was asked about an invalid backend %s"
898                 backend)))
899
900 (defun spam-backend-function (backend classification type)
901   "Get the BACKEND function for CLASSIFICATION and TYPE.
902 TYPE is 'registration or 'unregistration.
903 CLASSIFICATION is 'ham or 'spam."
904   (if (and
905        (spam-classification-valid-p classification)
906        (spam-backend-function-type-valid-p type))
907       (let ((retrieval
908              (intern
909               (format "spam-backend-%s-%s-function"
910                       classification
911                       type))))
912         (funcall retrieval backend))
913     (gnus-error
914      5
915      "%s was passed invalid backend %s, classification %s, or type %s"
916      "spam-backend-function"
917      backend
918      classification
919      type)))
920
921 (defun spam-backend-article-list-property (classification
922                                            &optional unregister)
923   "Property name of article list with CLASSIFICATION and UNREGISTER."
924   (let* ((r (if unregister "unregister" "register"))
925          (prop (format "%s-%s" classification r)))
926     prop))
927
928 (defun spam-backend-get-article-todo-list (backend
929                                            classification
930                                            &optional unregister)
931   "Get the articles to be processed for BACKEND and CLASSIFICATION.
932 With UNREGISTER, get articles to be unregistered.
933 This is a temporary storage function - nothing here persists."
934   (get
935    backend
936    (intern (spam-backend-article-list-property classification unregister))))
937
938 (defun spam-backend-put-article-todo-list (backend classification list
939                                                    &optional unregister)
940   "Set the LIST of articles to be processed for BACKEND and CLASSIFICATION.
941 With UNREGISTER, set articles to be unregistered.
942 This is a temporary storage function - nothing here persists."
943   (put
944    backend
945    (intern (spam-backend-article-list-property classification unregister))
946    list))
947
948 (defun spam-backend-ham-registration-function (backend)
949   "Get the ham registration function for BACKEND."
950   (get backend 'hrf))
951
952 (defun spam-backend-spam-registration-function (backend)
953   "Get the spam registration function for BACKEND."
954   (get backend 'srf))
955
956 (defun spam-backend-ham-unregistration-function (backend)
957   "Get the ham unregistration function for BACKEND."
958   (get backend 'huf))
959
960 (defun spam-backend-spam-unregistration-function (backend)
961   "Get the spam unregistration function for BACKEND."
962   (get backend 'suf))
963
964 (defun spam-backend-statistical-p (backend)
965   "Is BACKEND statistical?"
966   (get backend 'statistical))
967
968 (defun spam-backend-mover-p (backend)
969   "Is BACKEND a mover?"
970   (get backend 'mover))
971
972 (defun spam-install-backend-alias (backend alias)
973   "Add ALIAS to an existing BACKEND.
974 The previous backend settings for ALIAS are erased."
975
976   ;; install alias with no properties at first
977   (spam-install-backend-super alias)
978
979   (dolist (property (spam-backend-properties))
980     (put alias property (get backend property))))
981
982 (defun spam-install-checkonly-backend (backend check)
983   "Install a BACKEND than can only CHECK for spam."
984   (spam-install-backend-super backend 'check check))
985
986 (defun spam-install-mover-backend (backend hrf srf huf suf)
987   "Install a BACKEND than can move articles at summary exit.
988 Accepts ham registration function HRF, spam registration function
989 SRF, ham unregistration function HUF, spam unregistration
990 function SUF.  The backend has no incoming check and can't be
991 statistical."
992   (spam-install-backend-super
993    backend
994    'hrf hrf 'srf srf 'huf huf 'suf suf 'mover t))
995
996 (defun spam-install-nocheck-backend (backend hrf srf huf suf)
997   "Install a BACKEND than has no check.
998 Accepts ham registration function HRF, spam registration function
999 SRF, ham unregistration function HUF, spam unregistration
1000 function SUF.  The backend has no incoming check and can't be
1001 statistical (it could be, but in practice that doesn't happen)."
1002   (spam-install-backend-super
1003    backend
1004    'hrf hrf 'srf srf 'huf huf 'suf suf))
1005
1006 (defun spam-install-backend (backend check hrf srf huf suf)
1007   "Install a BACKEND.
1008 Accepts incoming CHECK, ham registration function HRF, spam
1009 registration function SRF, ham unregistration function HUF, spam
1010 unregistration function SUF.  The backend won't be
1011 statistical (use `spam-install-statistical-backend' for that)."
1012   (spam-install-backend-super
1013    backend
1014    'check check 'hrf hrf 'srf srf 'huf huf 'suf suf))
1015
1016 (defun spam-install-statistical-backend (backend check hrf srf huf suf)
1017   "Install a BACKEND.
1018 Accepts incoming CHECK, ham registration function HRF, spam
1019 registration function SRF, ham unregistration function HUF, spam
1020 unregistration function SUF.  The backend will be
1021 statistical (use `spam-install-backend' for non-statistical
1022 backends)."
1023   (spam-install-backend-super
1024    backend
1025    'check check 'statistical t 'hrf hrf 'srf srf 'huf huf 'suf suf))
1026
1027 (defun spam-install-statistical-checkonly-backend (backend check)
1028   "Install a statistical BACKEND than can only CHECK for spam."
1029   (spam-install-backend-super
1030    backend
1031    'check check 'statistical t))
1032
1033 ;;}}}
1034
1035 ;;{{{ backend installations
1036 (spam-install-checkonly-backend 'spam-use-blackholes
1037                                 'spam-check-blackholes)
1038
1039 (spam-install-checkonly-backend 'spam-use-hashcash
1040                                 'spam-check-hashcash)
1041
1042 (spam-install-checkonly-backend 'spam-use-spamassassin-headers
1043                                 'spam-check-spamassassin-headers)
1044
1045 (spam-install-checkonly-backend 'spam-use-bogofilter-headers
1046                                 'spam-check-bogofilter-headers)
1047
1048 (spam-install-checkonly-backend 'spam-use-bsfilter-headers
1049                                 'spam-check-bsfilter-headers)
1050
1051 (spam-install-checkonly-backend 'spam-use-gmane-xref
1052                                 'spam-check-gmane-xref)
1053
1054 (spam-install-checkonly-backend 'spam-use-regex-headers
1055                                 'spam-check-regex-headers)
1056
1057 (spam-install-statistical-checkonly-backend 'spam-use-regex-body
1058                                             'spam-check-regex-body)
1059
1060 ;; TODO: NOTE: spam-use-ham-copy is now obsolete, use (ham spam-use-copy)
1061 (spam-install-mover-backend 'spam-use-move
1062                             'spam-move-ham-routine
1063                             'spam-move-spam-routine
1064                             nil
1065                             nil)
1066
1067 (spam-install-nocheck-backend 'spam-use-copy
1068                               'spam-copy-ham-routine
1069                               'spam-copy-spam-routine
1070                               nil
1071                               nil)
1072
1073 (spam-install-nocheck-backend 'spam-use-gmane
1074                               'spam-report-gmane-unregister-routine
1075                               'spam-report-gmane-register-routine
1076                               'spam-report-gmane-register-routine
1077                               'spam-report-gmane-unregister-routine)
1078
1079 (spam-install-nocheck-backend 'spam-use-resend
1080                               'spam-report-resend-register-ham-routine
1081                               'spam-report-resend-register-routine
1082                               nil
1083                               nil)
1084
1085 (spam-install-backend 'spam-use-BBDB
1086                       'spam-check-BBDB
1087                       'spam-BBDB-register-routine
1088                       nil
1089                       'spam-BBDB-unregister-routine
1090                       nil)
1091
1092 (spam-install-backend-alias 'spam-use-BBDB 'spam-use-BBDB-exclusive)
1093
1094 (spam-install-backend 'spam-use-blacklist
1095                       'spam-check-blacklist
1096                       nil
1097                       'spam-blacklist-register-routine
1098                       nil
1099                       'spam-blacklist-unregister-routine)
1100
1101 (spam-install-backend 'spam-use-whitelist
1102                       'spam-check-whitelist
1103                       'spam-whitelist-register-routine
1104                       nil
1105                       'spam-whitelist-unregister-routine
1106                       nil)
1107
1108 (spam-install-statistical-backend 'spam-use-ifile
1109                                   'spam-check-ifile
1110                                   'spam-ifile-register-ham-routine
1111                                   'spam-ifile-register-spam-routine
1112                                   'spam-ifile-unregister-ham-routine
1113                                   'spam-ifile-unregister-spam-routine)
1114
1115 (spam-install-statistical-backend 'spam-use-spamoracle
1116                                   'spam-check-spamoracle
1117                                   'spam-spamoracle-learn-ham
1118                                   'spam-spamoracle-learn-spam
1119                                   'spam-spamoracle-unlearn-ham
1120                                   'spam-spamoracle-unlearn-spam)
1121
1122 (spam-install-statistical-backend 'spam-use-stat
1123                                   'spam-check-stat
1124                                   'spam-stat-register-ham-routine
1125                                   'spam-stat-register-spam-routine
1126                                   'spam-stat-unregister-ham-routine
1127                                   'spam-stat-unregister-spam-routine)
1128
1129 (spam-install-statistical-backend 'spam-use-spamassassin
1130                                   'spam-check-spamassassin
1131                                   'spam-spamassassin-register-ham-routine
1132                                   'spam-spamassassin-register-spam-routine
1133                                   'spam-spamassassin-unregister-ham-routine
1134                                   'spam-spamassassin-unregister-spam-routine)
1135
1136 (spam-install-statistical-backend 'spam-use-bogofilter
1137                                   'spam-check-bogofilter
1138                                   'spam-bogofilter-register-ham-routine
1139                                   'spam-bogofilter-register-spam-routine
1140                                   'spam-bogofilter-unregister-ham-routine
1141                                   'spam-bogofilter-unregister-spam-routine)
1142
1143 (spam-install-statistical-backend 'spam-use-bsfilter
1144                                   'spam-check-bsfilter
1145                                   'spam-bsfilter-register-ham-routine
1146                                   'spam-bsfilter-register-spam-routine
1147                                   'spam-bsfilter-unregister-ham-routine
1148                                   'spam-bsfilter-unregister-spam-routine)
1149
1150 (spam-install-statistical-backend 'spam-use-crm114
1151                                   'spam-check-crm114
1152                                   'spam-crm114-register-ham-routine
1153                                   'spam-crm114-register-spam-routine
1154                                   'spam-crm114-unregister-ham-routine
1155                                   'spam-crm114-unregister-spam-routine)
1156 ;;}}}
1157
1158 ;;{{{ scoring and summary formatting
1159 (defun spam-necessary-extra-headers ()
1160   "Return the extra headers spam.el thinks are necessary."
1161   (let (list)
1162     (when (or spam-use-spamassassin
1163               spam-use-spamassassin-headers
1164               spam-use-regex-headers)
1165       (push 'X-Spam-Status list))
1166     (when (or spam-use-bogofilter
1167               spam-use-regex-headers)
1168       (push 'X-Bogosity list))
1169     (when (or spam-use-crm114
1170               spam-use-regex-headers)
1171       (push 'X-CRM114-Status list))
1172     list))
1173
1174 (defun spam-user-format-function-S (headers)
1175   (when headers
1176     (format "%3.2f"
1177             (spam-summary-score headers spam-summary-score-preferred-header))))
1178
1179 (defun spam-article-sort-by-spam-status (h1 h2)
1180   "Sort articles by score."
1181   (let (result)
1182     (dolist (header (spam-necessary-extra-headers))
1183       (let ((s1 (spam-summary-score h1 header))
1184             (s2 (spam-summary-score h2 header)))
1185       (unless (= s1 s2)
1186         (setq result (< s1 s2))
1187         (return))))
1188     result))
1189
1190 (defvar spam-spamassassin-score-regexp
1191   ".*\\b\\(?:score\\|hits\\)=\\(-?[0-9.]+\\)"
1192   "Regexp matching SpamAssassin score header.
1193 The first group must match the number.")
1194
1195 (defun spam-extra-header-to-number (header headers)
1196   "Transform an extra HEADER to a number, using list of HEADERS.
1197 Note this has to be fast."
1198   (let ((header-content (gnus-extra-header header headers)))
1199     (if header-content
1200         (cond
1201          ((eq header 'X-Spam-Status)
1202           (string-to-number (gnus-replace-in-string
1203                              header-content
1204                              spam-spamassassin-score-regexp
1205                              "\\1")))
1206          ;; for CRM checking, it's probably faster to just do the string match
1207          ((string-match "( pR: \\([0-9.-]+\\)" header-content)
1208           (- (string-to-number (match-string 1 header-content))))
1209          ((eq header 'X-Bogosity)
1210           (string-to-number (gnus-replace-in-string
1211                              (gnus-replace-in-string
1212                               header-content
1213                               ".*spamicity=" "")
1214                              ",.*" "")))
1215          (t nil))
1216       nil)))
1217
1218 (defun spam-summary-score (headers &optional specific-header)
1219   "Score an article for the summary buffer, as fast as possible.
1220 With SPECIFIC-HEADER, returns only that header's score.
1221 Will not return a nil score."
1222   (let (score)
1223     (dolist (header
1224              (if specific-header
1225                  (list specific-header)
1226                (spam-necessary-extra-headers)))
1227       (setq score
1228             (spam-extra-header-to-number header headers))
1229       (when score
1230         (return)))
1231     (or score 0)))
1232
1233 (defun spam-generic-score (&optional recheck)
1234   "Invoke whatever scoring method we can."
1235   (interactive "P")
1236   (cond
1237    ((or spam-use-spamassassin spam-use-spamassassin-headers)
1238     (spam-spamassassin-score recheck))
1239    ((or spam-use-bsfilter spam-use-bsfilter-headers)
1240     (spam-bsfilter-score recheck))
1241    (spam-use-crm114
1242     (spam-crm114-score))
1243    (t (spam-bogofilter-score recheck))))
1244 ;;}}}
1245
1246 ;;{{{ set up widening, processor checks
1247
1248 ;;; set up IMAP widening if it's necessary
1249 (defun spam-setup-widening ()
1250   (when (spam-widening-needed-p)
1251     (setq nnimap-split-download-body-default t)))
1252
1253 (defun spam-widening-needed-p (&optional force-symbols)
1254   (let (found)
1255     (dolist (backend (spam-backend-list))
1256       (when (and (spam-backend-statistical-p backend)
1257                  (or (symbol-value backend)
1258                      (memq backend force-symbols)))
1259         (setq found backend)))
1260     found))
1261
1262 (defvar spam-list-of-processors
1263   ;; note the nil processors are not defined in gnus.el
1264   '((gnus-group-spam-exit-processor-bogofilter   spam spam-use-bogofilter)
1265     (gnus-group-spam-exit-processor-bsfilter     spam spam-use-bsfilter)
1266     (gnus-group-spam-exit-processor-blacklist    spam spam-use-blacklist)
1267     (gnus-group-spam-exit-processor-ifile        spam spam-use-ifile)
1268     (gnus-group-spam-exit-processor-stat         spam spam-use-stat)
1269     (gnus-group-spam-exit-processor-spamoracle   spam spam-use-spamoracle)
1270     (gnus-group-spam-exit-processor-spamassassin spam spam-use-spamassassin)
1271     (gnus-group-spam-exit-processor-report-gmane spam spam-use-gmane) ;; Buggy?
1272     (gnus-group-ham-exit-processor-ifile         ham spam-use-ifile)
1273     (gnus-group-ham-exit-processor-bogofilter    ham spam-use-bogofilter)
1274     (gnus-group-ham-exit-processor-bsfilter      ham spam-use-bsfilter)
1275     (gnus-group-ham-exit-processor-stat          ham spam-use-stat)
1276     (gnus-group-ham-exit-processor-whitelist     ham spam-use-whitelist)
1277     (gnus-group-ham-exit-processor-BBDB          ham spam-use-BBDB)
1278     (gnus-group-ham-exit-processor-copy          ham spam-use-ham-copy)
1279     (gnus-group-ham-exit-processor-spamassassin  ham spam-use-spamassassin)
1280     (gnus-group-ham-exit-processor-spamoracle    ham spam-use-spamoracle))
1281   "The OBSOLETE `spam-list-of-processors' list.
1282 This list contains pairs associating the obsolete ham/spam exit
1283 processor variables with a classification and a spam-use-*
1284 variable.  When the processor variable is nil, just the
1285 classification and spam-use-* check variable are used.  This is
1286 superseded by the new spam backend code, so it's only consulted
1287 for backwards compatibility.")
1288 (make-obsolete-variable 'spam-list-of-processors nil "22.1")
1289
1290 (defun spam-group-processor-p (group backend &optional classification)
1291   "Checks if GROUP has a BACKEND with CLASSIFICATION registered.
1292 Also accepts the obsolete processors, which can be found in
1293 gnus.el and in spam-list-of-processors.  In the case of mover
1294 backends, checks the setting of `spam-summary-exit-behavior' in
1295 addition to the set values for the group."
1296   (if (and (stringp group)
1297            (symbolp backend))
1298       (let ((old-style (assq backend spam-list-of-processors))
1299             (parameters (nth 0 (gnus-parameter-spam-process group)))
1300             found)
1301         (if old-style  ; old-style processor
1302             (spam-group-processor-p group (nth 2 old-style) (nth 1 old-style))
1303           ;; now search for the parameter
1304           (dolist (parameter parameters)
1305             (when (and (null found)
1306                        (listp parameter)
1307                        (eq classification (nth 0 parameter))
1308                        (eq backend (nth 1 parameter)))
1309               (setq found t)))
1310
1311           ;; now, if the parameter was not found, do the
1312           ;; spam-summary-exit-behavior-logic for mover backends
1313           (unless found
1314             (when (spam-backend-mover-p backend)
1315               (setq
1316                found
1317                (cond
1318                 ((eq spam-summary-exit-behavior 'move-all) t)
1319                 ((eq spam-summary-exit-behavior 'move-none) nil)
1320                 ((eq spam-summary-exit-behavior 'default)
1321                  (or (eq classification 'spam) ;move spam out of all groups
1322                      ;; move ham out of spam groups
1323                      (and (eq classification 'ham)
1324                           (spam-group-spam-contents-p group))))
1325                 (t (gnus-error 5 "Unknown spam-summary-exit-behavior: %s"
1326                                spam-summary-exit-behavior))))))
1327
1328           found))
1329     nil))
1330
1331 ;;}}}
1332
1333 ;;{{{ Summary entry and exit processing.
1334
1335 (defun spam-mark-junk-as-spam-routine ()
1336   ;; check the global list of group names spam-junk-mailgroups and the
1337   ;; group parameters
1338   (when (spam-group-spam-contents-p gnus-newsgroup-name)
1339     (gnus-message 6 "Marking %s articles as spam"
1340                   (if spam-mark-only-unseen-as-spam
1341                       "unseen"
1342                     "unread"))
1343     (let ((articles (if spam-mark-only-unseen-as-spam
1344                         gnus-newsgroup-unseen
1345                       gnus-newsgroup-unreads)))
1346       (if spam-mark-new-messages-in-spam-group-as-spam
1347           (dolist (article articles)
1348             (gnus-summary-mark-article article gnus-spam-mark))
1349         (gnus-message 9 "Did not mark new messages as spam.")))))
1350
1351 (defun spam-summary-prepare ()
1352   (setq spam-old-articles
1353         (list (cons 'ham (spam-list-articles gnus-newsgroup-articles 'ham))
1354               (cons 'spam (spam-list-articles gnus-newsgroup-articles 'spam))))
1355   (spam-mark-junk-as-spam-routine))
1356
1357 ;; The spam processors are invoked for any group, spam or ham or neither
1358 (defun spam-summary-prepare-exit ()
1359   (unless gnus-group-is-exiting-without-update-p
1360     (gnus-message 6 "Exiting summary buffer and applying spam rules")
1361
1362     ;; before we begin, remove any article limits
1363 ;    (ignore-errors
1364 ;      (gnus-summary-pop-limit t))
1365
1366     ;; first of all, unregister any articles that are no longer ham or spam
1367     ;; we have to iterate over the processors, or else we'll be too slow
1368     (dolist (classification (spam-classifications))
1369       (let* ((old-articles (cdr-safe (assq classification spam-old-articles)))
1370              (new-articles (spam-list-articles
1371                             gnus-newsgroup-articles
1372                             classification))
1373              (changed-articles (spam-set-difference new-articles old-articles)))
1374         ;; now that we have the changed articles, we go through the processors
1375         (dolist (backend (spam-backend-list))
1376           (let (unregister-list)
1377             (dolist (article changed-articles)
1378               (let ((id (spam-fetch-field-message-id-fast article)))
1379                 (when (spam-log-unregistration-needed-p
1380                        id 'process classification backend)
1381                   (push article unregister-list))))
1382             ;; call spam-register-routine with specific articles to unregister,
1383             ;; when there are articles to unregister and the check is enabled
1384             (when (and unregister-list (symbol-value backend))
1385               (spam-backend-put-article-todo-list backend
1386                                                   classification
1387                                                   unregister-list
1388                                                   t))))))
1389
1390     ;; do the non-moving backends first, then the moving ones
1391     (dolist (backend-type '(non-mover mover))
1392       (dolist (classification (spam-classifications))
1393         (dolist (backend (spam-backend-list backend-type))
1394           (when (spam-group-processor-p
1395                  gnus-newsgroup-name
1396                  backend
1397                  classification)
1398             (spam-backend-put-article-todo-list backend
1399                                                 classification
1400                                                 (spam-list-articles
1401                                                  gnus-newsgroup-articles
1402                                                  classification))))))
1403
1404     (spam-resolve-registrations-routine) ; do the registrations now
1405
1406     ;; we mark all the leftover spam articles as expired at the end
1407     (dolist (article (spam-list-articles
1408                       gnus-newsgroup-articles
1409                       'spam))
1410       (gnus-summary-mark-article article gnus-expirable-mark)))
1411
1412   (setq spam-old-articles nil))
1413
1414 ;;}}}
1415
1416 ;;{{{ spam-use-move and spam-use-copy backend support functions
1417
1418 (defun spam-copy-or-move-routine (copy groups articles classification)
1419
1420   (when (and (car-safe groups) (listp (car-safe groups)))
1421     (setq groups (pop groups)))
1422
1423   (unless (listp groups)
1424     (setq groups (list groups)))
1425
1426     ;; remove the current process mark
1427   (gnus-summary-kill-process-mark)
1428
1429   (let ((backend-supports-deletions
1430          (gnus-check-backend-function
1431           'request-move-article gnus-newsgroup-name))
1432         (respool-method (gnus-find-method-for-group gnus-newsgroup-name))
1433         article mark deletep respool valid-move-destinations)
1434
1435     (when (member 'respool groups)
1436       (setq respool t)                  ; boolean for later
1437       (setq groups '("fake"))) ; when respooling, groups are dynamic so fake it
1438
1439     ;; exclude invalid move destinations
1440     (dolist (group groups)
1441       (unless
1442           (or
1443            (and
1444             (eq classification 'spam)
1445             (spam-group-spam-contents-p gnus-newsgroup-name)
1446             (spam-group-spam-contents-p group)
1447             (gnus-message
1448              3
1449              "Sorry, can't move spam from spam group %s to spam group %s"
1450              gnus-newsgroup-name
1451              group))
1452            (and
1453             (eq classification 'ham)
1454             (spam-group-ham-contents-p gnus-newsgroup-name)
1455             (spam-group-ham-contents-p group)
1456             (gnus-message
1457              3
1458              "Sorry, can't move ham from ham group %s to ham group %s"
1459              gnus-newsgroup-name
1460              group)))
1461         (push group valid-move-destinations)))
1462
1463     (setq groups (nreverse valid-move-destinations))
1464
1465     ;; now do the actual move
1466     (dolist (group groups)
1467
1468       (when (and articles (stringp group))
1469
1470         ;; first, mark the article with the process mark and, if needed,
1471         ;; the unread or expired mark (for ham and spam respectively)
1472         (dolist (article articles)
1473           (when (and (eq classification 'ham)
1474                      spam-mark-ham-unread-before-move-from-spam-group)
1475             (gnus-message 9 "Marking ham article %d unread before move"
1476                           article)
1477             (gnus-summary-mark-article article gnus-unread-mark))
1478           (when (and (eq classification 'spam)
1479                      (not copy))
1480             (gnus-message 9 "Marking spam article %d expirable before move"
1481                           article)
1482             (gnus-summary-mark-article article gnus-expirable-mark))
1483           (gnus-summary-set-process-mark article)
1484
1485           (if respool              ; respooling is with a "fake" group
1486               (let ((spam-split-disabled
1487                      (or spam-split-disabled
1488                          (and (eq classification 'ham)
1489                               spam-disable-spam-split-during-ham-respool))))
1490                 (gnus-message 9 "Respooling article %d with method %s"
1491                               article respool-method)
1492                 (gnus-summary-respool-article nil respool-method))
1493             ;; else, we are not respooling
1494             (if (or (not backend-supports-deletions)
1495                     (> (length groups) 1))
1496                 (progn              ; if copying, copy and set deletep
1497                   (gnus-message 9 "Copying article %d to group %s"
1498                                 article group)
1499                   (gnus-summary-copy-article nil group)
1500                   (setq deletep t))
1501               (gnus-message 9 "Moving article %d to group %s"
1502                             article group)
1503               (gnus-summary-move-article nil group)))))) ; else move articles
1504
1505     ;; now delete the articles, unless a) copy is t, and there was a copy done
1506     ;;                                 b) a move was done to a single group
1507     ;;                                 c) backend-supports-deletions is nil
1508     (unless copy
1509       (when (and deletep backend-supports-deletions)
1510         (dolist (article articles)
1511           (gnus-summary-set-process-mark article)
1512           (gnus-message 9 "Deleting article %d" article))
1513         (when articles
1514           (let ((gnus-novice-user nil)) ; don't ask me if I'm sure
1515             (gnus-summary-delete-article nil)))))
1516     (gnus-summary-yank-process-mark)
1517     (length articles)))
1518
1519 (defun spam-copy-spam-routine (articles)
1520   (spam-copy-or-move-routine
1521    t
1522    (gnus-parameter-spam-process-destination gnus-newsgroup-name)
1523    articles
1524    'spam))
1525
1526 (defun spam-move-spam-routine (articles)
1527   (spam-copy-or-move-routine
1528    nil
1529    (gnus-parameter-spam-process-destination gnus-newsgroup-name)
1530    articles
1531    'spam))
1532
1533 (defun spam-copy-ham-routine (articles)
1534   (spam-copy-or-move-routine
1535    t
1536    (gnus-parameter-ham-process-destination gnus-newsgroup-name)
1537    articles
1538    'ham))
1539
1540 (defun spam-move-ham-routine (articles)
1541   (spam-copy-or-move-routine
1542    nil
1543    (gnus-parameter-ham-process-destination gnus-newsgroup-name)
1544    articles
1545    'ham))
1546
1547 ;;}}}
1548
1549 ;;{{{ article and field retrieval code
1550 (defun spam-get-article-as-string (article)
1551   (when (numberp article)
1552     (with-temp-buffer
1553       (gnus-request-article-this-buffer
1554        article
1555        gnus-newsgroup-name)
1556       (buffer-string))))
1557
1558 ;; disabled for now
1559 ;; (defun spam-get-article-as-filename (article)
1560 ;;   (let ((article-filename))
1561 ;;     (when (numberp article)
1562 ;;       (nnml-possibly-change-directory
1563 ;;        (gnus-group-real-name gnus-newsgroup-name))
1564 ;;       (setq article-filename (expand-file-name
1565 ;;                              (int-to-string article) nnml-current-directory)))
1566 ;;     (if (file-exists-p article-filename)
1567 ;;      article-filename
1568 ;;       nil)))
1569
1570 (defun spam-fetch-field-fast (article field &optional prepared-data-header)
1571   "Fetch a FIELD for ARTICLE with the internal `gnus-data-list' function.
1572 When PREPARED-DATA-HEADER is given, don't look in the Gnus data.
1573 When FIELD is 'number, ARTICLE can be any number (since we want
1574 to find it out)."
1575   (when (numberp article)
1576     (let* ((data-header (or prepared-data-header
1577                             (spam-fetch-article-header article))))
1578       (cond
1579        ((not (arrayp data-header))
1580         (gnus-message 6 "Article %d has a nil data header" article))
1581        ((equal field 'number)
1582         (mail-header-number data-header))
1583        ((equal field 'from)
1584         (mail-header-from data-header))
1585        ((equal field 'message-id)
1586         (mail-header-message-id data-header))
1587        ((equal field 'subject)
1588         (mail-header-subject data-header))
1589        ((equal field 'references)
1590         (mail-header-references data-header))
1591        ((equal field 'date)
1592         (mail-header-date data-header))
1593        ((equal field 'xref)
1594         (mail-header-xref data-header))
1595        ((equal field 'extra)
1596         (mail-header-extra data-header))
1597        (t
1598         (gnus-error
1599          5
1600          "spam-fetch-field-fast: unknown field %s requested"
1601          field)
1602         nil)))))
1603
1604 (defun spam-fetch-field-from-fast (article &optional prepared-data-header)
1605   (spam-fetch-field-fast article 'from prepared-data-header))
1606
1607 (defun spam-fetch-field-subject-fast (article &optional prepared-data-header)
1608   (spam-fetch-field-fast article 'subject prepared-data-header))
1609
1610 (defun spam-fetch-field-message-id-fast (article &optional prepared-data-header)
1611   (spam-fetch-field-fast article 'message-id prepared-data-header))
1612
1613 (defun spam-generate-fake-headers (article)
1614   (let ((dh (spam-fetch-article-header article)))
1615     (if dh
1616         (concat
1617          (format
1618           ;; 80-character limit makes for strange constructs
1619           (concat "From: %s\nSubject: %s\nMessage-ID: %s\n"
1620                   "Date: %s\nReferences: %s\nXref: %s\n")
1621           (spam-fetch-field-fast article 'from dh)
1622           (spam-fetch-field-fast article 'subject dh)
1623           (spam-fetch-field-fast article 'message-id dh)
1624           (spam-fetch-field-fast article 'date dh)
1625           (spam-fetch-field-fast article 'references dh)
1626           (spam-fetch-field-fast article 'xref dh))
1627          (when (spam-fetch-field-fast article 'extra dh)
1628            (format "%s\n" (spam-fetch-field-fast article 'extra dh))))
1629       (gnus-message
1630        5
1631        "spam-generate-fake-headers: article %d didn't have a valid header"
1632        article))))
1633
1634 (defun spam-fetch-article-header (article)
1635   (with-current-buffer gnus-summary-buffer
1636     (gnus-read-header article)
1637     (nth 3 (assq article gnus-newsgroup-data))))
1638 ;;}}}
1639
1640 ;;{{{ Spam determination.
1641
1642 (defun spam-split (&rest specific-checks)
1643   "Split this message into the `spam' group if it is spam.
1644 This function can be used as an entry in the variable `nnmail-split-fancy',
1645 for example like this: (: spam-split).  It can take checks as
1646 parameters.  A string as a parameter will set the
1647 `spam-split-group' to that string.
1648
1649 See the Info node `(gnus)Fancy Mail Splitting' for more details."
1650   (interactive)
1651   (setq spam-split-last-successful-check nil)
1652   (unless spam-split-disabled
1653     (let ((spam-split-group-choice spam-split-group))
1654       (dolist (check specific-checks)
1655         (when (stringp check)
1656           (setq spam-split-group-choice check)
1657           (setq specific-checks (delq check specific-checks))))
1658
1659       (let ((spam-split-group spam-split-group-choice)
1660             (widening-needed-check (spam-widening-needed-p specific-checks)))
1661         (save-excursion
1662           (save-restriction
1663             (when widening-needed-check
1664               (widen)
1665               (gnus-message 8 "spam-split: widening the buffer (%s requires it)"
1666                             widening-needed-check))
1667             (let ((backends (spam-backend-list))
1668                   decision)
1669               (while (and backends (not decision))
1670                 (let* ((backend (pop backends))
1671                        (check-function (spam-backend-check backend))
1672                        (spam-split-group (if spam-split-symbolic-return
1673                                              'spam
1674                                            spam-split-group)))
1675                   (when (or
1676                          ;; either, given specific checks, this is one of them
1677                          (memq backend specific-checks)
1678                          ;; or, given no specific checks, spam-use-CHECK is set
1679                          (and (null specific-checks) (symbol-value backend)))
1680                     (gnus-message 6 "spam-split: calling the %s function"
1681                                   check-function)
1682                     (setq decision (funcall check-function))
1683                     ;; if we got a decision at all, save the current check
1684                     (when decision
1685                       (setq spam-split-last-successful-check backend))
1686
1687                     (when (eq decision 'spam)
1688                       (unless spam-split-symbolic-return
1689                         (gnus-error
1690                          5
1691                          (format "spam-split got %s but %s is nil"
1692                                  decision
1693                                  spam-split-symbolic-return)))))))
1694               (if (eq decision t)
1695                   (if spam-split-symbolic-return-positive 'ham nil)
1696                 decision))))))))
1697
1698 (defun spam-find-spam ()
1699   "Detect spam in the current newsgroup using `spam-split'."
1700   (interactive)
1701
1702   (let* ((group gnus-newsgroup-name)
1703          (autodetect (gnus-parameter-spam-autodetect group))
1704          (methods (gnus-parameter-spam-autodetect-methods group))
1705          (first-method (nth 0 methods))
1706          (articles (if spam-autodetect-recheck-messages
1707                        gnus-newsgroup-articles
1708                      gnus-newsgroup-unseen))
1709          article-cannot-be-faked)
1710
1711
1712     (dolist (backend methods)
1713       (when (spam-backend-statistical-p backend)
1714         (setq article-cannot-be-faked t)
1715         (return)))
1716
1717     (when (memq 'default methods)
1718       (setq article-cannot-be-faked t))
1719
1720     (when (and autodetect
1721                (not (equal first-method 'none)))
1722       (mapcar
1723        (lambda (article)
1724          (let ((id (spam-fetch-field-message-id-fast article))
1725                (subject (spam-fetch-field-subject-fast article))
1726                (sender (spam-fetch-field-from-fast article))
1727                registry-lookup)
1728
1729            (unless id
1730              (gnus-message 6 "Article %d has no message ID!" article))
1731
1732            (when (and id spam-log-to-registry)
1733              (setq registry-lookup (spam-log-registration-type id 'incoming))
1734              (when registry-lookup
1735                (gnus-message
1736                 9
1737                 "spam-find-spam: message %s was already registered incoming"
1738                 id)))
1739
1740            (let* ((spam-split-symbolic-return t)
1741                   (spam-split-symbolic-return-positive t)
1742                   (fake-headers (spam-generate-fake-headers article))
1743                   (split-return
1744                    (or registry-lookup
1745                        (with-temp-buffer
1746                          (if article-cannot-be-faked
1747                              (gnus-request-article-this-buffer
1748                               article
1749                               group)
1750                            ;; else, we fake the article
1751                            (when fake-headers (insert fake-headers)))
1752                          (if (or (null first-method)
1753                                  (equal first-method 'default))
1754                              (spam-split)
1755                            (apply 'spam-split methods))))))
1756              (if (equal split-return 'spam)
1757                  (gnus-summary-mark-article article gnus-spam-mark))
1758
1759              (when (and id split-return spam-log-to-registry)
1760                (when (zerop (gnus-registry-group-count id))
1761                  (gnus-registry-handle-action id nil group subject sender))
1762
1763                (unless registry-lookup
1764                  (spam-log-processing-to-registry
1765                   id
1766                   'incoming
1767                   split-return
1768                   spam-split-last-successful-check
1769                   group))))))
1770        articles))))
1771
1772 ;;}}}
1773
1774 ;;{{{ registration/unregistration functions
1775
1776 (defun spam-resolve-registrations-routine ()
1777   "Go through the backends and register or unregister articles as needed."
1778   (dolist (backend-type '(non-mover mover))
1779     (dolist (classification (spam-classifications))
1780       (dolist (backend (spam-backend-list backend-type))
1781         (let ((rlist (spam-backend-get-article-todo-list
1782                       backend classification))
1783               (ulist (spam-backend-get-article-todo-list
1784                       backend classification t))
1785               (delcount 0))
1786
1787           ;; clear the old lists right away
1788           (spam-backend-put-article-todo-list backend
1789                                               classification
1790                                               nil
1791                                               nil)
1792           (spam-backend-put-article-todo-list backend
1793                                               classification
1794                                               nil
1795                                               t)
1796
1797           ;; eliminate duplicates
1798           (dolist (article (copy-sequence ulist))
1799             (when (memq article rlist)
1800               (incf delcount)
1801               (setq rlist (delq article rlist))
1802               (setq ulist (delq article ulist))))
1803
1804           (unless (zerop delcount)
1805             (gnus-message
1806              9
1807              "%d messages did not have to unregister and then register"
1808              delcount))
1809
1810           ;; unregister articles
1811           (unless (zerop (length ulist))
1812             (let ((num (spam-unregister-routine classification backend ulist)))
1813               (when (> num 0)
1814                 (gnus-message
1815                  6
1816                  "%d %s messages were unregistered by backend %s."
1817                  num
1818                  classification
1819                  backend))))
1820
1821             ;; register articles
1822             (unless (zerop (length rlist))
1823               (let ((num (spam-register-routine classification backend rlist)))
1824                 (when (> num 0)
1825                   (gnus-message
1826                    6
1827                    "%d %s messages were registered by backend %s."
1828                    num
1829                    classification
1830                    backend)))))))))
1831
1832 (defun spam-unregister-routine (classification
1833                                 backend
1834                                 specific-articles)
1835   (spam-register-routine classification backend specific-articles t))
1836
1837 (defun spam-register-routine (classification
1838                               backend
1839                               specific-articles
1840                               &optional unregister)
1841   (when (and (spam-classification-valid-p classification)
1842              (spam-backend-valid-p backend))
1843     (let* ((register-function
1844             (spam-backend-function backend classification 'registration))
1845            (unregister-function
1846             (spam-backend-function backend classification 'unregistration))
1847            (run-function (if unregister
1848                              unregister-function
1849                            register-function))
1850            (log-function (if unregister
1851                              'spam-log-undo-registration
1852                            'spam-log-processing-to-registry))
1853            article articles)
1854
1855       (when run-function
1856         ;; make list of articles, using specific-articles if given
1857         (setq articles (or specific-articles
1858                            (spam-list-articles
1859                             gnus-newsgroup-articles
1860                             classification)))
1861         ;; process them
1862         (when (> (length articles) 0)
1863           (gnus-message 5 "%s %d %s articles as %s using backend %s"
1864                         (if unregister "Unregistering" "Registering")
1865                         (length articles)
1866                         (if specific-articles "specific" "")
1867                         classification
1868                         backend)
1869           (funcall run-function articles)
1870           ;; now log all the registrations (or undo them, depending on
1871           ;; unregister)
1872           (dolist (article articles)
1873             (funcall log-function
1874                      (spam-fetch-field-message-id-fast article)
1875                      'process
1876                      classification
1877                      backend
1878                      gnus-newsgroup-name))))
1879       ;; return the number of articles processed
1880       (length articles))))
1881
1882 ;;; log a ham- or spam-processor invocation to the registry
1883 (defun spam-log-processing-to-registry (id type classification backend group)
1884   (when spam-log-to-registry
1885     (if (and (stringp id)
1886              (stringp group)
1887              (spam-process-type-valid-p type)
1888              (spam-classification-valid-p classification)
1889              (spam-backend-valid-p backend))
1890         (let ((cell-list (gnus-registry-get-id-key id type))
1891               (cell (list classification backend group)))
1892           (push cell cell-list)
1893           (gnus-registry-set-id-key id type cell-list))
1894
1895       (gnus-error
1896        7
1897        (format
1898         "%s call with bad ID, type, classification, spam-backend, or group"
1899         "spam-log-processing-to-registry")))))
1900
1901 ;;; check if a ham- or spam-processor registration has been done
1902 (defun spam-log-registered-p (id type)
1903   (when spam-log-to-registry
1904     (if (and (stringp id)
1905              (spam-process-type-valid-p type))
1906         (gnus-registry-get-id-key id type)
1907       (progn
1908         (gnus-error
1909          7
1910          (format "%s called with bad ID, type, classification, or spam-backend"
1911                  "spam-log-registered-p"))
1912         nil))))
1913
1914 ;;; check what a ham- or spam-processor registration says
1915 ;;; returns nil if conflicting registrations are found
1916 (defun spam-log-registration-type (id type)
1917   (let ((count 0)
1918         decision)
1919     (dolist (reg (spam-log-registered-p id type))
1920       (let ((classification (nth 0 reg)))
1921         (when (spam-classification-valid-p classification)
1922           (when (and decision
1923                      (not (eq classification decision)))
1924             (setq count (+ 1 count)))
1925           (setq decision classification))))
1926     (if (< 0 count)
1927         nil
1928       decision)))
1929
1930
1931 ;;; check if a ham- or spam-processor registration needs to be undone
1932 (defun spam-log-unregistration-needed-p (id type classification backend)
1933   (when spam-log-to-registry
1934     (if (and (stringp id)
1935              (spam-process-type-valid-p type)
1936              (spam-classification-valid-p classification)
1937              (spam-backend-valid-p backend))
1938         (let ((cell-list (gnus-registry-get-id-key id type))
1939               found)
1940           (dolist (cell cell-list)
1941             (unless found
1942               (when (and (eq classification (nth 0 cell))
1943                          (eq backend (nth 1 cell)))
1944                 (setq found t))))
1945           found)
1946       (progn
1947         (gnus-error
1948          7
1949          (format "%s called with bad ID, type, classification, or spam-backend"
1950                  "spam-log-unregistration-needed-p"))
1951         nil))))
1952
1953
1954 ;;; undo a ham- or spam-processor registration (the group is not used)
1955 (defun spam-log-undo-registration (id type classification backend
1956                                       &optional group)
1957   (when (and spam-log-to-registry
1958              (spam-log-unregistration-needed-p id type classification backend))
1959     (if (and (stringp id)
1960              (spam-process-type-valid-p type)
1961              (spam-classification-valid-p classification)
1962              (spam-backend-valid-p backend))
1963         (let ((cell-list (gnus-registry-get-id-key id type))
1964               new-cell-list found)
1965           (dolist (cell cell-list)
1966             (unless (and (eq classification (nth 0 cell))
1967                          (eq backend (nth 1 cell)))
1968               (push cell new-cell-list)))
1969           (gnus-registry-set-id-key id type new-cell-list))
1970       (progn
1971         (gnus-error 7 (format
1972                        "%s call with bad ID, type, spam-backend, or group"
1973                        "spam-log-undo-registration"))
1974         nil))))
1975
1976 ;;}}}
1977
1978 ;;{{{ backend functions
1979
1980 ;;{{{ Gmane xrefs
1981 (defun spam-check-gmane-xref ()
1982   (let ((header (or
1983                  (message-fetch-field "Xref")
1984                  (message-fetch-field "Newsgroups"))))
1985     (when header                        ; return nil when no header
1986       (when (string-match spam-gmane-xref-spam-group
1987                           header)
1988           spam-split-group))))
1989
1990 ;;}}}
1991
1992 ;;{{{ Regex body
1993
1994 (defun spam-check-regex-body ()
1995   (let ((spam-regex-headers-ham spam-regex-body-ham)
1996         (spam-regex-headers-spam spam-regex-body-spam))
1997     (spam-check-regex-headers t)))
1998
1999 ;;}}}
2000
2001 ;;{{{ Regex headers
2002
2003 (defun spam-check-regex-headers (&optional body)
2004   (let ((type (if body "body" "header"))
2005         ret found)
2006     (dolist (h-regex spam-regex-headers-ham)
2007       (unless found
2008         (goto-char (point-min))
2009         (when (re-search-forward h-regex nil t)
2010           (message "Ham regex %s search positive." type)
2011           (setq found t))))
2012     (dolist (s-regex spam-regex-headers-spam)
2013       (unless found
2014         (goto-char (point-min))
2015         (when (re-search-forward s-regex nil t)
2016           (message "Spam regex %s search positive." type)
2017           (setq found t)
2018           (setq ret spam-split-group))))
2019     ret))
2020
2021 ;;}}}
2022
2023 ;;{{{ Blackholes.
2024
2025 (defun spam-reverse-ip-string (ip)
2026   (when (stringp ip)
2027     (mapconcat 'identity
2028                (nreverse (split-string ip "\\."))
2029                ".")))
2030
2031 (defun spam-check-blackholes ()
2032   "Check the Received headers for blackholed relays."
2033   (let ((headers (message-fetch-field "received"))
2034         ips matches)
2035     (when headers
2036       (with-temp-buffer
2037         (insert headers)
2038         (goto-char (point-min))
2039         (gnus-message 6 "Checking headers for relay addresses")
2040         (while (re-search-forward
2041                 "\\([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\\)" nil t)
2042           (gnus-message 9 "Blackhole search found host IP %s." (match-string 1))
2043           (push (spam-reverse-ip-string (match-string 1))
2044                 ips)))
2045       (dolist (server spam-blackhole-servers)
2046         (dolist (ip ips)
2047           (unless (and spam-blackhole-good-server-regex
2048                        ;; match against the reversed (again) IP string
2049                        (string-match
2050                         spam-blackhole-good-server-regex
2051                         (spam-reverse-ip-string ip)))
2052             (unless matches
2053               (let ((query-string (concat ip "." server)))
2054                 (if spam-use-dig
2055                     (let ((query-result (query-dig query-string)))
2056                       (when query-result
2057                         (gnus-message 6 "(DIG): positive blackhole check ā€˜%sā€™"
2058                                       query-result)
2059                         (push (list ip server query-result)
2060                               matches)))
2061                   ;; else, if not using dig.el
2062                   (when (dns-query query-string)
2063                     (gnus-message 6 "positive blackhole check")
2064                     (push (list ip server (dns-query query-string 'TXT))
2065                           matches)))))))))
2066     (when matches
2067       spam-split-group)))
2068 ;;}}}
2069
2070 ;;{{{ Hashcash.
2071
2072 (defun spam-check-hashcash ()
2073   "Check the headers for hashcash payments."
2074   (ignore-errors (mail-check-payment)))  ;mail-check-payment returns a boolean
2075
2076 ;;}}}
2077
2078 ;;{{{ BBDB
2079
2080 ;;; original idea for spam-check-BBDB from Alexander Kotelnikov
2081 ;;; <sacha@giotto.sj.ru>
2082
2083 ;; all this is done inside a condition-case to trap errors
2084
2085 ;; Autoloaded in message, which we require.
2086 (declare-function gnus-extract-address-components "gnus-util" (from))
2087
2088 (eval-and-compile
2089   (condition-case nil
2090       (progn
2091         (require 'bbdb)
2092         (require 'bbdb-com))
2093     (file-error
2094      ;; `bbdb-records' should not be bound as an autoload function
2095      ;; before loading bbdb because of `bbdb-hashtable-size'.
2096      (defalias 'bbdb-buffer 'ignore)
2097      (defalias 'bbdb-create-internal 'ignore)
2098      (defalias 'bbdb-records 'ignore)
2099      (defalias 'spam-BBDB-register-routine 'ignore)
2100      (defalias 'spam-enter-ham-BBDB 'ignore)
2101      (defalias 'spam-exists-in-BBDB-p 'ignore)
2102      (defalias 'bbdb-gethash 'ignore)
2103      nil)))
2104
2105 (eval-and-compile
2106   (when (featurep 'bbdb-com)
2107     ;; when the BBDB changes, we want to clear out our cache
2108     (defun spam-clear-cache-BBDB (&rest immaterial)
2109       (spam-clear-cache 'spam-use-BBDB))
2110
2111     (add-hook 'bbdb-change-hook 'spam-clear-cache-BBDB)
2112
2113     (defun spam-enter-ham-BBDB (addresses &optional remove)
2114       "Enter an address into the BBDB; implies ham (non-spam) sender"
2115       (dolist (from addresses)
2116         (when (stringp from)
2117           (let* ((parsed-address (gnus-extract-address-components from))
2118                  (name (or (nth 0 parsed-address) "Ham Sender"))
2119                  (remove-function (if remove
2120                                       'bbdb-delete-record-internal
2121                                     'ignore))
2122                  (net-address (nth 1 parsed-address))
2123                  (record (and net-address
2124                               (spam-exists-in-BBDB-p net-address))))
2125             (when net-address
2126               (gnus-message 6 "%s address %s %s BBDB"
2127                             (if remove "Deleting" "Adding")
2128                             from
2129                             (if remove "from" "to"))
2130               (if record
2131                   (funcall remove-function record)
2132                 (bbdb-create-internal name nil net-address nil nil
2133                                       "ham sender added by spam.el")))))))
2134
2135     (defun spam-BBDB-register-routine (articles &optional unregister)
2136       (let (addresses)
2137         (dolist (article articles)
2138           (when (stringp (spam-fetch-field-from-fast article))
2139             (push (spam-fetch-field-from-fast article) addresses)))
2140         ;; now do the register/unregister action
2141         (spam-enter-ham-BBDB addresses unregister)))
2142
2143     (defun spam-BBDB-unregister-routine (articles)
2144       (spam-BBDB-register-routine articles t))
2145
2146     (defsubst spam-exists-in-BBDB-p (net)
2147       (when (and (stringp net) (not (zerop (length net))))
2148         (bbdb-records)
2149         (bbdb-gethash (downcase net))))
2150
2151     (defun spam-check-BBDB ()
2152       "Mail from people in the BBDB is classified as ham or non-spam"
2153       (let ((net (message-fetch-field "from")))
2154         (when net
2155           (setq net (nth 1 (gnus-extract-address-components net)))
2156           (if (spam-exists-in-BBDB-p net)
2157               t
2158             (if spam-use-BBDB-exclusive
2159                 spam-split-group
2160               nil)))))))
2161
2162 ;;}}}
2163
2164 ;;{{{ ifile
2165
2166 ;;; check the ifile backend; return nil if the mail was NOT classified
2167 ;;; as spam
2168
2169
2170 (defun spam-get-ifile-database-parameter ()
2171   "Return the command-line parameter for ifile's database.
2172 See `spam-ifile-database'."
2173   (if spam-ifile-database
2174       (format "--db-file=%s" spam-ifile-database)
2175     nil))
2176
2177 (defun spam-check-ifile ()
2178   "Check the ifile backend for the classification of this message."
2179   (let ((article-buffer-name (buffer-name))
2180         category return)
2181     (with-temp-buffer
2182       (let ((temp-buffer-name (buffer-name))
2183             (db-param (spam-get-ifile-database-parameter)))
2184         (with-current-buffer article-buffer-name
2185           (apply 'call-process-region
2186                  (point-min) (point-max) spam-ifile-program
2187                  nil temp-buffer-name nil "-c"
2188                  (if db-param `(,db-param "-q") `("-q"))))
2189         ;; check the return now (we're back in the temp buffer)
2190         (goto-char (point-min))
2191         (if (not (eobp))
2192             (setq category (buffer-substring (point) (point-at-eol))))
2193         (when (not (zerop (length category))) ; we need a category here
2194           (if spam-ifile-all-categories
2195               (setq return category)
2196             ;; else, if spam-ifile-all-categories is not set...
2197             (when (string-equal spam-ifile-spam-category category)
2198               (setq return spam-split-group)))))) ; note return is nil otherwise
2199     return))
2200
2201 (defun spam-ifile-register-with-ifile (articles category &optional unregister)
2202   "Register an article, given as a string, with a category.
2203 Uses `gnus-newsgroup-name' if category is nil (for ham registration)."
2204   (let ((category (or category gnus-newsgroup-name))
2205         (add-or-delete-option (if unregister "-d" "-i"))
2206         (db (spam-get-ifile-database-parameter))
2207         parameters)
2208     (with-temp-buffer
2209       (dolist (article articles)
2210         (let ((article-string (spam-get-article-as-string article)))
2211           (when (stringp article-string)
2212             (insert article-string))))
2213       (apply 'call-process-region
2214              (point-min) (point-max) spam-ifile-program
2215              nil nil nil
2216              add-or-delete-option category
2217              (if db `(,db "-h") `("-h"))))))
2218
2219 (defun spam-ifile-register-spam-routine (articles &optional unregister)
2220   (spam-ifile-register-with-ifile articles spam-ifile-spam-category unregister))
2221
2222 (defun spam-ifile-unregister-spam-routine (articles)
2223   (spam-ifile-register-spam-routine articles t))
2224
2225 (defun spam-ifile-register-ham-routine (articles &optional unregister)
2226   (spam-ifile-register-with-ifile articles spam-ifile-ham-category unregister))
2227
2228 (defun spam-ifile-unregister-ham-routine (articles)
2229   (spam-ifile-register-ham-routine articles t))
2230
2231 ;;}}}
2232
2233 ;;{{{ spam-stat
2234
2235 (require 'spam-stat)
2236
2237 (defun spam-check-stat ()
2238   "Check the spam-stat backend for the classification of this message"
2239   (let ((spam-stat-split-fancy-spam-group spam-split-group) ; override
2240         (spam-stat-buffer (buffer-name)) ; stat the current buffer
2241         category return)
2242     (spam-stat-split-fancy)))
2243
2244 (defun spam-stat-register-spam-routine (articles &optional unregister)
2245   (dolist (article articles)
2246     (let ((article-string (spam-get-article-as-string article)))
2247       (with-temp-buffer
2248         (insert article-string)
2249         (if unregister
2250             (spam-stat-buffer-change-to-non-spam)
2251           (spam-stat-buffer-is-spam))))))
2252
2253 (defun spam-stat-unregister-spam-routine (articles)
2254   (spam-stat-register-spam-routine articles t))
2255
2256 (defun spam-stat-register-ham-routine (articles &optional unregister)
2257   (dolist (article articles)
2258     (let ((article-string (spam-get-article-as-string article)))
2259       (with-temp-buffer
2260         (insert article-string)
2261         (if unregister
2262             (spam-stat-buffer-change-to-spam)
2263           (spam-stat-buffer-is-non-spam))))))
2264
2265 (defun spam-stat-unregister-ham-routine (articles)
2266   (spam-stat-register-ham-routine articles t))
2267
2268 (defun spam-maybe-spam-stat-load ()
2269   (when spam-use-stat (spam-stat-load)))
2270
2271 (defun spam-maybe-spam-stat-save ()
2272   (when spam-use-stat (spam-stat-save)))
2273
2274 ;;}}}
2275
2276 ;;{{{ Blacklists and whitelists.
2277
2278 (defvar spam-whitelist-cache nil)
2279 (defvar spam-blacklist-cache nil)
2280
2281 (defun spam-kill-whole-line ()
2282   (beginning-of-line)
2283   (let ((kill-whole-line t))
2284     (kill-line)))
2285
2286 ;;; address can be a list, too
2287 (defun spam-enter-whitelist (address &optional remove)
2288   "Enter ADDRESS (list or single) into the whitelist.
2289 With a non-nil REMOVE, remove them."
2290   (interactive "sAddress: ")
2291   (spam-enter-list address spam-whitelist remove)
2292   (setq spam-whitelist-cache nil)
2293   (spam-clear-cache 'spam-use-whitelist))
2294
2295 ;;; address can be a list, too
2296 (defun spam-enter-blacklist (address &optional remove)
2297   "Enter ADDRESS (list or single) into the blacklist.
2298 With a non-nil REMOVE, remove them."
2299   (interactive "sAddress: ")
2300   (spam-enter-list address spam-blacklist remove)
2301   (setq spam-blacklist-cache nil)
2302   (spam-clear-cache 'spam-use-whitelist))
2303
2304 (defun spam-enter-list (addresses file &optional remove)
2305   "Enter ADDRESSES into the given FILE.
2306 Either the whitelist or the blacklist files can be used.
2307 With a non-nil REMOVE, remove the ADDRESSES."
2308   (if (stringp addresses)
2309       (spam-enter-list (list addresses) file remove)
2310     ;; else, we have a list of addresses here
2311     (unless (file-exists-p (file-name-directory file))
2312       (make-directory (file-name-directory file) t))
2313     (with-current-buffer
2314        (find-file-noselect file)
2315       (dolist (a addresses)
2316         (when (stringp a)
2317           (goto-char (point-min))
2318           (if (re-search-forward (regexp-quote a) nil t)
2319               ;; found the address
2320               (when remove
2321                 (spam-kill-whole-line))
2322             ;; else, the address was not found
2323             (unless remove
2324               (goto-char (point-max))
2325               (unless (bobp)
2326                 (insert "\n"))
2327               (insert a "\n")))))
2328       (save-buffer))))
2329
2330 (defun spam-filelist-build-cache (type)
2331   (let ((cache (if (eq type 'spam-use-blacklist)
2332                    spam-blacklist-cache
2333                  spam-whitelist-cache))
2334         parsed-cache)
2335     (unless (gethash type spam-caches)
2336       (while cache
2337         (let ((address (pop cache)))
2338           (unless (zerop (length address)) ; 0 for a nil address too
2339             (setq address (regexp-quote address))
2340             ;; fix regexp-quote's treatment of user-intended regexes
2341             (while (string-match "\\\\\\*" address)
2342               (setq address (replace-match ".*" t t address))))
2343           (push address parsed-cache)))
2344       (puthash type parsed-cache spam-caches))))
2345
2346 (defun spam-filelist-check-cache (type from)
2347   (when (stringp from)
2348     (spam-filelist-build-cache type)
2349     (let (found)
2350       (dolist (address (gethash type spam-caches))
2351         (when (and address (string-match address from))
2352           (setq found t)
2353           (return)))
2354       found)))
2355
2356 ;;; returns t if the sender is in the whitelist, nil or
2357 ;;; spam-split-group otherwise
2358 (defun spam-check-whitelist ()
2359   ;; FIXME!  Should it detect when file timestamps change?
2360   (unless spam-whitelist-cache
2361     (setq spam-whitelist-cache (spam-parse-list spam-whitelist)))
2362   (if (spam-from-listed-p 'spam-use-whitelist)
2363       t
2364     (if spam-use-whitelist-exclusive
2365         spam-split-group
2366       nil)))
2367
2368 (defun spam-check-blacklist ()
2369   ;; FIXME!  Should it detect when file timestamps change?
2370   (unless spam-blacklist-cache
2371     (setq spam-blacklist-cache (spam-parse-list spam-blacklist)))
2372   (and (spam-from-listed-p 'spam-use-blacklist)
2373        spam-split-group))
2374
2375 (defun spam-parse-list (file)
2376   (when (file-readable-p file)
2377     (let (contents address)
2378       (with-temp-buffer
2379         (insert-file-contents file)
2380         (while (not (eobp))
2381           (setq address (buffer-substring (point) (point-at-eol)))
2382           (forward-line 1)
2383           ;; insert the e-mail address if detected, otherwise the raw data
2384           (unless (zerop (length address))
2385             (let ((pure-address
2386                    (nth 1 (gnus-extract-address-components address))))
2387               (push (or pure-address address) contents)))))
2388       (nreverse contents))))
2389
2390 (defun spam-from-listed-p (type)
2391   (let ((from (message-fetch-field "from"))
2392         found)
2393     (spam-filelist-check-cache type from)))
2394
2395 (defun spam-filelist-register-routine (articles blacklist &optional unregister)
2396   (let ((de-symbol (if blacklist 'spam-use-whitelist 'spam-use-blacklist))
2397         (declassification (if blacklist 'ham 'spam))
2398         (enter-function
2399          (if blacklist 'spam-enter-blacklist 'spam-enter-whitelist))
2400         (remove-function
2401          (if blacklist 'spam-enter-whitelist 'spam-enter-blacklist))
2402         from addresses unregister-list article-unregister-list)
2403     (dolist (article articles)
2404       (let ((from (spam-fetch-field-from-fast article))
2405             (id (spam-fetch-field-message-id-fast article))
2406             sender-ignored)
2407         (when (stringp from)
2408           (dolist (ignore-regex spam-blacklist-ignored-regexes)
2409             (when (and (not sender-ignored)
2410                        (stringp ignore-regex)
2411                        (string-match ignore-regex from))
2412               (setq sender-ignored t)))
2413           ;; remember the messages we need to unregister, unless remove is set
2414           (when (and
2415                  (null unregister)
2416                  (spam-log-unregistration-needed-p
2417                   id 'process declassification de-symbol))
2418             (push article article-unregister-list)
2419             (push from unregister-list))
2420           (unless sender-ignored
2421             (push from addresses)))))
2422
2423     (if unregister
2424         (funcall enter-function addresses t) ; unregister all these addresses
2425       ;; else, register normally and unregister what we need to
2426       (funcall remove-function unregister-list t)
2427       (dolist (article article-unregister-list)
2428         (spam-log-undo-registration
2429          (spam-fetch-field-message-id-fast article)
2430          'process
2431          declassification
2432          de-symbol))
2433       (funcall enter-function addresses nil))))
2434
2435 (defun spam-blacklist-unregister-routine (articles)
2436   (spam-blacklist-register-routine articles t))
2437
2438 (defun spam-blacklist-register-routine (articles &optional unregister)
2439   (spam-filelist-register-routine articles t unregister))
2440
2441 (defun spam-whitelist-unregister-routine (articles)
2442   (spam-whitelist-register-routine articles t))
2443
2444 (defun spam-whitelist-register-routine (articles &optional unregister)
2445   (spam-filelist-register-routine articles nil unregister))
2446
2447 ;;}}}
2448
2449 ;;{{{ Spam-report glue (gmane and resend reporting)
2450 (defun spam-report-gmane-register-routine (articles)
2451   (when articles
2452     (apply 'spam-report-gmane-spam articles)))
2453
2454 (defun spam-report-gmane-unregister-routine (articles)
2455   (when articles
2456     (apply 'spam-report-gmane-ham articles)))
2457
2458 (defun spam-report-resend-register-ham-routine (articles)
2459   (spam-report-resend-register-routine articles t))
2460
2461 (defvar spam-report-resend-to)
2462
2463 (defun spam-report-resend-register-routine (articles &optional ham)
2464   (require 'spam-report)
2465   (let* ((resend-to-gp
2466           (if ham
2467               (gnus-parameter-ham-resend-to gnus-newsgroup-name)
2468             (gnus-parameter-spam-resend-to gnus-newsgroup-name)))
2469          (spam-report-resend-to (or (car-safe resend-to-gp)
2470                                     spam-report-resend-to)))
2471     (spam-report-resend articles ham)))
2472
2473 ;;}}}
2474
2475 ;;{{{ Bogofilter
2476 (defun spam-check-bogofilter-headers (&optional score)
2477   (let ((header (message-fetch-field spam-bogofilter-header)))
2478     (when header                        ; return nil when no header
2479       (if score                         ; scoring mode
2480           (if (string-match "spamicity=\\([0-9.]+\\)" header)
2481               (match-string 1 header)
2482             "0")
2483         ;; spam detection mode
2484         (when (string-match spam-bogofilter-bogosity-positive-spam-header
2485                             header)
2486           spam-split-group)))))
2487
2488 ;; return something sensible if the score can't be determined
2489 (defun spam-bogofilter-score (&optional recheck)
2490   "Get the Bogofilter spamicity score."
2491   (interactive "P")
2492   (save-window-excursion
2493     (gnus-summary-show-article t)
2494     (set-buffer gnus-article-buffer)
2495     (let ((score (or (unless recheck
2496                        (spam-check-bogofilter-headers t))
2497                      (spam-check-bogofilter t))))
2498       (gnus-summary-show-article)
2499       (message "Spamicity score %s" score)
2500       (or score "0"))))
2501
2502 (defun spam-verify-bogofilter ()
2503   "Verify the Bogofilter version is sufficient."
2504   (when (eq spam-bogofilter-valid 'unknown)
2505     (setq spam-bogofilter-valid
2506           (not (string-match "^bogofilter version 0\\.\\([0-9]\\|1[01]\\)\\."
2507                              (shell-command-to-string
2508                               (format "%s -V" spam-bogofilter-program))))))
2509   spam-bogofilter-valid)
2510
2511 (defun spam-check-bogofilter (&optional score)
2512   "Check the Bogofilter backend for the classification of this message."
2513   (if (spam-verify-bogofilter)
2514       (let ((article-buffer-name (buffer-name))
2515             (db spam-bogofilter-database-directory)
2516             return)
2517         (with-temp-buffer
2518           (let ((temp-buffer-name (buffer-name)))
2519             (with-current-buffer article-buffer-name
2520               (apply 'call-process-region
2521                      (point-min) (point-max)
2522                      spam-bogofilter-program
2523                      nil temp-buffer-name nil
2524                      (if db `("-d" ,db "-v") `("-v"))))
2525             (setq return (spam-check-bogofilter-headers score))))
2526         return)
2527     (gnus-error 5 "`spam.el' doesn't support obsolete bogofilter versions")))
2528
2529 (defun spam-bogofilter-register-with-bogofilter (articles
2530                                                  spam
2531                                                  &optional unregister)
2532   "Register an article, given as a string, as spam or non-spam."
2533   (if (spam-verify-bogofilter)
2534       (dolist (article articles)
2535         (let ((article-string (spam-get-article-as-string article))
2536               (db spam-bogofilter-database-directory)
2537               (switch (if unregister
2538                           (if spam
2539                               spam-bogofilter-spam-strong-switch
2540                             spam-bogofilter-ham-strong-switch)
2541                         (if spam
2542                             spam-bogofilter-spam-switch
2543                           spam-bogofilter-ham-switch))))
2544           (when (stringp article-string)
2545             (with-temp-buffer
2546               (insert article-string)
2547
2548               (apply 'call-process-region
2549                      (point-min) (point-max)
2550                      spam-bogofilter-program
2551                      nil nil nil switch
2552                      (if db `("-d" ,db "-v") `("-v")))))))
2553     (gnus-error 5 "`spam.el' doesn't support obsolete bogofilter versions")))
2554
2555 (defun spam-bogofilter-register-spam-routine (articles &optional unregister)
2556   (spam-bogofilter-register-with-bogofilter articles t unregister))
2557
2558 (defun spam-bogofilter-unregister-spam-routine (articles)
2559   (spam-bogofilter-register-spam-routine articles t))
2560
2561 (defun spam-bogofilter-register-ham-routine (articles &optional unregister)
2562   (spam-bogofilter-register-with-bogofilter articles nil unregister))
2563
2564 (defun spam-bogofilter-unregister-ham-routine (articles)
2565   (spam-bogofilter-register-ham-routine articles t))
2566
2567
2568 ;;}}}
2569
2570 ;;{{{ spamoracle
2571 (defun spam-check-spamoracle ()
2572   "Run spamoracle on an article to determine whether it's spam."
2573   (let ((article-buffer-name (buffer-name)))
2574     (with-temp-buffer
2575       (let ((temp-buffer-name (buffer-name)))
2576         (with-current-buffer article-buffer-name
2577           (let ((status
2578                  (apply 'call-process-region
2579                         (point-min) (point-max)
2580                         spam-spamoracle-binary
2581                         nil temp-buffer-name nil
2582                         (if spam-spamoracle-database
2583                             `("-f" ,spam-spamoracle-database "mark")
2584                           '("mark")))))
2585             (if (eq 0 status)
2586                 (progn
2587                   (set-buffer temp-buffer-name)
2588                   (goto-char (point-min))
2589                   (when (re-search-forward "^X-Spam: yes;" nil t)
2590                     spam-split-group))
2591               (error "Error running spamoracle: %s" status))))))))
2592
2593 (defun spam-spamoracle-learn (articles article-is-spam-p &optional unregister)
2594   "Run spamoracle in training mode."
2595   (with-temp-buffer
2596     (let ((temp-buffer-name (buffer-name)))
2597       (save-excursion
2598         (goto-char (point-min))
2599         (dolist (article articles)
2600           (insert (spam-get-article-as-string article)))
2601         (let* ((arg (if (spam-xor unregister article-is-spam-p)
2602                         "-spam"
2603                       "-good"))
2604                (status
2605                 (apply 'call-process-region
2606                        (point-min) (point-max)
2607                        spam-spamoracle-binary
2608                        nil temp-buffer-name nil
2609                        (if spam-spamoracle-database
2610                            `("-f" ,spam-spamoracle-database
2611                              "add" ,arg)
2612                          `("add" ,arg)))))
2613           (unless (eq 0 status)
2614             (error "Error running spamoracle: %s" status)))))))
2615
2616 (defun spam-spamoracle-learn-ham (articles &optional unregister)
2617   (spam-spamoracle-learn articles nil unregister))
2618
2619 (defun spam-spamoracle-unlearn-ham (articles &optional unregister)
2620   (spam-spamoracle-learn-ham articles t))
2621
2622 (defun spam-spamoracle-learn-spam (articles &optional unregister)
2623   (spam-spamoracle-learn articles t unregister))
2624
2625 (defun spam-spamoracle-unlearn-spam (articles &optional unregister)
2626   (spam-spamoracle-learn-spam articles t))
2627
2628 ;;}}}
2629
2630 ;;{{{ SpamAssassin
2631 ;;; based mostly on the bogofilter code
2632 (defun spam-check-spamassassin-headers (&optional score)
2633   "Check the SpamAssassin headers for the classification of this message."
2634   (if score                             ; scoring mode
2635       (let ((header (message-fetch-field spam-spamassassin-spam-status-header)))
2636         (when header
2637           (if (string-match spam-spamassassin-score-regexp header)
2638               (match-string 1 header)
2639             "0")))
2640     ;; spam detection mode
2641     (let ((header (message-fetch-field spam-spamassassin-spam-flag-header)))
2642           (when header                  ; return nil when no header
2643             (when (string-match spam-spamassassin-positive-spam-flag-header
2644                                 header)
2645               spam-split-group)))))
2646
2647 (defun spam-check-spamassassin (&optional score)
2648   "Check the SpamAssassin backend for the classification of this message."
2649   (let ((article-buffer-name (buffer-name)))
2650     (with-temp-buffer
2651       (let ((temp-buffer-name (buffer-name)))
2652         (with-current-buffer article-buffer-name
2653           (apply 'call-process-region
2654                  (point-min) (point-max) spam-assassin-program
2655                  nil temp-buffer-name nil spam-spamassassin-arguments))
2656         ;; check the return now (we're back in the temp buffer)
2657         (goto-char (point-min))
2658         (spam-check-spamassassin-headers score)))))
2659
2660 ;; return something sensible if the score can't be determined
2661 (defun spam-spamassassin-score (&optional recheck)
2662   "Get the SpamAssassin score"
2663   (interactive "P")
2664   (save-window-excursion
2665     (gnus-summary-show-article t)
2666     (set-buffer gnus-article-buffer)
2667     (let ((score (or (unless recheck
2668                        (spam-check-spamassassin-headers t))
2669                      (spam-check-spamassassin t))))
2670       (gnus-summary-show-article)
2671       (message "SpamAssassin score %s" score)
2672       (or score "0"))))
2673
2674 (defun spam-spamassassin-register-with-sa-learn (articles spam
2675                                                  &optional unregister)
2676   "Register articles with spamassassin's sa-learn as spam or non-spam."
2677   (if articles
2678       (let ((action (if unregister spam-sa-learn-unregister-switch
2679                       (if spam spam-sa-learn-spam-switch
2680                         spam-sa-learn-ham-switch)))
2681             (summary-buffer-name (buffer-name)))
2682         (with-temp-buffer
2683           ;; group the articles into mbox format
2684           (dolist (article articles)
2685             (let (article-string)
2686               (with-current-buffer summary-buffer-name
2687                 (setq article-string (spam-get-article-as-string article)))
2688               (when (stringp article-string)
2689                 ;; mbox separator
2690                 (insert (concat "From nobody " (current-time-string) "\n"))
2691                 (insert article-string)
2692                 (insert "\n"))))
2693           ;; call sa-learn on all messages at the same time
2694           (apply 'call-process-region
2695                  (point-min) (point-max)
2696                  spam-sa-learn-program
2697                  nil nil nil "--mbox"
2698                  (if spam-sa-learn-rebuild
2699                      (list action)
2700                    `("--no-rebuild" ,action)))))))
2701
2702 (defun spam-spamassassin-register-spam-routine (articles &optional unregister)
2703   (spam-spamassassin-register-with-sa-learn articles t unregister))
2704
2705 (defun spam-spamassassin-register-ham-routine (articles &optional unregister)
2706   (spam-spamassassin-register-with-sa-learn articles nil unregister))
2707
2708 (defun spam-spamassassin-unregister-spam-routine (articles)
2709   (spam-spamassassin-register-with-sa-learn articles t t))
2710
2711 (defun spam-spamassassin-unregister-ham-routine (articles)
2712   (spam-spamassassin-register-with-sa-learn articles nil t))
2713
2714 ;;}}}
2715
2716 ;;{{{ Bsfilter
2717 ;;; based mostly on the bogofilter code
2718 (defun spam-check-bsfilter-headers (&optional score)
2719   (if score
2720       (or (nnmail-fetch-field spam-bsfilter-probability-header)
2721           "0")
2722     (let ((header (nnmail-fetch-field spam-bsfilter-header)))
2723       (when header ; return nil when no header
2724         (when (string-match "YES" header)
2725           spam-split-group)))))
2726
2727 ;; return something sensible if the score can't be determined
2728 (defun spam-bsfilter-score (&optional recheck)
2729   "Get the Bsfilter spamicity score."
2730   (interactive "P")
2731   (save-window-excursion
2732     (gnus-summary-show-article t)
2733     (set-buffer gnus-article-buffer)
2734     (let ((score (or (unless recheck
2735                        (spam-check-bsfilter-headers t))
2736                      (spam-check-bsfilter t))))
2737       (gnus-summary-show-article)
2738       (message "Spamicity score %s" score)
2739       (or score "0"))))
2740
2741 (defun spam-check-bsfilter (&optional score)
2742   "Check the Bsfilter backend for the classification of this message."
2743   (let ((article-buffer-name (buffer-name))
2744         (dir spam-bsfilter-database-directory)
2745         return)
2746     (with-temp-buffer
2747       (let ((temp-buffer-name (buffer-name)))
2748         (with-current-buffer article-buffer-name
2749           (apply 'call-process-region
2750                  (point-min) (point-max)
2751                  spam-bsfilter-program
2752                  nil temp-buffer-name nil
2753                  "--pipe"
2754                  "--insert-flag"
2755                  "--insert-probability"
2756                  (when dir
2757                    (list "--homedir" dir))))
2758         (setq return (spam-check-bsfilter-headers score))))
2759     return))
2760
2761 (defun spam-bsfilter-register-with-bsfilter (articles
2762                                              spam
2763                                              &optional unregister)
2764   "Register an article, given as a string, as spam or non-spam."
2765   (dolist (article articles)
2766     (let ((article-string (spam-get-article-as-string article))
2767           (switch (if unregister
2768                       (if spam
2769                           spam-bsfilter-spam-strong-switch
2770                         spam-bsfilter-ham-strong-switch)
2771                     (if spam
2772                         spam-bsfilter-spam-switch
2773                       spam-bsfilter-ham-switch))))
2774       (when (stringp article-string)
2775         (with-temp-buffer
2776           (insert article-string)
2777           (apply 'call-process-region
2778                  (point-min) (point-max)
2779                  spam-bsfilter-program
2780                  nil nil nil switch
2781                  "--update"
2782                  (when spam-bsfilter-database-directory
2783                    (list "--homedir"
2784                          spam-bsfilter-database-directory))))))))
2785
2786 (defun spam-bsfilter-register-spam-routine (articles &optional unregister)
2787   (spam-bsfilter-register-with-bsfilter articles t unregister))
2788
2789 (defun spam-bsfilter-unregister-spam-routine (articles)
2790   (spam-bsfilter-register-spam-routine articles t))
2791
2792 (defun spam-bsfilter-register-ham-routine (articles &optional unregister)
2793   (spam-bsfilter-register-with-bsfilter articles nil unregister))
2794
2795 (defun spam-bsfilter-unregister-ham-routine (articles)
2796   (spam-bsfilter-register-ham-routine articles t))
2797
2798 ;;}}}
2799
2800 ;;{{{ CRM114 Mailfilter
2801 (defun spam-check-crm114-headers (&optional score)
2802   (let ((header (message-fetch-field spam-crm114-header)))
2803     (when header                        ; return nil when no header
2804       (if score                         ; scoring mode
2805           (if (string-match "( pR: \\([0-9.-]+\\)" header)
2806               (match-string 1 header)
2807             "0")
2808         ;; spam detection mode
2809         (when (string-match spam-crm114-positive-spam-header
2810                             header)
2811           spam-split-group)))))
2812
2813 ;; return something sensible if the score can't be determined
2814 (defun spam-crm114-score ()
2815   "Get the CRM114 Mailfilter pR."
2816   (interactive)
2817   (save-window-excursion
2818     (gnus-summary-show-article t)
2819     (set-buffer gnus-article-buffer)
2820     (let ((score (or (spam-check-crm114-headers t)
2821                      (spam-check-crm114 t))))
2822       (gnus-summary-show-article)
2823       (message "pR: %s" score)
2824       (or score "0"))))
2825
2826 (defun spam-check-crm114 (&optional score)
2827   "Check the CRM114 Mailfilter backend for the classification of this message."
2828   (let ((article-buffer-name (buffer-name))
2829         (db spam-crm114-database-directory)
2830         return)
2831     (with-temp-buffer
2832       (let ((temp-buffer-name (buffer-name)))
2833         (with-current-buffer article-buffer-name
2834           (apply 'call-process-region
2835                  (point-min) (point-max)
2836                  spam-crm114-program
2837                  nil temp-buffer-name nil
2838                  (when db (list (concat "--fileprefix=" db)))))
2839         (setq return (spam-check-crm114-headers score))))
2840     return))
2841
2842 (defun spam-crm114-register-with-crm114 (articles
2843                                          spam
2844                                          &optional unregister)
2845   "Register an article, given as a string, as spam or non-spam."
2846   (dolist (article articles)
2847     (let ((article-string (spam-get-article-as-string article))
2848           (db spam-crm114-database-directory)
2849           (switch (if unregister
2850                       (if spam
2851                           spam-crm114-spam-strong-switch
2852                         spam-crm114-ham-strong-switch)
2853                     (if spam
2854                         spam-crm114-spam-switch
2855                       spam-crm114-ham-switch))))
2856       (when (stringp article-string)
2857         (with-temp-buffer
2858           (insert article-string)
2859
2860           (apply 'call-process-region
2861                  (point-min) (point-max)
2862                  spam-crm114-program
2863                  nil nil nil
2864                  (when db (list switch (concat "--fileprefix=" db)))))))))
2865
2866 (defun spam-crm114-register-spam-routine (articles &optional unregister)
2867   (spam-crm114-register-with-crm114 articles t unregister))
2868
2869 (defun spam-crm114-unregister-spam-routine (articles)
2870   (spam-crm114-register-spam-routine articles t))
2871
2872 (defun spam-crm114-register-ham-routine (articles &optional unregister)
2873   (spam-crm114-register-with-crm114 articles nil unregister))
2874
2875 (defun spam-crm114-unregister-ham-routine (articles)
2876   (spam-crm114-register-ham-routine articles t))
2877
2878 ;;}}}
2879
2880 ;;}}}
2881
2882 ;;{{{ Hooks
2883
2884 ;;;###autoload
2885 (defun spam-initialize (&rest symbols)
2886   "Install the spam.el hooks and do other initialization.
2887 When SYMBOLS is given, set those variables to t.  This is so you
2888 can call `spam-initialize' before you set spam-use-* variables on
2889 explicitly, and matters only if you need the extra headers
2890 installed through `spam-necessary-extra-headers'."
2891   (interactive)
2892
2893   (when spam-install-hooks
2894     (dolist (var symbols)
2895       (set var t))
2896
2897     (dolist (header (spam-necessary-extra-headers))
2898       (add-to-list 'nnmail-extra-headers header)
2899       (add-to-list 'gnus-extra-headers header))
2900
2901     ;; TODO: How do we redo this every time the `spam' face is customized?
2902     (push '((eq mark gnus-spam-mark) . spam)
2903           gnus-summary-highlight)
2904     ;; Add hooks for loading and saving the spam stats
2905     (add-hook 'gnus-save-newsrc-hook 'spam-maybe-spam-stat-save)
2906     (add-hook 'gnus-get-top-new-news-hook 'spam-maybe-spam-stat-load)
2907     (add-hook 'gnus-startup-hook 'spam-maybe-spam-stat-load)
2908     (add-hook 'gnus-summary-prepare-exit-hook 'spam-summary-prepare-exit)
2909     (add-hook 'gnus-summary-prepare-hook 'spam-summary-prepare)
2910     (add-hook 'gnus-get-new-news-hook 'spam-setup-widening)
2911     (add-hook 'gnus-summary-prepared-hook 'spam-find-spam)
2912     ;; Don't install things more than once.
2913     (setq spam-install-hooks nil)))
2914
2915 (defun spam-unload-hook ()
2916   "Uninstall the spam.el hooks."
2917   (interactive)
2918   (remove-hook 'gnus-save-newsrc-hook 'spam-maybe-spam-stat-save)
2919   (remove-hook 'gnus-get-top-new-news-hook 'spam-maybe-spam-stat-load)
2920   (remove-hook 'gnus-startup-hook 'spam-maybe-spam-stat-load)
2921   (remove-hook 'gnus-summary-prepare-exit-hook 'spam-summary-prepare-exit)
2922   (remove-hook 'gnus-summary-prepare-hook 'spam-summary-prepare)
2923   (remove-hook 'gnus-get-new-news-hook 'spam-setup-widening)
2924   (remove-hook 'gnus-summary-prepare-hook 'spam-find-spam))
2925
2926 (add-hook 'spam-unload-hook 'spam-unload-hook)
2927
2928 ;;}}}
2929
2930 (provide 'spam)
2931
2932 ;;; spam.el ends here