cgit.sxemacs.org Git - gnus/blob - lisp/spam.el

   1 ;;; spam.el --- Identifying spam
   2
   3 ;; Copyright (C) 2002-2015 Free Software Foundation, Inc.
   4
   5 ;; Author: Lars Magne Ingebrigtsen <larsi@gnus.org>
   6 ;; Maintainer: Ted Zlatanov <tzz@lifelogs.com>
   7 ;; Keywords: network, spam, mail, bogofilter, BBDB, dspam, dig, whitelist, blacklist, gmane, hashcash, spamassassin, bsfilter, ifile, stat, crm114, spamoracle
   8
   9 ;; This file is part of GNU Emacs.
  10
  11 ;; GNU Emacs is free software: you can redistribute it and/or modify
  12 ;; it under the terms of the GNU General Public License as published by
  13 ;; the Free Software Foundation, either version 3 of the License, or
  14 ;; (at your option) any later version.
  15
  16 ;; GNU Emacs is distributed in the hope that it will be useful,
  17 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 ;; GNU General Public License for more details.
  20
  21 ;; You should have received a copy of the GNU General Public License
  22 ;; along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.
  23
  24 ;;; Commentary:
  25
  26 ;;; This module addresses a few aspects of spam control under Gnus.  Page
  27 ;;; breaks are used for grouping declarations and documentation relating to
  28 ;;; each particular aspect.
  29
  30 ;;; The integration with Gnus is not yet complete.  See various `FIXME'
  31 ;;; comments, below, for supplementary explanations or discussions.
  32
  33 ;;; Several TODO items are marked as such
  34
  35 ;; TODO: cross-server splitting, remote processing, training through files
  36
  37 ;;; Code:
  38
  39 ;;{{{ compilation directives and autoloads/requires
  40
  41 (eval-when-compile (require 'cl))
  42
  43 (require 'message)              ;for the message-fetch-field functions
  44 (require 'gnus-sum)
  45 (require 'gnus-uu)                      ; because of key prefix issues
  46 ;;; for the definitions of group content classification and spam processors
  47 (require 'gnus)
  48
  49 (eval-when-compile (require 'hashcash))
  50
  51 ;; for nnimap-split-download-body-default
  52 (eval-when-compile (require 'nnimap))
  53
  54 ;; autoload query-dig
  55 (autoload 'query-dig "dig")
  56
  57 ;; autoload spam-report
  58 (autoload 'spam-report-gmane "spam-report")
  59 (autoload 'spam-report-gmane-spam "spam-report")
  60 (autoload 'spam-report-gmane-ham "spam-report")
  61 (autoload 'spam-report-resend "spam-report")
  62
  63 ;; autoload gnus-registry
  64 (autoload 'gnus-registry-group-count "gnus-registry")
  65 (autoload 'gnus-registry-get-id-key "gnus-registry")
  66 (autoload 'gnus-registry-set-id-key "gnus-registry")
  67 (autoload 'gnus-registry-handle-action "gnus-registry")
  68
  69 ;; autoload dns-query
  70 (autoload 'dns-query "dns")
  71
  72 ;;}}}
  73
  74 ;;{{{ Main parameters.
  75 (defvar spam-backends nil
  76   "List of spam.el backends with all the pertinent data.
  77 Populated by `spam-install-backend-super'.")
  78
  79 (defgroup spam nil
  80   "Spam configuration."
  81   :version "22.1"
  82   :group 'mail
  83   :group 'news)
  84
  85 (defcustom spam-summary-exit-behavior 'default
  86   "Exit behavior at the time of summary exit.
  87 Note that setting the `spam-use-move' or `spam-use-copy' backends on
  88 a group through group/topic parameters overrides this mechanism."
  89   :type '(choice
  90           (const
  91            :tag "Move spam out of all groups and ham out of spam groups"
  92            default)
  93           (const
  94            :tag "Move spam out of all groups and ham out of all groups"
  95            move-all)
  96           (const
  97            :tag "Never move spam or ham out of any groups"
  98            move-none))
  99   :group 'spam)
 100
 101 (defcustom spam-directory (nnheader-concat gnus-directory "spam/")
 102   "Directory for spam whitelists and blacklists."
 103   :type 'directory
 104   :group 'spam)
 105
 106 (defcustom spam-mark-new-messages-in-spam-group-as-spam t
 107   "Whether new messages in a spam group should get the spam-mark."
 108   :type 'boolean
 109   ;; :version "22.1" ;; Gnus 5.10.8 / No Gnus 0.3
 110   :group 'spam)
 111
 112 (defcustom spam-log-to-registry nil
 113   "Whether spam/ham processing should be logged in the registry."
 114   :type 'boolean
 115   :group 'spam)
 116
 117 (defcustom spam-split-symbolic-return nil
 118   "Whether `spam-split' should work with symbols or group names."
 119   :type 'boolean
 120   :group 'spam)
 121
 122 (defcustom spam-split-symbolic-return-positive nil
 123   "Whether `spam-split' should ALWAYS work with symbols or group names.
 124 Do not set this if you use `spam-split' in a fancy split method."
 125   :type 'boolean
 126   :group 'spam)
 127
 128 (defcustom spam-mark-only-unseen-as-spam t
 129   "Whether only unseen articles should be marked as spam in spam groups.
 130 When nil, all unread articles in a spam group are marked as
 131 spam.  Set this if you want to leave an article unread in a spam group
 132 without losing it to the automatic spam-marking process."
 133   :type 'boolean
 134   :group 'spam)
 135
 136 (defcustom spam-mark-ham-unread-before-move-from-spam-group nil
 137   "Whether ham should be marked unread before it's moved.
 138 The article is moved out of a spam group according to `ham-process-destination'.
 139 This variable is an official entry in the international Longest Variable Name
 140 Competition."
 141   :type 'boolean
 142   :group 'spam)
 143
 144 (defcustom spam-disable-spam-split-during-ham-respool nil
 145   "Whether `spam-split' should be ignored while resplitting ham.
 146 This is useful to prevent ham from ending up in the same spam
 147 group after the resplit.  Don't set this to t if you have `spam-split' as the
 148 last rule in your split configuration."
 149   :type 'boolean
 150   :group 'spam)
 151
 152 (defcustom spam-autodetect-recheck-messages nil
 153   "Should spam.el recheck all messages when autodetecting?
 154 Normally this is nil, so only unseen messages will be checked."
 155   :type 'boolean
 156   :group 'spam)
 157
 158 (defcustom spam-whitelist (expand-file-name "whitelist" spam-directory)
 159   "The location of the whitelist.
 160 The file format is one regular expression per line.
 161 The regular expression is matched against the address."
 162   :type 'file
 163   :group 'spam)
 164
 165 (defcustom spam-blacklist (expand-file-name "blacklist" spam-directory)
 166   "The location of the blacklist.
 167 The file format is one regular expression per line.
 168 The regular expression is matched against the address."
 169   :type 'file
 170   :group 'spam)
 171
 172 (defcustom spam-use-dig t
 173   "Whether `query-dig' should be used instead of `dns-query'."
 174   :type 'boolean
 175   :group 'spam)
 176
 177 (defcustom spam-use-gmane-xref nil
 178   "Whether the Gmane spam xref should be used by `spam-split'."
 179   :type 'boolean
 180   :group 'spam)
 181
 182 (defcustom spam-use-blacklist nil
 183   "Whether the blacklist should be used by `spam-split'."
 184   :type 'boolean
 185   :group 'spam)
 186
 187 (defcustom spam-blacklist-ignored-regexes nil
 188   "Regular expressions that the blacklist should ignore."
 189   :type '(repeat (regexp :tag "Regular expression to ignore when blacklisting"))
 190   :group 'spam)
 191
 192 (defcustom spam-use-whitelist nil
 193   "Whether the whitelist should be used by `spam-split'."
 194   :type 'boolean
 195   :group 'spam)
 196
 197 (defcustom spam-use-whitelist-exclusive nil
 198   "Whether whitelist-exclusive should be used by `spam-split'.
 199 Exclusive whitelisting means that all messages from senders not in the whitelist
 200 are considered spam."
 201   :type 'boolean
 202   :group 'spam)
 203
 204 (defcustom spam-use-blackholes nil
 205   "Whether blackholes should be used by `spam-split'."
 206   :type 'boolean
 207   :group 'spam)
 208
 209 (defcustom spam-use-hashcash nil
 210   "Whether hashcash payments should be detected by `spam-split'."
 211   :type 'boolean
 212   :group 'spam)
 213
 214 (defcustom spam-use-regex-headers nil
 215   "Whether a header regular expression match should be used by `spam-split'.
 216 Also see the variables `spam-regex-headers-spam' and `spam-regex-headers-ham'."
 217   :type 'boolean
 218   :group 'spam)
 219
 220 (defcustom spam-use-regex-body nil
 221   "Whether a body regular expression match should be used by `spam-split'.
 222 Also see the variables `spam-regex-body-spam' and `spam-regex-body-ham'."
 223   :type 'boolean
 224   :group 'spam)
 225
 226 (defcustom spam-use-bogofilter-headers nil
 227   "Whether bogofilter headers should be used by `spam-split'.
 228 Enable this if you pre-process messages with Bogofilter BEFORE Gnus sees them."
 229   :type 'boolean
 230   :group 'spam)
 231
 232 (defcustom spam-use-bogofilter nil
 233   "Whether bogofilter should be invoked by `spam-split'.
 234 Enable this if you want Gnus to invoke Bogofilter on new messages."
 235   :type 'boolean
 236   :group 'spam)
 237
 238 (defcustom spam-use-bsfilter-headers nil
 239   "Whether bsfilter headers should be used by `spam-split'.
 240 Enable this if you pre-process messages with Bsfilter BEFORE Gnus sees them."
 241   :type 'boolean
 242   :group 'spam)
 243
 244 (defcustom spam-use-bsfilter nil
 245   "Whether bsfilter should be invoked by `spam-split'.
 246 Enable this if you want Gnus to invoke Bsfilter on new messages."
 247   :type 'boolean
 248   :group 'spam)
 249
 250 (defcustom spam-use-BBDB nil
 251   "Whether BBDB should be used by `spam-split'."
 252   :type 'boolean
 253   :group 'spam)
 254
 255 (defcustom spam-use-BBDB-exclusive nil
 256   "Whether BBDB-exclusive should be used by `spam-split'.
 257 Exclusive BBDB means that all messages from senders not in the BBDB are
 258 considered spam."
 259   :type 'boolean
 260   :group 'spam)
 261
 262 (defcustom spam-use-ifile nil
 263   "Whether ifile should be used by `spam-split'."
 264   :type 'boolean
 265   :group 'spam)
 266
 267 (defcustom spam-use-stat nil
 268   "Whether `spam-stat' should be used by `spam-split'."
 269   :type 'boolean
 270   :group 'spam)
 271
 272 (defcustom spam-use-spamoracle nil
 273   "Whether spamoracle should be used by `spam-split'."
 274   :type 'boolean
 275   :group 'spam)
 276
 277 (defcustom spam-use-spamassassin nil
 278   "Whether spamassassin should be invoked by `spam-split'.
 279 Enable this if you want Gnus to invoke SpamAssassin on new messages."
 280   :type 'boolean
 281   :group 'spam)
 282
 283 (defcustom spam-use-spamassassin-headers nil
 284   "Whether spamassassin headers should be checked by `spam-split'.
 285 Enable this if you pre-process messages with SpamAssassin BEFORE Gnus sees
 286 them."
 287   :type 'boolean
 288   :group 'spam)
 289
 290 (defcustom spam-use-crm114 nil
 291   "Whether the CRM114 Mailfilter should be used by `spam-split'."
 292   :type 'boolean
 293   :group 'spam)
 294
 295 (defcustom spam-install-hooks (or
 296                                spam-use-dig
 297                                spam-use-gmane-xref
 298                                spam-use-blacklist
 299                                spam-use-whitelist
 300                                spam-use-whitelist-exclusive
 301                                spam-use-blackholes
 302                                spam-use-hashcash
 303                                spam-use-regex-headers
 304                                spam-use-regex-body
 305                                spam-use-bogofilter
 306                                spam-use-bogofilter-headers
 307                                spam-use-spamassassin
 308                                spam-use-spamassassin-headers
 309                                spam-use-bsfilter
 310                                spam-use-bsfilter-headers
 311                                spam-use-BBDB
 312                                spam-use-BBDB-exclusive
 313                                spam-use-ifile
 314                                spam-use-stat
 315                                spam-use-spamoracle
 316                                spam-use-crm114)
 317   "Whether the spam hooks should be installed.
 318 Default to t if one of the spam-use-* variables is set."
 319   :group 'spam
 320   :type 'boolean)
 321
 322 (defcustom spam-split-group "spam"
 323   "Group name where incoming spam should be put by `spam-split'."
 324   :type 'string
 325   :group 'spam)
 326
 327 ;;; TODO: deprecate this variable, it's confusing since it's a list of strings,
 328 ;;; not regular expressions
 329 (defcustom spam-junk-mailgroups (cons
 330                                  spam-split-group
 331                                  '("mail.junk" "poste.pourriel"))
 332   "Mailgroups with spam contents.
 333 All unmarked article in such group receive the spam mark on group entry."
 334   :type '(repeat (string :tag "Group"))
 335   :group 'spam)
 336
 337
 338 (defcustom spam-gmane-xref-spam-group "gmane.spam.detected"
 339   "The group where spam xrefs can be found on Gmane.
 340 Only meaningful if you enable `spam-use-gmane-xref'."
 341   :type 'string
 342   :group 'spam)
 343
 344 (defcustom spam-blackhole-servers '("bl.spamcop.net" "relays.ordb.org"
 345                                     "dev.null.dk" "relays.visi.com")
 346   "List of blackhole servers.
 347 Only meaningful if you enable `spam-use-blackholes'."
 348   :type '(repeat (string :tag "Server"))
 349   :group 'spam)
 350
 351 (defcustom spam-blackhole-good-server-regex nil
 352   "String matching IP addresses that should not be checked in the blackholes.
 353 Only meaningful if you enable `spam-use-blackholes'."
 354   :type '(radio (const nil) regexp)
 355   :group 'spam)
 356
 357 (defface spam
 358   '((((class color) (type tty) (background dark))
 359      (:foreground "gray80" :background "gray50"))
 360     (((class color) (type tty) (background light))
 361      (:foreground "gray50" :background "gray80"))
 362     (((class color) (background dark))
 363      (:foreground "ivory2"))
 364     (((class color) (background light))
 365      (:foreground "ivory4"))
 366     (t :inverse-video t))
 367   "Face for spam-marked articles."
 368   :group 'spam)
 369 ;; backward-compatibility alias
 370 (put 'spam-face 'face-alias 'spam)
 371 (put 'spam-face 'obsolete-face "22.1")
 372
 373 (defcustom spam-face 'spam
 374   "Face for spam-marked articles."
 375   :type 'face
 376   :group 'spam)
 377
 378 (defcustom spam-regex-headers-spam '("^X-Spam-Flag: YES")
 379   "Regular expression for positive header spam matches.
 380 Only meaningful if you enable `spam-use-regex-headers'."
 381   :type '(repeat (regexp :tag "Regular expression to match spam header"))
 382   :group 'spam)
 383
 384 (defcustom spam-regex-headers-ham '("^X-Spam-Flag: NO")
 385   "Regular expression for positive header ham matches.
 386 Only meaningful if you enable `spam-use-regex-headers'."
 387   :type '(repeat (regexp :tag "Regular expression to match ham header"))
 388   :group 'spam)
 389
 390 (defcustom spam-regex-body-spam '()
 391   "Regular expression for positive body spam matches.
 392 Only meaningful if you enable `spam-use-regex-body'."
 393   :type '(repeat (regexp :tag "Regular expression to match spam body"))
 394   :group 'spam)
 395
 396 (defcustom spam-regex-body-ham '()
 397   "Regular expression for positive body ham matches.
 398 Only meaningful if you enable `spam-use-regex-body'."
 399   :type '(repeat (regexp :tag "Regular expression to match ham body"))
 400   :group 'spam)
 401
 402 (defcustom spam-summary-score-preferred-header nil
 403   "Preferred header to use for `spam-summary-score'."
 404   :type '(choice :tag "Header name"
 405           (symbol :tag "SpamAssassin etc" X-Spam-Status)
 406           (symbol :tag "Bogofilter"       X-Bogosity)
 407           (const  :tag "No preference, take best guess." nil))
 408   :group 'spam)
 409
 410 (defgroup spam-ifile nil
 411   "Spam ifile configuration."
 412   :group 'spam)
 413
 414 (make-obsolete-variable 'spam-ifile-path 'spam-ifile-program
 415                         "Gnus 5.10.9 (Emacs 22.1)")
 416 (defcustom spam-ifile-program (executable-find "ifile")
 417   "Name of the ifile program."
 418   :type '(choice (file :tag "Location of ifile")
 419                  (const :tag "ifile is not installed"))
 420   :group 'spam-ifile)
 421
 422 (make-obsolete-variable 'spam-ifile-database-path 'spam-ifile-database
 423                         "Gnus 5.10.9 (Emacs 22.1)")
 424 (defcustom spam-ifile-database nil
 425   "File name of the ifile database."
 426   :type '(choice (file :tag "Location of the ifile database")
 427                  (const :tag "Use the default"))
 428   :group 'spam-ifile)
 429
 430 (defcustom spam-ifile-spam-category "spam"
 431   "Name of the spam ifile category."
 432   :type 'string
 433   :group 'spam-ifile)
 434
 435 (defcustom spam-ifile-ham-category nil
 436   "Name of the ham ifile category.
 437 If nil, the current group name will be used."
 438   :type '(choice (string :tag "Use a fixed category")
 439                  (const :tag "Use the current group name"))
 440   :group 'spam-ifile)
 441
 442 (defcustom spam-ifile-all-categories nil
 443   "Whether the ifile check will return all categories, or just spam.
 444 Set this to t if you want to use the `spam-split' invocation of ifile as
 445 your main source of newsgroup names."
 446   :type 'boolean
 447   :group 'spam-ifile)
 448
 449 (defgroup spam-bogofilter nil
 450   "Spam bogofilter configuration."
 451   :group 'spam)
 452
 453 (make-obsolete-variable 'spam-bogofilter-path 'spam-bogofilter-program
 454                         "Gnus 5.10.9 (Emacs 22.1)")
 455 (defcustom spam-bogofilter-program (executable-find "bogofilter")
 456   "Name of the Bogofilter program."
 457   :type '(choice (file :tag "Location of bogofilter")
 458                  (const :tag "Bogofilter is not installed"))
 459   :group 'spam-bogofilter)
 460
 461 (defvar spam-bogofilter-valid 'unknown "Is the bogofilter version valid?")
 462
 463 (defcustom spam-bogofilter-header "X-Bogosity"
 464   "The header that Bogofilter inserts in messages."
 465   :type 'string
 466   :group 'spam-bogofilter)
 467
 468 (defcustom spam-bogofilter-spam-switch "-s"
 469   "The switch that Bogofilter uses to register spam messages."
 470   :type 'string
 471   :group 'spam-bogofilter)
 472
 473 (defcustom spam-bogofilter-ham-switch "-n"
 474   "The switch that Bogofilter uses to register ham messages."
 475   :type 'string
 476   :group 'spam-bogofilter)
 477
 478 (defcustom spam-bogofilter-spam-strong-switch "-S"
 479   "The switch that Bogofilter uses to unregister ham messages."
 480   :type 'string
 481   :group 'spam-bogofilter)
 482
 483 (defcustom spam-bogofilter-ham-strong-switch "-N"
 484   "The switch that Bogofilter uses to unregister spam messages."
 485   :type 'string
 486   :group 'spam-bogofilter)
 487
 488 (defcustom spam-bogofilter-bogosity-positive-spam-header "^\\(Yes\\|Spam\\)"
 489   "The regex on `spam-bogofilter-header' for positive spam identification."
 490   :type 'regexp
 491   :group 'spam-bogofilter)
 492
 493 (defcustom spam-bogofilter-database-directory nil
 494   "Location of the Bogofilter database.
 495 When nil, use the default location."
 496   :type '(choice (directory
 497                   :tag "Location of the Bogofilter database directory")
 498                  (const :tag "Use the default"))
 499   :group 'spam-bogofilter)
 500
 501 (defgroup spam-bsfilter nil
 502   "Spam bsfilter configuration."
 503   :group 'spam)
 504
 505 (make-obsolete-variable 'spam-bsfilter-path 'spam-bsfilter-program
 506                         "Gnus 5.10.9 (Emacs 22.1)")
 507 (defcustom spam-bsfilter-program (executable-find "bsfilter")
 508   "Name of the Bsfilter program."
 509   :type '(choice (file :tag "Location of bsfilter")
 510                  (const :tag "Bsfilter is not installed"))
 511   :group 'spam-bsfilter)
 512
 513 (defcustom spam-bsfilter-header "X-Spam-Flag"
 514   "The header inserted by Bsfilter to flag spam."
 515   :type 'string
 516   :group 'spam-bsfilter)
 517
 518 (defcustom spam-bsfilter-probability-header "X-Spam-Probability"
 519   "The header that Bsfilter inserts in messages."
 520   :type 'string
 521   :group 'spam-bsfilter)
 522
 523 (defcustom spam-bsfilter-spam-switch "--add-spam"
 524   "The switch that Bsfilter uses to register spam messages."
 525   :type 'string
 526   :group 'spam-bsfilter)
 527
 528 (defcustom spam-bsfilter-ham-switch "--add-clean"
 529   "The switch that Bsfilter uses to register ham messages."
 530   :type 'string
 531   :group 'spam-bsfilter)
 532
 533 (defcustom spam-bsfilter-spam-strong-switch "--sub-spam"
 534   "The switch that Bsfilter uses to unregister ham messages."
 535   :type 'string
 536   :group 'spam-bsfilter)
 537
 538 (defcustom spam-bsfilter-ham-strong-switch "--sub-clean"
 539   "The switch that Bsfilter uses to unregister spam messages."
 540   :type 'string
 541   :group 'spam-bsfilter)
 542
 543 (defcustom spam-bsfilter-database-directory nil
 544   "Directory path of the Bsfilter databases."
 545   :type '(choice (directory
 546                   :tag "Location of the Bsfilter database directory")
 547                  (const :tag "Use the default"))
 548   :group 'spam-bsfilter)
 549
 550 (defgroup spam-spamoracle nil
 551   "Spam spamoracle configuration."
 552   :group 'spam)
 553
 554 (defcustom spam-spamoracle-database nil
 555   "Location of spamoracle database file.
 556 When nil, use the default spamoracle database."
 557   :type '(choice (directory :tag "Location of spamoracle database file.")
 558                  (const :tag "Use the default"))
 559   :group 'spam-spamoracle)
 560
 561 (defcustom spam-spamoracle-binary (executable-find "spamoracle")
 562   "Location of the spamoracle binary."
 563   :type '(choice (directory :tag "Location of the spamoracle binary")
 564                  (const :tag "Use the default"))
 565   :group 'spam-spamoracle)
 566
 567 (defgroup spam-spamassassin nil
 568   "Spam SpamAssassin configuration."
 569   :group 'spam)
 570
 571 (make-obsolete-variable 'spam-spamassassin-path
 572   'spam-spamassassin-program "Gnus 5.10.9 (Emacs 22.1)")
 573 (defcustom spam-assassin-program (executable-find "spamassassin")
 574   "Name of the spamassassin program.
 575 Hint: set this to \"spamc\" if you have spamd running.  See the spamc and
 576 spamd man pages for more information on these programs."
 577   :type '(choice (file :tag "Location of spamc")
 578                  (const :tag "spamassassin is not installed"))
 579   :group 'spam-spamassassin)
 580
 581 (defcustom spam-spamassassin-arguments ()
 582   "Arguments to pass to the spamassassin executable.
 583 This must be a list.  For example, `(\"-C\" \"configfile\")'."
 584   :type '(restricted-sexp :match-alternatives (listp))
 585   :group 'spam-spamassassin)
 586
 587 (defcustom spam-spamassassin-spam-flag-header "X-Spam-Flag"
 588   "The header inserted by SpamAssassin to flag spam."
 589   :type 'string
 590   :group 'spam-spamassassin)
 591
 592 (defcustom spam-spamassassin-positive-spam-flag-header "YES"
 593   "The regex on `spam-spamassassin-spam-flag-header' for positive spam
 594 identification"
 595   :type 'string
 596   :group 'spam-spamassassin)
 597
 598 (defcustom spam-spamassassin-spam-status-header "X-Spam-Status"
 599   "The header inserted by SpamAssassin, giving extended scoring information"
 600   :type 'string
 601   :group 'spam-spamassassin)
 602
 603 (make-obsolete-variable 'spam-sa-learn-path 'spam-sa-learn-program
 604                         "Gnus 5.10.9 (Emacs 22.1)")
 605 (defcustom spam-sa-learn-program (executable-find "sa-learn")
 606   "Name of the sa-learn program."
 607   :type '(choice (file :tag "Location of spamassassin")
 608                  (const :tag "spamassassin is not installed"))
 609   :group 'spam-spamassassin)
 610
 611 (defcustom spam-sa-learn-rebuild t
 612   "Whether sa-learn should rebuild the database every time it is called
 613 Enable this if you want sa-learn to rebuild the database automatically.  Doing
 614 this will slightly increase the running time of the spam registration process.
 615 If you choose not to do this, you will have to run \"sa-learn --rebuild\" in
 616 order for SpamAssassin to recognize the new registered spam."
 617   :type 'boolean
 618   :group 'spam-spamassassin)
 619
 620 (defcustom spam-sa-learn-spam-switch "--spam"
 621   "The switch that sa-learn uses to register spam messages."
 622   :type 'string
 623   :group 'spam-spamassassin)
 624
 625 (defcustom spam-sa-learn-ham-switch "--ham"
 626   "The switch that sa-learn uses to register ham messages."
 627   :type 'string
 628   :group 'spam-spamassassin)
 629
 630 (defcustom spam-sa-learn-unregister-switch "--forget"
 631   "The switch that sa-learn uses to unregister messages messages."
 632   :type 'string
 633   :group 'spam-spamassassin)
 634
 635 (defgroup spam-crm114 nil
 636   "Spam CRM114 Mailfilter configuration."
 637   :group 'spam)
 638
 639 (defcustom spam-crm114-program (executable-find "mailfilter.crm")
 640   "File path of the CRM114 Mailfilter executable program."
 641   :type '(choice (file :tag "Location of CRM114 Mailfilter")
 642          (const :tag "CRM114 Mailfilter is not installed"))
 643   :group 'spam-crm114)
 644
 645 (defcustom spam-crm114-header "X-CRM114-Status"
 646   "The header that CRM114 Mailfilter inserts in messages."
 647   :type 'string
 648   :group 'spam-crm114)
 649
 650 (defcustom spam-crm114-spam-switch "--learnspam"
 651   "The switch that CRM114 Mailfilter uses to register spam messages."
 652   :type 'string
 653   :group 'spam-crm114)
 654
 655 (defcustom spam-crm114-ham-switch "--learnnonspam"
 656   "The switch that CRM114 Mailfilter uses to register ham messages."
 657   :type 'string
 658   :group 'spam-crm114)
 659
 660 (defcustom spam-crm114-spam-strong-switch "--unlearn"
 661   "The switch that CRM114 Mailfilter uses to unregister ham messages."
 662   :type 'string
 663   :group 'spam-crm114)
 664
 665 (defcustom spam-crm114-ham-strong-switch "--unlearn"
 666   "The switch that CRM114 Mailfilter uses to unregister spam messages."
 667   :type 'string
 668   :group 'spam-crm114)
 669
 670 (defcustom spam-crm114-positive-spam-header "^SPAM"
 671   "The regex on `spam-crm114-header' for positive spam identification."
 672   :type 'regexp
 673   :group 'spam-crm114)
 674
 675 (defcustom spam-crm114-database-directory nil
 676   "Directory path of the CRM114 Mailfilter databases."
 677   :type '(choice (directory
 678           :tag "Location of the CRM114 Mailfilter database directory")
 679          (const :tag "Use the default"))
 680   :group 'spam-crm114)
 681
 682 ;;; Key bindings for spam control.
 683
 684 (gnus-define-keys gnus-summary-mode-map
 685   "St" spam-generic-score
 686   "Sx" gnus-summary-mark-as-spam
 687   "Mst" spam-generic-score
 688   "Msx" gnus-summary-mark-as-spam
 689   "\M-d" gnus-summary-mark-as-spam
 690   "$" gnus-summary-mark-as-spam)
 691
 692 (defvar spam-cache-lookups t
 693   "Whether spam.el will try to cache lookups using `spam-caches'.")
 694
 695 (defvar spam-caches (make-hash-table
 696                      :size 10
 697                      :test 'equal)
 698   "Cache of spam detection entries.")
 699
 700 (defvar spam-old-articles nil
 701   "List of old ham and spam articles, generated when a group is entered.")
 702
 703 (defvar spam-split-disabled nil
 704   "If non-nil, `spam-split' is disabled, and always returns nil.")
 705
 706 (defvar spam-split-last-successful-check nil
 707   "Internal variable.
 708 `spam-split' will set this to nil or a spam-use-XYZ check if it
 709 finds ham or spam.")
 710
 711 ;; internal variables for backends
 712 ;; TODO: find a way to create these on the fly in spam-install-backend-super
 713 (defvar spam-use-copy nil)
 714 (defvar spam-use-move nil)
 715 (defvar spam-use-gmane nil)
 716 (defvar spam-use-resend nil)
 717
 718 ;;}}}
 719
 720 ;;{{{ convenience functions
 721
 722 (defun spam-clear-cache (symbol)
 723   "Clear the `spam-caches' entry for a check."
 724   (remhash symbol spam-caches))
 725
 726 (defun spam-xor (a b)
 727   "Logical A xor B."
 728   (and (or a b) (not (and a b))))
 729
 730 (defun spam-set-difference (list1 list2)
 731   "Return a set difference of LIST1 and LIST2.
 732 When either list is nil, the other is returned."
 733   (if (and list1 list2)
 734       ;; we have two non-nil lists
 735       (progn
 736         (dolist (item (append list1 list2))
 737           (when (and (memq item list1) (memq item list2))
 738             (setq list1 (delq item list1))
 739             (setq list2 (delq item list2))))
 740         (append list1 list2))
 741     ;; if either of the lists was nil, return the other one
 742     (if list1 list1 list2)))
 743
 744 (defun spam-group-ham-mark-p (group mark &optional spam)
 745   "Checks if MARK is considered a ham mark in GROUP."
 746   (when (stringp group)
 747     (let* ((marks (spam-group-ham-marks group spam))
 748            (marks (if (symbolp mark)
 749                       marks
 750                     (mapcar 'symbol-value marks))))
 751       (memq mark marks))))
 752
 753 (defun spam-group-spam-mark-p (group mark)
 754   "Checks if MARK is considered a spam mark in GROUP."
 755   (spam-group-ham-mark-p group mark t))
 756
 757 (defun spam-group-ham-marks (group &optional spam)
 758   "In GROUP, get all the ham marks."
 759   (when (stringp group)
 760     (let* ((marks (if spam
 761                       (gnus-parameter-spam-marks group)
 762                     (gnus-parameter-ham-marks group)))
 763            (marks (car marks))
 764            (marks (if (listp (car marks)) (car marks) marks)))
 765       marks)))
 766
 767 (defun spam-group-spam-marks (group)
 768   "In GROUP, get all the spam marks."
 769   (spam-group-ham-marks group t))
 770
 771 (defun spam-group-spam-contents-p (group)
 772   "Is GROUP a spam group?"
 773   (if (and (stringp group) (< 0 (length group)))
 774       (or (member group spam-junk-mailgroups)
 775           (memq 'gnus-group-spam-classification-spam
 776                 (gnus-parameter-spam-contents group)))
 777     nil))
 778
 779 (defun spam-group-ham-contents-p (group)
 780   "Is GROUP a ham group?"
 781   (if (stringp group)
 782       (memq 'gnus-group-spam-classification-ham
 783             (gnus-parameter-spam-contents group))
 784     nil))
 785
 786 (defun spam-classifications ()
 787   "Return list of valid classifications"
 788   '(spam ham))
 789
 790 (defun spam-classification-valid-p (classification)
 791   "Is CLASSIFICATION a valid spam/ham classification?"
 792   (memq classification (spam-classifications)))
 793
 794 (defun spam-backend-properties ()
 795   "Return list of valid classifications."
 796   '(statistical mover check hrf srf huf suf))
 797
 798 (defun spam-backend-property-valid-p (property)
 799   "Is PROPERTY a valid backend property?"
 800   (memq property (spam-backend-properties)))
 801
 802 (defun spam-backend-function-type-valid-p (type)
 803   (or (eq type 'registration)
 804       (eq type 'unregistration)))
 805
 806 (defun spam-process-type-valid-p (process-type)
 807   (or (eq process-type 'incoming)
 808       (eq process-type 'process)))
 809
 810 (defun spam-list-articles (articles classification)
 811   (let ((mark-check (if (eq classification 'spam)
 812                         'spam-group-spam-mark-p
 813                       'spam-group-ham-mark-p))
 814         alist mark-cache-yes mark-cache-no)
 815     (dolist (article articles)
 816       (let ((mark (gnus-summary-article-mark article)))
 817         (unless (or (memq mark mark-cache-yes)
 818                     (memq mark mark-cache-no))
 819           (if (funcall mark-check
 820                        gnus-newsgroup-name
 821                        mark)
 822               (push mark mark-cache-yes)
 823             (push mark mark-cache-no)))
 824         (when (memq mark mark-cache-yes)
 825           (push article alist))))
 826     alist))
 827
 828 ;;}}}
 829
 830 ;;{{{ backend installation functions and procedures
 831
 832 (defun spam-install-backend-super (backend &rest properties)
 833   "Install BACKEND for spam.el.
 834 Accepts incoming CHECK, ham registration function HRF, spam
 835 registration function SRF, ham unregistration function HUF, spam
 836 unregistration function SUF, and an indication whether the
 837 backend is STATISTICAL."
 838   (setq spam-backends (add-to-list 'spam-backends backend))
 839   (while properties
 840     (let ((property (pop properties))
 841           (value (pop properties)))
 842       (if (spam-backend-property-valid-p property)
 843           (put backend property value)
 844         (gnus-error
 845          5
 846          "spam-install-backend-super got an invalid property %s"
 847          property)))))
 848
 849 (defun spam-backend-list (&optional type)
 850   "Return a list of all the backend symbols, constrained by TYPE.
 851 When TYPE is 'non-mover, only non-mover backends are returned.
 852 When TYPE is 'mover, only mover backends are returned."
 853   (let (list)
 854     (dolist (backend spam-backends)
 855       (when (or
 856              (null type)                ;either no type was requested
 857              ;; or the type is 'mover and the backend is a mover
 858              (and
 859               (eq type 'mover)
 860               (spam-backend-mover-p backend))
 861              ;; or the type is 'non-mover and the backend is not a mover
 862              (and
 863               (eq type 'non-mover)
 864               (not (spam-backend-mover-p backend))))
 865         (push backend list)))
 866       list))
 867
 868 (defun spam-backend-check (backend)
 869   "Get the check function for BACKEND.
 870 Each individual check may return nil, t, or a mailgroup name.
 871 The value nil means that the check does not yield a decision, and
 872 so, that further checks are needed.  The value t means that the
 873 message is definitely not spam, and that further spam checks
 874 should be inhibited.  Otherwise, a mailgroup name or the symbol
 875 'spam (depending on `spam-split-symbolic-return') is returned where
 876 the mail should go, and further checks are also inhibited.  The
 877 usual mailgroup name is the value of `spam-split-group', meaning
 878 that the message is definitely a spam."
 879   (get backend 'check))
 880
 881 (defun spam-backend-valid-p (backend)
 882   "Is BACKEND valid?"
 883   (member backend (spam-backend-list)))
 884
 885 (defun spam-backend-info (backend)
 886   "Return information about BACKEND."
 887   (if (spam-backend-valid-p backend)
 888       (let (info)
 889         (setq info (format "Backend %s has the following properties:\n"
 890                            backend))
 891         (dolist (property (spam-backend-properties))
 892           (setq info (format "%s%s=%s\n"
 893                              info
 894                              property
 895                              (get backend property))))
 896         info)
 897     (gnus-error 5 "spam-backend-info was asked about an invalid backend %s"
 898                 backend)))
 899
 900 (defun spam-backend-function (backend classification type)
 901   "Get the BACKEND function for CLASSIFICATION and TYPE.
 902 TYPE is 'registration or 'unregistration.
 903 CLASSIFICATION is 'ham or 'spam."
 904   (if (and
 905        (spam-classification-valid-p classification)
 906        (spam-backend-function-type-valid-p type))
 907       (let ((retrieval
 908              (intern
 909               (format "spam-backend-%s-%s-function"
 910                       classification
 911                       type))))
 912         (funcall retrieval backend))
 913     (gnus-error
 914      5
 915      "%s was passed invalid backend %s, classification %s, or type %s"
 916      "spam-backend-function"
 917      backend
 918      classification
 919      type)))
 920
 921 (defun spam-backend-article-list-property (classification
 922                                            &optional unregister)
 923   "Property name of article list with CLASSIFICATION and UNREGISTER."
 924   (let* ((r (if unregister "unregister" "register"))
 925          (prop (format "%s-%s" classification r)))
 926     prop))
 927
 928 (defun spam-backend-get-article-todo-list (backend
 929                                            classification
 930                                            &optional unregister)
 931   "Get the articles to be processed for BACKEND and CLASSIFICATION.
 932 With UNREGISTER, get articles to be unregistered.
 933 This is a temporary storage function - nothing here persists."
 934   (get
 935    backend
 936    (intern (spam-backend-article-list-property classification unregister))))
 937
 938 (defun spam-backend-put-article-todo-list (backend classification list
 939                                                    &optional unregister)
 940   "Set the LIST of articles to be processed for BACKEND and CLASSIFICATION.
 941 With UNREGISTER, set articles to be unregistered.
 942 This is a temporary storage function - nothing here persists."
 943   (put
 944    backend
 945    (intern (spam-backend-article-list-property classification unregister))
 946    list))
 947
 948 (defun spam-backend-ham-registration-function (backend)
 949   "Get the ham registration function for BACKEND."
 950   (get backend 'hrf))
 951
 952 (defun spam-backend-spam-registration-function (backend)
 953   "Get the spam registration function for BACKEND."
 954   (get backend 'srf))
 955
 956 (defun spam-backend-ham-unregistration-function (backend)
 957   "Get the ham unregistration function for BACKEND."
 958   (get backend 'huf))
 959
 960 (defun spam-backend-spam-unregistration-function (backend)
 961   "Get the spam unregistration function for BACKEND."
 962   (get backend 'suf))
 963
 964 (defun spam-backend-statistical-p (backend)
 965   "Is BACKEND statistical?"
 966   (get backend 'statistical))
 967
 968 (defun spam-backend-mover-p (backend)
 969   "Is BACKEND a mover?"
 970   (get backend 'mover))
 971
 972 (defun spam-install-backend-alias (backend alias)
 973   "Add ALIAS to an existing BACKEND.
 974 The previous backend settings for ALIAS are erased."
 975
 976   ;; install alias with no properties at first
 977   (spam-install-backend-super alias)
 978
 979   (dolist (property (spam-backend-properties))
 980     (put alias property (get backend property))))
 981
 982 (defun spam-install-checkonly-backend (backend check)
 983   "Install a BACKEND than can only CHECK for spam."
 984   (spam-install-backend-super backend 'check check))
 985
 986 (defun spam-install-mover-backend (backend hrf srf huf suf)
 987   "Install a BACKEND than can move articles at summary exit.
 988 Accepts ham registration function HRF, spam registration function
 989 SRF, ham unregistration function HUF, spam unregistration
 990 function SUF.  The backend has no incoming check and can't be
 991 statistical."
 992   (spam-install-backend-super
 993    backend
 994    'hrf hrf 'srf srf 'huf huf 'suf suf 'mover t))
 995
 996 (defun spam-install-nocheck-backend (backend hrf srf huf suf)
 997   "Install a BACKEND than has no check.
 998 Accepts ham registration function HRF, spam registration function
 999 SRF, ham unregistration function HUF, spam unregistration
1000 function SUF.  The backend has no incoming check and can't be
1001 statistical (it could be, but in practice that doesn't happen)."
1002   (spam-install-backend-super
1003    backend
1004    'hrf hrf 'srf srf 'huf huf 'suf suf))
1005
1006 (defun spam-install-backend (backend check hrf srf huf suf)
1007   "Install a BACKEND.
1008 Accepts incoming CHECK, ham registration function HRF, spam
1009 registration function SRF, ham unregistration function HUF, spam
1010 unregistration function SUF.  The backend won't be
1011 statistical (use `spam-install-statistical-backend' for that)."
1012   (spam-install-backend-super
1013    backend
1014    'check check 'hrf hrf 'srf srf 'huf huf 'suf suf))
1015
1016 (defun spam-install-statistical-backend (backend check hrf srf huf suf)
1017   "Install a BACKEND.
1018 Accepts incoming CHECK, ham registration function HRF, spam
1019 registration function SRF, ham unregistration function HUF, spam
1020 unregistration function SUF.  The backend will be
1021 statistical (use `spam-install-backend' for non-statistical
1022 backends)."
1023   (spam-install-backend-super
1024    backend
1025    'check check 'statistical t 'hrf hrf 'srf srf 'huf huf 'suf suf))
1026
1027 (defun spam-install-statistical-checkonly-backend (backend check)
1028   "Install a statistical BACKEND than can only CHECK for spam."
1029   (spam-install-backend-super
1030    backend
1031    'check check 'statistical t))
1032
1033 ;;}}}
1034
1035 ;;{{{ backend installations
1036 (spam-install-checkonly-backend 'spam-use-blackholes
1037                                 'spam-check-blackholes)
1038
1039 (spam-install-checkonly-backend 'spam-use-hashcash
1040                                 'spam-check-hashcash)
1041
1042 (spam-install-checkonly-backend 'spam-use-spamassassin-headers
1043                                 'spam-check-spamassassin-headers)
1044
1045 (spam-install-checkonly-backend 'spam-use-bogofilter-headers
1046                                 'spam-check-bogofilter-headers)
1047
1048 (spam-install-checkonly-backend 'spam-use-bsfilter-headers
1049                                 'spam-check-bsfilter-headers)
1050
1051 (spam-install-checkonly-backend 'spam-use-gmane-xref
1052                                 'spam-check-gmane-xref)
1053
1054 (spam-install-checkonly-backend 'spam-use-regex-headers
1055                                 'spam-check-regex-headers)
1056
1057 (spam-install-statistical-checkonly-backend 'spam-use-regex-body
1058                                             'spam-check-regex-body)
1059
1060 ;; TODO: NOTE: spam-use-ham-copy is now obsolete, use (ham spam-use-copy)
1061 (spam-install-mover-backend 'spam-use-move
1062                             'spam-move-ham-routine
1063                             'spam-move-spam-routine
1064                             nil
1065                             nil)
1066
1067 (spam-install-nocheck-backend 'spam-use-copy
1068                               'spam-copy-ham-routine
1069                               'spam-copy-spam-routine
1070                               nil
1071                               nil)
1072
1073 (spam-install-nocheck-backend 'spam-use-gmane
1074                               'spam-report-gmane-unregister-routine
1075                               'spam-report-gmane-register-routine
1076                               'spam-report-gmane-register-routine
1077                               'spam-report-gmane-unregister-routine)
1078
1079 (spam-install-nocheck-backend 'spam-use-resend
1080                               'spam-report-resend-register-ham-routine
1081                               'spam-report-resend-register-routine
1082                               nil
1083                               nil)
1084
1085 (spam-install-backend 'spam-use-BBDB
1086                       'spam-check-BBDB
1087                       'spam-BBDB-register-routine
1088                       nil
1089                       'spam-BBDB-unregister-routine
1090                       nil)
1091
1092 (spam-install-backend-alias 'spam-use-BBDB 'spam-use-BBDB-exclusive)
1093
1094 (spam-install-backend 'spam-use-blacklist
1095                       'spam-check-blacklist
1096                       nil
1097                       'spam-blacklist-register-routine
1098                       nil
1099                       'spam-blacklist-unregister-routine)
1100
1101 (spam-install-backend 'spam-use-whitelist
1102                       'spam-check-whitelist
1103                       'spam-whitelist-register-routine
1104                       nil
1105                       'spam-whitelist-unregister-routine
1106                       nil)
1107
1108 (spam-install-statistical-backend 'spam-use-ifile
1109                                   'spam-check-ifile
1110                                   'spam-ifile-register-ham-routine
1111                                   'spam-ifile-register-spam-routine
1112                                   'spam-ifile-unregister-ham-routine
1113                                   'spam-ifile-unregister-spam-routine)
1114
1115 (spam-install-statistical-backend 'spam-use-spamoracle
1116                                   'spam-check-spamoracle
1117                                   'spam-spamoracle-learn-ham
1118                                   'spam-spamoracle-learn-spam
1119                                   'spam-spamoracle-unlearn-ham
1120                                   'spam-spamoracle-unlearn-spam)
1121
1122 (spam-install-statistical-backend 'spam-use-stat
1123                                   'spam-check-stat
1124                                   'spam-stat-register-ham-routine
1125                                   'spam-stat-register-spam-routine
1126                                   'spam-stat-unregister-ham-routine
1127                                   'spam-stat-unregister-spam-routine)
1128
1129 (spam-install-statistical-backend 'spam-use-spamassassin
1130                                   'spam-check-spamassassin
1131                                   'spam-spamassassin-register-ham-routine
1132                                   'spam-spamassassin-register-spam-routine
1133                                   'spam-spamassassin-unregister-ham-routine
1134                                   'spam-spamassassin-unregister-spam-routine)
1135
1136 (spam-install-statistical-backend 'spam-use-bogofilter
1137                                   'spam-check-bogofilter
1138                                   'spam-bogofilter-register-ham-routine
1139                                   'spam-bogofilter-register-spam-routine
1140                                   'spam-bogofilter-unregister-ham-routine
1141                                   'spam-bogofilter-unregister-spam-routine)
1142
1143 (spam-install-statistical-backend 'spam-use-bsfilter
1144                                   'spam-check-bsfilter
1145                                   'spam-bsfilter-register-ham-routine
1146                                   'spam-bsfilter-register-spam-routine
1147                                   'spam-bsfilter-unregister-ham-routine
1148                                   'spam-bsfilter-unregister-spam-routine)
1149
1150 (spam-install-statistical-backend 'spam-use-crm114
1151                                   'spam-check-crm114
1152                                   'spam-crm114-register-ham-routine
1153                                   'spam-crm114-register-spam-routine
1154                                   'spam-crm114-unregister-ham-routine
1155                                   'spam-crm114-unregister-spam-routine)
1156 ;;}}}
1157
1158 ;;{{{ scoring and summary formatting
1159 (defun spam-necessary-extra-headers ()
1160   "Return the extra headers spam.el thinks are necessary."
1161   (let (list)
1162     (when (or spam-use-spamassassin
1163               spam-use-spamassassin-headers
1164               spam-use-regex-headers)
1165       (push 'X-Spam-Status list))
1166     (when (or spam-use-bogofilter
1167               spam-use-regex-headers)
1168       (push 'X-Bogosity list))
1169     (when (or spam-use-crm114
1170               spam-use-regex-headers)
1171       (push 'X-CRM114-Status list))
1172     list))
1173
1174 (defun spam-user-format-function-S (headers)
1175   (when headers
1176     (format "%3.2f"
1177             (spam-summary-score headers spam-summary-score-preferred-header))))
1178
1179 (defun spam-article-sort-by-spam-status (h1 h2)
1180   "Sort articles by score."
1181   (let (result)
1182     (dolist (header (spam-necessary-extra-headers))
1183       (let ((s1 (spam-summary-score h1 header))
1184             (s2 (spam-summary-score h2 header)))
1185       (unless (= s1 s2)
1186         (setq result (< s1 s2))
1187         (return))))
1188     result))
1189
1190 (defvar spam-spamassassin-score-regexp
1191   ".*\\b\\(?:score\\|hits\\)=\\(-?[0-9.]+\\)"
1192   "Regexp matching SpamAssassin score header.
1193 The first group must match the number.")
1194
1195 (defun spam-extra-header-to-number (header headers)
1196   "Transform an extra HEADER to a number, using list of HEADERS.
1197 Note this has to be fast."
1198   (let ((header-content (gnus-extra-header header headers)))
1199     (if header-content
1200         (cond
1201          ((eq header 'X-Spam-Status)
1202           (string-to-number (gnus-replace-in-string
1203                              header-content
1204                              spam-spamassassin-score-regexp
1205                              "\\1")))
1206          ;; for CRM checking, it's probably faster to just do the string match
1207          ((string-match "( pR: \\([0-9.-]+\\)" header-content)
1208           (- (string-to-number (match-string 1 header-content))))
1209          ((eq header 'X-Bogosity)
1210           (string-to-number (gnus-replace-in-string
1211                              (gnus-replace-in-string
1212                               header-content
1213                               ".*spamicity=" "")
1214                              ",.*" "")))
1215          (t nil))
1216       nil)))
1217
1218 (defun spam-summary-score (headers &optional specific-header)
1219   "Score an article for the summary buffer, as fast as possible.
1220 With SPECIFIC-HEADER, returns only that header's score.
1221 Will not return a nil score."
1222   (let (score)
1223     (dolist (header
1224              (if specific-header
1225                  (list specific-header)
1226                (spam-necessary-extra-headers)))
1227       (setq score
1228             (spam-extra-header-to-number header headers))
1229       (when score
1230         (return)))
1231     (or score 0)))
1232
1233 (defun spam-generic-score (&optional recheck)
1234   "Invoke whatever scoring method we can."
1235   (interactive "P")
1236   (cond
1237    ((or spam-use-spamassassin spam-use-spamassassin-headers)
1238     (spam-spamassassin-score recheck))
1239    ((or spam-use-bsfilter spam-use-bsfilter-headers)
1240     (spam-bsfilter-score recheck))
1241    (spam-use-crm114
1242     (spam-crm114-score))
1243    (t (spam-bogofilter-score recheck))))
1244 ;;}}}
1245
1246 ;;{{{ set up widening, processor checks
1247
1248 ;;; set up IMAP widening if it's necessary
1249 (defun spam-setup-widening ()
1250   (when (spam-widening-needed-p)
1251     (setq nnimap-split-download-body-default t)))
1252
1253 (defun spam-widening-needed-p (&optional force-symbols)
1254   (let (found)
1255     (dolist (backend (spam-backend-list))
1256       (when (and (spam-backend-statistical-p backend)
1257                  (or (symbol-value backend)
1258                      (memq backend force-symbols)))
1259         (setq found backend)))
1260     found))
1261
1262 (defvar spam-list-of-processors
1263   ;; note the nil processors are not defined in gnus.el
1264   '((gnus-group-spam-exit-processor-bogofilter   spam spam-use-bogofilter)
1265     (gnus-group-spam-exit-processor-bsfilter     spam spam-use-bsfilter)
1266     (gnus-group-spam-exit-processor-blacklist    spam spam-use-blacklist)
1267     (gnus-group-spam-exit-processor-ifile        spam spam-use-ifile)
1268     (gnus-group-spam-exit-processor-stat         spam spam-use-stat)
1269     (gnus-group-spam-exit-processor-spamoracle   spam spam-use-spamoracle)
1270     (gnus-group-spam-exit-processor-spamassassin spam spam-use-spamassassin)
1271     (gnus-group-spam-exit-processor-report-gmane spam spam-use-gmane) ;; Buggy?
1272     (gnus-group-ham-exit-processor-ifile         ham spam-use-ifile)
1273     (gnus-group-ham-exit-processor-bogofilter    ham spam-use-bogofilter)
1274     (gnus-group-ham-exit-processor-bsfilter      ham spam-use-bsfilter)
1275     (gnus-group-ham-exit-processor-stat          ham spam-use-stat)
1276     (gnus-group-ham-exit-processor-whitelist     ham spam-use-whitelist)
1277     (gnus-group-ham-exit-processor-BBDB          ham spam-use-BBDB)
1278     (gnus-group-ham-exit-processor-copy          ham spam-use-ham-copy)
1279     (gnus-group-ham-exit-processor-spamassassin  ham spam-use-spamassassin)
1280     (gnus-group-ham-exit-processor-spamoracle    ham spam-use-spamoracle))
1281   "The OBSOLETE `spam-list-of-processors' list.
1282 This list contains pairs associating the obsolete ham/spam exit
1283 processor variables with a classification and a spam-use-*
1284 variable.  When the processor variable is nil, just the
1285 classification and spam-use-* check variable are used.  This is
1286 superseded by the new spam backend code, so it's only consulted
1287 for backwards compatibility.")
1288 (make-obsolete-variable 'spam-list-of-processors nil "22.1")
1289
1290 (defun spam-group-processor-p (group backend &optional classification)
1291   "Checks if GROUP has a BACKEND with CLASSIFICATION registered.
1292 Also accepts the obsolete processors, which can be found in
1293 gnus.el and in spam-list-of-processors.  In the case of mover
1294 backends, checks the setting of `spam-summary-exit-behavior' in
1295 addition to the set values for the group."
1296   (if (and (stringp group)
1297            (symbolp backend))
1298       (let ((old-style (assq backend spam-list-of-processors))
1299             (parameters (nth 0 (gnus-parameter-spam-process group)))
1300             found)
1301         (if old-style  ; old-style processor
1302             (spam-group-processor-p group (nth 2 old-style) (nth 1 old-style))
1303           ;; now search for the parameter
1304           (dolist (parameter parameters)
1305             (when (and (null found)
1306                        (listp parameter)
1307                        (eq classification (nth 0 parameter))
1308                        (eq backend (nth 1 parameter)))
1309               (setq found t)))
1310
1311           ;; now, if the parameter was not found, do the
1312           ;; spam-summary-exit-behavior-logic for mover backends
1313           (unless found
1314             (when (spam-backend-mover-p backend)
1315               (setq
1316                found
1317                (cond
1318                 ((eq spam-summary-exit-behavior 'move-all) t)
1319                 ((eq spam-summary-exit-behavior 'move-none) nil)
1320                 ((eq spam-summary-exit-behavior 'default)
1321                  (or (eq classification 'spam) ;move spam out of all groups
1322                      ;; move ham out of spam groups
1323                      (and (eq classification 'ham)
1324                           (spam-group-spam-contents-p group))))
1325                 (t (gnus-error 5 "Unknown spam-summary-exit-behavior: %s"
1326                                spam-summary-exit-behavior))))))
1327
1328           found))
1329     nil))
1330
1331 ;;}}}
1332
1333 ;;{{{ Summary entry and exit processing.
1334
1335 (defun spam-mark-junk-as-spam-routine ()
1336   ;; check the global list of group names spam-junk-mailgroups and the
1337   ;; group parameters
1338   (when (spam-group-spam-contents-p gnus-newsgroup-name)
1339     (gnus-message 6 "Marking %s articles as spam"
1340                   (if spam-mark-only-unseen-as-spam
1341                       "unseen"
1342                     "unread"))
1343     (let ((articles (if spam-mark-only-unseen-as-spam
1344                         gnus-newsgroup-unseen
1345                       gnus-newsgroup-unreads)))
1346       (if spam-mark-new-messages-in-spam-group-as-spam
1347           (dolist (article articles)
1348             (gnus-summary-mark-article article gnus-spam-mark))
1349         (gnus-message 9 "Did not mark new messages as spam.")))))
1350
1351 (defun spam-summary-prepare ()
1352   (setq spam-old-articles
1353         (list (cons 'ham (spam-list-articles gnus-newsgroup-articles 'ham))
1354               (cons 'spam (spam-list-articles gnus-newsgroup-articles 'spam))))
1355   (spam-mark-junk-as-spam-routine))
1356
1357 ;; The spam processors are invoked for any group, spam or ham or neither
1358 (defun spam-summary-prepare-exit ()
1359   (unless gnus-group-is-exiting-without-update-p
1360     (gnus-message 6 "Exiting summary buffer and applying spam rules")
1361
1362     ;; before we begin, remove any article limits
1363 ;    (ignore-errors
1364 ;      (gnus-summary-pop-limit t))
1365
1366     ;; first of all, unregister any articles that are no longer ham or spam
1367     ;; we have to iterate over the processors, or else we'll be too slow
1368     (dolist (classification (spam-classifications))
1369       (let* ((old-articles (cdr-safe (assq classification spam-old-articles)))
1370              (new-articles (spam-list-articles
1371                             gnus-newsgroup-articles
1372                             classification))
1373              (changed-articles (spam-set-difference new-articles old-articles)))
1374         ;; now that we have the changed articles, we go through the processors
1375         (dolist (backend (spam-backend-list))
1376           (let (unregister-list)
1377             (dolist (article changed-articles)
1378               (let ((id (spam-fetch-field-message-id-fast article)))
1379                 (when (spam-log-unregistration-needed-p
1380                        id 'process classification backend)
1381                   (push article unregister-list))))
1382             ;; call spam-register-routine with specific articles to unregister,
1383             ;; when there are articles to unregister and the check is enabled
1384             (when (and unregister-list (symbol-value backend))
1385               (spam-backend-put-article-todo-list backend
1386                                                   classification
1387                                                   unregister-list
1388                                                   t))))))
1389
1390     ;; do the non-moving backends first, then the moving ones
1391     (dolist (backend-type '(non-mover mover))
1392       (dolist (classification (spam-classifications))
1393         (dolist (backend (spam-backend-list backend-type))
1394           (when (spam-group-processor-p
1395                  gnus-newsgroup-name
1396                  backend
1397                  classification)
1398             (spam-backend-put-article-todo-list backend
1399                                                 classification
1400                                                 (spam-list-articles
1401                                                  gnus-newsgroup-articles
1402                                                  classification))))))
1403
1404     (spam-resolve-registrations-routine) ; do the registrations now
1405
1406     ;; we mark all the leftover spam articles as expired at the end
1407     (dolist (article (spam-list-articles
1408                       gnus-newsgroup-articles
1409                       'spam))
1410       (gnus-summary-mark-article article gnus-expirable-mark)))
1411
1412   (setq spam-old-articles nil))
1413
1414 ;;}}}
1415
1416 ;;{{{ spam-use-move and spam-use-copy backend support functions
1417
1418 (defun spam-copy-or-move-routine (copy groups articles classification)
1419
1420   (when (and (car-safe groups) (listp (car-safe groups)))
1421     (setq groups (pop groups)))
1422
1423   (unless (listp groups)
1424     (setq groups (list groups)))
1425
1426     ;; remove the current process mark
1427   (gnus-summary-kill-process-mark)
1428
1429   (let ((backend-supports-deletions
1430          (gnus-check-backend-function
1431           'request-move-article gnus-newsgroup-name))
1432         (respool-method (gnus-find-method-for-group gnus-newsgroup-name))
1433         article mark deletep respool valid-move-destinations)
1434
1435     (when (member 'respool groups)
1436       (setq respool t)                  ; boolean for later
1437       (setq groups '("fake"))) ; when respooling, groups are dynamic so fake it
1438
1439     ;; exclude invalid move destinations
1440     (dolist (group groups)
1441       (unless
1442           (or
1443            (and
1444             (eq classification 'spam)
1445             (spam-group-spam-contents-p gnus-newsgroup-name)
1446             (spam-group-spam-contents-p group)
1447             (gnus-message
1448              3
1449              "Sorry, can't move spam from spam group %s to spam group %s"
1450              gnus-newsgroup-name
1451              group))
1452            (and
1453             (eq classification 'ham)
1454             (spam-group-ham-contents-p gnus-newsgroup-name)
1455             (spam-group-ham-contents-p group)
1456             (gnus-message
1457              3
1458              "Sorry, can't move ham from ham group %s to ham group %s"
1459              gnus-newsgroup-name
1460              group)))
1461         (push group valid-move-destinations)))
1462
1463     (setq groups (nreverse valid-move-destinations))
1464
1465     ;; now do the actual move
1466     (dolist (group groups)
1467
1468       (when (and articles (stringp group))
1469
1470         ;; first, mark the article with the process mark and, if needed,
1471         ;; the unread or expired mark (for ham and spam respectively)
1472         (dolist (article articles)
1473           (when (and (eq classification 'ham)
1474                      spam-mark-ham-unread-before-move-from-spam-group)
1475             (gnus-message 9 "Marking ham article %d unread before move"
1476                           article)
1477             (gnus-summary-mark-article article gnus-unread-mark))
1478           (when (and (eq classification 'spam)
1479                      (not copy))
1480             (gnus-message 9 "Marking spam article %d expirable before move"
1481                           article)
1482             (gnus-summary-mark-article article gnus-expirable-mark))
1483           (gnus-summary-set-process-mark article)
1484
1485           (if respool              ; respooling is with a "fake" group
1486               (let ((spam-split-disabled
1487                      (or spam-split-disabled
1488                          (and (eq classification 'ham)
1489                               spam-disable-spam-split-during-ham-respool))))
1490                 (gnus-message 9 "Respooling article %d with method %s"
1491                               article respool-method)
1492                 (gnus-summary-respool-article nil respool-method))
1493             ;; else, we are not respooling
1494             (if (or (not backend-supports-deletions)
1495                     (> (length groups) 1))
1496                 (progn              ; if copying, copy and set deletep
1497                   (gnus-message 9 "Copying article %d to group %s"
1498                                 article group)
1499                   (gnus-summary-copy-article nil group)
1500                   (setq deletep t))
1501               (gnus-message 9 "Moving article %d to group %s"
1502                             article group)
1503               (gnus-summary-move-article nil group)))))) ; else move articles
1504
1505     ;; now delete the articles, unless a) copy is t, and there was a copy done
1506     ;;                                 b) a move was done to a single group
1507     ;;                                 c) backend-supports-deletions is nil
1508     (unless copy
1509       (when (and deletep backend-supports-deletions)
1510         (dolist (article articles)
1511           (gnus-summary-set-process-mark article)
1512           (gnus-message 9 "Deleting article %d" article))
1513         (when articles
1514           (let ((gnus-novice-user nil)) ; don't ask me if I'm sure
1515             (gnus-summary-delete-article nil)))))
1516     (gnus-summary-yank-process-mark)
1517     (length articles)))
1518
1519 (defun spam-copy-spam-routine (articles)
1520   (spam-copy-or-move-routine
1521    t
1522    (gnus-parameter-spam-process-destination gnus-newsgroup-name)
1523    articles
1524    'spam))
1525
1526 (defun spam-move-spam-routine (articles)
1527   (spam-copy-or-move-routine
1528    nil
1529    (gnus-parameter-spam-process-destination gnus-newsgroup-name)
1530    articles
1531    'spam))
1532
1533 (defun spam-copy-ham-routine (articles)
1534   (spam-copy-or-move-routine
1535    t
1536    (gnus-parameter-ham-process-destination gnus-newsgroup-name)
1537    articles
1538    'ham))
1539
1540 (defun spam-move-ham-routine (articles)
1541   (spam-copy-or-move-routine
1542    nil
1543    (gnus-parameter-ham-process-destination gnus-newsgroup-name)
1544    articles
1545    'ham))
1546
1547 ;;}}}
1548
1549 ;;{{{ article and field retrieval code
1550 (defun spam-get-article-as-string (article)
1551   (when (numberp article)
1552     (with-temp-buffer
1553       (gnus-request-article-this-buffer
1554        article
1555        gnus-newsgroup-name)
1556       (buffer-string))))
1557
1558 ;; disabled for now
1559 ;; (defun spam-get-article-as-filename (article)
1560 ;;   (let ((article-filename))
1561 ;;     (when (numberp article)
1562 ;;       (nnml-possibly-change-directory
1563 ;;        (gnus-group-real-name gnus-newsgroup-name))
1564 ;;       (setq article-filename (expand-file-name
1565 ;;                              (int-to-string article) nnml-current-directory)))
1566 ;;     (if (file-exists-p article-filename)
1567 ;;      article-filename
1568 ;;       nil)))
1569
1570 (defun spam-fetch-field-fast (article field &optional prepared-data-header)
1571   "Fetch a FIELD for ARTICLE with the internal `gnus-data-list' function.
1572 When PREPARED-DATA-HEADER is given, don't look in the Gnus data.
1573 When FIELD is 'number, ARTICLE can be any number (since we want
1574 to find it out)."
1575   (when (numberp article)
1576     (let* ((data-header (or prepared-data-header
1577                             (spam-fetch-article-header article))))
1578       (cond
1579        ((not (arrayp data-header))
1580         (gnus-message 6 "Article %d has a nil data header" article))
1581        ((equal field 'number)
1582         (mail-header-number data-header))
1583        ((equal field 'from)
1584         (mail-header-from data-header))
1585        ((equal field 'message-id)
1586         (mail-header-message-id data-header))
1587        ((equal field 'subject)
1588         (mail-header-subject data-header))
1589        ((equal field 'references)
1590         (mail-header-references data-header))
1591        ((equal field 'date)
1592         (mail-header-date data-header))
1593        ((equal field 'xref)
1594         (mail-header-xref data-header))
1595        ((equal field 'extra)
1596         (mail-header-extra data-header))
1597        (t
1598         (gnus-error
1599          5
1600          "spam-fetch-field-fast: unknown field %s requested"
1601          field)
1602         nil)))))
1603
1604 (defun spam-fetch-field-from-fast (article &optional prepared-data-header)
1605   (spam-fetch-field-fast article 'from prepared-data-header))
1606
1607 (defun spam-fetch-field-subject-fast (article &optional prepared-data-header)
1608   (spam-fetch-field-fast article 'subject prepared-data-header))
1609
1610 (defun spam-fetch-field-message-id-fast (article &optional prepared-data-header)
1611   (spam-fetch-field-fast article 'message-id prepared-data-header))
1612
1613 (defun spam-generate-fake-headers (article)
1614   (let ((dh (spam-fetch-article-header article)))
1615     (if dh
1616         (concat
1617          (format
1618           ;; 80-character limit makes for strange constructs
1619           (concat "From: %s\nSubject: %s\nMessage-ID: %s\n"
1620                   "Date: %s\nReferences: %s\nXref: %s\n")
1621           (spam-fetch-field-fast article 'from dh)
1622           (spam-fetch-field-fast article 'subject dh)
1623           (spam-fetch-field-fast article 'message-id dh)
1624           (spam-fetch-field-fast article 'date dh)
1625           (spam-fetch-field-fast article 'references dh)
1626           (spam-fetch-field-fast article 'xref dh))
1627          (when (spam-fetch-field-fast article 'extra dh)
1628            (format "%s\n" (spam-fetch-field-fast article 'extra dh))))
1629       (gnus-message
1630        5
1631        "spam-generate-fake-headers: article %d didn't have a valid header"
1632        article))))
1633
1634 (defun spam-fetch-article-header (article)
1635   (with-current-buffer gnus-summary-buffer
1636     (gnus-read-header article)
1637     (nth 3 (assq article gnus-newsgroup-data))))
1638 ;;}}}
1639
1640 ;;{{{ Spam determination.
1641
1642 (defun spam-split (&rest specific-checks)
1643   "Split this message into the `spam' group if it is spam.
1644 This function can be used as an entry in the variable `nnmail-split-fancy',
1645 for example like this: (: spam-split).  It can take checks as
1646 parameters.  A string as a parameter will set the
1647 `spam-split-group' to that string.
1648
1649 See the Info node `(gnus)Fancy Mail Splitting' for more details."
1650   (interactive)
1651   (setq spam-split-last-successful-check nil)
1652   (unless spam-split-disabled
1653     (let ((spam-split-group-choice spam-split-group))
1654       (dolist (check specific-checks)
1655         (when (stringp check)
1656           (setq spam-split-group-choice check)
1657           (setq specific-checks (delq check specific-checks))))
1658
1659       (let ((spam-split-group spam-split-group-choice)
1660             (widening-needed-check (spam-widening-needed-p specific-checks)))
1661         (save-excursion
1662           (save-restriction
1663             (when widening-needed-check
1664               (widen)
1665               (gnus-message 8 "spam-split: widening the buffer (%s requires it)"
1666                             widening-needed-check))
1667             (let ((backends (spam-backend-list))
1668                   decision)
1669               (while (and backends (not decision))
1670                 (let* ((backend (pop backends))
1671                        (check-function (spam-backend-check backend))
1672                        (spam-split-group (if spam-split-symbolic-return
1673                                              'spam
1674                                            spam-split-group)))
1675                   (when (or
1676                          ;; either, given specific checks, this is one of them
1677                          (memq backend specific-checks)
1678                          ;; or, given no specific checks, spam-use-CHECK is set
1679                          (and (null specific-checks) (symbol-value backend)))
1680                     (gnus-message 6 "spam-split: calling the %s function"
1681                                   check-function)
1682                     (setq decision (funcall check-function))
1683                     ;; if we got a decision at all, save the current check
1684                     (when decision
1685                       (setq spam-split-last-successful-check backend))
1686
1687                     (when (eq decision 'spam)
1688                       (unless spam-split-symbolic-return
1689                         (gnus-error
1690                          5
1691                          (format "spam-split got %s but %s is nil"
1692                                  decision
1693                                  spam-split-symbolic-return)))))))
1694               (if (eq decision t)
1695                   (if spam-split-symbolic-return-positive 'ham nil)
1696                 decision))))))))
1697
1698 (defun spam-find-spam ()
1699   "Detect spam in the current newsgroup using `spam-split'."
1700   (interactive)
1701
1702   (let* ((group gnus-newsgroup-name)
1703          (autodetect (gnus-parameter-spam-autodetect group))
1704          (methods (gnus-parameter-spam-autodetect-methods group))
1705          (first-method (nth 0 methods))
1706          (articles (if spam-autodetect-recheck-messages
1707                        gnus-newsgroup-articles
1708                      gnus-newsgroup-unseen))
1709          article-cannot-be-faked)
1710
1711
1712     (dolist (backend methods)
1713       (when (spam-backend-statistical-p backend)
1714         (setq article-cannot-be-faked t)
1715         (return)))
1716
1717     (when (memq 'default methods)
1718       (setq article-cannot-be-faked t))
1719
1720     (when (and autodetect
1721                (not (equal first-method 'none)))
1722       (mapcar
1723        (lambda (article)
1724          (let ((id (spam-fetch-field-message-id-fast article))
1725                (subject (spam-fetch-field-subject-fast article))
1726                (sender (spam-fetch-field-from-fast article))
1727                registry-lookup)
1728
1729            (unless id
1730              (gnus-message 6 "Article %d has no message ID!" article))
1731
1732            (when (and id spam-log-to-registry)
1733              (setq registry-lookup (spam-log-registration-type id 'incoming))
1734              (when registry-lookup
1735                (gnus-message
1736                 9
1737                 "spam-find-spam: message %s was already registered incoming"
1738                 id)))
1739
1740            (let* ((spam-split-symbolic-return t)
1741                   (spam-split-symbolic-return-positive t)
1742                   (fake-headers (spam-generate-fake-headers article))
1743                   (split-return
1744                    (or registry-lookup
1745                        (with-temp-buffer
1746                          (if article-cannot-be-faked
1747                              (gnus-request-article-this-buffer
1748                               article
1749                               group)
1750                            ;; else, we fake the article
1751                            (when fake-headers (insert fake-headers)))
1752                          (if (or (null first-method)
1753                                  (equal first-method 'default))
1754                              (spam-split)
1755                            (apply 'spam-split methods))))))
1756              (if (equal split-return 'spam)
1757                  (gnus-summary-mark-article article gnus-spam-mark))
1758
1759              (when (and id split-return spam-log-to-registry)
1760                (when (zerop (gnus-registry-group-count id))
1761                  (gnus-registry-handle-action id nil group subject sender))
1762
1763                (unless registry-lookup
1764                  (spam-log-processing-to-registry
1765                   id
1766                   'incoming
1767                   split-return
1768                   spam-split-last-successful-check
1769                   group))))))
1770        articles))))
1771
1772 ;;}}}
1773
1774 ;;{{{ registration/unregistration functions
1775
1776 (defun spam-resolve-registrations-routine ()
1777   "Go through the backends and register or unregister articles as needed."
1778   (dolist (backend-type '(non-mover mover))
1779     (dolist (classification (spam-classifications))
1780       (dolist (backend (spam-backend-list backend-type))
1781         (let ((rlist (spam-backend-get-article-todo-list
1782                       backend classification))
1783               (ulist (spam-backend-get-article-todo-list
1784                       backend classification t))
1785               (delcount 0))
1786
1787           ;; clear the old lists right away
1788           (spam-backend-put-article-todo-list backend
1789                                               classification
1790                                               nil
1791                                               nil)
1792           (spam-backend-put-article-todo-list backend
1793                                               classification
1794                                               nil
1795                                               t)
1796
1797           ;; eliminate duplicates
1798           (dolist (article (copy-sequence ulist))
1799             (when (memq article rlist)
1800               (incf delcount)
1801               (setq rlist (delq article rlist))
1802               (setq ulist (delq article ulist))))
1803
1804           (unless (zerop delcount)
1805             (gnus-message
1806              9
1807              "%d messages did not have to unregister and then register"
1808              delcount))
1809
1810           ;; unregister articles
1811           (unless (zerop (length ulist))
1812             (let ((num (spam-unregister-routine classification backend ulist)))
1813               (when (> num 0)
1814                 (gnus-message
1815                  6
1816                  "%d %s messages were unregistered by backend %s."
1817                  num
1818                  classification
1819                  backend))))
1820
1821             ;; register articles
1822             (unless (zerop (length rlist))
1823               (let ((num (spam-register-routine classification backend rlist)))
1824                 (when (> num 0)
1825                   (gnus-message
1826                    6
1827                    "%d %s messages were registered by backend %s."
1828                    num
1829                    classification
1830                    backend)))))))))
1831
1832 (defun spam-unregister-routine (classification
1833                                 backend
1834                                 specific-articles)
1835   (spam-register-routine classification backend specific-articles t))
1836
1837 (defun spam-register-routine (classification
1838                               backend
1839                               specific-articles
1840                               &optional unregister)
1841   (when (and (spam-classification-valid-p classification)
1842              (spam-backend-valid-p backend))
1843     (let* ((register-function
1844             (spam-backend-function backend classification 'registration))
1845            (unregister-function
1846             (spam-backend-function backend classification 'unregistration))
1847            (run-function (if unregister
1848                              unregister-function
1849                            register-function))
1850            (log-function (if unregister
1851                              'spam-log-undo-registration
1852                            'spam-log-processing-to-registry))
1853            article articles)
1854
1855       (when run-function
1856         ;; make list of articles, using specific-articles if given
1857         (setq articles (or specific-articles
1858                            (spam-list-articles
1859                             gnus-newsgroup-articles
1860                             classification)))
1861         ;; process them
1862         (when (> (length articles) 0)
1863           (gnus-message 5 "%s %d %s articles as %s using backend %s"
1864                         (if unregister "Unregistering" "Registering")
1865                         (length articles)
1866                         (if specific-articles "specific" "")
1867                         classification
1868                         backend)
1869           (funcall run-function articles)
1870           ;; now log all the registrations (or undo them, depending on
1871           ;; unregister)
1872           (dolist (article articles)
1873             (funcall log-function
1874                      (spam-fetch-field-message-id-fast article)
1875                      'process
1876                      classification
1877                      backend
1878                      gnus-newsgroup-name))))
1879       ;; return the number of articles processed
1880       (length articles))))
1881
1882 ;;; log a ham- or spam-processor invocation to the registry
1883 (defun spam-log-processing-to-registry (id type classification backend group)
1884   (when spam-log-to-registry
1885     (if (and (stringp id)
1886              (stringp group)
1887              (spam-process-type-valid-p type)
1888              (spam-classification-valid-p classification)
1889              (spam-backend-valid-p backend))
1890         (let ((cell-list (gnus-registry-get-id-key id type))
1891               (cell (list classification backend group)))
1892           (push cell cell-list)
1893           (gnus-registry-set-id-key id type cell-list))
1894
1895       (gnus-error
1896        7
1897        (format
1898         "%s call with bad ID, type, classification, spam-backend, or group"
1899         "spam-log-processing-to-registry")))))
1900
1901 ;;; check if a ham- or spam-processor registration has been done
1902 (defun spam-log-registered-p (id type)
1903   (when spam-log-to-registry
1904     (if (and (stringp id)
1905              (spam-process-type-valid-p type))
1906         (gnus-registry-get-id-key id type)
1907       (progn
1908         (gnus-error
1909          7
1910          (format "%s called with bad ID, type, classification, or spam-backend"
1911                  "spam-log-registered-p"))
1912         nil))))
1913
1914 ;;; check what a ham- or spam-processor registration says
1915 ;;; returns nil if conflicting registrations are found
1916 (defun spam-log-registration-type (id type)
1917   (let ((count 0)
1918         decision)
1919     (dolist (reg (spam-log-registered-p id type))
1920       (let ((classification (nth 0 reg)))
1921         (when (spam-classification-valid-p classification)
1922           (when (and decision
1923                      (not (eq classification decision)))
1924             (setq count (+ 1 count)))
1925           (setq decision classification))))
1926     (if (< 0 count)
1927         nil
1928       decision)))
1929
1930
1931 ;;; check if a ham- or spam-processor registration needs to be undone
1932 (defun spam-log-unregistration-needed-p (id type classification backend)
1933   (when spam-log-to-registry
1934     (if (and (stringp id)
1935              (spam-process-type-valid-p type)
1936              (spam-classification-valid-p classification)
1937              (spam-backend-valid-p backend))
1938         (let ((cell-list (gnus-registry-get-id-key id type))
1939               found)
1940           (dolist (cell cell-list)
1941             (unless found
1942               (when (and (eq classification (nth 0 cell))
1943                          (eq backend (nth 1 cell)))
1944                 (setq found t))))
1945           found)
1946       (progn
1947         (gnus-error
1948          7
1949          (format "%s called with bad ID, type, classification, or spam-backend"
1950                  "spam-log-unregistration-needed-p"))
1951         nil))))
1952
1953
1954 ;;; undo a ham- or spam-processor registration (the group is not used)
1955 (defun spam-log-undo-registration (id type classification backend
1956                                       &optional group)
1957   (when (and spam-log-to-registry
1958              (spam-log-unregistration-needed-p id type classification backend))
1959     (if (and (stringp id)
1960              (spam-process-type-valid-p type)
1961              (spam-classification-valid-p classification)
1962              (spam-backend-valid-p backend))
1963         (let ((cell-list (gnus-registry-get-id-key id type))
1964               new-cell-list found)
1965           (dolist (cell cell-list)
1966             (unless (and (eq classification (nth 0 cell))
1967                          (eq backend (nth 1 cell)))
1968               (push cell new-cell-list)))
1969           (gnus-registry-set-id-key id type new-cell-list))
1970       (progn
1971         (gnus-error 7 (format
1972                        "%s call with bad ID, type, spam-backend, or group"
1973                        "spam-log-undo-registration"))
1974         nil))))
1975
1976 ;;}}}
1977
1978 ;;{{{ backend functions
1979
1980 ;;{{{ Gmane xrefs
1981 (defun spam-check-gmane-xref ()
1982   (let ((header (or
1983                  (message-fetch-field "Xref")
1984                  (message-fetch-field "Newsgroups"))))
1985     (when header                        ; return nil when no header
1986       (when (string-match spam-gmane-xref-spam-group
1987                           header)
1988           spam-split-group))))
1989
1990 ;;}}}
1991
1992 ;;{{{ Regex body
1993
1994 (defun spam-check-regex-body ()
1995   (let ((spam-regex-headers-ham spam-regex-body-ham)
1996         (spam-regex-headers-spam spam-regex-body-spam))
1997     (spam-check-regex-headers t)))
1998
1999 ;;}}}
2000
2001 ;;{{{ Regex headers
2002
2003 (defun spam-check-regex-headers (&optional body)
2004   (let ((type (if body "body" "header"))
2005         ret found)
2006     (dolist (h-regex spam-regex-headers-ham)
2007       (unless found
2008         (goto-char (point-min))
2009         (when (re-search-forward h-regex nil t)
2010           (message "Ham regex %s search positive." type)
2011           (setq found t))))
2012     (dolist (s-regex spam-regex-headers-spam)
2013       (unless found
2014         (goto-char (point-min))
2015         (when (re-search-forward s-regex nil t)
2016           (message "Spam regex %s search positive." type)
2017           (setq found t)
2018           (setq ret spam-split-group))))
2019     ret))
2020
2021 ;;}}}
2022
2023 ;;{{{ Blackholes.
2024
2025 (defun spam-reverse-ip-string (ip)
2026   (when (stringp ip)
2027     (mapconcat 'identity
2028                (nreverse (split-string ip "\\."))
2029                ".")))
2030
2031 (defun spam-check-blackholes ()
2032   "Check the Received headers for blackholed relays."
2033   (let ((headers (message-fetch-field "received"))
2034         ips matches)
2035     (when headers
2036       (with-temp-buffer
2037         (insert headers)
2038         (goto-char (point-min))
2039         (gnus-message 6 "Checking headers for relay addresses")
2040         (while (re-search-forward
2041                 "\\([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\\)" nil t)
2042           (gnus-message 9 "Blackhole search found host IP %s." (match-string 1))
2043           (push (spam-reverse-ip-string (match-string 1))
2044                 ips)))
2045       (dolist (server spam-blackhole-servers)
2046         (dolist (ip ips)
2047           (unless (and spam-blackhole-good-server-regex
2048                        ;; match against the reversed (again) IP string
2049                        (string-match
2050                         spam-blackhole-good-server-regex
2051                         (spam-reverse-ip-string ip)))
2052             (unless matches
2053               (let ((query-string (concat ip "." server)))
2054                 (if spam-use-dig
2055                     (let ((query-result (query-dig query-string)))
2056                       (when query-result
2057                         (gnus-message 6 "(DIG): positive blackhole check '%s'"
2058                                       query-result)
2059                         (push (list ip server query-result)
2060                               matches)))
2061                   ;; else, if not using dig.el
2062                   (when (dns-query query-string)
2063                     (gnus-message 6 "positive blackhole check")
2064                     (push (list ip server (dns-query query-string 'TXT))
2065                           matches)))))))))
2066     (when matches
2067       spam-split-group)))
2068 ;;}}}
2069
2070 ;;{{{ Hashcash.
2071
2072 (defun spam-check-hashcash ()
2073   "Check the headers for hashcash payments."
2074   (ignore-errors (mail-check-payment)))  ;mail-check-payment returns a boolean
2075
2076 ;;}}}
2077
2078 ;;{{{ BBDB
2079
2080 ;;; original idea for spam-check-BBDB from Alexander Kotelnikov
2081 ;;; <sacha@giotto.sj.ru>
2082
2083 ;; all this is done inside a condition-case to trap errors
2084
2085 ;; Autoloaded in message, which we require.
2086 (declare-function gnus-extract-address-components "gnus-util" (from))
2087
2088 (eval-and-compile
2089   (condition-case nil
2090       (progn
2091         (require 'bbdb)
2092         (require 'bbdb-com))
2093     (file-error
2094      ;; `bbdb-records' should not be bound as an autoload function
2095      ;; before loading bbdb because of `bbdb-hashtable-size'.
2096      (defalias 'bbdb-buffer 'ignore)
2097      (defalias 'bbdb-create-internal 'ignore)
2098      (defalias 'bbdb-records 'ignore)
2099      (defalias 'spam-BBDB-register-routine 'ignore)
2100      (defalias 'spam-enter-ham-BBDB 'ignore)
2101      (defalias 'spam-exists-in-BBDB-p 'ignore)
2102      (defalias 'bbdb-gethash 'ignore)
2103      nil)))
2104
2105 (eval-and-compile
2106   (when (featurep 'bbdb-com)
2107     ;; when the BBDB changes, we want to clear out our cache
2108     (defun spam-clear-cache-BBDB (&rest immaterial)
2109       (spam-clear-cache 'spam-use-BBDB))
2110
2111     (add-hook 'bbdb-change-hook 'spam-clear-cache-BBDB)
2112
2113     (defun spam-enter-ham-BBDB (addresses &optional remove)
2114       "Enter an address into the BBDB; implies ham (non-spam) sender"
2115       (dolist (from addresses)
2116         (when (stringp from)
2117           (let* ((parsed-address (gnus-extract-address-components from))
2118                  (name (or (nth 0 parsed-address) "Ham Sender"))
2119                  (remove-function (if remove
2120                                       'bbdb-delete-record-internal
2121                                     'ignore))
2122                  (net-address (nth 1 parsed-address))
2123                  (record (and net-address
2124                               (spam-exists-in-BBDB-p net-address))))
2125             (when net-address
2126               (gnus-message 6 "%s address %s %s BBDB"
2127                             (if remove "Deleting" "Adding")
2128                             from
2129                             (if remove "from" "to"))
2130               (if record
2131                   (funcall remove-function record)
2132                 (bbdb-create-internal name nil net-address nil nil
2133                                       "ham sender added by spam.el")))))))
2134
2135     (defun spam-BBDB-register-routine (articles &optional unregister)
2136       (let (addresses)
2137         (dolist (article articles)
2138           (when (stringp (spam-fetch-field-from-fast article))
2139             (push (spam-fetch-field-from-fast article) addresses)))
2140         ;; now do the register/unregister action
2141         (spam-enter-ham-BBDB addresses unregister)))
2142
2143     (defun spam-BBDB-unregister-routine (articles)
2144       (spam-BBDB-register-routine articles t))
2145
2146     (defsubst spam-exists-in-BBDB-p (net)
2147       (when (and (stringp net) (not (zerop (length net))))
2148         (bbdb-records)
2149         (bbdb-gethash (downcase net))))
2150
2151     (defun spam-check-BBDB ()
2152       "Mail from people in the BBDB is classified as ham or non-spam"
2153       (let ((net (message-fetch-field "from")))
2154         (when net
2155           (setq net (nth 1 (gnus-extract-address-components net)))
2156           (if (spam-exists-in-BBDB-p net)
2157               t
2158             (if spam-use-BBDB-exclusive
2159                 spam-split-group
2160               nil)))))))
2161
2162 ;;}}}
2163
2164 ;;{{{ ifile
2165
2166 ;;; check the ifile backend; return nil if the mail was NOT classified
2167 ;;; as spam
2168
2169
2170 (defun spam-get-ifile-database-parameter ()
2171   "Return the command-line parameter for ifile's database.
2172 See `spam-ifile-database'."
2173   (if spam-ifile-database
2174       (format "--db-file=%s" spam-ifile-database)
2175     nil))
2176
2177 (defun spam-check-ifile ()
2178   "Check the ifile backend for the classification of this message."
2179   (let ((article-buffer-name (buffer-name))
2180         category return)
2181     (with-temp-buffer
2182       (let ((temp-buffer-name (buffer-name))
2183             (db-param (spam-get-ifile-database-parameter)))
2184         (with-current-buffer article-buffer-name
2185           (apply 'call-process-region
2186                  (point-min) (point-max) spam-ifile-program
2187                  nil temp-buffer-name nil "-c"
2188                  (if db-param `(,db-param "-q") `("-q"))))
2189         ;; check the return now (we're back in the temp buffer)
2190         (goto-char (point-min))
2191         (if (not (eobp))
2192             (setq category (buffer-substring (point) (point-at-eol))))
2193         (when (not (zerop (length category))) ; we need a category here
2194           (if spam-ifile-all-categories
2195               (setq return category)
2196             ;; else, if spam-ifile-all-categories is not set...
2197             (when (string-equal spam-ifile-spam-category category)
2198               (setq return spam-split-group)))))) ; note return is nil otherwise
2199     return))
2200
2201 (defun spam-ifile-register-with-ifile (articles category &optional unregister)
2202   "Register an article, given as a string, with a category.
2203 Uses `gnus-newsgroup-name' if category is nil (for ham registration)."
2204   (let ((category (or category gnus-newsgroup-name))
2205         (add-or-delete-option (if unregister "-d" "-i"))
2206         (db (spam-get-ifile-database-parameter))
2207         parameters)
2208     (with-temp-buffer
2209       (dolist (article articles)
2210         (let ((article-string (spam-get-article-as-string article)))
2211           (when (stringp article-string)
2212             (insert article-string))))
2213       (apply 'call-process-region
2214              (point-min) (point-max) spam-ifile-program
2215              nil nil nil
2216              add-or-delete-option category
2217              (if db `(,db "-h") `("-h"))))))
2218
2219 (defun spam-ifile-register-spam-routine (articles &optional unregister)
2220   (spam-ifile-register-with-ifile articles spam-ifile-spam-category unregister))
2221
2222 (defun spam-ifile-unregister-spam-routine (articles)
2223   (spam-ifile-register-spam-routine articles t))
2224
2225 (defun spam-ifile-register-ham-routine (articles &optional unregister)
2226   (spam-ifile-register-with-ifile articles spam-ifile-ham-category unregister))
2227
2228 (defun spam-ifile-unregister-ham-routine (articles)
2229   (spam-ifile-register-ham-routine articles t))
2230
2231 ;;}}}
2232
2233 ;;{{{ spam-stat
2234
2235 (eval-when-compile
2236   (autoload 'spam-stat-buffer-change-to-non-spam "spam-stat")
2237   (autoload 'spam-stat-buffer-change-to-spam "spam-stat")
2238   (autoload 'spam-stat-buffer-is-non-spam "spam-stat")
2239   (autoload 'spam-stat-buffer-is-spam "spam-stat")
2240   (autoload 'spam-stat-load "spam-stat")
2241   (autoload 'spam-stat-save "spam-stat")
2242   (autoload 'spam-stat-split-fancy "spam-stat"))
2243
2244 (require 'spam-stat)
2245
2246 (defun spam-check-stat ()
2247   "Check the spam-stat backend for the classification of this message"
2248   (let ((spam-stat-split-fancy-spam-group spam-split-group) ; override
2249         (spam-stat-buffer (buffer-name)) ; stat the current buffer
2250         category return)
2251     (spam-stat-split-fancy)))
2252
2253 (defun spam-stat-register-spam-routine (articles &optional unregister)
2254   (dolist (article articles)
2255     (let ((article-string (spam-get-article-as-string article)))
2256       (with-temp-buffer
2257         (insert article-string)
2258         (if unregister
2259             (spam-stat-buffer-change-to-non-spam)
2260           (spam-stat-buffer-is-spam))))))
2261
2262 (defun spam-stat-unregister-spam-routine (articles)
2263   (spam-stat-register-spam-routine articles t))
2264
2265 (defun spam-stat-register-ham-routine (articles &optional unregister)
2266   (dolist (article articles)
2267     (let ((article-string (spam-get-article-as-string article)))
2268       (with-temp-buffer
2269         (insert article-string)
2270         (if unregister
2271             (spam-stat-buffer-change-to-spam)
2272           (spam-stat-buffer-is-non-spam))))))
2273
2274 (defun spam-stat-unregister-ham-routine (articles)
2275   (spam-stat-register-ham-routine articles t))
2276
2277 (defun spam-maybe-spam-stat-load ()
2278   (when spam-use-stat (spam-stat-load)))
2279
2280 (defun spam-maybe-spam-stat-save ()
2281   (when spam-use-stat (spam-stat-save)))
2282
2283 ;;}}}
2284
2285 ;;{{{ Blacklists and whitelists.
2286
2287 (defvar spam-whitelist-cache nil)
2288 (defvar spam-blacklist-cache nil)
2289
2290 (defun spam-kill-whole-line ()
2291   (beginning-of-line)
2292   (let ((kill-whole-line t))
2293     (kill-line)))
2294
2295 ;;; address can be a list, too
2296 (defun spam-enter-whitelist (address &optional remove)
2297   "Enter ADDRESS (list or single) into the whitelist.
2298 With a non-nil REMOVE, remove them."
2299   (interactive "sAddress: ")
2300   (spam-enter-list address spam-whitelist remove)
2301   (setq spam-whitelist-cache nil)
2302   (spam-clear-cache 'spam-use-whitelist))
2303
2304 ;;; address can be a list, too
2305 (defun spam-enter-blacklist (address &optional remove)
2306   "Enter ADDRESS (list or single) into the blacklist.
2307 With a non-nil REMOVE, remove them."
2308   (interactive "sAddress: ")
2309   (spam-enter-list address spam-blacklist remove)
2310   (setq spam-blacklist-cache nil)
2311   (spam-clear-cache 'spam-use-whitelist))
2312
2313 (defun spam-enter-list (addresses file &optional remove)
2314   "Enter ADDRESSES into the given FILE.
2315 Either the whitelist or the blacklist files can be used.
2316 With a non-nil REMOVE, remove the ADDRESSES."
2317   (if (stringp addresses)
2318       (spam-enter-list (list addresses) file remove)
2319     ;; else, we have a list of addresses here
2320     (unless (file-exists-p (file-name-directory file))
2321       (make-directory (file-name-directory file) t))
2322     (with-current-buffer
2323        (find-file-noselect file)
2324       (dolist (a addresses)
2325         (when (stringp a)
2326           (goto-char (point-min))
2327           (if (re-search-forward (regexp-quote a) nil t)
2328               ;; found the address
2329               (when remove
2330                 (spam-kill-whole-line))
2331             ;; else, the address was not found
2332             (unless remove
2333               (goto-char (point-max))
2334               (unless (bobp)
2335                 (insert "\n"))
2336               (insert a "\n")))))
2337       (save-buffer))))
2338
2339 (defun spam-filelist-build-cache (type)
2340   (let ((cache (if (eq type 'spam-use-blacklist)
2341                    spam-blacklist-cache
2342                  spam-whitelist-cache))
2343         parsed-cache)
2344     (unless (gethash type spam-caches)
2345       (while cache
2346         (let ((address (pop cache)))
2347           (unless (zerop (length address)) ; 0 for a nil address too
2348             (setq address (regexp-quote address))
2349             ;; fix regexp-quote's treatment of user-intended regexes
2350             (while (string-match "\\\\\\*" address)
2351               (setq address (replace-match ".*" t t address))))
2352           (push address parsed-cache)))
2353       (puthash type parsed-cache spam-caches))))
2354
2355 (defun spam-filelist-check-cache (type from)
2356   (when (stringp from)
2357     (spam-filelist-build-cache type)
2358     (let (found)
2359       (dolist (address (gethash type spam-caches))
2360         (when (and address (string-match address from))
2361           (setq found t)
2362           (return)))
2363       found)))
2364
2365 ;;; returns t if the sender is in the whitelist, nil or
2366 ;;; spam-split-group otherwise
2367 (defun spam-check-whitelist ()
2368   ;; FIXME!  Should it detect when file timestamps change?
2369   (unless spam-whitelist-cache
2370     (setq spam-whitelist-cache (spam-parse-list spam-whitelist)))
2371   (if (spam-from-listed-p 'spam-use-whitelist)
2372       t
2373     (if spam-use-whitelist-exclusive
2374         spam-split-group
2375       nil)))
2376
2377 (defun spam-check-blacklist ()
2378   ;; FIXME!  Should it detect when file timestamps change?
2379   (unless spam-blacklist-cache
2380     (setq spam-blacklist-cache (spam-parse-list spam-blacklist)))
2381   (and (spam-from-listed-p 'spam-use-blacklist)
2382        spam-split-group))
2383
2384 (defun spam-parse-list (file)
2385   (when (file-readable-p file)
2386     (let (contents address)
2387       (with-temp-buffer
2388         (insert-file-contents file)
2389         (while (not (eobp))
2390           (setq address (buffer-substring (point) (point-at-eol)))
2391           (forward-line 1)
2392           ;; insert the e-mail address if detected, otherwise the raw data
2393           (unless (zerop (length address))
2394             (let ((pure-address
2395                    (nth 1 (gnus-extract-address-components address))))
2396               (push (or pure-address address) contents)))))
2397       (nreverse contents))))
2398
2399 (defun spam-from-listed-p (type)
2400   (let ((from (message-fetch-field "from"))
2401         found)
2402     (spam-filelist-check-cache type from)))
2403
2404 (defun spam-filelist-register-routine (articles blacklist &optional unregister)
2405   (let ((de-symbol (if blacklist 'spam-use-whitelist 'spam-use-blacklist))
2406         (declassification (if blacklist 'ham 'spam))
2407         (enter-function
2408          (if blacklist 'spam-enter-blacklist 'spam-enter-whitelist))
2409         (remove-function
2410          (if blacklist 'spam-enter-whitelist 'spam-enter-blacklist))
2411         from addresses unregister-list article-unregister-list)
2412     (dolist (article articles)
2413       (let ((from (spam-fetch-field-from-fast article))
2414             (id (spam-fetch-field-message-id-fast article))
2415             sender-ignored)
2416         (when (stringp from)
2417           (dolist (ignore-regex spam-blacklist-ignored-regexes)
2418             (when (and (not sender-ignored)
2419                        (stringp ignore-regex)
2420                        (string-match ignore-regex from))
2421               (setq sender-ignored t)))
2422           ;; remember the messages we need to unregister, unless remove is set
2423           (when (and
2424                  (null unregister)
2425                  (spam-log-unregistration-needed-p
2426                   id 'process declassification de-symbol))
2427             (push article article-unregister-list)
2428             (push from unregister-list))
2429           (unless sender-ignored
2430             (push from addresses)))))
2431
2432     (if unregister
2433         (funcall enter-function addresses t) ; unregister all these addresses
2434       ;; else, register normally and unregister what we need to
2435       (funcall remove-function unregister-list t)
2436       (dolist (article article-unregister-list)
2437         (spam-log-undo-registration
2438          (spam-fetch-field-message-id-fast article)
2439          'process
2440          declassification
2441          de-symbol))
2442       (funcall enter-function addresses nil))))
2443
2444 (defun spam-blacklist-unregister-routine (articles)
2445   (spam-blacklist-register-routine articles t))
2446
2447 (defun spam-blacklist-register-routine (articles &optional unregister)
2448   (spam-filelist-register-routine articles t unregister))
2449
2450 (defun spam-whitelist-unregister-routine (articles)
2451   (spam-whitelist-register-routine articles t))
2452
2453 (defun spam-whitelist-register-routine (articles &optional unregister)
2454   (spam-filelist-register-routine articles nil unregister))
2455
2456 ;;}}}
2457
2458 ;;{{{ Spam-report glue (gmane and resend reporting)
2459 (defun spam-report-gmane-register-routine (articles)
2460   (when articles
2461     (apply 'spam-report-gmane-spam articles)))
2462
2463 (defun spam-report-gmane-unregister-routine (articles)
2464   (when articles
2465     (apply 'spam-report-gmane-ham articles)))
2466
2467 (defun spam-report-resend-register-ham-routine (articles)
2468   (spam-report-resend-register-routine articles t))
2469
2470 (defvar spam-report-resend-to)
2471
2472 (defun spam-report-resend-register-routine (articles &optional ham)
2473   (require 'spam-report)
2474   (let* ((resend-to-gp
2475           (if ham
2476               (gnus-parameter-ham-resend-to gnus-newsgroup-name)
2477             (gnus-parameter-spam-resend-to gnus-newsgroup-name)))
2478          (spam-report-resend-to (or (car-safe resend-to-gp)
2479                                     spam-report-resend-to)))
2480     (spam-report-resend articles ham)))
2481
2482 ;;}}}
2483
2484 ;;{{{ Bogofilter
2485 (defun spam-check-bogofilter-headers (&optional score)
2486   (let ((header (message-fetch-field spam-bogofilter-header)))
2487     (when header                        ; return nil when no header
2488       (if score                         ; scoring mode
2489           (if (string-match "spamicity=\\([0-9.]+\\)" header)
2490               (match-string 1 header)
2491             "0")
2492         ;; spam detection mode
2493         (when (string-match spam-bogofilter-bogosity-positive-spam-header
2494                             header)
2495           spam-split-group)))))
2496
2497 ;; return something sensible if the score can't be determined
2498 (defun spam-bogofilter-score (&optional recheck)
2499   "Get the Bogofilter spamicity score."
2500   (interactive "P")
2501   (save-window-excursion
2502     (gnus-summary-show-article t)
2503     (set-buffer gnus-article-buffer)
2504     (let ((score (or (unless recheck
2505                        (spam-check-bogofilter-headers t))
2506                      (spam-check-bogofilter t))))
2507       (gnus-summary-show-article)
2508       (message "Spamicity score %s" score)
2509       (or score "0"))))
2510
2511 (defun spam-verify-bogofilter ()
2512   "Verify the Bogofilter version is sufficient."
2513   (when (eq spam-bogofilter-valid 'unknown)
2514     (setq spam-bogofilter-valid
2515           (not (string-match "^bogofilter version 0\\.\\([0-9]\\|1[01]\\)\\."
2516                              (shell-command-to-string
2517                               (format "%s -V" spam-bogofilter-program))))))
2518   spam-bogofilter-valid)
2519
2520 (defun spam-check-bogofilter (&optional score)
2521   "Check the Bogofilter backend for the classification of this message."
2522   (if (spam-verify-bogofilter)
2523       (let ((article-buffer-name (buffer-name))
2524             (db spam-bogofilter-database-directory)
2525             return)
2526         (with-temp-buffer
2527           (let ((temp-buffer-name (buffer-name)))
2528             (with-current-buffer article-buffer-name
2529               (apply 'call-process-region
2530                      (point-min) (point-max)
2531                      spam-bogofilter-program
2532                      nil temp-buffer-name nil
2533                      (if db `("-d" ,db "-v") `("-v"))))
2534             (setq return (spam-check-bogofilter-headers score))))
2535         return)
2536     (gnus-error 5 "`spam.el' doesn't support obsolete bogofilter versions")))
2537
2538 (defun spam-bogofilter-register-with-bogofilter (articles
2539                                                  spam
2540                                                  &optional unregister)
2541   "Register an article, given as a string, as spam or non-spam."
2542   (if (spam-verify-bogofilter)
2543       (dolist (article articles)
2544         (let ((article-string (spam-get-article-as-string article))
2545               (db spam-bogofilter-database-directory)
2546               (switch (if unregister
2547                           (if spam
2548                               spam-bogofilter-spam-strong-switch
2549                             spam-bogofilter-ham-strong-switch)
2550                         (if spam
2551                             spam-bogofilter-spam-switch
2552                           spam-bogofilter-ham-switch))))
2553           (when (stringp article-string)
2554             (with-temp-buffer
2555               (insert article-string)
2556
2557               (apply 'call-process-region
2558                      (point-min) (point-max)
2559                      spam-bogofilter-program
2560                      nil nil nil switch
2561                      (if db `("-d" ,db "-v") `("-v")))))))
2562     (gnus-error 5 "`spam.el' doesn't support obsolete bogofilter versions")))
2563
2564 (defun spam-bogofilter-register-spam-routine (articles &optional unregister)
2565   (spam-bogofilter-register-with-bogofilter articles t unregister))
2566
2567 (defun spam-bogofilter-unregister-spam-routine (articles)
2568   (spam-bogofilter-register-spam-routine articles t))
2569
2570 (defun spam-bogofilter-register-ham-routine (articles &optional unregister)
2571   (spam-bogofilter-register-with-bogofilter articles nil unregister))
2572
2573 (defun spam-bogofilter-unregister-ham-routine (articles)
2574   (spam-bogofilter-register-ham-routine articles t))
2575
2576
2577 ;;}}}
2578
2579 ;;{{{ spamoracle
2580 (defun spam-check-spamoracle ()
2581   "Run spamoracle on an article to determine whether it's spam."
2582   (let ((article-buffer-name (buffer-name)))
2583     (with-temp-buffer
2584       (let ((temp-buffer-name (buffer-name)))
2585         (with-current-buffer article-buffer-name
2586           (let ((status
2587                  (apply 'call-process-region
2588                         (point-min) (point-max)
2589                         spam-spamoracle-binary
2590                         nil temp-buffer-name nil
2591                         (if spam-spamoracle-database
2592                             `("-f" ,spam-spamoracle-database "mark")
2593                           '("mark")))))
2594             (if (eq 0 status)
2595                 (progn
2596                   (set-buffer temp-buffer-name)
2597                   (goto-char (point-min))
2598                   (when (re-search-forward "^X-Spam: yes;" nil t)
2599                     spam-split-group))
2600               (error "Error running spamoracle: %s" status))))))))
2601
2602 (defun spam-spamoracle-learn (articles article-is-spam-p &optional unregister)
2603   "Run spamoracle in training mode."
2604   (with-temp-buffer
2605     (let ((temp-buffer-name (buffer-name)))
2606       (save-excursion
2607         (goto-char (point-min))
2608         (dolist (article articles)
2609           (insert (spam-get-article-as-string article)))
2610         (let* ((arg (if (spam-xor unregister article-is-spam-p)
2611                         "-spam"
2612                       "-good"))
2613                (status
2614                 (apply 'call-process-region
2615                        (point-min) (point-max)
2616                        spam-spamoracle-binary
2617                        nil temp-buffer-name nil
2618                        (if spam-spamoracle-database
2619                            `("-f" ,spam-spamoracle-database
2620                              "add" ,arg)
2621                          `("add" ,arg)))))
2622           (unless (eq 0 status)
2623             (error "Error running spamoracle: %s" status)))))))
2624
2625 (defun spam-spamoracle-learn-ham (articles &optional unregister)
2626   (spam-spamoracle-learn articles nil unregister))
2627
2628 (defun spam-spamoracle-unlearn-ham (articles &optional unregister)
2629   (spam-spamoracle-learn-ham articles t))
2630
2631 (defun spam-spamoracle-learn-spam (articles &optional unregister)
2632   (spam-spamoracle-learn articles t unregister))
2633
2634 (defun spam-spamoracle-unlearn-spam (articles &optional unregister)
2635   (spam-spamoracle-learn-spam articles t))
2636
2637 ;;}}}
2638
2639 ;;{{{ SpamAssassin
2640 ;;; based mostly on the bogofilter code
2641 (defun spam-check-spamassassin-headers (&optional score)
2642   "Check the SpamAssassin headers for the classification of this message."
2643   (if score                             ; scoring mode
2644       (let ((header (message-fetch-field spam-spamassassin-spam-status-header)))
2645         (when header
2646           (if (string-match spam-spamassassin-score-regexp header)
2647               (match-string 1 header)
2648             "0")))
2649     ;; spam detection mode
2650     (let ((header (message-fetch-field spam-spamassassin-spam-flag-header)))
2651           (when header                  ; return nil when no header
2652             (when (string-match spam-spamassassin-positive-spam-flag-header
2653                                 header)
2654               spam-split-group)))))
2655
2656 (defun spam-check-spamassassin (&optional score)
2657   "Check the SpamAssassin backend for the classification of this message."
2658   (let ((article-buffer-name (buffer-name)))
2659     (with-temp-buffer
2660       (let ((temp-buffer-name (buffer-name)))
2661         (with-current-buffer article-buffer-name
2662           (apply 'call-process-region
2663                  (point-min) (point-max) spam-assassin-program
2664                  nil temp-buffer-name nil spam-spamassassin-arguments))
2665         ;; check the return now (we're back in the temp buffer)
2666         (goto-char (point-min))
2667         (spam-check-spamassassin-headers score)))))
2668
2669 ;; return something sensible if the score can't be determined
2670 (defun spam-spamassassin-score (&optional recheck)
2671   "Get the SpamAssassin score"
2672   (interactive "P")
2673   (save-window-excursion
2674     (gnus-summary-show-article t)
2675     (set-buffer gnus-article-buffer)
2676     (let ((score (or (unless recheck
2677                        (spam-check-spamassassin-headers t))
2678                      (spam-check-spamassassin t))))
2679       (gnus-summary-show-article)
2680       (message "SpamAssassin score %s" score)
2681       (or score "0"))))
2682
2683 (defun spam-spamassassin-register-with-sa-learn (articles spam
2684                                                  &optional unregister)
2685   "Register articles with spamassassin's sa-learn as spam or non-spam."
2686   (if articles
2687       (let ((action (if unregister spam-sa-learn-unregister-switch
2688                       (if spam spam-sa-learn-spam-switch
2689                         spam-sa-learn-ham-switch)))
2690             (summary-buffer-name (buffer-name)))
2691         (with-temp-buffer
2692           ;; group the articles into mbox format
2693           (dolist (article articles)
2694             (let (article-string)
2695               (with-current-buffer summary-buffer-name
2696                 (setq article-string (spam-get-article-as-string article)))
2697               (when (stringp article-string)
2698                 ;; mbox separator
2699                 (insert (concat "From nobody " (current-time-string) "\n"))
2700                 (insert article-string)
2701                 (insert "\n"))))
2702           ;; call sa-learn on all messages at the same time
2703           (apply 'call-process-region
2704                  (point-min) (point-max)
2705                  spam-sa-learn-program
2706                  nil nil nil "--mbox"
2707                  (if spam-sa-learn-rebuild
2708                      (list action)
2709                    `("--no-rebuild" ,action)))))))
2710
2711 (defun spam-spamassassin-register-spam-routine (articles &optional unregister)
2712   (spam-spamassassin-register-with-sa-learn articles t unregister))
2713
2714 (defun spam-spamassassin-register-ham-routine (articles &optional unregister)
2715   (spam-spamassassin-register-with-sa-learn articles nil unregister))
2716
2717 (defun spam-spamassassin-unregister-spam-routine (articles)
2718   (spam-spamassassin-register-with-sa-learn articles t t))
2719
2720 (defun spam-spamassassin-unregister-ham-routine (articles)
2721   (spam-spamassassin-register-with-sa-learn articles nil t))
2722
2723 ;;}}}
2724
2725 ;;{{{ Bsfilter
2726 ;;; based mostly on the bogofilter code
2727 (defun spam-check-bsfilter-headers (&optional score)
2728   (if score
2729       (or (nnmail-fetch-field spam-bsfilter-probability-header)
2730           "0")
2731     (let ((header (nnmail-fetch-field spam-bsfilter-header)))
2732       (when header ; return nil when no header
2733         (when (string-match "YES" header)
2734           spam-split-group)))))
2735
2736 ;; return something sensible if the score can't be determined
2737 (defun spam-bsfilter-score (&optional recheck)
2738   "Get the Bsfilter spamicity score."
2739   (interactive "P")
2740   (save-window-excursion
2741     (gnus-summary-show-article t)
2742     (set-buffer gnus-article-buffer)
2743     (let ((score (or (unless recheck
2744                        (spam-check-bsfilter-headers t))
2745                      (spam-check-bsfilter t))))
2746       (gnus-summary-show-article)
2747       (message "Spamicity score %s" score)
2748       (or score "0"))))
2749
2750 (defun spam-check-bsfilter (&optional score)
2751   "Check the Bsfilter backend for the classification of this message."
2752   (let ((article-buffer-name (buffer-name))
2753         (dir spam-bsfilter-database-directory)
2754         return)
2755     (with-temp-buffer
2756       (let ((temp-buffer-name (buffer-name)))
2757         (with-current-buffer article-buffer-name
2758           (apply 'call-process-region
2759                  (point-min) (point-max)
2760                  spam-bsfilter-program
2761                  nil temp-buffer-name nil
2762                  "--pipe"
2763                  "--insert-flag"
2764                  "--insert-probability"
2765                  (when dir
2766                    (list "--homedir" dir))))
2767         (setq return (spam-check-bsfilter-headers score))))
2768     return))
2769
2770 (defun spam-bsfilter-register-with-bsfilter (articles
2771                                              spam
2772                                              &optional unregister)
2773   "Register an article, given as a string, as spam or non-spam."
2774   (dolist (article articles)
2775     (let ((article-string (spam-get-article-as-string article))
2776           (switch (if unregister
2777                       (if spam
2778                           spam-bsfilter-spam-strong-switch
2779                         spam-bsfilter-ham-strong-switch)
2780                     (if spam
2781                         spam-bsfilter-spam-switch
2782                       spam-bsfilter-ham-switch))))
2783       (when (stringp article-string)
2784         (with-temp-buffer
2785           (insert article-string)
2786           (apply 'call-process-region
2787                  (point-min) (point-max)
2788                  spam-bsfilter-program
2789                  nil nil nil switch
2790                  "--update"
2791                  (when spam-bsfilter-database-directory
2792                    (list "--homedir"
2793                          spam-bsfilter-database-directory))))))))
2794
2795 (defun spam-bsfilter-register-spam-routine (articles &optional unregister)
2796   (spam-bsfilter-register-with-bsfilter articles t unregister))
2797
2798 (defun spam-bsfilter-unregister-spam-routine (articles)
2799   (spam-bsfilter-register-spam-routine articles t))
2800
2801 (defun spam-bsfilter-register-ham-routine (articles &optional unregister)
2802   (spam-bsfilter-register-with-bsfilter articles nil unregister))
2803
2804 (defun spam-bsfilter-unregister-ham-routine (articles)
2805   (spam-bsfilter-register-ham-routine articles t))
2806
2807 ;;}}}
2808
2809 ;;{{{ CRM114 Mailfilter
2810 (defun spam-check-crm114-headers (&optional score)
2811   (let ((header (message-fetch-field spam-crm114-header)))
2812     (when header                        ; return nil when no header
2813       (if score                         ; scoring mode
2814           (if (string-match "( pR: \\([0-9.-]+\\)" header)
2815               (match-string 1 header)
2816             "0")
2817         ;; spam detection mode
2818         (when (string-match spam-crm114-positive-spam-header
2819                             header)
2820           spam-split-group)))))
2821
2822 ;; return something sensible if the score can't be determined
2823 (defun spam-crm114-score ()
2824   "Get the CRM114 Mailfilter pR."
2825   (interactive)
2826   (save-window-excursion
2827     (gnus-summary-show-article t)
2828     (set-buffer gnus-article-buffer)
2829     (let ((score (or (spam-check-crm114-headers t)
2830                      (spam-check-crm114 t))))
2831       (gnus-summary-show-article)
2832       (message "pR: %s" score)
2833       (or score "0"))))
2834
2835 (defun spam-check-crm114 (&optional score)
2836   "Check the CRM114 Mailfilter backend for the classification of this message."
2837   (let ((article-buffer-name (buffer-name))
2838         (db spam-crm114-database-directory)
2839         return)
2840     (with-temp-buffer
2841       (let ((temp-buffer-name (buffer-name)))
2842         (with-current-buffer article-buffer-name
2843           (apply 'call-process-region
2844                  (point-min) (point-max)
2845                  spam-crm114-program
2846                  nil temp-buffer-name nil
2847                  (when db (list (concat "--fileprefix=" db)))))
2848         (setq return (spam-check-crm114-headers score))))
2849     return))
2850
2851 (defun spam-crm114-register-with-crm114 (articles
2852                                          spam
2853                                          &optional unregister)
2854   "Register an article, given as a string, as spam or non-spam."
2855   (dolist (article articles)
2856     (let ((article-string (spam-get-article-as-string article))
2857           (db spam-crm114-database-directory)
2858           (switch (if unregister
2859                       (if spam
2860                           spam-crm114-spam-strong-switch
2861                         spam-crm114-ham-strong-switch)
2862                     (if spam
2863                         spam-crm114-spam-switch
2864                       spam-crm114-ham-switch))))
2865       (when (stringp article-string)
2866         (with-temp-buffer
2867           (insert article-string)
2868
2869           (apply 'call-process-region
2870                  (point-min) (point-max)
2871                  spam-crm114-program
2872                  nil nil nil
2873                  (when db (list switch (concat "--fileprefix=" db)))))))))
2874
2875 (defun spam-crm114-register-spam-routine (articles &optional unregister)
2876   (spam-crm114-register-with-crm114 articles t unregister))
2877
2878 (defun spam-crm114-unregister-spam-routine (articles)
2879   (spam-crm114-register-spam-routine articles t))
2880
2881 (defun spam-crm114-register-ham-routine (articles &optional unregister)
2882   (spam-crm114-register-with-crm114 articles nil unregister))
2883
2884 (defun spam-crm114-unregister-ham-routine (articles)
2885   (spam-crm114-register-ham-routine articles t))
2886
2887 ;;}}}
2888
2889 ;;}}}
2890
2891 ;;{{{ Hooks
2892
2893 ;;;###autoload
2894 (defun spam-initialize (&rest symbols)
2895   "Install the spam.el hooks and do other initialization.
2896 When SYMBOLS is given, set those variables to t.  This is so you
2897 can call `spam-initialize' before you set spam-use-* variables on
2898 explicitly, and matters only if you need the extra headers
2899 installed through `spam-necessary-extra-headers'."
2900   (interactive)
2901
2902   (when spam-install-hooks
2903     (dolist (var symbols)
2904       (set var t))
2905
2906     (dolist (header (spam-necessary-extra-headers))
2907       (add-to-list 'nnmail-extra-headers header)
2908       (add-to-list 'gnus-extra-headers header))
2909
2910     ;; TODO: How do we redo this every time the `spam' face is customized?
2911     (push '((eq mark gnus-spam-mark) . spam)
2912           gnus-summary-highlight)
2913     ;; Add hooks for loading and saving the spam stats
2914     (add-hook 'gnus-save-newsrc-hook 'spam-maybe-spam-stat-save)
2915     (add-hook 'gnus-get-top-new-news-hook 'spam-maybe-spam-stat-load)
2916     (add-hook 'gnus-startup-hook 'spam-maybe-spam-stat-load)
2917     (add-hook 'gnus-summary-prepare-exit-hook 'spam-summary-prepare-exit)
2918     (add-hook 'gnus-summary-prepare-hook 'spam-summary-prepare)
2919     (add-hook 'gnus-get-new-news-hook 'spam-setup-widening)
2920     (add-hook 'gnus-summary-prepared-hook 'spam-find-spam)
2921     ;; Don't install things more than once.
2922     (setq spam-install-hooks nil)))
2923
2924 (defun spam-unload-hook ()
2925   "Uninstall the spam.el hooks."
2926   (interactive)
2927   (remove-hook 'gnus-save-newsrc-hook 'spam-maybe-spam-stat-save)
2928   (remove-hook 'gnus-get-top-new-news-hook 'spam-maybe-spam-stat-load)
2929   (remove-hook 'gnus-startup-hook 'spam-maybe-spam-stat-load)
2930   (remove-hook 'gnus-summary-prepare-exit-hook 'spam-summary-prepare-exit)
2931   (remove-hook 'gnus-summary-prepare-hook 'spam-summary-prepare)
2932   (remove-hook 'gnus-get-new-news-hook 'spam-setup-widening)
2933   (remove-hook 'gnus-summary-prepare-hook 'spam-find-spam))
2934
2935 (add-hook 'spam-unload-hook 'spam-unload-hook)
2936
2937 ;;}}}
2938
2939 (provide 'spam)
2940
2941 ;;; spam.el ends here