Clarify comment.

[gnus] / lisp / spam-stat.el

;;; spam-stat.el --- detecting spam based on statistics

;; Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
;;   2010  Free Software Foundation, Inc.

;; Author: Alex Schroeder <alex@gnu.org>
;; Keywords: network
;; URL: http://www.emacswiki.org/cgi-bin/wiki.pl?SpamStat

;; This file is part of GNU Emacs.

;; GNU Emacs is free software: you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation, either version 3 of the License, or
;; (at your option) any later version.

;; GNU Emacs is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;; GNU General Public License for more details.

;; You should have received a copy of the GNU General Public License
;; along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.

;;; Commentary:

;; This implements spam analysis according to Paul Graham in "A Plan
;; for Spam".  The basis for all this is a statistical distribution of
;; words for your spam and non-spam mails.  We need this information
;; in a hash-table so that the analysis can use the information when
;; looking at your mails.  Therefore, before you begin, you need tons
;; of mails (Graham uses 4000 non-spam and 4000 spam mails for his
;; experiments).
;;
;; The main interface to using spam-stat, are the following functions:
;;
;; `spam-stat-buffer-is-spam' -- called in a buffer, that buffer is
;; considered to be a new spam mail; use this for new mail that has
;; not been processed before
;;
;; `spam-stat-buffer-is-non-spam' -- called in a buffer, that buffer
;; is considered to be a new non-spam mail; use this for new mail that
;; has not been processed before
;;
;; `spam-stat-buffer-change-to-spam' -- called in a buffer, that
;; buffer is no longer considered to be normal mail but spam; use this
;; to change the status of a mail that has already been processed as
;; non-spam
;;
;; `spam-stat-buffer-change-to-non-spam' -- called in a buffer, that
;; buffer is no longer considered to be spam but normal mail; use this
;; to change the status of a mail that has already been processed as
;; spam
;;
;; `spam-stat-save' -- save the hash table to the file; the filename
;; used is stored in the variable `spam-stat-file'
;;
;; `spam-stat-load' -- load the hash table from a file; the filename
;; used is stored in the variable `spam-stat-file'
;;
;; `spam-stat-score-word' -- return the spam score for a word
;;
;; `spam-stat-score-buffer' -- return the spam score for a buffer
;;
;; `spam-stat-split-fancy' -- for fancy mail splitting; add
;; the rule (: spam-stat-split-fancy) to `nnmail-split-fancy'
;;
;; This requires the following in your ~/.gnus file:
;;
;; (require 'spam-stat)
;; (spam-stat-load)

;;; Testing:

;; Typical test will involve calls to the following functions:
;;
;; Reset: (spam-stat-reset)
;; Learn spam: (spam-stat-process-spam-directory "~/Mail/mail/spam")
;; Learn non-spam: (spam-stat-process-non-spam-directory "~/Mail/mail/misc")
;; Save table: (spam-stat-save)
;; File size: (nth 7 (file-attributes spam-stat-file))
;; Number of words: (hash-table-count spam-stat)
;; Test spam: (spam-stat-test-directory "~/Mail/mail/spam")
;; Test non-spam: (spam-stat-test-directory "~/Mail/mail/misc")
;; Reduce table size: (spam-stat-reduce-size)
;; Save table: (spam-stat-save)
;; File size: (nth 7 (file-attributes spam-stat-file))
;; Number of words: (hash-table-count spam-stat)
;; Test spam: (spam-stat-test-directory "~/Mail/mail/spam")
;; Test non-spam: (spam-stat-test-directory "~/Mail/mail/misc")

;;; Dictionary Creation:

;; Typically, you will filter away mailing lists etc. using specific
;; rules in `nnmail-split-fancy'.  Somewhere among these rules, you
;; will filter spam.  Here is how you would create your dictionary:

;; Reset: (spam-stat-reset)
;; Learn spam: (spam-stat-process-spam-directory "~/Mail/mail/spam")
;; Learn non-spam: (spam-stat-process-non-spam-directory "~/Mail/mail/misc")
;; Repeat for any other non-spam group you need...
;; Reduce table size: (spam-stat-reduce-size)
;; Save table: (spam-stat-save)

;;; Todo:

;; Speed it up.  Integrate with Gnus such that it uses spam and expiry
;; marks to call the appropriate functions when leaving the summary
;; buffer and saves the hash table when leaving Gnus.  More testing:
;; More mails, disabling SpamAssassin, double checking algorithm, find
;; improved algorithm.

;;; Thanks:

;; Ted Zlatanov <tzz@lifelogs.com>
;; Jesper Harder <harder@myrealbox.com>
;; Dan Schmidt <dfan@dfan.org>
;; Lasse Rasinen <lrasinen@iki.fi>
;; Milan Zamazal <pdm@zamazal.org>

\f

;;; Code:
(require 'mail-parse)

(defvar gnus-original-article-buffer)

(defgroup spam-stat nil
  "Statistical spam detection for Emacs.
Use the functions to build a dictionary of words and their statistical
distribution in spam and non-spam mails.  Then use a function to determine
whether a buffer contains spam or not."
  :version "22.1"
  :group 'gnus)

(defcustom spam-stat-file "~/.spam-stat.el"
  "File used to save and load the dictionary.
See `spam-stat-to-hash-table' for the format of the file."
  :type 'file
  :group 'spam-stat)

(defcustom spam-stat-install-hooks t
  "Whether spam-stat should install its hooks in Gnus.
This is set to nil if you use spam-stat through spam.el."
  :type 'boolean
  :group 'spam-stat)

(defcustom spam-stat-unknown-word-score 0.2
  "The score to use for unknown words.
Also used for words that don't appear often enough."
  :type 'number
  :group 'spam-stat)

(defcustom spam-stat-max-word-length 15
  "Only words shorter than this will be considered."
  :type 'integer
  :group 'spam-stat)

(defcustom spam-stat-max-buffer-length 10240
  "Only the beginning of buffers will be analyzed.
This variable says how many characters this will be."
  :type 'integer
  :group 'spam-stat)

(defcustom spam-stat-split-fancy-spam-group "mail.spam"
  "Name of the group where spam should be stored.
If `spam-stat-split-fancy' is used in fancy splitting rules.  Has
no effect when spam-stat is invoked through spam.el."
  :type 'string
  :group 'spam-stat)

(defcustom spam-stat-split-fancy-spam-threshold 0.9
  "Spam score threshold in spam-stat-split-fancy."
  :type 'number
  :group 'spam-stat)

(defcustom spam-stat-washing-hook nil
  "Hook applied to each message before analysis."
  :type 'hook
  :group 'spam-stat)

(defcustom spam-stat-score-buffer-user-functions nil
  "List of additional scoring functions.
Called  one by one on the buffer.

If all of these functions return non-nil answers, these numerical
answers are added to the computed spam stat score on the buffer.  If
you defun such functions, make sure they don't return the buffer in a
narrowed state or such: use, for example, `save-excursion'.  Each of
your functions is also passed the initial spam-stat score which might
aid in your scoring.

Also be careful when defining such functions.  If they take a long
time, they will slow down your mail splitting.  Thus, if the buffer is
large, don't forget to use smaller regions, by wrapping your work in,
say, `with-spam-stat-max-buffer-size'."
  :type '(repeat sexp)
  :group 'spam-stat)

(defcustom spam-stat-process-directory-age 90
  "Max. age of files to be processed in directory, in days.
When using `spam-stat-process-spam-directory' or
`spam-stat-process-non-spam-directory', only files that have
been touched in this many days will be considered.  Without
this filter, re-training spam-stat with several thousand messages
will start to take a very long time."
  :type 'number
  :group 'spam-stat)

(defvar spam-stat-last-saved-at nil
  "Time stamp of last change of spam-stat-file on this run")

(defvar spam-stat-syntax-table
  (let ((table (copy-syntax-table text-mode-syntax-table)))
    (modify-syntax-entry ?- "w" table)
    (modify-syntax-entry ?_ "w" table)
    (modify-syntax-entry ?. "w" table)
    (modify-syntax-entry ?! "w" table)
    (modify-syntax-entry ?? "w" table)
    (modify-syntax-entry ?+ "w" table)
    table)
  "Syntax table used when processing mails for statistical analysis.
The important part is which characters are word constituents.")

(defvar spam-stat-dirty nil
  "Whether the spam-stat database needs saving.")

(defvar spam-stat-buffer nil
  "Buffer to use for scoring while splitting.
This is set by hooking into Gnus.")

(defvar spam-stat-buffer-name " *spam stat buffer*"
  "Name of the `spam-stat-buffer'.")

(defvar spam-stat-coding-system
  (if (mm-coding-system-p 'emacs-mule) 'emacs-mule 'raw-text)
  "Coding system used for `spam-stat-file'.")

;; Hooking into Gnus

(defun spam-stat-store-current-buffer ()
  "Store a copy of the current buffer in `spam-stat-buffer'."
  (let ((buf (current-buffer)))
    (with-current-buffer (get-buffer-create spam-stat-buffer-name)
      (erase-buffer)
      (insert-buffer-substring buf)
      (setq spam-stat-buffer (current-buffer)))))

(defun spam-stat-store-gnus-article-buffer ()
  "Store a copy of the current article in `spam-stat-buffer'.
This uses `gnus-article-buffer'."
  (with-current-buffer gnus-original-article-buffer
    (spam-stat-store-current-buffer)))

;; Data -- not using defstruct in order to save space and time

(defvar spam-stat (make-hash-table :test 'equal)
  "Hash table used to store the statistics.
Use `spam-stat-load' to load the file.
Every word is used as a key in this table.  The value is a vector.
Use `spam-stat-ngood', `spam-stat-nbad', `spam-stat-good',
`spam-stat-bad', and `spam-stat-score' to access this vector.")

(defvar spam-stat-ngood 0
  "The number of good mails in the dictionary.")

(defvar spam-stat-nbad 0
  "The number of bad mails in the dictionary.")

(defvar spam-stat-error-holder nil
  "A holder for condition-case errors while scoring buffers.")

(defsubst spam-stat-good (entry)
  "Return the number of times this word belongs to good mails."
  (aref entry 0))

(defsubst spam-stat-bad (entry)
  "Return the number of times this word belongs to bad mails."
  (aref entry 1))

(defsubst spam-stat-score (entry)
  "Set the score of this word."
  (if entry
      (aref entry 2)
    spam-stat-unknown-word-score))

(defsubst spam-stat-set-good (entry value)
  "Set the number of times this word belongs to good mails."
  (aset entry 0 value))

(defsubst spam-stat-set-bad (entry value)
  "Set the number of times this word belongs to bad mails."
  (aset entry 1 value))

(defsubst spam-stat-set-score (entry value)
  "Set the score of this word."
  (aset entry 2 value))

(defsubst spam-stat-make-entry (good bad)
  "Return a vector with the given properties."
  (let ((entry (vector good bad nil)))
    (spam-stat-set-score entry (spam-stat-compute-score entry))
    entry))

;; Computing

(defun spam-stat-compute-score (entry)
  "Compute the score of this word.  1.0 means spam."
   ;; promote all numbers to floats for the divisions
   (let* ((g (* 2.0 (spam-stat-good entry)))
          (b (float (spam-stat-bad entry))))
     (cond ((< (+ g b) 5)
            .2)
           ((= 0 spam-stat-ngood)
            .99)
           ((= 0 spam-stat-nbad)
            .01)
           (t
            (max .01
                 (min .99 (/ (/ b spam-stat-nbad)
                             (+ (/ g spam-stat-ngood)
                                (/ b spam-stat-nbad)))))))))

;; Parsing

(defmacro with-spam-stat-max-buffer-size (&rest body)
  "Narrow the buffer down to the first 4k characters, then evaluate BODY."
  `(save-restriction
     (when (> (- (point-max)
                 (point-min))
              spam-stat-max-buffer-length)
       (narrow-to-region (point-min)
                         (+ (point-min) spam-stat-max-buffer-length)))
     ,@body))

(defun spam-stat-buffer-words ()
  "Return a hash table of words and number of occurrences in the buffer."
  (run-hooks 'spam-stat-washing-hook)
  (with-spam-stat-max-buffer-size
   (with-syntax-table spam-stat-syntax-table
     (goto-char (point-min))
     (let ((result (make-hash-table :test 'equal))
           word count)