1 ;;; spam-stat.el --- detecting spam based on statistics
3 ;; Copyright (C) 2002 Alex Schroeder
5 ;; Author: Alex Schroeder <alex@gnu.org>
6 ;; Maintainer: Alex Schroeder <alex@gnu.org>
8 ;; Keywords: spam filtering gnus
9 ;; URL: http://www.emacswiki.org/cgi-bin/wiki.pl?SpamStat
11 ;; This file is NOT part of GNU Emacs.
13 ;; This is free software; you can redistribute it and/or modify it
14 ;; under the terms of the GNU General Public License as published by
15 ;; the Free Software Foundation; either version 2, or (at your option)
18 ;; This is distributed in the hope that it will be useful, but WITHOUT
19 ;; ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
20 ;; or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
21 ;; License for more details.
23 ;; You should have received a copy of the GNU General Public License
24 ;; along with GNU Emacs; see the file COPYING. If not, write to the
25 ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
26 ;; Boston, MA 02111-1307, USA.
30 ;; This implements spam analysis according to Paul Graham in "A Plan
31 ;; for Spam". The basis for all this is a statistical distribution of
32 ;; words for your spam and non-spam mails. We need this information
33 ;; in a hash-table so that the analysis can use the information when
34 ;; looking at your mails. Therefore, before you begin, you need tons
35 ;; of mails (Graham uses 4000 non-spam and 4000 spam mails for his
38 ;; The main interface to using spam-stat, are the following functions:
40 ;; `spam-stat-buffer-is-spam' -- called in a buffer, that buffer is
41 ;; considered to be a new spam mail; use this for new mail that has
42 ;; not been processed before
44 ;; `spam-stat-buffer-is-no-spam' -- called in a buffer, that buffer
45 ;; is considered to be a new non-spam mail; use this for new mail that
46 ;; has not been processed before
48 ;; `spam-stat-buffer-change-to-spam' -- called in a buffer, that
49 ;; buffer is no longer considered to be normal mail but spam; use this
50 ;; to change the status of a mail that has already been processed as
53 ;; `spam-stat-buffer-change-to-non-spam' -- called in a buffer, that
54 ;; buffer is no longer considered to be spam but normal mail; use this
55 ;; to change the status of a mail that has already been processed as
58 ;; `spam-stat-save' -- save the hash table to the file; the filename
59 ;; used is stored in the variable `spam-stat-file'
61 ;; `spam-stat-load' -- load the hash table from a file; the filename
62 ;; used is stored in the variable `spam-stat-file'
64 ;; `spam-stat-score-word' -- return the spam score for a word
66 ;; `spam-stat-score-buffer' -- return the spam score for a buffer
68 ;; `spam-stat-split-fancy' -- for fancy mail splitting; add
69 ;; the rule (: spam-stat-split-fancy) to `nnmail-split-fancy'
71 ;; This requires the following in your ~/.gnus file:
73 ;; (require 'spam-stat)
78 ;; Typical test will involve calls to the following functions:
80 ;; Reset: (setq spam-stat (make-hash-table :test 'equal))
81 ;; Learn spam: (spam-stat-process-spam-directory "~/Mail/mail/spam")
82 ;; Learn non-spam: (spam-stat-process-non-spam-directory "~/Mail/mail/misc")
83 ;; Save table: (spam-stat-save)
84 ;; File size: (nth 7 (file-attributes spam-stat-file))
85 ;; Number of words: (hash-table-count spam-stat)
86 ;; Test spam: (spam-stat-test-directory "~/Mail/mail/spam")
87 ;; Test non-spam: (spam-stat-test-directory "~/Mail/mail/misc")
88 ;; Reduce table size: (spam-stat-reduce-size)
89 ;; Save table: (spam-stat-save)
90 ;; File size: (nth 7 (file-attributes spam-stat-file))
91 ;; Number of words: (hash-table-count spam-stat)
92 ;; Test spam: (spam-stat-test-directory "~/Mail/mail/spam")
93 ;; Test non-spam: (spam-stat-test-directory "~/Mail/mail/misc")
95 ;;; Dictionary Creation:
97 ;; Typically, you will filter away mailing lists etc. using specific
98 ;; rules in `nnmail-split-fancy'. Somewhere among these rules, you
99 ;; will filter spam. Here is how you would create your dictionary:
101 ;; Reset: (setq spam-stat (make-hash-table :test 'equal))
102 ;; Learn spam: (spam-stat-process-spam-directory "~/Mail/mail/spam")
103 ;; Learn non-spam: (spam-stat-process-non-spam-directory "~/Mail/mail/misc")
104 ;; Repeat for any other non-spam group you need...
105 ;; Reduce table size: (spam-stat-reduce-size)
106 ;; Save table: (spam-stat-save)
110 ;; Speed it up. Integrate with Gnus such that it uses spam and expiry
111 ;; marks to call the appropriate functions when leaving the summary
112 ;; buffer and saves the hash table when leaving Gnus. More testing:
113 ;; More mails, disabling SpamAssassin, double checking algorithm, find
114 ;; improved algorithm.
118 ;; Ted Zlatanov <tzz@lifelogs.com>
119 ;; Jesper Harder <harder@myrealbox.com>
120 ;; Dan Schmidt <dfan@dfan.org>
126 (defgroup spam-stat nil
127 "Statistical spam detection for Emacs.
128 Use the functions to build a dictionary of words and their statistical
129 distribution in spam and non-spam mails. Then use a function to determine
130 wether a buffer contains spam or not."
133 (defcustom spam-stat-file "~/.spam-stat.el"
134 "File used to save and load the dictionary.
135 See `spam-stat-to-hash-table' for the format of the file."
139 (defcustom spam-stat-unknown-word-score 0.2
140 "The score to use for unknown words.
141 Also used for words that don't appear often enough."
145 (defcustom spam-stat-max-word-length 15
146 "Only words shorter than this will be considered."
150 (defcustom spam-stat-max-buffer-length 10240
151 "Only the beginning of buffers will be analyzed.
152 This variable says how many characters this will be."
156 (defcustom spam-stat-split-fancy-spam-group "mail.spam"
157 "Name of the group where spam should be stored, if
158 `spam-stat-split-fancy' is used in fancy splitting rules."
162 (defvar spam-stat-syntax-table
163 (let ((table (copy-syntax-table text-mode-syntax-table)))
164 (modify-syntax-entry ?- "w" table)
165 (modify-syntax-entry ?_ "w" table)
166 (modify-syntax-entry ?. "w" table)
167 (modify-syntax-entry ?! "w" table)
168 (modify-syntax-entry ?? "w" table)
169 (modify-syntax-entry ?+ "w" table)
171 "Syntax table used when processing mails for statistical analysis.
172 The important part is which characters are word constituents.")
174 (defvar spam-stat-buffer nil
175 "Buffer to use for scoring while splitting.
176 This is set by hooking into Gnus.")
178 (defvar spam-stat-buffer-name " *spam stat buffer*"
179 "Name of the `spam-stat-buffer'.")
181 ;; Functions missing in Emacs 20
183 (when (memq nil (mapcar 'fboundp
184 '(gethash hash-table-count make-hash-table
187 (unless (fboundp 'puthash)
188 ;; alias puthash is missing from Emacs 20 cl-extra.el
189 (defalias 'puthash 'cl-puthash)))
192 (unless (fboundp 'with-syntax-table)
193 ;; Imported from Emacs 21.2
194 (defmacro with-syntax-table (table &rest body) "\
195 Evaluate BODY with syntax table of current buffer set to a copy of TABLE.
196 The syntax table of the current buffer is saved, BODY is evaluated, and the
197 saved table is restored, even in case of an abnormal exit.
198 Value is what BODY returns."
199 (let ((old-table (make-symbol "table"))
200 (old-buffer (make-symbol "buffer")))
201 `(let ((,old-table (syntax-table))
202 (,old-buffer (current-buffer)))
205 (set-syntax-table (copy-syntax-table ,table))
208 (set-buffer ,old-buffer)
209 (set-syntax-table ,old-table))))))))
213 (defun spam-stat-store-current-buffer ()
214 "Store a copy of the current buffer in `spam-stat-buffer'."
216 (let ((str (buffer-string)))
217 (set-buffer (get-buffer-create spam-stat-buffer-name))
220 (setq spam-stat-buffer (current-buffer)))))
222 (defun spam-stat-store-gnus-article-buffer ()
223 "Store a copy of the current article in `spam-stat-buffer'.
224 This uses `gnus-article-buffer'."
226 (set-buffer gnus-original-article-buffer)
227 (spam-stat-store-current-buffer)))
229 (add-hook 'nnmail-prepare-incoming-message-hook
230 'spam-stat-store-current-buffer)
231 (add-hook 'gnus-select-article-hook
232 'spam-stat-store-gnus-article-buffer)
234 ;; Data -- not using defstruct in order to save space and time
236 (defvar spam-stat (make-hash-table :test 'equal)
237 "Hash table used to store the statistics.
238 Use `spam-stat-load' to load the file.
239 Every word is used as a key in this table. The value is a vector.
240 Use `spam-stat-ngood', `spam-stat-nbad', `spam-stat-good',
241 `spam-stat-bad', and `spam-stat-score' to access this vector.")
243 (defvar spam-stat-ngood 0
244 "The number of good mails in the dictionary.")
246 (defvar spam-stat-nbad 0
247 "The number of bad mails in the dictionary.")
249 (defsubst spam-stat-good (entry)
250 "Return the number of times this word belongs to good mails."
253 (defsubst spam-stat-bad (entry)
254 "Return the number of times this word belongs to bad mails."
257 (defsubst spam-stat-score (entry)
258 "Set the score of this word."
261 spam-stat-unknown-word-score))
263 (defsubst spam-stat-set-good (entry value)
264 "Set the number of times this word belongs to good mails."
265 (aset entry 0 value))
267 (defsubst spam-stat-set-bad (entry value)
268 "Set the number of times this word belongs to bad mails."
269 (aset entry 1 value))
271 (defsubst spam-stat-set-score (entry value)
272 "Set the score of this word."
273 (aset entry 2 value))
275 (defsubst spam-stat-make-entry (good bad)
276 "Return a vector with the given properties."
277 (let ((entry (vector good bad nil)))
278 (spam-stat-set-score entry (spam-stat-compute-score entry))
283 (defun spam-stat-compute-score (entry)
284 "Compute the score of this word. 1.0 means spam."
285 ;; promote all numbers to floats for the divisions
286 (let* ((g (* 2.0 (spam-stat-good entry)))
287 (b (float (spam-stat-bad entry))))
290 ((= 0 spam-stat-ngood)
292 ((= 0 spam-stat-nbad)
296 (min .99 (/ (/ b spam-stat-nbad)
297 (+ (/ g spam-stat-ngood)
298 (/ b spam-stat-nbad)))))))))
302 (defmacro with-spam-stat-max-buffer-size (&rest body)
303 "Narrows the buffer down to the first 4k characters, then evaluates BODY."
305 (when (> (- (point-max)
307 spam-stat-max-buffer-length)
308 (narrow-to-region (point-min)
309 (+ (point-min) spam-stat-max-buffer-length)))
312 (defun spam-stat-buffer-words ()
313 "Return a hash table of words and number of occurences in the buffer."
314 (with-spam-stat-max-buffer-size
315 (with-syntax-table spam-stat-syntax-table
316 (goto-char (point-min))
317 (let ((result (make-hash-table :test 'equal))
319 (while (re-search-forward "\\w+" nil t)
320 (setq word (match-string-no-properties 0)
321 count (1+ (gethash word result 0)))
322 (when (< (length word) spam-stat-max-word-length)
323 (puthash word count result)))
326 (defun spam-stat-buffer-is-spam ()
327 "Consider current buffer to be a new spam mail."
328 (setq spam-stat-nbad (1+ spam-stat-nbad))
331 (let ((entry (gethash word spam-stat)))
333 (spam-stat-set-bad entry (+ count (spam-stat-bad entry)))
334 (setq entry (spam-stat-make-entry 0 count)))
335 (spam-stat-set-score entry (spam-stat-compute-score entry))
336 (puthash word entry spam-stat)))
337 (spam-stat-buffer-words)))
339 (defun spam-stat-buffer-is-non-spam ()
340 "Consider current buffer to be a new non-spam mail."
341 (setq spam-stat-ngood (1+ spam-stat-ngood))
344 (let ((entry (gethash word spam-stat)))
346 (spam-stat-set-good entry (+ count (spam-stat-good entry)))
347 (setq entry (spam-stat-make-entry count 0)))
348 (spam-stat-set-score entry (spam-stat-compute-score entry))
349 (puthash word entry spam-stat)))
350 (spam-stat-buffer-words)))
352 (defun spam-stat-buffer-change-to-spam ()
353 "Consider current buffer no longer normal mail but spam."
354 (setq spam-stat-nbad (1+ spam-stat-nbad)
355 spam-stat-ngood (1- spam-stat-ngood))
358 (let ((entry (gethash word spam-stat)))
360 (error "This buffer has unknown words in it.")
361 (spam-stat-set-good entry (- (spam-stat-good entry) count))
362 (spam-stat-set-bad entry (+ (spam-stat-bad entry) count))
363 (spam-stat-set-score entry (spam-stat-compute-score entry))
364 (puthash word entry spam-stat))))
365 (spam-stat-buffer-words)))
367 (defun spam-stat-buffer-change-to-non-spam ()
368 "Consider current buffer no longer spam but normal mail."
369 (setq spam-stat-nbad (1- spam-stat-nbad)
370 spam-stat-ngood (1+ spam-stat-ngood))
373 (let ((entry (gethash word spam-stat)))
375 (error "This buffer has unknown words in it.")
376 (spam-stat-set-good entry (+ (spam-stat-good entry) count))
377 (spam-stat-set-bad entry (- (spam-stat-bad entry) count))
378 (spam-stat-set-score entry (spam-stat-compute-score entry))
379 (puthash word entry spam-stat))))
380 (spam-stat-buffer-words)))
382 ;; Saving and Loading
384 (defun spam-stat-save ()
385 "Save the `spam-stat' hash table as lisp file."
388 (let ((standard-output (current-buffer)))
389 (insert "(setq spam-stat (spam-stat-to-hash-table '(")
390 (maphash (lambda (word entry)
392 (spam-stat-good entry)
393 (spam-stat-bad entry))))
395 (insert ")) spam-stat-ngood "
396 (number-to-string spam-stat-ngood)
398 (number-to-string spam-stat-nbad)
400 (write-file spam-stat-file)))
402 (defun spam-stat-load ()
403 "Read the `spam-stat' hash table from disk."
404 (load-file spam-stat-file))
406 (defun spam-stat-to-hash-table (entries)
407 "Turn list ENTRIES into a hash table and store as `spam-stat'.
408 Every element in ENTRIES has the form \(WORD GOOD BAD) where WORD is
409 the word string, NGOOD is the number of good mails it has appeared in,
410 NBAD is the number of bad mails it has appeared in, GOOD is the number
411 of times it appeared in good mails, and BAD is the number of times it
412 has appeared in bad mails."
413 (let ((table (make-hash-table :test 'equal)))
416 (spam-stat-make-entry (nth 1 l) (nth 2 l))
421 (defun spam-stat-reset ()
422 "Reset `spam-stat' to an empty hash-table.
423 This deletes all the statistics."
425 (setq spam-stat (make-hash-table :test 'equal)))
429 (defvar spam-stat-score-data nil
430 "Raw data used in the last run of `spam-stat-score-buffer'.")
432 (defsubst spam-stat-score-word (word)
433 "Return score for WORD.
434 The default score for unknown words is stored in
435 `spam-stat-unknown-word-score'."
436 (spam-stat-score (gethash word spam-stat)))
438 (defun spam-stat-buffer-words-with-scores ()
439 "Process current buffer, return the 15 most conspicuous words.
440 These are the words whose spam-stat differs the most from 0.5.
441 The list returned contains elements of the form \(WORD SCORE DIFF),
442 where DIFF is the difference between SCORE and 0.5."
443 (with-spam-stat-max-buffer-size
444 (with-syntax-table spam-stat-syntax-table
445 (let (result word score)
446 (maphash (lambda (word ignore)
447 (setq score (spam-stat-score-word word)
448 result (cons (list word score (abs (- score 0.5)))
450 (spam-stat-buffer-words))
451 (setq result (sort result (lambda (a b) (< (nth 2 b) (nth 2 a)))))
452 (setcdr (nthcdr 14 result) nil)
455 (defun spam-stat-score-buffer ()
456 "Return a score describing the spam-probability for this buffer."
457 (setq spam-stat-score-data (spam-stat-buffer-words-with-scores))
458 (let* ((probs (mapcar (lambda (e) (cadr e)) spam-stat-score-data))
459 (prod (apply #'* probs)))
460 (/ prod (+ prod (apply #'* (mapcar #'(lambda (x) (- 1 x))
463 (defun spam-stat-split-fancy ()
464 "Return the name of the spam group if the current mail is spam.
465 Use this function on `nnmail-split-fancy'. If you are interested in
466 the raw data used for the last run of `spam-stat-score-buffer',
467 check the variable `spam-stat-score-data'."
470 (set-buffer spam-stat-buffer)
471 (goto-char (point-min))
472 (when (> (spam-stat-score-buffer) 0.9)
473 (when (boundp 'nnmail-split-trace)
474 (mapc (lambda (entry)
475 (push entry nnmail-split-trace))
476 spam-stat-score-data))
477 spam-stat-split-fancy-spam-group))
478 (error (message "Error in spam-stat-split-fancy: %S" var)
483 (defun spam-stat-process-directory (dir func)
484 "Process all the regular files in directory DIR using function FUNC."
485 (let* ((files (directory-files dir t "^[^.]"))
486 (max (/ (length files) 100.0))
490 (when (and (file-readable-p f)
492 (setq count (1+ count))
493 (message "Reading %s: %.2f%%" dir (/ count max))
494 (insert-file-contents f)
498 (defun spam-stat-process-spam-directory (dir)
499 "Process all the regular files in directory DIR as spam."
501 (spam-stat-process-directory dir 'spam-stat-buffer-is-spam))
503 (defun spam-stat-process-non-spam-directory (dir)
504 "Process all the regular files in directory DIR as non-spam."
506 (spam-stat-process-directory dir 'spam-stat-buffer-is-non-spam))
508 (defun spam-stat-count ()
509 "Return size of `spam-stat'."
511 (hash-table-count spam-stat))
513 (defun spam-stat-test-directory (dir)
514 "Test all the regular files in directory DIR for spam.
515 If the result is 1.0, then all files are considered spam.
516 If the result is 0.0, non of the files is considered spam.
517 You can use this to determine error rates."
519 (let* ((files (directory-files dir t "^[^.]"))
520 (total (length files))
522 (max (/ total 100.0)); float
526 (when (and (file-readable-p f)
528 (setq count (1+ count))
529 (message "Reading %.2f%%, score %.2f%%"
530 (/ count max) (/ score count))
531 (insert-file-contents f)
532 (when (> (spam-stat-score-buffer) 0.9)
533 (setq score (1+ score)))
535 (message "Final score: %d / %d = %f" score total (/ score total))))
537 ;; Shrinking the dictionary
539 (defun spam-stat-reduce-size (&optional count)
540 "Reduce the size of `spam-stat'.
541 This removes all words that occur less than COUNT from the dictionary.
544 (setq count (or count 5))
545 (maphash (lambda (key entry)
546 (when (< (+ (spam-stat-good entry)
547 (spam-stat-bad entry))
549 (remhash key spam-stat)))
554 ;;; spam-stat.el ends here