gnus-art.el (article-date-ut): Work properly even when
[gnus] / lisp / nnfolder.el
1 ;;; nnfolder.el --- mail folder access for Gnus
2
3 ;; Copyright (C) 1995-2011 Free Software Foundation, Inc.
4
5 ;; Author: Simon Josefsson <simon@josefsson.org> (adding MARKS)
6 ;;      ShengHuo Zhu <zsh@cs.rochester.edu> (adding NOV)
7 ;;      Scott Byer <byer@mv.us.adobe.com>
8 ;;      Lars Magne Ingebrigtsen <larsi@gnus.org>
9 ;;      Masanobu UMEDA <umerin@flab.flab.fujitsu.junet>
10 ;; Keywords: mail
11
12 ;; This file is part of GNU Emacs.
13
14 ;; GNU Emacs is free software: you can redistribute it and/or modify
15 ;; it under the terms of the GNU General Public License as published by
16 ;; the Free Software Foundation, either version 3 of the License, or
17 ;; (at your option) any later version.
18
19 ;; GNU Emacs is distributed in the hope that it will be useful,
20 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
21 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
22 ;; GNU General Public License for more details.
23
24 ;; You should have received a copy of the GNU General Public License
25 ;; along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.
26
27 ;;; Commentary:
28
29 ;;; Code:
30
31 ;; For Emacs <22.2 and XEmacs.
32 (eval-and-compile
33   (unless (fboundp 'declare-function) (defmacro declare-function (&rest r))))
34
35 (require 'nnheader)
36 (require 'message)
37 (require 'nnmail)
38 (require 'nnoo)
39 (eval-when-compile (require 'cl))
40 (require 'gnus)
41 (require 'gnus-util)
42 (require 'gnus-range)
43
44 ;; FIXME not explicitly used in this file.
45 (autoload 'gnus-article-unpropagatable-p "gnus-sum")
46
47 (nnoo-declare nnfolder)
48
49 (defvoo nnfolder-directory (expand-file-name message-directory)
50   "The name of the nnfolder directory.")
51
52 (defvoo nnfolder-nov-directory nil
53   "The name of the nnfolder NOV directory.
54 If nil, `nnfolder-directory' is used.")
55
56 (defvoo nnfolder-marks-directory nil
57   "The name of the nnfolder MARKS directory.
58 If nil, `nnfolder-directory' is used.")
59
60 (defvoo nnfolder-active-file
61     (nnheader-concat nnfolder-directory "active")
62   "The name of the active file.")
63
64 ;; I renamed this variable to something more in keeping with the general GNU
65 ;; style. -SLB
66
67 (defvoo nnfolder-ignore-active-file nil
68   "If non-nil, the active file is ignored.
69 This causes nnfolder to do some extra work in order to determine the
70 true active ranges of an mbox file.  Note that the active file is
71 still saved, but its values are not used.  This costs some extra time
72 when scanning an mbox when opening it.")
73
74 (defvoo nnfolder-distrust-mbox nil
75   "If non-nil, the folder will be distrusted.
76 This means that nnfolder will not trust the user with respect to
77 inserting unaccounted for mail in the middle of an mbox file.  This
78 can greatly slow down scans, which now must scan the entire file for
79 unmarked messages.  When nil, scans occur forward from the last marked
80 message, a huge time saver for large mailboxes.")
81
82 (defvoo nnfolder-newsgroups-file
83     (concat (file-name-as-directory nnfolder-directory) "newsgroups")
84   "Mail newsgroups description file.")
85
86 (defvoo nnfolder-get-new-mail t
87   "If non-nil, nnfolder will check the incoming mail file and split the mail.")
88
89 (defvoo nnfolder-prepare-save-mail-hook nil
90   "Hook run narrowed to an article before saving.")
91
92 (defvoo nnfolder-save-buffer-hook nil
93   "Hook run before saving the nnfolder mbox buffer.")
94
95
96 (defvoo nnfolder-inhibit-expiry nil
97   "If non-nil, inhibit expiry.")
98
99 \f
100
101 (defconst nnfolder-version "nnfolder 2.0"
102   "nnfolder version.")
103
104 (defconst nnfolder-article-marker "X-Gnus-Article-Number: "
105   "String used to demarcate what the article number for a message is.")
106
107 (defvoo nnfolder-current-group nil)
108 (defvoo nnfolder-current-buffer nil)
109 (defvoo nnfolder-status-string "")
110 (defvoo nnfolder-group-alist nil)
111 (defvoo nnfolder-buffer-alist nil)
112 (defvoo nnfolder-scantime-alist nil)
113 (defvoo nnfolder-active-timestamp nil)
114 (defvoo nnfolder-active-file-coding-system mm-text-coding-system)
115 (defvoo nnfolder-active-file-coding-system-for-write
116     nnmail-active-file-coding-system)
117 (defvoo nnfolder-file-coding-system mm-text-coding-system)
118 (defvoo nnfolder-file-coding-system-for-write nnheader-file-coding-system
119   "Coding system for save nnfolder file.
120 if nil, `nnfolder-file-coding-system' is used.") ; FIXME: fill-in the doc-string of this variable
121
122 (defvoo nnfolder-nov-is-evil nil
123   "If non-nil, Gnus will never generate and use nov databases for mail groups.
124 Using nov databases will speed up header fetching considerably.
125 This variable shouldn't be flipped much.  If you have, for some reason,
126 set this to t, and want to set it to nil again, you should always run
127 the `nnfolder-generate-active-file' command.  The function will go
128 through all nnfolder directories and generate nov databases for them
129 all.  This may very well take some time.")
130
131 (defvoo nnfolder-nov-file-suffix ".nov")
132
133 (defvoo nnfolder-nov-buffer-alist nil)
134
135 (defvar nnfolder-nov-buffer-file-name nil)
136
137 (defvoo nnfolder-marks-is-evil nil
138   "If non-nil, Gnus will never generate and use marks file for mail groups.
139 Using marks files makes it possible to backup and restore mail groups
140 separately from `.newsrc.eld'.  If you have, for some reason, set
141 this to t, and want to set it to nil again, you should always remove
142 the corresponding marks file (usually base nnfolder file name
143 concatenated with `.mrk', but see `nnfolder-marks-file-suffix') for
144 the group.  Then the marks file will be regenerated properly by Gnus.")
145
146 (defvoo nnfolder-marks nil)
147
148 (defvoo nnfolder-marks-file-suffix ".mrk")
149
150 (defvar nnfolder-marks-modtime (gnus-make-hashtable))
151
152 \f
153
154 ;;; Interface functions
155
156 (nnoo-define-basics nnfolder)
157
158 (deffoo nnfolder-retrieve-headers (articles &optional group server fetch-old)
159   (with-current-buffer nntp-server-buffer
160     (erase-buffer)
161     (let (article start stop num)
162       (nnfolder-possibly-change-group group server)
163       (when nnfolder-current-buffer
164         (set-buffer nnfolder-current-buffer)
165         (goto-char (point-min))
166         (if (stringp (car articles))
167             'headers
168           (if (nnfolder-retrieve-headers-with-nov articles fetch-old)
169               'nov
170             (setq articles (gnus-sorted-intersection
171                             ;; Is ARTICLES sorted?
172                             (sort articles '<)
173                             (nnfolder-existing-articles)))
174             (while (setq article (pop articles))
175               (set-buffer nnfolder-current-buffer)
176               (cond ((nnfolder-goto-article article)
177                      (setq start (point))
178                      (setq stop (if (search-forward "\n\n" nil t)
179                                     (1- (point))
180                                   (point-max)))
181                      (set-buffer nntp-server-buffer)
182                      (insert (format "221 %d Article retrieved.\n" article))
183                      (insert-buffer-substring nnfolder-current-buffer
184                                               start stop)
185                      (goto-char (point-max))
186                      (insert ".\n"))
187
188                     ;; If we couldn't find this article, skip over ranges
189                     ;; of missing articles so we don't search the whole file
190                     ;; for each of them.
191                     ((numberp article)
192                      (setq start (point))
193                      (and
194                       ;; Check that we are either at BOF or after an
195                       ;; article with a lower number.  We do this so we
196                       ;; won't be confused by out-of-order article numbers,
197                       ;; as caused by active file bogosity.
198                       (cond
199                        ((bobp))
200                        ((search-backward (concat "\n" nnfolder-article-marker)
201                                          nil t)
202                         (goto-char (match-end 0))
203                         (setq num (string-to-number
204                                    (buffer-substring
205                                     (point) (point-at-eol))))
206                         (goto-char start)
207                         (< num article)))
208                       ;; Check that we are before an article with a
209                       ;; higher number.
210                       (search-forward (concat "\n" nnfolder-article-marker)
211                                       nil t)
212                       (progn
213                         (setq num (string-to-number
214                                    (buffer-substring
215                                     (point) (point-at-eol))))
216                         (> num article))
217                       ;; Discard any article numbers before the one we're
218                       ;; now looking at.
219                       (while (and articles
220                                   (< (car articles) num))
221                         (setq articles (cdr articles))))
222                      (goto-char start))))
223             (set-buffer nntp-server-buffer)
224             (nnheader-fold-continuation-lines)
225             'headers))))))
226
227 (deffoo nnfolder-open-server (server &optional defs)
228   (nnoo-change-server 'nnfolder server defs)
229   (nnmail-activate 'nnfolder t)
230   (gnus-make-directory nnfolder-directory)
231   (unless (or gnus-nov-is-evil nnfolder-nov-is-evil)
232     (and nnfolder-nov-directory
233          (gnus-make-directory nnfolder-nov-directory)))
234   (unless nnfolder-marks-is-evil
235     (and nnfolder-marks-directory
236          (gnus-make-directory nnfolder-marks-directory)))
237   (cond
238    ((not (file-exists-p nnfolder-directory))
239     (nnfolder-close-server)
240     (nnheader-report 'nnfolder "Couldn't create directory: %s"
241                      nnfolder-directory))
242    ((not (file-directory-p (file-truename nnfolder-directory)))
243     (nnfolder-close-server)
244     (nnheader-report 'nnfolder "Not a directory: %s" nnfolder-directory))
245    (t
246     (nnmail-activate 'nnfolder)
247     (nnheader-report 'nnfolder "Opened server %s using directory %s"
248                      server nnfolder-directory)
249     t)))
250
251 (deffoo nnfolder-request-close ()
252   (let ((alist nnfolder-buffer-alist))
253     (while alist
254       (nnfolder-close-group (caar alist) nil t)
255       (setq alist (cdr alist))))
256   (nnoo-close-server 'nnfolder)
257   (setq nnfolder-buffer-alist nil
258         nnfolder-group-alist nil))
259
260 (deffoo nnfolder-request-article (article &optional group server buffer)
261   (nnfolder-possibly-change-group group server)
262   (with-current-buffer nnfolder-current-buffer
263     (goto-char (point-min))
264     (when (nnfolder-goto-article article)
265       (let (start stop)
266         (setq start (point))
267         (forward-line 1)
268         (unless (and (nnmail-search-unix-mail-delim)
269                      (forward-line -1))
270           (goto-char (point-max)))
271         (setq stop (point))
272         (let ((nntp-server-buffer (or buffer nntp-server-buffer)))
273           (set-buffer nntp-server-buffer)
274           (erase-buffer)
275           (insert-buffer-substring nnfolder-current-buffer start stop)
276           (goto-char (point-min))
277           (while (looking-at "From ")
278             (delete-char 5)
279             (insert "X-From-Line: ")
280             (forward-line 1))
281           (if (numberp article)
282               (cons nnfolder-current-group article)
283             (goto-char (point-min))
284             (cons nnfolder-current-group
285                   (if (search-forward (concat "\n" nnfolder-article-marker)
286                                       nil t)
287                       (string-to-number (buffer-substring
288                                       (point) (point-at-eol)))
289                     -1))))))))
290
291 (deffoo nnfolder-request-group (group &optional server dont-check info)
292   (nnfolder-possibly-change-group group server t)
293   (save-excursion
294     (cond ((not (assoc group nnfolder-group-alist))
295            (nnheader-report 'nnfolder "No such group: %s" group))
296           ((file-directory-p (nnfolder-group-pathname group))
297            (nnheader-report 'nnfolder "%s is a directory"
298                             (file-name-as-directory
299                              (let ((nnmail-pathname-coding-system nil))
300                                (nnfolder-group-pathname group)))))
301           (dont-check
302            (nnheader-report 'nnfolder "Selected group %s" group)
303            t)
304           (t
305            (let* ((active (assoc group nnfolder-group-alist))
306                   (group (car active))
307                   (range (cadr active)))
308              (cond
309               ((null active)
310                (nnheader-report 'nnfolder "No such group: %s" group))
311               ((null nnfolder-current-group)
312                (nnheader-report 'nnfolder "Empty group: %s" group))
313               (t
314                (nnheader-report 'nnfolder "Selected group %s" group)
315                (nnheader-insert "211 %d %d %d %s\n"
316                                 (1+ (- (cdr range) (car range)))
317                                 (car range) (cdr range) group))))))))
318
319 (deffoo nnfolder-request-scan (&optional group server)
320   (nnfolder-possibly-change-group nil server)
321   (when nnfolder-get-new-mail
322     (nnfolder-possibly-change-group group server)
323     (nnmail-get-new-mail
324      'nnfolder 'nnfolder-save-all-buffers
325      nnfolder-directory group)))
326
327 (defun nnfolder-save-all-buffers ()
328   (let ((bufs nnfolder-buffer-alist))
329     (save-excursion
330       (while bufs
331         (if (not (gnus-buffer-live-p (nth 1 (car bufs))))
332             (setq nnfolder-buffer-alist
333                   (delq (car bufs) nnfolder-buffer-alist))
334           (set-buffer (nth 1 (car bufs)))
335           (nnfolder-save-buffer)
336           (kill-buffer (current-buffer)))
337         (setq bufs (cdr bufs))))))
338
339 ;; Don't close the buffer if we're not shutting down the server.  This way,
340 ;; we can keep the buffer in the group buffer cache, and not have to grovel
341 ;; over the buffer again unless we add new mail to it or modify it in some
342 ;; way.
343
344 (deffoo nnfolder-close-group (group &optional server force)
345   ;; Make sure we _had_ the group open.
346   (when (or (assoc group nnfolder-buffer-alist)
347             (equal group nnfolder-current-group))
348     (let ((inf (assoc group nnfolder-buffer-alist)))
349       (when inf
350         (when (and nnfolder-current-group
351                    nnfolder-current-buffer)
352           (push (list nnfolder-current-group nnfolder-current-buffer)
353                 nnfolder-buffer-alist))
354         (setq nnfolder-buffer-alist
355               (delq inf nnfolder-buffer-alist))
356         (setq nnfolder-current-buffer (cadr inf)
357               nnfolder-current-group (car inf))))
358     (when (and nnfolder-current-buffer
359                (buffer-name nnfolder-current-buffer))
360       (with-current-buffer nnfolder-current-buffer
361         ;; If the buffer was modified, write the file out now.
362         (nnfolder-save-buffer)
363         ;; If we're shutting the server down, we need to kill the
364         ;; buffer and remove it from the open buffer list.  Or, of
365         ;; course, if we're trying to minimize our space impact.
366         (kill-buffer (current-buffer))
367         (setq nnfolder-buffer-alist (delq (assoc group nnfolder-buffer-alist)
368                                           nnfolder-buffer-alist)))))
369   (setq nnfolder-current-group nil
370         nnfolder-current-buffer nil)
371   t)
372
373 (deffoo nnfolder-request-create-group (group &optional server args)
374   (nnfolder-possibly-change-group nil server)
375   (nnmail-activate 'nnfolder)
376   (cond ((zerop (length group))
377          (nnheader-report 'nnfolder "Invalid (empty) group name"))
378         ((file-directory-p (nnfolder-group-pathname group))
379          (nnheader-report 'nnfolder "%s is a directory"
380                           (file-name-as-directory
381                            (let ((nnmail-pathname-coding-system nil))
382                              (nnfolder-group-pathname group)))))
383         ((assoc group nnfolder-group-alist)
384          t)
385         (t
386          (push (list group (cons 1 0)) nnfolder-group-alist)
387          (nnfolder-save-active nnfolder-group-alist nnfolder-active-file)
388          (save-current-buffer
389            (nnfolder-read-folder group))
390          t)))
391
392 (deffoo nnfolder-request-list (&optional server)
393   (nnfolder-possibly-change-group nil server)
394   (save-excursion
395     (let ((nnmail-file-coding-system nnfolder-active-file-coding-system))
396       (nnmail-find-file nnfolder-active-file)
397       (setq nnfolder-group-alist (nnmail-get-active)))
398     t))
399
400 (deffoo nnfolder-request-newgroups (date &optional server)
401   (nnfolder-possibly-change-group nil server)
402   (nnfolder-request-list server))
403
404 (deffoo nnfolder-request-list-newsgroups (&optional server)
405   (nnfolder-possibly-change-group nil server)
406   (save-excursion
407     (let ((nnmail-file-coding-system nnfolder-file-coding-system))
408       (nnmail-find-file nnfolder-newsgroups-file))))
409
410 ;; Return a list consisting of all article numbers existing in the
411 ;; current folder.
412
413 (defun nnfolder-existing-articles ()
414   (save-excursion
415     (when nnfolder-current-buffer
416       (set-buffer nnfolder-current-buffer)
417       (goto-char (point-min))
418       (let ((marker (concat "\n" nnfolder-article-marker))
419             (number "[0-9]+")
420             numbers)
421         (while (and (search-forward marker nil t)
422                     (re-search-forward number nil t))
423           (let ((newnum (string-to-number (match-string 0))))
424             (if (nnmail-within-headers-p)
425                 (push newnum numbers))))
426       ;; The article numbers are increasing, so this result is sorted.
427         (nreverse numbers)))))
428
429 (autoload 'gnus-request-group "gnus-int")
430 (declare-function gnus-request-create-group "gnus-int"
431                   (group &optional gnus-command-method args))
432
433 (deffoo nnfolder-request-expire-articles (articles newsgroup
434                                                    &optional server force)
435   (nnfolder-possibly-change-group newsgroup server)
436   (let ((is-old t)
437         ;; The articles we have deleted so far.
438         (deleted-articles nil)
439         ;; The articles that really exist and will
440         ;; be expired if they are old enough.
441         (maybe-expirable
442          (gnus-sorted-intersection articles (nnfolder-existing-articles)))
443         target)
444     (nnmail-activate 'nnfolder)
445
446     (with-current-buffer nnfolder-current-buffer
447       ;; Since messages are sorted in arrival order and expired in the
448       ;; same order, we can stop as soon as we find a message that is
449       ;; too old.
450       (while (and maybe-expirable is-old)
451         (goto-char (point-min))
452         (when (and (nnfolder-goto-article (car maybe-expirable))
453                    (search-forward (concat "\n" nnfolder-article-marker)
454                                    nil t))
455           (forward-sexp)
456           (when (setq is-old
457                       (nnmail-expired-article-p
458                        newsgroup
459                        (buffer-substring
460                         (point) (progn (end-of-line) (point)))
461                        force nnfolder-inhibit-expiry))
462             (setq target nnmail-expiry-target)
463             (unless (eq target 'delete)
464               (with-temp-buffer
465                 (nnfolder-request-article (car maybe-expirable)
466                                           newsgroup server (current-buffer))
467                 (let ((nnfolder-current-directory nil))
468                   (when (functionp target)
469                     (setq target (funcall target newsgroup)))
470                   (when (and target (not (eq target 'delete)))
471                     (if (or (gnus-request-group target)
472                             (gnus-request-create-group target))
473                         (nnmail-expiry-target-group target newsgroup)
474                       (setq target nil)))))
475               (nnfolder-possibly-change-group newsgroup server))
476             (when target
477               (nnheader-message 5 "Deleting article %d in %s..."
478                                 (car maybe-expirable) newsgroup)
479               (nnfolder-delete-mail)
480               (unless (or gnus-nov-is-evil nnfolder-nov-is-evil)
481                 (nnfolder-nov-delete-article newsgroup (car maybe-expirable)))
482               ;; Must remember which articles were actually deleted
483               (push (car maybe-expirable) deleted-articles))))
484         (setq maybe-expirable (cdr maybe-expirable)))
485       (unless nnfolder-inhibit-expiry
486         (nnheader-message 5 "Deleting articles...done"))
487       (nnfolder-save-buffer)
488       (nnfolder-adjust-min-active newsgroup)
489       (nnfolder-save-active nnfolder-group-alist nnfolder-active-file)
490       (nnfolder-save-all-buffers)
491       (gnus-sorted-difference articles (nreverse deleted-articles)))))
492
493 (deffoo nnfolder-request-move-article (article group server accept-form
494                                                &optional last move-is-internal)
495   (save-excursion
496     (let ((buf (get-buffer-create " *nnfolder move*"))
497           result)
498       (and
499        (nnfolder-request-article article group server)
500        (with-current-buffer buf
501          (erase-buffer)
502          (insert-buffer-substring nntp-server-buffer)
503          (goto-char (point-min))
504          (while (re-search-forward
505                  (concat "^" nnfolder-article-marker)
506                  (save-excursion (and (search-forward "\n\n" nil t) (point)))
507                  t)
508            (gnus-delete-line))
509          (setq result (eval accept-form))
510          (kill-buffer buf)
511          result)
512        (save-excursion
513          (nnfolder-possibly-change-group group server)
514          (set-buffer nnfolder-current-buffer)
515          (goto-char (point-min))
516          (when (nnfolder-goto-article article)
517            (nnfolder-delete-mail))
518          (unless (or gnus-nov-is-evil nnfolder-nov-is-evil)
519            (nnfolder-nov-delete-article group article))
520          (when last
521            (nnfolder-save-buffer)
522            (nnfolder-adjust-min-active group)
523            (nnfolder-save-active nnfolder-group-alist nnfolder-active-file))))
524       result)))
525
526 (deffoo nnfolder-request-accept-article (group &optional server last)
527   (save-excursion
528     (nnfolder-possibly-change-group group server)
529     (nnmail-check-syntax)
530     (let ((buf (current-buffer))
531           result art-group)
532       (goto-char (point-min))
533       (when (looking-at "X-From-Line: ")
534         (replace-match "From ")
535         (while (progn (forward-line) (looking-at "[ \t]"))
536           (delete-char -1)))
537       (with-temp-buffer
538         (let ((nnmail-file-coding-system nnfolder-active-file-coding-system)
539               (nntp-server-buffer (current-buffer)))
540           (nnmail-find-file nnfolder-active-file)
541           (setq nnfolder-group-alist (nnmail-parse-active))))
542       (save-excursion
543         (goto-char (point-min))
544         (if (search-forward "\n\n" nil t)
545             (forward-line -1)
546           (goto-char (point-max)))
547         (while (re-search-backward (concat "^" nnfolder-article-marker) nil t)
548           (delete-region (point) (progn (forward-line 1) (point))))
549         (when nnmail-cache-accepted-message-ids
550           (nnmail-cache-insert (nnmail-fetch-field "message-id")
551                                group
552                                (nnmail-fetch-field "subject")
553                                (nnmail-fetch-field "from")))
554         (setq result (if (stringp group)
555                          (list (cons group (nnfolder-active-number group)))
556                        (setq art-group
557                              (nnmail-article-group 'nnfolder-active-number))))
558         (if (and (null result)
559                  (yes-or-no-p "Moved to `junk' group; delete article? "))
560             (setq result 'junk)
561           (setq result
562                 (car (nnfolder-save-mail result)))))
563       (when last
564         (save-excursion
565           (nnfolder-possibly-change-folder (or (caar art-group) group))
566           (nnfolder-save-buffer)
567           (when nnmail-cache-accepted-message-ids
568             (nnmail-cache-close))))
569       (nnfolder-save-active nnfolder-group-alist nnfolder-active-file)
570       (unless result
571         (nnheader-report 'nnfolder "Couldn't store article"))
572       result)))
573
574 (deffoo nnfolder-request-replace-article (article group buffer)
575   (nnfolder-possibly-change-group group)
576   (with-current-buffer buffer
577     (goto-char (point-min))
578     (if (not (looking-at "X-From-Line: "))
579         (insert "From nobody " (current-time-string) "\n")
580       (replace-match "From ")
581       (forward-line 1)
582       (while (looking-at "[ \t]")
583         (delete-char -1)
584         (forward-line 1)))
585     (nnfolder-normalize-buffer)
586     (set-buffer&nb