1 ;;; nnsoup.el --- SOUP access for Gnus
2 ;; Copyright (C) 1995,96 Free Software Foundation, Inc.
4 ;; Author: Lars Magne Ingebrigtsen <larsi@ifi.uio.no>
5 ;; Masanobu UMEDA <umerin@flab.flab.fujitsu.junet>
6 ;; Keywords: news, mail
8 ;; This file is part of GNU Emacs.
10 ;; GNU Emacs is free software; you can redistribute it and/or modify
11 ;; it under the terms of the GNU General Public License as published by
12 ;; the Free Software Foundation; either version 2, or (at your option)
15 ;; GNU Emacs is distributed in the hope that it will be useful,
16 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
17 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 ;; GNU General Public License for more details.
20 ;; You should have received a copy of the GNU General Public License
21 ;; along with GNU Emacs; see the file COPYING. If not, write to the
22 ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
23 ;; Boston, MA 02111-1307, USA.
34 (eval-when-compile (require 'cl))
38 (defvoo nnsoup-directory "~/SOUP/"
39 "*SOUP packet directory.")
41 (defvoo nnsoup-tmp-directory "/tmp/"
42 "*Where nnsoup will store temporary files.")
44 (defvoo nnsoup-replies-directory (concat nnsoup-directory "replies/")
45 "*Directory where outgoing packets will be composed.")
47 (defvoo nnsoup-replies-format-type ?n
48 "*Format of the replies packages.")
50 (defvoo nnsoup-replies-index-type ?n
51 "*Index type of the replies packages.")
53 (defvoo nnsoup-active-file (concat nnsoup-directory "active")
56 (defvoo nnsoup-packer "tar cf - %s | gzip > $HOME/Soupin%d.tgz"
57 "Format string command for packing a SOUP packet.
58 The SOUP files will be inserted where the %s is in the string.
59 This string MUST contain both %s and %d. The file number will be
60 inserted where %d appears.")
62 (defvoo nnsoup-unpacker "gunzip -c %s | tar xvf -"
63 "*Format string command for unpacking a SOUP packet.
64 The SOUP packet file name will be inserted at the %s.")
66 (defvoo nnsoup-packet-directory "~/"
67 "*Where nnsoup will look for incoming packets.")
69 (defvoo nnsoup-packet-regexp "Soupout"
70 "*Regular expression matching SOUP packets in `nnsoup-packet-directory'.")
74 (defconst nnsoup-version "nnsoup 0.0"
77 (defvoo nnsoup-status-string "")
78 (defvoo nnsoup-group-alist nil)
79 (defvoo nnsoup-current-prefix 0)
80 (defvoo nnsoup-replies-list nil)
81 (defvoo nnsoup-buffers nil)
82 (defvoo nnsoup-current-group nil)
83 (defvoo nnsoup-group-alist-touched nil)
87 ;;; Interface functions.
89 (nnoo-define-basics nnsoup)
91 (deffoo nnsoup-retrieve-headers (sequence &optional group server fetch-old)
92 (nnsoup-possibly-change-group group)
94 (set-buffer nntp-server-buffer)
96 (let ((areas (cddr (assoc nnsoup-current-group nnsoup-group-alist)))
99 useful-areas this-area-seq msg-buf)
100 (if (stringp (car sequence))
101 ;; We don't support fetching by Message-ID.
103 ;; We go through all the areas and find which files the
104 ;; articles in SEQUENCE come from.
105 (while (and areas sequence)
106 ;; Peel off areas that are below sequence.
107 (while (and areas (< (cdaar areas) (car sequence)))
108 (setq areas (cdr areas)))
110 ;; This is a useful area.
111 (push (car areas) useful-areas)
112 (setq this-area-seq nil)
113 ;; We take note whether this MSG has a corresponding IDX
115 (when (or (= (gnus-soup-encoding-index
116 (gnus-soup-area-encoding (nth 1 (car areas)))) ?n)
119 (gnus-soup-area-prefix (nth 1 (car areas)))))))
121 ;; We assign the portion of `sequence' that is relevant to
122 ;; this MSG packet to this packet.
123 (while (and sequence (<= (car sequence) (cdaar areas)))
124 (push (car sequence) this-area-seq)
125 (setq sequence (cdr sequence)))
126 (setcar useful-areas (cons (nreverse this-area-seq)
127 (car useful-areas)))))
129 ;; We now have a list of article numbers and corresponding
131 (setq useful-areas (nreverse useful-areas))
133 ;; Two different approaches depending on whether all the MSG
134 ;; files have corresponding IDX files. If they all do, we
135 ;; simply return the relevant IDX files and let Gnus sort out
136 ;; what lines are relevant. If some of the IDX files are
137 ;; missing, we must return HEADs for all the articles.
139 ;; We have IDX files for all areas.
142 (goto-char (point-max))
144 (number (car (nth 1 (car useful-areas))))
145 (index-buffer (nnsoup-index-buffer
146 (gnus-soup-area-prefix
147 (nth 2 (car useful-areas))))))
149 (insert-buffer-substring index-buffer)
151 ;; We have to remove the index number entires and
152 ;; insert article numbers instead.
153 (while (looking-at "[0-9]+")
154 (replace-match (int-to-string number) t t)
157 (setq useful-areas (cdr useful-areas)))
161 (setq articles (caar useful-areas)
162 useful-areas (cdr useful-areas))
165 (nnsoup-narrow-to-article
166 (car articles) (cdar useful-areas) 'head))
167 (goto-char (point-max))
168 (insert (format "221 %d Article retrieved.\n" (car articles)))
169 (insert-buffer-substring msg-buf)
170 (goto-char (point-max))
172 (setq articles (cdr articles))))
174 (nnheader-fold-continuation-lines)
177 (deffoo nnsoup-open-server (server &optional defs)
178 (nnoo-change-server 'nnsoup server defs)
179 (when (not (file-exists-p nnsoup-directory))
181 (make-directory nnsoup-directory t)
184 ((not (file-exists-p nnsoup-directory))
185 (nnsoup-close-server)
186 (nnheader-report 'nnsoup "Couldn't create directory: %s" nnsoup-directory))
187 ((not (file-directory-p (file-truename nnsoup-directory)))
188 (nnsoup-close-server)
189 (nnheader-report 'nnsoup "Not a directory: %s" nnsoup-directory))
191 (nnsoup-read-active-file)
192 (nnheader-report 'nnsoup "Opened server %s using directory %s"
193 server nnsoup-directory)
196 (deffoo nnsoup-request-close ()
197 (nnsoup-write-active-file)
198 (nnsoup-write-replies)
199 (gnus-soup-save-areas)
200 ;; Kill all nnsoup buffers.
202 (while nnsoup-buffers
203 (setq buffer (cdr (pop nnsoup-buffers)))
206 (kill-buffer buffer))))
207 (setq nnsoup-group-alist nil
208 nnsoup-group-alist-touched nil
209 nnsoup-current-group nil
210 nnsoup-replies-list nil)
211 (nnoo-close-server 'nnoo)
214 (deffoo nnsoup-request-article (id &optional newsgroup server buffer)
215 (nnsoup-possibly-change-group newsgroup)
218 (set-buffer (or buffer nntp-server-buffer))
220 (when (and (not (stringp id))
221 (setq buf (nnsoup-narrow-to-article id)))
222 (insert-buffer-substring buf)
225 (deffoo nnsoup-request-group (group &optional server dont-check)
226 (nnsoup-possibly-change-group group)
229 (let ((active (cadr (assoc group nnsoup-group-alist))))
231 (nnheader-report 'nnsoup "No such group: %s" group)
234 (max (1+ (- (cdr active) (car active))) 0)
235 (car active) (cdr active) group)))))
237 (deffoo nnsoup-request-type (group &optional article)
238 (nnsoup-possibly-change-group group)
241 (let ((kind (gnus-soup-encoding-kind
242 (gnus-soup-area-encoding
243 (nth 1 (nnsoup-article-to-area
244 article nnsoup-current-group))))))
245 (cond ((= kind ?m) 'mail)
249 (deffoo nnsoup-close-group (group &optional server)
250 ;; Kill all nnsoup buffers.
251 (let ((buffers nnsoup-buffers)
254 (when (equal (car (setq elem (pop buffers))) group)
255 (setq nnsoup-buffers (delq elem nnsoup-buffers))
256 (and (cdr elem) (buffer-name (cdr elem))
257 (kill-buffer (cdr elem))))))
260 (deffoo nnsoup-request-list (&optional server)
262 (set-buffer nntp-server-buffer)
264 (let ((alist nnsoup-group-alist)
265 (standard-output (current-buffer))
267 (while (setq entry (pop alist))
268 (insert (car entry) " ")
269 (princ (cdadr entry))
271 (princ (caadr entry))
275 (deffoo nnsoup-request-scan (group &optional server)
276 (nnsoup-unpack-packets))
278 (deffoo nnsoup-request-newgroups (date &optional server)
279 (nnsoup-request-list))
281 (deffoo nnsoup-request-list-newsgroups (&optional server)
284 (deffoo nnsoup-request-post (&optional server)
285 (nnsoup-store-reply "news")
288 (deffoo nnsoup-request-mail (&optional server)
289 (nnsoup-store-reply "mail")
292 (deffoo nnsoup-request-expire-articles (articles group &optional server force)
293 (nnsoup-possibly-change-group group)
294 (let* ((total-infolist (assoc group nnsoup-group-alist))
295 (active (cadr total-infolist))
296 (infolist (cddr total-infolist))
297 info range-list mod-time prefix)
299 (setq info (pop infolist)
300 range-list (gnus-uncompress-range (car info))
301 prefix (gnus-soup-area-prefix (nth 1 info)))
302 (when ;; All the articles in this file are marked for expiry.
303 (and (or (setq mod-time (nth 5 (file-attributes
304 (nnsoup-file prefix))))
305 (setq mod-time (nth 5 (file-attributes
306 (nnsoup-file prefix t)))))
307 (gnus-sublist-p articles range-list)
308 ;; This file is old enough.
309 (nnmail-expired-article-p group mod-time force))
310 ;; Ok, we delete this file.
311 (when (condition-case nil
314 5 "Deleting %s in group %s..." (nnsoup-file prefix)
316 (when (file-exists-p (nnsoup-file prefix))
317 (delete-file (nnsoup-file prefix)))
319 5 "Deleting %s in group %s..." (nnsoup-file prefix t)
321 (when (file-exists-p (nnsoup-file prefix t))
322 (delete-file (nnsoup-file prefix t)))
325 (setcdr (cdr total-infolist) (delq info (cddr total-infolist)))
326 (setq articles (gnus-sorted-complement articles range-list))))
328 (setcdr (cdr total-infolist) (delq info (cddr total-infolist)))))
329 (if (cddr total-infolist)
330 (setcar active (caaadr (cdr total-infolist)))
331 (setcar active (1+ (cdr active))))
332 (nnsoup-write-active-file t)
333 ;; Return the articles that weren't expired.
337 ;;; Internal functions
339 (defun nnsoup-possibly-change-group (group &optional force)
341 (setq nnsoup-current-group group)
344 (defun nnsoup-read-active-file ()
345 (setq nnsoup-group-alist)
346 (when (file-exists-p nnsoup-active-file)
348 (load nnsoup-active-file t t t)
350 ;; Be backwards compatible.
351 (when (and nnsoup-group-alist
352 (not (atom (caadar nnsoup-group-alist))))
353 (let ((alist nnsoup-group-alist)
355 (while (setq e (cdr (setq entry (pop alist))))
360 (setcdr entry (cons (cons min max) (cdr entry)))))
361 (setq nnsoup-group-alist-touched t))
364 (defun nnsoup-write-active-file (&optional force)
365 (when (and nnsoup-group-alist
367 nnsoup-group-alist-touched))
368 (setq nnsoup-group-alist-touched nil)
369 (nnheader-temp-write nnsoup-active-file
370 (let ((standard-output (current-buffer)))
371 (prin1 `(setq nnsoup-group-alist ',nnsoup-group-alist))
373 (prin1 `(setq nnsoup-current-prefix ,nnsoup-current-prefix))
376 (defun nnsoup-next-prefix ()
377 "Return the next free prefix."
379 (while (or (file-exists-p
380 (nnsoup-file (setq prefix (int-to-string
381 nnsoup-current-prefix))))
382 (file-exists-p (nnsoup-file prefix t)))
383 (incf nnsoup-current-prefix))
384 (incf nnsoup-current-prefix)
387 (defun nnsoup-read-areas ()
389 (set-buffer nntp-server-buffer)
390 (let ((areas (gnus-soup-parse-areas (concat nnsoup-tmp-directory "AREAS")))
391 entry number area lnum cur-prefix file)
392 ;; Go through all areas in the new AREAS file.
393 (while (setq area (pop areas))
394 ;; Change the name to the permanent name and move the files.
395 (setq cur-prefix (nnsoup-next-prefix))
396 (message "Incorporating file %s..." cur-prefix)
398 (setq file (concat nnsoup-tmp-directory
399 (gnus-soup-area-prefix area) ".IDX")))
400 (rename-file file (nnsoup-file cur-prefix)))
402 (setq file (concat nnsoup-tmp-directory
403 (gnus-soup-area-prefix area) ".MSG")))
404 (rename-file file (nnsoup-file cur-prefix t))
405 (gnus-soup-set-area-prefix area cur-prefix)
406 ;; Find the number of new articles in this area.
407 (setq number (nnsoup-number-of-articles area))
408 (if (not (setq entry (assoc (gnus-soup-area-name area)
409 nnsoup-group-alist)))
410 ;; If this is a new area (group), we just add this info to
412 (push (list (gnus-soup-area-name area)
414 (list (cons 1 number) area))
416 ;; There are already articles in this group, so we add this
417 ;; info to the end of the entry.
418 (nconc entry (list (list (cons (1+ (setq lnum (cdadr entry)))
421 (setcdr (cadr entry) (+ lnum number))))))
422 (nnsoup-write-active-file t)
423 (delete-file (concat nnsoup-tmp-directory "AREAS"))))
425 (defun nnsoup-number-of-articles (area)
428 ;; If the number is in the area info, we just return it.
429 ((gnus-soup-area-number area)
430 (gnus-soup-area-number area))
431 ;; If there is an index file, we just count the lines.
432 ((/= (gnus-soup-encoding-index (gnus-soup-area-encoding area)) ?n)
433 (set-buffer (nnsoup-index-buffer (gnus-soup-area-prefix area)))
434 (count-lines (point-min) (point-max)))
435 ;; We do it the hard way - re-searching through the message
438 (set-buffer (nnsoup-message-buffer (gnus-soup-area-prefix area)))
439 (goto-char (point-min))
440 (let ((regexp (nnsoup-header (gnus-soup-encoding-format
441 (gnus-soup-area-encoding area))))
443 (while (re-search-forward regexp nil t)
447 (defun nnsoup-index-buffer (prefix &optional message)
448 (let* ((file (concat prefix (if message ".MSG" ".IDX")))
449 (buffer-name (concat " *nnsoup " file "*")))
450 (or (get-buffer buffer-name) ; File aready loaded.
451 (when (file-exists-p (concat nnsoup-directory file))
452 (save-excursion ; Load the file.
453 (set-buffer (get-buffer-create buffer-name))
454 (buffer-disable-undo (current-buffer))
455 (push (cons nnsoup-current-group (current-buffer)) nnsoup-buffers)
456 (insert-file-contents (concat nnsoup-directory file))
457 (current-buffer))))))
459 (defun nnsoup-file (prefix &optional message)
461 (concat nnsoup-directory prefix (if message ".MSG" ".IDX"))))
463 (defun nnsoup-message-buffer (prefix)
464 (nnsoup-index-buffer prefix 'msg))
466 (defun nnsoup-unpack-packets ()
467 "Unpack all packets in `nnsoup-packet-directory'."
468 (let ((packets (directory-files
469 nnsoup-packet-directory t nnsoup-packet-regexp))
471 (while (setq packet (pop packets))
472 (message (format "nnsoup: unpacking %s..." packet))
473 (if (not (gnus-soup-unpack-packet
474 nnsoup-tmp-directory nnsoup-unpacker packet))
475 (message "Couldn't unpack %s" packet)
478 (message "Unpacking...done")))))
480 (defun nnsoup-narrow-to-article (article &optional area head)
481 (let* ((area (or area (nnsoup-article-to-area article nnsoup-current-group)))
482 (prefix (and area (gnus-soup-area-prefix (nth 1 area))))
483 (msg-buf (and prefix (nnsoup-index-buffer prefix 'msg)))
488 ;; There is no MSG file.
492 ;; We use the index file to find out where the article begins and ends.
493 ((and (= (gnus-soup-encoding-index
494 (gnus-soup-area-encoding (nth 1 area)))
496 (file-exists-p (nnsoup-file prefix)))
497 (set-buffer (nnsoup-index-buffer prefix))
499 (goto-char (point-min))
500 (forward-line (- article (caar area)))
501 (setq beg (read (current-buffer)))
503 (if (looking-at "[0-9]+")
505 (setq end (read (current-buffer)))
508 (let ((format (gnus-soup-encoding-format
509 (gnus-soup-area-encoding (nth 1 area)))))
511 (if (or (= format ?n) (= format ?m))
512 (setq end (progn (forward-line -1) (point))))))
513 (set-buffer msg-buf))
515 (narrow-to-region beg (or end (point-max))))
519 (goto-char (point-min))
520 (let ((header (nnsoup-header
521 (gnus-soup-encoding-format
522 (gnus-soup-area-encoding (nth 1 area))))))
523 (re-search-forward header nil t (- article (caar area)))
526 (if (re-search-forward header nil t)
529 (goto-char (point-min))
534 (if (search-forward "\n\n" nil t)
539 (defun nnsoup-header (format)
542 "^#! *rnews +[0-9]+ *$")
544 (concat "^" rmail-unix-mail-delimiter))
548 (error "Unknown format: %c" format))))
551 (defun nnsoup-pack-replies ()
552 "Make an outbound package of SOUP replies."
554 ;; Write all data buffers.
555 (gnus-soup-save-areas)
556 ;; Write the active file.
557 (nnsoup-write-active-file)
558 ;; Write the REPLIES file.
559 (nnsoup-write-replies)
560 ;; Pack all these files into a SOUP packet.
561 (gnus-soup-pack nnsoup-replies-directory nnsoup-packer))
563 (defun nnsoup-write-replies ()
564 "Write the REPLIES file."
565 (when nnsoup-replies-list
566 (gnus-soup-write-replies nnsoup-replies-directory nnsoup-replies-list)
567 (setq nnsoup-replies-list nil)))
569 (defun nnsoup-article-to-area (article group)
570 "Return the area that ARTICLE in GROUP is located in."
571 (let ((areas (cddr (assoc group nnsoup-group-alist))))
572 (while (and areas (< (cdaar areas) article))
573 (setq areas (cdr areas)))
574 (and areas (car areas))))
576 (defvar nnsoup-old-functions
577 (list message-send-mail-function message-send-news-function))
580 (defun nnsoup-set-variables ()
581 "Use the SOUP methods for posting news and mailing mail."
583 (setq message-send-news-function 'nnsoup-request-post)
584 (setq message-send-mail-function 'nnsoup-request-mail))
587 (defun nnsoup-revert-variables ()
588 "Revert posting and mailing methods to the standard Emacs methods."
590 (setq message-send-mail-function (car nnsoup-old-functions))
591 (setq message-send-news-function (cadr nnsoup-old-functions)))
593 (defun nnsoup-store-reply (kind)
594 ;; Mostly stolen from `message.el'.
595 (require 'mail-utils)
596 (let ((tembuf (generate-new-buffer " message temp"))
597 (case-fold-search nil)
598 (news (message-news-p))
599 (resend-to-addresses (mail-fetch-field "resent-to"))
601 (mailbuf (current-buffer)))
606 (insert-buffer-substring mailbuf)
607 ;; Remove some headers.
609 (message-narrow-to-headers)
610 ;; Remove some headers.
611 (message-remove-header message-ignored-mail-headers t)
612 (if (equal kind "mail")
613 (message-generate-headers message-required-mail-headers)
614 (message-generate-headers message-required-news-headers)))
615 (goto-char (point-max))
616 ;; require one newline at the end.
617 (or (= (preceding-char) ?\n)
621 (or (mail-fetch-field "cc")
622 (mail-fetch-field "to")))
623 (message-insert-courtesy-copy))
624 (let ((case-fold-search t))
625 ;; Change header-delimiter to be what sendmail expects.
626 (goto-char (point-min))
628 (concat "^" (regexp-quote mail-header-separator) "\n"))
631 (setq delimline (point-marker))
632 ;; Insert an extra newline if we need it to work around
633 ;; Sun's bug that swallows newlines.
634 (goto-char (1+ delimline))
635 (when (eval message-mailer-swallows-blank-line)
639 nnsoup-replies-directory
640 (nnsoup-kind-to-prefix kind) nil nnsoup-replies-format-type
641 nnsoup-replies-index-type))
643 (when (and msg-buf (bufferp msg-buf))
646 (goto-char (point-min))
647 (while (re-search-forward "^#! *rnews" nil t)
649 (message "Stored %d messages" num)))
650 (nnsoup-write-replies)
651 (kill-buffer tembuf))))))
653 (defun nnsoup-kind-to-prefix (kind)
654 (unless nnsoup-replies-list
655 (setq nnsoup-replies-list
656 (gnus-soup-parse-replies
657 (concat nnsoup-replies-directory "REPLIES"))))
658 (let ((replies nnsoup-replies-list))
660 (not (string= kind (gnus-soup-reply-kind (car replies)))))
661 (setq replies (cdr replies)))
663 (gnus-soup-reply-prefix (car replies))
664 (setq nnsoup-replies-list
665 (cons (vector (gnus-soup-unique-prefix nnsoup-replies-directory)
668 nnsoup-replies-format-type
669 nnsoup-replies-index-type
670 (if (string= kind "news")
672 nnsoup-replies-list))
673 (gnus-soup-reply-prefix (car nnsoup-replies-list)))))
675 (defun nnsoup-make-active ()
676 "(Re-)create the SOUP active file."
678 (let ((files (sort (directory-files nnsoup-directory t "IDX$")
680 (< (progn (string-match "/\\([0-9]+\\)\\." f1)
681 (string-to-int (match-string 1 f1)))
682 (progn (string-match "/\\([0-9]+\\)\\." f2)
683 (string-to-int (match-string 1 f2)))))))
684 active group lines ident elem min)
685 (set-buffer (get-buffer-create " *nnsoup work*"))
686 (buffer-disable-undo (current-buffer))
688 (message "Doing %s..." (car files))
690 (insert-file-contents (car files))
691 (goto-char (point-min))
692 (if (not (re-search-forward "^[^\t]*\t[^\t]*\t[^\t]*\t[^\t]*\t[^\t]*\t[^\t]*\t[^\t]*\t[^\t]*\t *\\(Xref: \\)? *[^ ]* \\([^ ]+\\):[0-9]" nil t))
693 (setq group "unknown")
694 (setq group (match-string 2)))
695 (setq lines (count-lines (point-min) (point-max)))
696 (setq ident (progn (string-match
697 "/\\([0-9]+\\)\\." (car files))
699 (car files) (match-beginning 1)
701 (if (not (setq elem (assoc group active)))
702 (push (list group (cons 1 lines)
704 (vector ident group "ncm" "" lines)))
708 (list (cons (1+ (setq min (cdadr elem)))
710 (vector ident group "ncm" "" lines))))
711 (setcdr (cadr elem) (+ min lines)))
712 (setq files (cdr files)))
714 (setq nnsoup-group-alist active)
715 (nnsoup-write-active-file t)))
717 (defun nnsoup-delete-unreferenced-message-files ()
718 "Delete any *.MSG and *.IDX files that aren't known by nnsoup."
720 (let* ((known (apply 'nconc (mapcar
724 (gnus-soup-area-prefix (cadr area)))
726 nnsoup-group-alist)))
727 (regexp "\\.MSG$\\|\\.IDX$")
728 (files (directory-files nnsoup-directory nil regexp))
730 ;; Find all files that aren't known by nnsoup.
731 (while (setq file (pop files))
732 (string-match regexp file)
733 (unless (member (substring file 0 (match-beginning 0)) known)
734 (push file non-files)))
735 ;; Sort and delete the files.
736 (setq non-files (sort non-files 'string<))
737 (map-y-or-n-p "Delete file %s? "
738 (lambda (file) (delete-file (concat nnsoup-directory file)))
743 ;;; nnsoup.el ends here