1 ;;; nnvirtual.el --- virtual newsgroups access for Gnus
2 ;; Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001
3 ;; Free Software Foundation, Inc.
5 ;; Author: David Moore <dmoore@ucsd.edu>
6 ;; Lars Magne Ingebrigtsen <larsi@gnus.org>
7 ;; Masanobu UMEDA <umerin@flab.flab.fujitsu.junet>
10 ;; This file is part of GNU Emacs.
12 ;; GNU Emacs is free software; you can redistribute it and/or modify
13 ;; it under the terms of the GNU General Public License as published by
14 ;; the Free Software Foundation; either version 2, or (at your option)
17 ;; GNU Emacs is distributed in the hope that it will be useful,
18 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
19 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 ;; GNU General Public License for more details.
22 ;; You should have received a copy of the GNU General Public License
23 ;; along with GNU Emacs; see the file COPYING. If not, write to the
24 ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
25 ;; Boston, MA 02111-1307, USA.
29 ;; The other access methods (nntp, nnspool, etc) are general news
30 ;; access methods. This module relies on Gnus and can not be used
43 (eval-when-compile (require 'cl))
45 (nnoo-declare nnvirtual)
47 (defvoo nnvirtual-always-rescan t
48 "*If non-nil, always scan groups for unread articles when entering a group.
49 If this variable is nil and you read articles in a component group
50 after the virtual group has been activated, the read articles from the
51 component group will show up when you enter the virtual group.")
53 (defvoo nnvirtual-component-regexp nil
54 "*Regexp to match component groups.")
56 (defvoo nnvirtual-component-groups nil
57 "Component group in this nnvirtual group.")
61 (defconst nnvirtual-version "nnvirtual 1.1")
63 (defvoo nnvirtual-current-group nil)
65 (defvoo nnvirtual-mapping-table nil
66 "Table of rules on how to map between component group and article number to virtual article number.")
68 (defvoo nnvirtual-mapping-offsets nil
69 "Table indexed by component group to an offset to be applied to article numbers in that group.")
71 (defvoo nnvirtual-mapping-len 0
72 "Number of articles in this virtual group.")
74 (defvoo nnvirtual-mapping-reads nil
75 "Compressed sequence of read articles on the virtual group as computed from the unread status of individual component groups.")
77 (defvoo nnvirtual-mapping-marks nil
78 "Compressed marks alist for the virtual group as computed from the marks of individual component groups.")
80 (defvoo nnvirtual-info-installed nil
81 "T if we have already installed the group info for this group, and shouldn't blast over it again.")
83 (defvoo nnvirtual-status-string "")
86 (autoload 'gnus-cache-articles-in-group "gnus-cache"))
90 ;;; Interface functions.
92 (nnoo-define-basics nnvirtual)
95 (deffoo nnvirtual-retrieve-headers (articles &optional newsgroup
97 (when (nnvirtual-possibly-change-server server)
99 (set-buffer nntp-server-buffer)
101 (if (stringp (car articles))
103 (let ((vbuf (nnheader-set-temp-buffer
104 (get-buffer-create " *virtual headers*")))
105 (carticles (nnvirtual-partition-sequence articles))
106 (system-name (system-name))
107 cgroup carticle article result prefix)
109 (setq cgroup (caar carticles))
110 (setq articles (cdar carticles))
114 (gnus-find-method-for-group cgroup) t)
115 (gnus-request-group cgroup t)
116 (setq prefix (gnus-group-real-prefix cgroup))
117 ;; FIX FIX FIX we want to check the cache!
118 ;; This is probably evil if people have set
119 ;; gnus-use-cache to nil themselves, but I
120 ;; have no way of finding the true value of it.
121 (let ((gnus-use-cache t))
122 (setq result (gnus-retrieve-headers
123 articles cgroup nil))))
124 (set-buffer nntp-server-buffer)
125 ;; If we got HEAD headers, we convert them into NOV
126 ;; headers. This is slow, inefficient and, come to think
127 ;; of it, downright evil. So sue me. I couldn't be
128 ;; bothered to write a header parse routine that could
129 ;; parse a mixed HEAD/NOV buffer.
130 (when (eq result 'headers)
131 (nnvirtual-convert-headers))
132 (goto-char (point-min))
134 (delete-region (point)
136 (setq carticle (read nntp-server-buffer))
139 ;; We remove this article from the articles list, if
140 ;; anything is left in the articles list after going through
141 ;; the entire buffer, then those articles have been
142 ;; expired or canceled, so we appropriately update the
143 ;; component group below. They should be coming up
144 ;; generally in order, so this shouldn't be slow.
145 (setq articles (delq carticle articles))
147 (setq article (nnvirtual-reverse-map-article cgroup carticle))
149 ;; This line has no reverse mapping, that means it
150 ;; was an extra article reference returned by nntp.
153 (delete-region (point) (progn (forward-line 1) (point))))
154 ;; Otherwise insert the virtual article number,
155 ;; and clean up the xrefs.
156 (princ article nntp-server-buffer)
157 (nnvirtual-update-xref-header cgroup carticle
163 (goto-char (point-max))
164 (insert-buffer-substring nntp-server-buffer))
165 ;; Anything left in articles is expired or canceled.
166 ;; Could be smart and not tell it about articles already known?
168 (gnus-group-make-articles-read cgroup articles))
171 ;; The headers are ready for reading, so they are inserted into
172 ;; the nntp-server-buffer, which is where Gnus expects to find
176 (set-buffer nntp-server-buffer)
178 (insert-buffer-substring vbuf)
179 ;; FIX FIX FIX, we should be able to sort faster than
180 ;; this if needed, since each cgroup is sorted, we just
182 (sort-numeric-fields 1 (point-min) (point-max))
184 (kill-buffer vbuf)))))))
187 (defvoo nnvirtual-last-accessed-component-group nil)
189 (deffoo nnvirtual-request-article (article &optional group server buffer)
190 (when (nnvirtual-possibly-change-server server)
191 (if (stringp article)
192 ;; This is a fetch by Message-ID.
194 ((not nnvirtual-last-accessed-component-group)
196 'nnvirtual "Don't know what server to request from"))
201 (let* ((gnus-override-method nil)
202 (method (gnus-find-method-for-group
203 nnvirtual-last-accessed-component-group)))
204 (funcall (gnus-get-function method 'request-article)
205 article nil (nth 1 method) buffer)))))
206 ;; This is a fetch by number.
207 (let* ((amap (nnvirtual-map-article article))
211 (nnheader-report 'nnvirtual "No such article: %s" article))
212 ((not (gnus-check-group cgroup))
214 'nnvirtual "Can't open server where %s exists" cgroup))
215 ((not (gnus-request-group cgroup t))
216 (nnheader-report 'nnvirtual "Can't open component group %s" cgroup))
218 (setq nnvirtual-last-accessed-component-group cgroup)
222 ;; We bind this here to avoid double decoding.
223 (let ((gnus-article-decode-hook nil))
224 (gnus-request-article-this-buffer (cdr amap) cgroup)))
225 (gnus-request-article (cdr amap) cgroup))))))))
228 (deffoo nnvirtual-open-server (server &optional defs)
229 (unless (assq 'nnvirtual-component-regexp defs)
230 (push `(nnvirtual-component-regexp ,server)
232 (nnoo-change-server 'nnvirtual server defs)
233 (if nnvirtual-component-groups
235 (setq nnvirtual-mapping-table nil
236 nnvirtual-mapping-offsets nil
237 nnvirtual-mapping-len 0
238 nnvirtual-mapping-reads nil
239 nnvirtual-mapping-marks nil
240 nnvirtual-info-installed nil)
241 (when nnvirtual-component-regexp
242 ;; Go through the newsrc alist and find all component groups.
243 (let ((newsrc (cdr gnus-newsrc-alist))
245 (while (setq group (car (pop newsrc)))
246 (when (string-match nnvirtual-component-regexp group) ; Match
247 ;; Add this group to the list of component groups.
248 (setq nnvirtual-component-groups
249 (cons group (delete group nnvirtual-component-groups)))))))
250 (if (not nnvirtual-component-groups)
251 (nnheader-report 'nnvirtual "No component groups: %s" server)
255 (deffoo nnvirtual-request-group (group &optional server dont-check)
256 (nnvirtual-possibly-change-server server)
257 (setq nnvirtual-component-groups
258 (delete (nnvirtual-current-group) nnvirtual-component-groups))
260 ((null nnvirtual-component-groups)
261 (setq nnvirtual-current-group nil)
262 (nnheader-report 'nnvirtual "No component groups in %s" group))
264 (setq nnvirtual-current-group group)
265 (when (or (not dont-check)
266 nnvirtual-always-rescan)
267 (nnvirtual-create-mapping)
268 (when nnvirtual-always-rescan
269 (nnvirtual-request-update-info
270 (nnvirtual-current-group)
271 (gnus-get-info (nnvirtual-current-group)))))
272 (nnheader-insert "211 %d 1 %d %s\n"
273 nnvirtual-mapping-len nnvirtual-mapping-len group))))
276 (deffoo nnvirtual-request-type (group &optional article)
279 (if (numberp article)
280 (let ((mart (nnvirtual-map-article article)))
282 (gnus-request-type (car mart) (cdr mart))))
284 nnvirtual-last-accessed-component-group nil))))
286 (deffoo nnvirtual-request-update-mark (group article mark)
287 (let* ((nart (nnvirtual-map-article article))
290 (memq mark gnus-auto-expirable-marks)
291 ;; The component group might be a virtual group.
292 (= mark (gnus-request-update-mark cgroup (cdr nart) mark))
293 (gnus-group-auto-expirable-p cgroup))
294 (setq mark gnus-expirable-mark)))
298 (deffoo nnvirtual-close-group (group &optional server)
299 (when (and (nnvirtual-possibly-change-server server)
300 (not (gnus-ephemeral-group-p (nnvirtual-current-group))))
301 (nnvirtual-update-read-and-marked t t))
305 (deffoo nnvirtual-request-list (&optional server)
306 (nnheader-report 'nnvirtual "LIST is not implemented."))
309 (deffoo nnvirtual-request-newgroups (date &optional server)
310 (nnheader-report 'nnvirtual "NEWGROUPS is not supported."))
313 (deffoo nnvirtual-request-list-newsgroups (&optional server)
314 (nnheader-report 'nnvirtual "LIST NEWSGROUPS is not implemented."))
317 (deffoo nnvirtual-request-update-info (group info &optional server)
318 (when (and (nnvirtual-possibly-change-server server)
319 (not nnvirtual-info-installed))
320 ;; Install the precomputed lists atomically, so the virtual group
321 ;; is not left in a half-way state in case of C-g.
323 (setcar (cddr info) nnvirtual-mapping-reads)
325 (setcar (nthcdr 3 info) nnvirtual-mapping-marks)
326 (when nnvirtual-mapping-marks
327 (setcdr (nthcdr 2 info) (list nnvirtual-mapping-marks))))
328 (setq nnvirtual-info-installed t))
332 (deffoo nnvirtual-catchup-group (group &optional server all)
333 (when (and (nnvirtual-possibly-change-server server)
334 (not (gnus-ephemeral-group-p (nnvirtual-current-group))))
335 ;; copy over existing marks first, in case they set anything
336 (nnvirtual-update-read-and-marked nil nil)
337 ;; do a catchup on all component groups
338 (let ((gnus-group-marked (copy-sequence nnvirtual-component-groups))
339 (gnus-expert-user t))
340 ;; Make sure all groups are activated.
343 (when (not (numberp (car (gnus-gethash g gnus-newsrc-hashtb))))
344 (gnus-activate-group g)))
345 nnvirtual-component-groups)
347 (set-buffer gnus-group-buffer)
348 (gnus-group-catchup-current nil all)))))
351 (deffoo nnvirtual-find-group-art (group article)
352 "Return the real group and article for virtual GROUP and ARTICLE."
353 (nnvirtual-map-article article))
356 (deffoo nnvirtual-request-post (&optional server)
357 (if (not gnus-message-group-art)
358 (nnheader-report 'nnvirtual "Can't post to an nnvirtual group")
359 (let ((group (car (nnvirtual-find-group-art
360 (car gnus-message-group-art)
361 (cdr gnus-message-group-art)))))
362 (gnus-request-post (gnus-find-method-for-group group)))))
365 (deffoo nnvirtual-request-expire-articles (articles group
366 &optional server force)
367 (nnvirtual-possibly-change-server server)
368 (setq nnvirtual-component-groups
369 (delete (nnvirtual-current-group) nnvirtual-component-groups))
371 (dolist (group nnvirtual-component-groups)
372 (setq unexpired (nconc unexpired
375 (nnvirtual-reverse-map-article
377 (gnus-uncompress-range
378 (gnus-group-expire-articles-1 group))))))
379 (sort (delq nil unexpired) '<)))
382 ;;; Internal functions.
384 (defun nnvirtual-convert-headers ()
385 "Convert HEAD headers into NOV headers."
387 (set-buffer nntp-server-buffer)
388 (let* ((dependencies (make-vector 100 0))
389 (headers (gnus-get-newsgroup-headers dependencies))
392 (while (setq header (pop headers))
393 (nnheader-insert-nov header)))))
396 (defun nnvirtual-update-xref-header (group article prefix system-name)
397 "Edit current NOV header in current buffer to have an xref to the component group, and also server prefix any existing xref lines."
398 ;; Move to beginning of Xref field, creating a slot if needed.
401 "[^\t]*\t[^\t]*\t[^\t]*\t[^\t]*\t[^\t]*\t[^\t]*\t[^\t]*\t")
402 (goto-char (match-end 0))
403 (unless (search-forward "\t" (gnus-point-at-eol) 'move)
406 ;; Remove any spaces at the beginning of the Xref field.
407 (while (eq (char-after (1- (point))) ? )
411 (insert "Xref: " system-name " " group ":")
412 (princ article (current-buffer))
415 ;; If there were existing xref lines, clean them up to have the correct
416 ;; component server prefix.
418 (narrow-to-region (point)
419 (or (search-forward "\t" (gnus-point-at-eol) t)
420 (gnus-point-at-eol)))
421 (goto-char (point-min))
422 (when (re-search-forward "Xref: *[^\n:0-9 ]+ *" nil t)
423 (replace-match "" t t))
424 (goto-char (point-min))
425 (when (re-search-forward
426 (concat (regexp-quote (gnus-group-real-name group)) ":[0-9]+")
428 (replace-match "" t t))
429 (unless (= (point) (point-max))
431 (when (not (string= "" prefix))
432 (while (re-search-forward "[^ ]+:[0-9]+" nil t)
434 (goto-char (match-beginning 0))
437 ;; Ensure a trailing \t.
439 (or (eq (char-after (1- (point))) ?\t)
443 (defun nnvirtual-possibly-change-server (server)
445 (nnoo-current-server-p 'nnvirtual server)
446 (nnvirtual-open-server server)))
449 (defun nnvirtual-update-read-and-marked (read-p update-p)
450 "Copy marks from the virtual group to the component groups.
451 If READ-P is not nil, update the (un)read status of the components.
452 If UPDATE-P is not nil, call gnus-group-update-group on the components."
453 (when nnvirtual-current-group
454 (let ((unreads (and read-p
455 (nnvirtual-partition-sequence
456 (gnus-list-of-unread-articles
457 (nnvirtual-current-group)))))
461 (if (eq (car ml) 'score)
464 (nnvirtual-partition-sequence (cdr ml)))))
465 (gnus-info-marks (gnus-get-info
466 (nnvirtual-current-group))))))
467 mark type groups carticles info entry)
469 ;; Ok, atomically move all of the (un)read info, clear any old
470 ;; marks, and move all of the current marks. This way if someone
471 ;; hits C-g, you won't leave the component groups in a half-way state.
474 ;; bind for workaround guns-update-read-articles
475 (let ((gnus-newsgroup-active nil))
476 (while (setq entry (pop unreads))
477 (gnus-update-read-articles (car entry) (cdr entry))))
479 ;; clear all existing marks on the component groups
480 (setq groups nnvirtual-component-groups)
482 (when (and (setq info (gnus-get-info (pop groups)))
483 (gnus-info-marks info))
486 (if (assq 'score (gnus-info-marks info))
487 (list (assq 'score (gnus-info-marks info)))
490 ;; Ok, currently type-marks is an assq list with keys of a mark type,
491 ;; with data of an assq list with keys of component group names
492 ;; and the articles which correspond to that key/group pair.
493 (while (setq mark (pop type-marks))
494 (setq type (car mark))
495 (setq groups (cdr mark))
496 (while (setq carticles (pop groups))
497 (gnus-add-marked-articles (car carticles) type (cdr carticles)
500 ;; possibly update the display, it is really slow
502 (setq groups nnvirtual-component-groups)
504 (gnus-group-update-group (pop groups) t))))))
507 (defun nnvirtual-current-group ()
508 "Return the prefixed name of the current nnvirtual group."
509 (concat "nnvirtual:" nnvirtual-current-group))
513 ;;; This is currently O(kn^2) to merge n lists of length k.
514 ;;; You could do it in O(knlogn), but we have a small n, and the
515 ;;; overhead of the other approach is probably greater.
516 (defun nnvirtual-merge-sorted-lists (&rest lists)
517 "Merge many sorted lists of numbers."
518 (if (null (cdr lists))
520 (sort (apply 'nconc lists) '<)))
523 ;;; We map between virtual articles and real articles in a manner
524 ;;; which keeps the size of the virtual active list the same as
525 ;;; the sum of the component active lists.
526 ;;; To achieve fair mixing of the groups, the last article in
527 ;;; each of N component groups will be in the the last N articles
528 ;;; in the virtual group.
530 ;;; If you have 3 components A, B and C, with articles 1-8, 1-5, and 6-7
531 ;;; resprectively, then the virtual article numbers look like:
533 ;;; 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
534 ;;; A1 A2 A3 A4 B1 A5 B2 A6 B3 A7 B4 C6 A8 B5 C7
536 ;;; To compute these mappings we generate a couple tables and then
537 ;;; do some fast operations on them. Tables for the example above:
539 ;;; Offsets - [(A 0) (B -3) (C -1)]
542 ;;; Mapping - ([ 3 0 1 3 0 ]
546 ;;; (note column 'e' is different in real algorithm, which is slightly
547 ;;; different than described here, but this gives you the methodology.)
549 ;;; The basic idea is this, when going from component->virtual, apply
550 ;;; the appropriate offset to the article number. Then search the first
551 ;;; column of the table for a row where 'a' is less than or equal to the
552 ;;; modified number. You can see that only group A can therefore go to
553 ;;; the first row, groups A and B to the second, and all to the last.
554 ;;; The third column of the table is telling us the number of groups
555 ;;; which might be able to reach that row (it might increase by more than
556 ;;; 1 if several groups have the same size).
557 ;;; Then column 'b' provides an additional offset you apply when you have
558 ;;; found the correct row. You then multiply by 'c' and add on the groups
559 ;;; _position_ in the offset table. The basic idea here is that on
560 ;;; any given row we are going to map back and forth using X'=X*c+Y and
561 ;;; X=(X'/c), Y=(X' mod c). Then once you've done this transformation,
562 ;;; you apply a final offset from column 'e' to give the virtual article.
564 ;;; Going the other direction, you instead search on column 'd' instead
565 ;;; of 'a', and apply everything in reverse order.
567 ;;; Convert component -> virtual:
568 ;;; set num = num - Offset(group)
569 ;;; find first row in Mapping where num <= 'a'
570 ;;; num = (num-'b')*c + Position(group) + 'e'
572 ;;; Convert virtual -> component:
573 ;;; find first row in Mapping where num <= 'd'
575 ;;; group_pos = num mod 'c'
576 ;;; num = (num / 'c') + 'b' + Offset(group_pos)
580 ;;; Well actually, you need to keep column e offset smaller by the 'c'
581 ;;; column for that line, and always add 1 more when going from
582 ;;; component -> virtual. Otherwise you run into a problem with
583 ;;; unique reverse mapping.
585 (defun nnvirtual-map-article (article)
586 "Return a cons of the component group and article corresponding to the given virtual ARTICLE."
587 (let ((table nnvirtual-mapping-table)
590 (> article (aref (car table) 3)))
591 (setq table (cdr table)))
594 (setq entry (car table))
595 (setq article (- article (aref entry 4) 1))
596 (setq group-pos (mod article (aref entry 2)))
597 (cons (car (aref nnvirtual-mapping-offsets group-pos))
598 (+ (/ article (aref entry 2))
600 (cdr (aref nnvirtual-mapping-offsets group-pos)))
606 (defun nnvirtual-reverse-map-article (group article)
607 "Return the virtual article number corresponding to the given component GROUP and ARTICLE."
608 (when (numberp article)
609 (let ((table nnvirtual-mapping-table)
612 (while (not (string= group (car (aref nnvirtual-mapping-offsets
614 (setq group-pos (1+ group-pos)))
615 (setq article (- article (cdr (aref nnvirtual-mapping-offsets
618 (> article (aref (car table) 0)))
619 (setq table (cdr table)))
620 (setq entry (car table))
623 (< group-pos (aref entry 2))) ; article not out of range below
626 (* (- article (aref entry 1))
632 (defsubst nnvirtual-reverse-map-sequence (group articles)
633 "Return list of virtual article numbers for all ARTICLES in GROUP.
634 The ARTICLES should be sorted, and can be a compressed sequence.
635 If any of the article numbers has no corresponding virtual article,
636 then it is left out of the result."
637 (when (numberp (cdr-safe articles))
638 (setq articles (list articles)))
639 (let (result a i j new-a)
640 (while (setq a (pop articles))
647 ;; If this is slow, you can optimize by moving article checking
648 ;; into here. You don't have to recompute the group-pos,
649 ;; nor scan the table every time.
650 (when (setq new-a (nnvirtual-reverse-map-article group i))
656 (defun nnvirtual-partition-sequence (articles)
657 "Return an association list of component article numbers.
658 These are indexed by elements of nnvirtual-component-groups, based on
659 the sequence ARTICLES of virtual article numbers. ARTICLES should be
660 sorted, and can be a compressed sequence. If any of the article
661 numbers has no corresponding component article, then it is left out of
663 (when (numberp (cdr-safe articles))
664 (setq articles (list articles)))
665 (let ((carticles (mapcar (lambda (g) (list g))
666 nnvirtual-component-groups))
668 (while (setq a (pop articles))
675 (when (setq article (nnvirtual-map-article i))
676 (setq entry (assoc (car article) carticles))
677 (setcdr entry (cons (cdr article) (cdr entry))))
679 (mapcar (lambda (x) (setcdr x (nreverse (cdr x))))