;;; nnweb.el --- retrieving articles via web search engines
-;; Copyright (C) 1996 Free Software Foundation, Inc.
-;; Author: Lars Magne Ingebrigtsen <larsi@ifi.uio.no>
+;; Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
+;; 2004, 2005, 2006 Free Software Foundation, Inc.
+
+;; Author: Lars Magne Ingebrigtsen <larsi@gnus.org>
;; Keywords: news
;; This file is part of GNU Emacs.
;; You should have received a copy of the GNU General Public License
;; along with GNU Emacs; see the file COPYING. If not, write to the
-;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-;; Boston, MA 02111-1307, USA.
+;; Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+;; Boston, MA 02110-1301, USA.
;;; Commentary:
-;; Note: You need to have `url' and `w3' installed for this
-;; backend to work.
+;; Note: You need to have `w3' installed for some functions to work.
;;; Code:
+(eval-when-compile (require 'cl))
+
(require 'nnoo)
(require 'message)
(require 'gnus-util)
-(require 'w3)
-(require 'url)
-(ignore-errors
- (require 'w3-forms))
+(require 'gnus)
+(require 'nnmail)
+(require 'mm-util)
+(require 'mm-url)
+(eval-and-compile
+ (ignore-errors
+ (require 'url)))
+(autoload 'w3-parse-buffer "w3-parse")
(nnoo-declare nnweb)
(defvoo nnweb-directory (nnheader-concat gnus-directory "nnweb/")
"Where nnweb will save its files.")
-(defvoo nnweb-type 'dejanews
- "What search engine type is being used.")
+(defvoo nnweb-type 'google
+ "What search engine type is being used.
+Valid types include `google', `dejanews', and `gmane'.")
(defvar nnweb-type-definition
- '((dejanews
- (article . nnweb-dejanews-wash-article)
- (map . nnweb-dejanews-create-mapping)
- (search . nnweb-dejanews-search)
- (address . "http://xp9.dejanews.com/dnquery.xp"))
- (reference
- (article . nnweb-reference-wash-article)
- (map . nnweb-reference-create-mapping)
- (search . nnweb-reference-search)
- (address . "http://www.reference.com/cgi-bin/pn/go"))
- (altavista
- (article . nnweb-altavista-wash-article)
- (map . nnweb-altavista-create-mapping)
- (search . nnweb-altavista-search)
- (address . "http://www.altavista.digital.com/cgi-bin/query")
- (id . "/cgi-bin/news?id@%s")))
+ '((google
+ (id . "http://www.google.com/groups?as_umsgid=%s&hl=en&dmode=source")
+ (result . "http://groups.google.com/group/%s/msg/%s?dmode=source")
+ (article . nnweb-google-wash-article)
+ (reference . identity)
+ (map . nnweb-google-create-mapping)
+ (search . nnweb-google-search)
+ (address . "http://groups.google.com/groups")
+ (base . "http://groups.google.com")
+ (identifier . nnweb-google-identity))
+ (dejanews ;; alias of google
+ (id . "http://www.google.com/groups?as_umsgid=%s&hl=en&dmode=source")
+ (result . "http://groups.google.com/group/%s/msg/%s?dmode=source")
+ (article . nnweb-google-wash-article)
+ (reference . identity)
+ (map . nnweb-google-create-mapping)
+ (search . nnweb-google-search)
+ (address . "http://groups.google.com/groups")
+ (base . "http://groups.google.com")
+ (identifier . nnweb-google-identity))
+ (gmane
+ (article . nnweb-gmane-wash-article)
+ (id . "http://gmane.org/view.php?group=%s")
+ (reference . identity)
+ (map . nnweb-gmane-create-mapping)
+ (search . nnweb-gmane-search)
+ (address . "http://search.gmane.org/nov.php")
+ (identifier . nnweb-gmane-identity)))
"Type-definition alist.")
(defvoo nnweb-search nil
- "Search string to feed to DejaNews.")
+ "Search string to feed to Google.")
-(defvoo nnweb-max-hits 100
+(defvoo nnweb-max-hits 999
"Maximum number of hits to display.")
(defvoo nnweb-ephemeral-p nil
(set-buffer nntp-server-buffer)
(erase-buffer)
(let (article header)
- (while (setq article (pop articles))
- (when (setq header (cadr (assq article nnweb-articles)))
- (nnheader-insert-nov header)))
+ (mm-with-unibyte-current-buffer
+ (while (setq article (pop articles))
+ (when (setq header (cadr (assq article nnweb-articles)))
+ (nnheader-insert-nov header))))
'nov)))
(deffoo nnweb-request-scan (&optional group server)
(nnweb-possibly-change-server group server)
+ (if nnweb-ephemeral-p
+ (setq nnweb-hashtb (gnus-make-hashtable 4095))
+ (unless nnweb-articles
+ (nnweb-read-overview group)))
(funcall (nnweb-definition 'map))
(unless nnweb-ephemeral-p
(nnweb-write-active)
(nnweb-write-overview group)))
(deffoo nnweb-request-group (group &optional server dont-check)
- (nnweb-possibly-change-server nil server)
- (when (and group
- (not (equal group nnweb-group))
- (not nnweb-ephemeral-p))
- (let ((info (assoc group nnweb-group-alist)))
- (setq nnweb-group group)
- (setq nnweb-type (nth 2 info))
- (setq nnweb-search (nth 3 info))
- (unless dont-check
- (nnweb-read-overview group))))
+ (nnweb-possibly-change-server group server)
+ (unless (or nnweb-ephemeral-p
+ dont-check
+ nnweb-articles)
+ (nnweb-read-overview group))
(cond
((not nnweb-articles)
(nnheader-report 'nnweb "No matching articles"))
(nnweb-possibly-change-server group server)
(save-excursion
(set-buffer (or buffer nntp-server-buffer))
- (let ((url (caddr (assq article nnweb-articles))))
+ (let* ((header (cadr (assq article nnweb-articles)))
+ (url (and header (mail-header-xref header))))
(when (or (and url
- (nnweb-fetch-url url))
+ (mm-with-unibyte-current-buffer
+ (mm-url-insert url)))
(and (stringp article)
+ (nnweb-definition 'id t)
(let ((fetch (nnweb-definition 'id))
- art)
+ art active)
(when (string-match "^<\\(.*\\)>$" article)
(setq art (match-string 1 article)))
- (and fetch
- art
- (nnweb-fetch-url
- (format fetch article))))))
+ (when (and fetch art)
+ (setq url (format fetch art))
+ (mm-with-unibyte-current-buffer
+ (mm-url-insert url))
+ (if (nnweb-definition 'reference t)
+ (setq article
+ (funcall (nnweb-definition
+ 'reference) article)))))))
(unless nnheader-callback-function
- (funcall (nnweb-definition 'article))
- (nnweb-decode-entities))
+ (funcall (nnweb-definition 'article)))
(nnheader-report 'nnweb "Fetched article %s" article)
- t))))
+ (cons group (and (numberp article) article))))))
(deffoo nnweb-close-server (&optional server)
(when (and (nnweb-server-opened server)
(nnweb-possibly-change-server nil server)
(save-excursion
(set-buffer nntp-server-buffer)
- (nnmail-generate-active nnweb-group-alist)
+ (nnmail-generate-active (list (assoc server nnweb-group-alist)))
t))
(deffoo nnweb-request-update-info (group info &optional server)
- (nnweb-possibly-change-server group server)
- ;;(setcar (cddr info) nil)
- )
+ (nnweb-possibly-change-server group server))
(deffoo nnweb-asynchronous-p ()
- t)
+ nil)
(deffoo nnweb-request-create-group (group &optional server args)
(nnweb-possibly-change-server nil server)
(nnweb-request-delete-group group)
- (push `(,group ,(cons 1 0) ,@args) nnweb-group-alist)
+ (push `(,group ,(cons 1 0)) nnweb-group-alist)
(nnweb-write-active)
t)
(deffoo nnweb-request-delete-group (group &optional force server)
(nnweb-possibly-change-server group server)
- (gnus-delete-assoc group nnweb-group-alist)
+ (gnus-pull group nnweb-group-alist t)
+ (nnweb-write-active)
(gnus-delete-file (nnweb-overview-file group))
t)
-
+
(nnoo-define-skeleton nnweb)
;;; Internal functions
(defun nnweb-read-overview (group)
"Read the overview of GROUP and build the map."
(when (file-exists-p (nnweb-overview-file group))
- (nnheader-temp-write nil
- (insert-file-contents (nnweb-overview-file group))
+ (mm-with-unibyte-buffer
+ (nnheader-insert-file-contents (nnweb-overview-file group))
(goto-char (point-min))
- (setq nnweb-hashtb (gnus-make-hashtable
- (count-lines (point-min) (point-max))))
(let (header)
(while (not (eobp))
(setq header (nnheader-parse-nov))
(defun nnweb-write-overview (group)
"Write the overview file for GROUP."
- (nnheader-temp-write (nnweb-overview-file group)
+ (with-temp-file (nnweb-overview-file group)
(let ((articles nnweb-articles))
(while articles
(nnheader-insert-nov (cadr (pop articles)))))))
(defun nnweb-set-hashtb (header data)
(gnus-sethash (nnweb-identifier (mail-header-xref header))
- data nnweb-hashtb))
+ data nnweb-hashtb))
(defun nnweb-get-hashtb (url)
(gnus-gethash (nnweb-identifier url) nnweb-hashtb))
(defun nnweb-identifier (ident)
- ident)
+ (funcall (nnweb-definition 'identifier) ident))
(defun nnweb-overview-file (group)
"Return the name of the overview file of GROUP."
(defun nnweb-write-active ()
"Save the active file."
- (nnheader-temp-write (nnheader-concat nnweb-directory "active")
+ (gnus-make-directory nnweb-directory)
+ (with-temp-file (nnheader-concat nnweb-directory "active")
(prin1 `(setq nnweb-group-alist ',nnweb-group-alist) (current-buffer))))
(defun nnweb-read-active ()
"Read the active file."
(load (nnheader-concat nnweb-directory "active") t t t))
-
-(defun nnweb-definition (type)
+
+(defun nnweb-definition (type &optional noerror)
"Return the definition of TYPE."
(let ((def (cdr (assq type (assq nnweb-type nnweb-type-definition)))))
- (unless def
+ (when (and (not def)
+ (not noerror))
(error "Undefined definition %s" type))
def))
(defun nnweb-possibly-change-server (&optional group server)
- (nnweb-init server)
(when server
(unless (nnweb-server-opened server)
- (nnweb-open-server server)))
+ (nnweb-open-server server))
+ (nnweb-init server))
+ (unless nnweb-group-alist
+ (nnweb-read-active))
+ (unless nnweb-hashtb
+ (setq nnweb-hashtb (gnus-make-hashtable 4095)))
(when group
- (when (and (not nnweb-ephemeral-p)
- (not (equal group nnweb-group)))
- (nnweb-request-group group nil t))))
+ (setq nnweb-group group)))
(defun nnweb-init (server)
"Initialize buffers and such."
(unless (gnus-buffer-live-p nnweb-buffer)
(setq nnweb-buffer
(save-excursion
- (nnheader-set-temp-buffer
- (format " *nnweb %s %s %s*" nnweb-type nnweb-search server))))))
-
-(defun nnweb-fetch-url (url)
- (save-excursion
- (if (not nnheader-callback-function)
- (let ((buf (current-buffer)))
- (save-excursion
- (set-buffer nnweb-buffer)
- (erase-buffer)
- (prog1
- (url-insert-file-contents url)
- (copy-to-buffer buf (point-min) (point-max)))))
- (nnweb-url-retrieve-asynch
- url 'nnweb-callback (current-buffer) nnheader-callback-function)
- t)))
-
-(defun nnweb-callback (buffer callback)
- (when (gnus-buffer-live-p url-working-buffer)
- (save-excursion
- (set-buffer url-working-buffer)
- (funcall (nnweb-definition 'article))
- (nnweb-decode-entities)
- (set-buffer buffer)
- (goto-char (point-max))
- (insert-buffer-substring url-working-buffer))
-
projects / gnus / blobdiff