;;; nnweb.el --- retrieving articles via web search engines
;; Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
-;; 2004, 2005 Free Software Foundation, Inc.
+;; 2004, 2005, 2006 Free Software Foundation, Inc.
;; Author: Lars Magne Ingebrigtsen <larsi@gnus.org>
;; Keywords: news
;; Note: You need to have `w3' installed for some functions to work.
-;; FIXME: Due to changes in the HTML output of Gmane, stuff related to
-;; Gmane web groups doesn't work anymore.
-
-;; FIXME: Solid web groups are currently broken because ARGS are no
-;; longer passed to nnweb-request-create-group from
-;; gnus-group-make-web-group
-
;;; Code:
(eval-when-compile (require 'cl))
(reference . identity)
(map . nnweb-gmane-create-mapping)
(search . nnweb-gmane-search)
- (address . "http://gmane.org/")
+ (address . "http://search.gmane.org/nov.php")
(identifier . nnweb-gmane-identity)))
"Type-definition alist.")
(deffoo nnweb-request-scan (&optional group server)
(nnweb-possibly-change-server group server)
(if nnweb-ephemeral-p
- (setq nnweb-hashtb (gnus-make-hashtable 4095)))
+ (setq nnweb-hashtb (gnus-make-hashtable 4095))
+ (unless nnweb-articles
+ (nnweb-read-overview group)))
(funcall (nnweb-definition 'map))
(unless nnweb-ephemeral-p
(nnweb-write-active)
(nnweb-write-overview group)))
(deffoo nnweb-request-group (group &optional server dont-check)
- (nnweb-possibly-change-server nil server)
- (when (and group
- (not (equal group nnweb-group))
- (not nnweb-ephemeral-p))
- (setq nnweb-group group
- nnweb-articles nil)
- (let ((info (assoc group nnweb-group-alist)))
- (when info
- (setq nnweb-type (nth 2 info))
- (setq nnweb-search (nth 3 info))
- (unless dont-check
- (nnweb-read-overview group)))))
+ (nnweb-possibly-change-server group server)
+ (unless (or nnweb-ephemeral-p
+ dont-check
+ nnweb-articles)
+ (nnweb-read-overview group))
(cond
((not nnweb-articles)
(nnheader-report 'nnweb "No matching articles"))
(nnweb-possibly-change-server nil server)
(save-excursion
(set-buffer nntp-server-buffer)
- (nnmail-generate-active nnweb-group-alist)
+ (nnmail-generate-active (list (assoc server nnweb-group-alist)))
t))
(deffoo nnweb-request-update-info (group info &optional server)
(deffoo nnweb-request-create-group (group &optional server args)
(nnweb-possibly-change-server nil server)
(nnweb-request-delete-group group)
- (push `(,group ,(cons 1 0) ,@args) nnweb-group-alist)
+ (push `(,group ,(cons 1 0)) nnweb-group-alist)
(nnweb-write-active)
t)
def))
(defun nnweb-possibly-change-server (&optional group server)
- (nnweb-init server)
(when server
(unless (nnweb-server-opened server)
- (nnweb-open-server server)))
+ (nnweb-open-server server))
+ (nnweb-init server))
(unless nnweb-group-alist
(nnweb-read-active))
(unless nnweb-hashtb
(setq nnweb-hashtb (gnus-make-hashtable 4095)))
(when group
- (when (and (not nnweb-ephemeral-p)
- (equal group nnweb-group))
- (nnweb-request-group group nil t))))
+ (setq nnweb-group group)))
(defun nnweb-init (server)
"Initialize buffers and such."
(defun nnweb-google-wash-article ()
;; We have Google's masked e-mail addresses here. :-/
- (let ((case-fold-search t))
+ (let ((case-fold-search t)
+ (start-re "<pre>\n *")
+ (end-re "\n *</pre>"))
(goto-char (point-min))
(if (save-excursion
(or (re-search-forward "The requested message.*could not be found."
nil t)
- (not (and (re-search-forward "^<pre>" nil t)
- (re-search-forward "^</pre>" nil t)))))
+ (not (and (re-search-forward start-re nil t)
+ (re-search-forward end-re nil t)))))
;; FIXME: Don't know how to indicate "not found".
;; Should this function throw an error? --rsteib
(progn
(gnus-message 3 "Requested article not found")
(erase-buffer))
(delete-region (point-min)
- (1+ (re-search-forward "^<pre>" nil t)))
+ (re-search-forward start-re))
(goto-char (point-min))
- (delete-region (- (re-search-forward "^</pre>" nil t) (length "</pre>"))
+ (delete-region (progn
+ (re-search-forward end-re)
+ (match-beginning 0))
(point-max))
(mm-url-decode-entities))))
Subject Score Date Newsgroups From
map url mid)
(unless active
- (push (list nnweb-group (setq active (cons 1 0))
- nnweb-type nnweb-search)
+ (push (list nnweb-group (setq active (cons 1 0)))
nnweb-group-alist))
;; Go through all the article hits on this page.
(goto-char (point-min))
- (while (re-search-forward
- "a +href=\"/group/\\([^>\"]+\\)/browse_thread/[^>]+#\\([0-9a-f]+\\)" nil t)
+ (while
+ (re-search-forward
+ "a +href=\"/group/\\([^>\"]+\\)/browse_thread/[^>]+#\\([0-9a-f]+\\)"
+ nil t)
(setq Newsgroups (match-string-no-properties 1)
- ;; Note: with groups-ng, mid is no longer a common
- ;; Message-ID, but some internal id.
+ ;; Note: Starting with Google Groups 2, `mid' is a Google-internal
+ ;; ID, not a proper Message-ID.
mid (match-string-no-properties 2)
url (format
(nnweb-definition 'result) Newsgroups mid))
(save-excursion
(set-buffer nnweb-buffer)
(erase-buffer)
+ (nnheader-message 7 "Searching google...")
(when (funcall (nnweb-definition 'search) nnweb-search)
(let ((more t)
(i 0))
(goto-char (point-min))
(incf i 100)
(if (or (not (re-search-forward
- "<td><a href=\"\n\\([^>\"]+\\)\"><img src=\"/img/nav_next" nil t))
+ "<a href=\"\n\\([^>\"]+\\)\"><img src=\"[^\"]+next"
+ nil t))
(>= i nnweb-max-hits))
(setq more nil)
;; Yup, there are more articles
(setq more (concat (nnweb-definition 'base) (match-string 1)))
(when more
(erase-buffer)
+ (nnheader-message 7 "Searching google...(%d)" i)
(mm-url-insert more))))
;; Return the articles in the right order.
+ (nnheader-message 7 "Searching google...done")
(setq nnweb-articles
(sort nnweb-articles 'car-less-than-car))))))
"Perform the search and create a number-to-url alist."
(save-excursion
(set-buffer nnweb-buffer)
- (erase-buffer)
- (when (funcall (nnweb-definition 'search) nnweb-search)
- (let ((more t)
- (case-fold-search t)
- (active (or (cadr (assoc nnweb-group nnweb-group-alist))
- (cons 1 0)))
- subject group url
- map)
- ;; Remove stuff from the beginning of results
- (goto-char (point-min))
- (search-forward "Search Results</h1><ul>" nil t)
- (delete-region (point-min) (point))
+ (let ((case-fold-search t)
+ (active (or (cadr (assoc nnweb-group nnweb-group-alist))
+ (cons 1 0)))
+ map)
+ (erase-buffer)
+ (nnheader-message 7 "Searching Gmane..." )
+ (when (funcall (nnweb-definition 'search) nnweb-search)
(goto-char (point-min))
- ;; Iterate over the actual hits
- (while (re-search-forward ".*href=\"\\([^\"]+\\)\">\\(.*\\)" nil t)
- (setq url (concat "http://gmane.org/" (match-string 1)))
- (setq subject (match-string 2))
- (unless (nnweb-get-hashtb url)
- (push
- (list
- (incf (cdr active))
- (make-full-mail-header
- (cdr active) (concat "(" group ") " subject) nil nil
- nil nil 0 0 url))
- map)
- (nnweb-set-hashtb (cadar map) (car map))))
- ;; Return the articles in the right order.
- (setq nnweb-articles
- (sort (nconc nnweb-articles map) 'car-less-than-car))))))
+ ;; Skip the status line
+ (forward-line 1)
+ ;; Thanks to Olly Betts we now have NOV lines in our buffer!
+ (while (not (eobp))
+ (unless (or (eolp) (looking-at "\x0d"))
+ (let ((header (nnheader-parse-nov)))
+ (let ((xref (mail-header-xref header))
+ (from (mail-header-from header))
+ (subject (mail-header-subject header))
+ (rfc2047-encoding-type 'mime))
+ (when (string-match " \\([^:]+\\):\\([0-9]+\\)" xref)
+ (mail-header-set-xref
+ header
+ (format "http://article.gmane.org/%s/%s/raw"
+ (match-string 1 xref)
+ (match-string 2 xref))))
+
+ ;; Add host part to gmane-encrypted addresses
+ (when (string-match "@$" from)
+ (mail-header-set-from header
+ (concat from "public.gmane.org")))
+
+ (mail-header-set-subject header
+ (rfc2047-encode-string subject))
+
+ (unless (nnweb-get-hashtb (mail-header-xref header))
+ (push
+ (list
+ (incf (cdr active))
+ header)
+ map)
+ (nnweb-set-hashtb (cadar map) (car map))))))
+ (forward-line 1)))
+ (nnheader-message 7 "Searching Gmane...done")
+ (setq nnweb-articles
+ (sort (nconc nnweb-articles map) 'car-less-than-car)))))
(defun nnweb-gmane-wash-article ()
(let ((case-fold-search t))
(goto-char (point-min))
- (search-forward "<!--X-Head-of-Message-->" nil t)
- (delete-region (point-min) (point))
- (goto-char (point-min))
- (while (looking-at "^<li><em>\\([^ ]+\\)</em>.*</li>")
- (replace-match "\\1\\2" t)
- (forward-line 1))
- (mm-url-remove-markup)))
+ (when (search-forward "<!--X-Head-of-Message-->" nil t)
+ (delete-region (point-min) (point))
+ (goto-char (point-min))
+ (while (looking-at "^<li><em>\\([^ ]+\\)</em>.*</li>")
+ (replace-match "\\1\\2" t)
+ (forward-line 1))
+ (mm-url-remove-markup))))
(defun nnweb-gmane-search (search)
(mm-url-insert
(nnweb-definition 'address)
"?"
(mm-url-encode-www-form-urlencoded
- `(("query" . ,search)))))
+ `(("query" . ,search)
+ ("HITSPERPAGE" . ,(number-to-string nnweb-max-hits))))))
(setq buffer-file-name nil)
+ (set-buffer-multibyte t)
+ (mm-decode-coding-region (point-min) (point-max) 'utf-8)
t)
-
(defun nnweb-gmane-identity (url)
"Return a unique identifier based on URL."
(if (string-match "group=\\(.+\\)" url)