;;; nnweb.el --- retrieving articles via web search engines
-;; Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002
-;; Free Software Foundation, Inc.
+
+;; Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
+;; 2004, 2005, 2006 Free Software Foundation, Inc.
;; Author: Lars Magne Ingebrigtsen <larsi@gnus.org>
;; Keywords: news
;; You should have received a copy of the GNU General Public License
;; along with GNU Emacs; see the file COPYING. If not, write to the
-;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-;; Boston, MA 02111-1307, USA.
+;; Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+;; Boston, MA 02110-1301, USA.
;;; Commentary:
;; Note: You need to have `w3' installed for some functions to work.
+;; FIXME: Due to changes in the HTML output of Gmane, stuff related to Gmane
+;; web groups (`gnus-group-make-web-group') doesn't work anymore.
+
;;; Code:
(eval-when-compile (require 'cl))
(defvar nnweb-type-definition
'((google
- (article . ignore)
- (id . "http://groups.google.com/groups?selm=%s&output=gplain")
+ (id . "http://www.google.com/groups?as_umsgid=%s&hl=en&dmode=source")
+ (result . "http://groups.google.com/group/%s/msg/%s?dmode=source")
+ (article . nnweb-google-wash-article)
(reference . identity)
(map . nnweb-google-create-mapping)
(search . nnweb-google-search)
(address . "http://groups.google.com/groups")
+ (base . "http://groups.google.com")
(identifier . nnweb-google-identity))
(dejanews ;; alias of google
- (article . ignore)
- (id . "http://groups.google.com/groups?selm=%s&output=gplain")
+ (id . "http://www.google.com/groups?as_umsgid=%s&hl=en&dmode=source")
+ (result . "http://groups.google.com/group/%s/msg/%s?dmode=source")
+ (article . nnweb-google-wash-article)
(reference . identity)
(map . nnweb-google-create-mapping)
(search . nnweb-google-search)
(address . "http://groups.google.com/groups")
+ (base . "http://groups.google.com")
(identifier . nnweb-google-identity))
(gmane
(article . nnweb-gmane-wash-article)
(defvoo nnweb-articles nil)
(defvoo nnweb-buffer nil)
-(defvoo nnweb-group-alist nil)
+(defvar nnweb-group-alist nil)
(defvoo nnweb-group nil)
(defvoo nnweb-hashtb nil)
(deffoo nnweb-request-scan (&optional group server)
(nnweb-possibly-change-server group server)
(if nnweb-ephemeral-p
- (setq nnweb-hashtb (gnus-make-hashtable 4095)))
+ (setq nnweb-hashtb (gnus-make-hashtable 4095))
+ (unless nnweb-articles
+ (nnweb-read-overview group)))
(funcall (nnweb-definition 'map))
(unless nnweb-ephemeral-p
(nnweb-write-active)
(nnweb-write-overview group)))
(deffoo nnweb-request-group (group &optional server dont-check)
- (nnweb-possibly-change-server nil server)
- (when (and group
- (not (equal group nnweb-group))
- (not nnweb-ephemeral-p))
- (setq nnweb-group group
- nnweb-articles nil)
- (let ((info (assoc group nnweb-group-alist)))
- (when info
- (setq nnweb-type (nth 2 info))
- (setq nnweb-search (nth 3 info))
- (unless dont-check
- (nnweb-read-overview group)))))
+ (nnweb-possibly-change-server group server)
+ (unless (or nnweb-ephemeral-p
+ dont-check
+ nnweb-articles)
+ (nnweb-read-overview group))
(cond
((not nnweb-articles)
(nnheader-report 'nnweb "No matching articles"))
(nnweb-possibly-change-server nil server)
(save-excursion
(set-buffer nntp-server-buffer)
- (nnmail-generate-active nnweb-group-alist)
+ (nnmail-generate-active (list (assoc server nnweb-group-alist)))
t))
(deffoo nnweb-request-update-info (group info &optional server)
(deffoo nnweb-request-create-group (group &optional server args)
(nnweb-possibly-change-server nil server)
(nnweb-request-delete-group group)
- (push `(,group ,(cons 1 0) ,@args) nnweb-group-alist)
+ (push `(,group ,(cons 1 0)) nnweb-group-alist)
(nnweb-write-active)
t)
def))
(defun nnweb-possibly-change-server (&optional group server)
- (nnweb-init server)
(when server
(unless (nnweb-server-opened server)
- (nnweb-open-server server)))
+ (nnweb-open-server server))
+ (nnweb-init server))
(unless nnweb-group-alist
(nnweb-read-active))
(unless nnweb-hashtb
(setq nnweb-hashtb (gnus-make-hashtable 4095)))
(when group
- (when (and (not nnweb-ephemeral-p)
- (equal group nnweb-group))
- (nnweb-request-group group nil t))))
+ (setq nnweb-group group)))
(defun nnweb-init (server)
"Initialize buffers and such."
(current-buffer))))))
;;;
-;;; Deja bought by google.com
+;;; groups.google.com
;;;
(defun nnweb-google-wash-article ()
- (let ((case-fold-search t) url)
- (goto-char (point-min))
- (re-search-forward "^<pre>" nil t)
- (narrow-to-region (point-min) (point))
- (search-backward "<table " nil t 2)
- (delete-region (point-min) (point))
- (if (re-search-forward "Search Result [0-9]+" nil t)
- (replace-match ""))
- (if (re-search-forward "View complete thread ([0-9]+ articles?)" nil t)
- (replace-match ""))
- (goto-char (point-min))
- (while (search-forward "<br>" nil t)
- (replace-match "\n"))
- (mm-url-remove-markup)
+ ;; We have Google's masked e-mail addresses here. :-/
+ (let ((case-fold-search t))
(goto-char (point-min))
- (while (re-search-forward "^[ \t]*\n" nil t)
- (replace-match ""))
- (goto-char (point-max))
- (insert "\n")
- (widen)
- (narrow-to-region (point) (point-max))
- (search-forward "</pre>" nil t)
- (delete-region (point) (point-max))
- (mm-url-remove-markup)
- (widen)))
+ (if (save-excursion
+ (or (re-search-forward "The requested message.*could not be found."
+ nil t)
+ (not (and (re-search-forward "^<pre>" nil t)
+ (re-search-forward "^</pre>" nil t)))))
+ ;; FIXME: Don't know how to indicate "not found".
+ ;; Should this function throw an error? --rsteib
+ (progn
+ (gnus-message 3 "Requested article not found")
+ (erase-buffer))
+ (delete-region (point-min)
+ (1+ (re-search-forward "^<pre>" nil t)))
+ (goto-char (point-min))
+ (delete-region (- (re-search-forward "^</pre>" nil t) (length "</pre>"))
+ (point-max))
+ (mm-url-decode-entities))))
(defun nnweb-google-parse-1 (&optional Message-ID)
+ "Parse search result in current buffer."
(let ((i 0)
(case-fold-search t)
(active (cadr (assoc nnweb-group nnweb-group-alist)))
Subject Score Date Newsgroups From
map url mid)
(unless active
- (push (list nnweb-group (setq active (cons 1 0))
- nnweb-type nnweb-search)
+ (push (list nnweb-group (setq active (cons 1 0)))
nnweb-group-alist))
;; Go through all the article hits on this page.
(goto-char (point-min))
- (while (re-search-forward
- "a href=/groups\\(\\?[^ \">]*selm=\\([^ &\">]+\\)\\)" nil t)
- (setq mid (match-string 2)
+ (while
+ (re-search-forward
+ "a +href=\"/group/\\([^>\"]+\\)/browse_thread/[^>]+#\\([0-9a-f]+\\)"
+ nil t)
+ (setq Newsgroups (match-string-no-properties 1)
+ ;; Note: Starting with Google Groups 2, `mid' is a Google-internal
+ ;; ID, not a proper Message-ID.
+ mid (match-string-no-properties 2)
url (format
- "http://groups.google.com/groups?selm=%s&output=gplain" mid))
+ (nnweb-definition 'result) Newsgroups mid))
(narrow-to-region (search-forward ">" nil t)
(search-forward "</a>" nil t))
(mm-url-remove-markup)
(setq Subject (buffer-string))
(goto-char (point-max))
(widen)
- (forward-line 1)
- (when (looking-at "<br><font[^>]+>")
- (goto-char (match-end 0)))
- (if (not (looking-at "<a[^>]+>"))
- (skip-chars-forward " \t")
- (narrow-to-region (point)
- (search-forward "</a>" nil t))
- (mm-url-remove-markup)
- (mm-url-decode-entities)
- (setq Newsgroups (buffer-string))
- (goto-char (point-max))
- (widen)
- (skip-chars-forward "- \t"))
+ (narrow-to-region (point)
+ (search-forward "</td" nil t))
+
+ (mm-url-remove-markup)
+ (mm-url-decode-entities)
+ (search-backward " - ")
(when (looking-at
- "\\([0-9]+\\)[/ ]\\([A-Za-z]+\\)[/ ]\\([0-9]+\\)[ \t]*by[ \t]*\\([^<]*\\) - <a")
+ " - \\([a-zA-Z]+\\) \\([0-9]+\\)\\(?: \\([0-9]\\{4\\}\\)\\)?, [^\n]+by \\([^<\n]+\\)\n")
(setq From (match-string 4)
Date (format "%s %s 00:00:00 %s"
- (match-string 2) (match-string 1)
- (match-string 3))))
+ (match-string 1)
+ (match-string 2)
+ (or (match-string 3)
+ (substring (current-time-string) -4)))))
+
+ (widen)
(forward-line 1)
(incf i)
(unless (nnweb-get-hashtb url)
(goto-char (point-min))
(incf i 100)
(if (or (not (re-search-forward
- "<td nowrap><a href=\\([^>]+\\).*<span class=b>Next</span>" nil t))
+ "<td><a href=\"\n\\([^>\"]+\\)\"><img src=\"/img/nav_next" nil t))
(>= i nnweb-max-hits))
(setq more nil)
;; Yup, there are more articles
- (setq more (concat "http://groups.google.com" (match-string 1)))
+ (setq more (concat (nnweb-definition 'base) (match-string 1)))
(when more
(erase-buffer)
(mm-url-insert more))))
"?"
(mm-url-encode-www-form-urlencoded
`(("q" . ,search)
- ("num". "100")
+ ("num" . "100")
("hq" . "")
- ("hl" . "")
+ ("hl" . "en")
("lr" . "")
("safe" . "off")
- ("sites" . "groups")))))
+ ("sites" . "groups")
+ ("filter" . "0")))))
t)
(defun nnweb-google-identity (url)
(defun nnweb-gmane-wash-article ()
(let ((case-fold-search t))
(goto-char (point-min))
- (re-search-forward "<!--X-Head-of-Message-->" nil t)
+ (search-forward "<!--X-Head-of-Message-->" nil t)
(delete-region (point-min) (point))
(goto-char (point-min))
(while (looking-at "^<li><em>\\([^ ]+\\)</em>.*</li>")
(nth 1 parse)
" "))
(insert ">\n")
- (mapcar 'nnweb-insert-html (nth 2 parse))
+ (mapc 'nnweb-insert-html (nth 2 parse))
(insert "</" (symbol-name (car parse)) ">\n")))
(defun nnweb-parse-find (type parse &optional maxdepth)
(provide 'nnweb)
+;;; arch-tag: f59307eb-c90f-479f-b7d2-dbd8bf51b697
;;; nnweb.el ends here