2001-12-06 10:00:00 ShengHuo ZHU <zsh@cs.rochester.edu>

[gnus] / lisp / nnweb.el

diff --git a/lisp/nnweb.el b/lisp/nnweb.el
index 64821e3..277e24e 100644 (file)
--- a/lisp/nnweb.el
+++ b/lisp/nnweb.el
@@ -55,17 +55,31 @@
 (defvoo nnweb-directory (nnheader-concat gnus-directory "nnweb/")
   "Where nnweb will save its files.")
 
-(defvoo nnweb-type 'dejanews
+(defvoo nnweb-type 'google
   "What search engine type is being used.
-Valid types include `dejanews', `dejanewsold', `reference',
+Valid types include `google', `dejanews', `dejanewsold', `reference',
 and `altavista'.")
 
 (defvar nnweb-type-definition
   '(
-    (dejanews ;; bought by google.com
-     (article . nnweb-google-wash-article)
-     (id . "http://groups.google.com/groups?as_umsgid=%s")
-     (reference . nnweb-google-reference)
+    (google
+     ;;(article . nnweb-google-wash-article)
+     ;;(id . "http://groups.google.com/groups?as_umsgid=%s")
+     (article . ignore)
+     (id . "http://groups.google.com/groups?selm=%s&output=gplain")
+     ;;(reference . nnweb-google-reference)
+     (reference . identity)
+     (map . nnweb-google-create-mapping)
+     (search . nnweb-google-search)
+     (address . "http://groups.google.com/groups")
+     (identifier . nnweb-google-identity))
+    (dejanews ;; alias of google
+     ;;(article . nnweb-google-wash-article)
+     ;;(id . "http://groups.google.com/groups?as_umsgid=%s")
+     (article . ignore)
+     (id . "http://groups.google.com/groups?selm=%s&output=gplain")
+     ;;(reference . nnweb-google-reference)
+     (reference . identity)
      (map . nnweb-google-create-mapping)
      (search . nnweb-google-search)
      (address . "http://groups.google.com/groups")
@@ -133,6 +147,8 @@ and `altavista'.")
 
 (deffoo nnweb-request-scan (&optional group server)
   (nnweb-possibly-change-server group server)
+  (if nnweb-ephemeral-p
+      (setq nnweb-hashtb (gnus-make-hashtable 4095)))
   (funcall (nnweb-definition 'map))
   (unless nnweb-ephemeral-p
     (nnweb-write-active)
@@ -143,7 +159,8 @@ and `altavista'.")
   (when (and group
             (not (equal group nnweb-group))
             (not nnweb-ephemeral-p))
-    (setq nnweb-group group)
+    (setq nnweb-group group
+         nnweb-articles nil)
     (let ((info (assoc group nnweb-group-alist)))
       (when info
        (setq nnweb-type (nth 2 info))
@@ -188,16 +205,15 @@ and `altavista'.")
                       (when (string-match "^<\\(.*\\)>$" article)
                         (setq art (match-string 1 article)))
                       (when (and fetch art)
-                        (setq url (format fetch article))
+                        (setq url (format fetch art))
                         (mm-with-unibyte-current-buffer
                           (nnweb-fetch-url url))
                         (if (nnweb-definition 'reference t)
                             (setq article
-                                  (funcall (nnweb-definition 
+                                  (funcall (nnweb-definition
                                             'reference) article)))))))
        (unless nnheader-callback-function
-         (funcall (nnweb-definition 'article))
-         (nnweb-decode-entities))
+         (funcall (nnweb-definition 'article)))
        (nnheader-report 'nnweb "Fetched article %s" article)
        (cons group (and (numberp article) article))))))
 
@@ -302,10 +318,11 @@ and `altavista'.")
       (nnweb-open-server server)))
   (unless nnweb-group-alist
     (nnweb-read-active))
+  (unless nnweb-hashtb
+    (setq nnweb-hashtb (gnus-make-hashtable 4095)))
   (when group
     (when (and (not nnweb-ephemeral-p)
-              (not (equal group nnweb-group)))
-      (setq nnweb-hashtb (gnus-make-hashtable 4095))
+              (equal group nnweb-group))
       (nnweb-request-group group nil t))))
 
 (defun nnweb-init (server)
@@ -571,6 +588,7 @@ and `altavista'.")
        (while (search-forward "," nil t)
          (replace-match " " t t)))
       (widen)
+      (nnweb-decode-entities)
       (set-marker body nil))))
 
 (defun nnweb-reference-search (search)
@@ -675,7 +693,8 @@ and `altavista'.")
       (while (re-search-forward "<A.*\\?id@\\([^\"]+\\)\">[0-9]+</A>" nil t)
        (replace-match "&lt;\\1&gt; " t)))
     (widen)
-    (nnweb-remove-markup)))
+    (nnweb-remove-markup)
+    (nnweb-decode-entities)))
 
 (defun nnweb-altavista-search (search &optional part)
   (url-insert-file-contents
@@ -703,9 +722,11 @@ and `altavista'.")
     (goto-char (point-min))
     (re-search-forward "^<pre>" nil t)
     (narrow-to-region (point-min) (point))
-    (search-backward "</table>" nil t 2)
+    (search-backward "<table " nil t 2)
     (delete-region (point-min) (point))
-    (if (search-forward "[view thread]" nil t)
+    (if (re-search-forward "Search Result [0-9]+" nil t)
+       (replace-match ""))
+    (if (re-search-forward "View complete thread ([0-9]+ articles?)" nil t)
        (replace-match ""))
     (goto-char (point-min))
     (while (search-forward "<br>" nil t)
@@ -728,17 +749,18 @@ and `altavista'.")
        (case-fold-search t)
        (active (cadr (assoc nnweb-group nnweb-group-alist)))
        Subject Score Date Newsgroups From
-       map url)
+       map url mid)
     (unless active
-      (push (list nnweb-group (setq active (cons 1 0)))
+      (push (list nnweb-group (setq active (cons 1 0))
+                 nnweb-type nnweb-search)
            nnweb-group-alist))
     ;; Go through all the article hits on this page.
     (goto-char (point-min))
     (while (re-search-forward
-           "a href=/groups\\(\\?[^ \">]*seld=[0-9]+[^ \">]*\\)" nil t)
-      (setq url
-           (concat (nnweb-definition 'address)
-                   (match-string 1)))
+           "a href=/groups\\(\\?[^ \">]*selm=\\([^ &\">]+\\)\\)" nil t)
+      (setq mid (match-string 2)
+           url (format 
+                "http://groups.google.com/groups?selm=%s&output=gplain" mid))
       (narrow-to-region (search-forward ">" nil t)
                        (search-forward "</a>" nil t))
       (nnweb-remove-markup)
@@ -746,7 +768,7 @@ and `altavista'.")
       (setq Subject (buffer-string))
       (goto-char (point-max))
       (widen)
-      (forward-line 2)
+      (forward-line 1)
       (when (looking-at "<br><font[^>]+>")
        (goto-char (match-end 0)))
       (if (not (looking-at "<a[^>]+>"))
@@ -759,8 +781,8 @@ and `altavista'.")
        (goto-char (point-max))
        (widen)
        (skip-chars-forward "- \t"))
-      (when (looking-at 
-            "\\([0-9]+/[A-Za-z]+/[0-9]+\\)[ \t]*by[ \t]*\\([^<]*\\) - <a")
+      (when (looking-at
+            "\\([0-9]+[/ ][A-Za-z]+[/ ][0-9]+\\)[ \t]*by[ \t]*\\([^<]*\\) - <a")
        (setq From (match-string 2)
              Date (match-string 1)))
       (forward-line 1)
@@ -771,9 +793,9 @@ and `altavista'.")
          (incf (cdr active))
          (make-full-mail-header
           (cdr active) (if Newsgroups
-                           (concat  "(" Newsgroups ") " Subject) 
+                           (concat  "(" Newsgroups ") " Subject)
                          Subject)
-          From Date Message-ID
+          From Date (or Message-ID mid)
           nil 0 0 url))
         map)
        (nnweb-set-hashtb (cadar map) (car map))))
@@ -781,7 +803,7 @@ and `altavista'.")
 
 (defun nnweb-google-reference (id)
   (let ((map (nnweb-google-parse-1 id)) header)
-    (setq nnweb-articles 
+    (setq nnweb-articles
          (nconc nnweb-articles map))
     (when (setq header (cadar map))
       (mm-with-unibyte-current-buffer
@@ -821,7 +843,7 @@ and `altavista'.")
 
 (defun nnweb-google-identity (url)
   "Return an unique identifier based on URL."
-  (if (string-match "seld=\\([0-9]+\\)" url)
+  (if (string-match "selm=\\([^ &>]+\\)" url)
       (match-string 1 url)
     url))
 
@@ -903,7 +925,7 @@ and `altavista'.")
   "Insert the contents from an URL in the current buffer.
 If FOLLOW-REFRESH is non-nil, redirect refresh url in META."
   (let ((name buffer-file-name))
-    (if follow-refresh
+    (if follow-refresh 
        (save-restriction
          (narrow-to-region (point) (point))
          (url-insert-file-contents url)
@@ -965,11 +987,6 @@ If FOLLOW-REFRESH is non-nil, redirect refresh url in META."
                 (listp (cdr element)))
        (nnweb-text-1 element)))))
 
-(defun nnweb-replace-in-string (string match newtext)
-  (while (string-match match string)
-    (setq string (replace-match newtext t t string)))
-  string)
-
 (provide 'nnweb)
 
 ;;; nnweb.el ends here