X-Git-Url: https://cgit.sxemacs.org/?a=blobdiff_plain;f=lisp%2Fnnweb.el;h=eacdc2f428faed24d1026f716d69938b592b11c4;hb=b28454eed83f245c4160228b076134ce930b320a;hp=40707a3ad3456e7520fdf159681cf40933d6bfe4;hpb=d0498ec691ac9cc3f6bdd9f4ba3ac26457cc3d8a;p=gnus diff --git a/lisp/nnweb.el b/lisp/nnweb.el index 40707a3ad..eacdc2f42 100644 --- a/lisp/nnweb.el +++ b/lisp/nnweb.el @@ -1,5 +1,5 @@ ;;; nnweb.el --- retrieving articles via web search engines -;; Copyright (C) 1996 Free Software Foundation, Inc. +;; Copyright (C) 1996,97 Free Software Foundation, Inc. ;; Author: Lars Magne Ingebrigtsen ;; Keywords: news @@ -31,9 +31,12 @@ (require 'nnoo) (require 'message) (require 'gnus-util) +(require 'gnus) (require 'w3) -(require 'w3-forms) (require 'url) +(require 'nnmail) +(ignore-errors + (require 'w3-forms)) (nnoo-declare nnweb) @@ -48,18 +51,21 @@ (article . nnweb-dejanews-wash-article) (map . nnweb-dejanews-create-mapping) (search . nnweb-dejanews-search) - (address . "http://search.dejanews.com/dnquery.xp")) + (address . "http://xp9.dejanews.com/dnquery.xp") + (identifier . nnweb-dejanews-identity)) (reference (article . nnweb-reference-wash-article) (map . nnweb-reference-create-mapping) (search . nnweb-reference-search) - (address . "http://www.reference.com/cgi-bin/pn/go")) + (address . "http://www.reference.com/cgi-bin/pn/go") + (identifier . identity)) (altavista (article . nnweb-altavista-wash-article) (map . nnweb-altavista-create-mapping) (search . nnweb-altavista-search) (address . "http://www.altavista.digital.com/cgi-bin/query") - (id . "/cgi-bin/news?id@%s"))) + (id . "/cgi-bin/news?id@%s") + (identifier . identity))) "Type-definition alist.") (defvoo nnweb-search nil @@ -103,7 +109,8 @@ (deffoo nnweb-request-group (group &optional server dont-check) (nnweb-possibly-change-server nil server) - (when (and (not (equal group nnweb-group)) + (when (and group + (not (equal group nnweb-group)) (not nnweb-ephemeral-p)) (let ((info (assoc group nnweb-group-alist))) (setq nnweb-group group) @@ -137,10 +144,12 @@ (nnweb-possibly-change-server group server) (save-excursion (set-buffer (or buffer nntp-server-buffer)) - (let ((url (caddr (assq article nnweb-articles)))) + (let* ((header (cadr (assq article nnweb-articles))) + (url (and header (mail-header-xref header)))) (when (or (and url (nnweb-fetch-url url)) (and (stringp article) + (nnweb-definition 'id t) (let ((fetch (nnweb-definition 'id)) art) (when (string-match "^<\\(.*\\)>$" article) @@ -191,7 +200,7 @@ (gnus-delete-assoc group nnweb-group-alist) (gnus-delete-file (nnweb-overview-file group)) t) - + (nnoo-define-skeleton nnweb) ;;; Internal functions @@ -200,7 +209,7 @@ "Read the overview of GROUP and build the map." (when (file-exists-p (nnweb-overview-file group)) (nnheader-temp-write nil - (insert-file-contents (nnweb-overview-file group)) + (nnheader-insert-file-contents (nnweb-overview-file group)) (goto-char (point-min)) (setq nnweb-hashtb (gnus-make-hashtable (count-lines (point-min) (point-max)))) @@ -209,7 +218,7 @@ (setq header (nnheader-parse-nov)) (forward-line 1) (push (list (mail-header-number header) - header (mail-header-xrefs header)) + header (mail-header-xref header)) nnweb-articles) (nnweb-set-hashtb header (car nnweb-articles))))))) @@ -221,14 +230,14 @@ (nnheader-insert-nov (cadr (pop articles))))))) (defun nnweb-set-hashtb (header data) - (gnus-sethasb (nnweb-identifier (mail-header-xrefs header)) + (gnus-sethash (nnweb-identifier (mail-header-xref header)) data nnweb-hashtb)) (defun nnweb-get-hashtb (url) (gnus-gethash (nnweb-identifier url) nnweb-hashtb)) (defun nnweb-identifier (ident) - ident) + (funcall (nnweb-definition 'identifier) ident)) (defun nnweb-overview-file (group) "Return the name of the overview file of GROUP." @@ -242,11 +251,12 @@ (defun nnweb-read-active () "Read the active file." (load (nnheader-concat nnweb-directory "active") t t t)) - -(defun nnweb-definition (type) + +(defun nnweb-definition (type &optional noerror) "Return the definition of TYPE." (let ((def (cdr (assq type (assq nnweb-type nnweb-type-definition))))) - (unless def + (when (and (not def) + (not noerror)) (error "Undefined definition %s" type)) def)) @@ -255,6 +265,8 @@ (when server (unless (nnweb-server-opened server) (nnweb-open-server server))) + (unless nnweb-group-alist + (nnweb-read-active)) (when group (when (and (not nnweb-ephemeral-p) (not (equal group nnweb-group))) @@ -311,16 +323,17 @@ (defun nnweb-encode-www-form-urlencoded (pairs) "Return PAIRS encoded for forms." - (mapconcat + (mapconcat (function (lambda (data) (concat (w3-form-encode-xwfu (car data)) "=" - (w3-form-encode-xwfu (cdr data))))) pairs "&")) + (w3-form-encode-xwfu (cdr data))))) + pairs "&")) (defun nnweb-fetch-form (url pairs) (let ((url-request-data (nnweb-encode-www-form-urlencoded pairs)) - (url-request-method 'POST) - (url-request-extra-headers + (url-request-method "POST") + (url-request-extra-headers '(("Content-type" . "application/x-www-form-urlencoded")))) (url-insert-file-contents url) (setq buffer-file-name nil)) @@ -367,8 +380,8 @@ (nnweb-decode-entities) (goto-char (point-min)) (while (re-search-forward "^ +[0-9]+\\." nil t) - (narrow-to-region - (point) + (narrow-to-region + (point) (cond ((re-search-forward "^ +[0-9]+\\." nil t) (match-beginning 0)) ((search-forward "\n\n" nil t) @@ -388,18 +401,17 @@ (widen) (when (string-match "#[0-9]+/[0-9]+ *$" Subject) (setq Subject (substring Subject 0 (match-beginning 0)))) + (incf i) (unless (nnweb-get-hashtb url) - (incf i) (push (list (incf (cdr active)) (make-full-mail-header (cdr active) (concat "(" Newsgroup ") " Subject) Author Date - (concat "<" (message-unique-id) "-" (int-to-string i) - "@dejanews>") - nil 0 (string-to-int Score) nil) - url) - map))) + (concat "<" (nnweb-identifier url) "@dejanews>") + nil 0 (string-to-int Score) url)) + map) + (nnweb-set-hashtb (cadar map) (car map)))) ;; See whether there is a "Get next 20 hits" button here. (if (or (not (re-search-forward "HREF=\"\\([^\"]+\\)\">Get next" nil t)) @@ -411,7 +423,8 @@ (url-insert-file-contents more))) ;; Return the articles in the right order. (setq nnweb-articles - (sort map (lambda (s1 s2) (< (car s1) (car s2))))))))) + (sort (nconc nnweb-articles map) + (lambda (s1 s2) (< (car s1) (car s2))))))))) (defun nnweb-dejanews-wash-article () (let ((case-fold-search t)) @@ -432,7 +445,7 @@ (replace-match "\n" t t)))) (defun nnweb-dejanews-search (search) - (nnweb-fetch-form + (nnweb-fetch-form (nnweb-definition 'address) `(("query" . ,search) ("defaultOp" . "AND") @@ -445,6 +458,12 @@ ("ageweight" . "1"))) t) +(defun nnweb-dejanews-identity (url) + "Return an unique identifier based on URL." + (if (string-match "recnum=\\([0-9]+\\)" url) + (match-string 1 url) + url)) + ;;; ;;; InReference ;;; @@ -470,8 +489,8 @@ ;(nnweb-decode-entities) (goto-char (point-min)) (while (re-search-forward "^ +[0-9]+\\." nil t) - (narrow-to-region - (point) + (narrow-to-region + (point) (if (re-search-forward "^$" nil t) (match-beginning 0) (point-max))) @@ -487,21 +506,22 @@ (set (intern (match-string 1)) (match-string 2))) (widen) (search-forward "" nil t) + (incf i) (unless (nnweb-get-hashtb url) - (incf i) (push (list (incf (cdr active)) (make-full-mail-header (cdr active) (concat "(" Newsgroups ") " Subject) From Date Message-ID - nil 0 (string-to-int Score) nil) - url) - map))) + nil 0 (string-to-int Score) url)) + map) + (nnweb-set-hashtb (cadar map) (car map)))) (setq more nil)) ;; Return the articles in the right order. (setq nnweb-articles - (sort map (lambda (s1 s2) (< (car s1) (car s2))))))))) + (sort (nconc nnweb-articles map) + (lambda (s1 s2) (< (car s1) (car s2))))))))) (defun nnweb-reference-wash-article () (let ((case-fold-search t)) @@ -545,10 +565,10 @@ (defun nnweb-reference-search (search) (prog1 (url-insert-file-contents - (concat + (concat (nnweb-definition 'address) "?" - (nnweb-encode-www-form-urlencoded + (nnweb-encode-www-form-urlencoded `(("search" . "advanced") ("querytext" . ,search) ("subj" . "") @@ -610,16 +630,16 @@ group (match-string 4) id (concat "<" (match-string 5) ">") from (match-string 6)) + (incf i) (unless (nnweb-get-hashtb url) - (incf i) (push (list (incf (cdr active)) (make-full-mail-header (cdr active) (concat "(" group ") " subject) from date - id nil 0 0 nil) - url) - map))) + id nil 0 0 url)) + map) + (nnweb-set-hashtb (cadar map) (car map)))) ;; See if we want more. (when (or (not nnweb-articles) (>= i nnweb-max-hits) @@ -628,13 +648,14 @@ (setq more nil))) ;; Return the articles in the right order. (setq nnweb-articles - (sort map (lambda (s1 s2) (< (car s1) (car s2)))))))))) + (sort (nconc nnweb-articles map) + (lambda (s1 s2) (< (car s1) (car s2)))))))))) (defun nnweb-altavista-wash-article () (goto-char (point-min)) (let ((case-fold-search t)) - (re-search-forward "^" nil t) - (delete-region (point-min) (match-beginning 0)) + (when (re-search-forward "^" nil t) + (delete-region (point-min) (match-beginning 0))) (goto-char (point-min)) (while (looking-at "\\([^ ]+\\) + +\\(.*\\)$") (replace-match "\\1: \\2" t) @@ -650,13 +671,13 @@ (defun nnweb-altavista-search (search &optional part) (prog1 (url-insert-file-contents - (concat + (concat (nnweb-definition 'address) "?" - (nnweb-encode-www-form-urlencoded + (nnweb-encode-www-form-urlencoded `(("pg" . "aq") ("what" . "news") - ,@(if part `(("stq" . ,(int-to-string (* part 30))))) + ,@(when part `(("stq" . ,(int-to-string (* part 30))))) ("fmt" . "d") ("q" . ,search) ("r" . "")