;;; mm-url.el --- a wrapper of url functions/commands for Gnus
-;; Copyright (C) 2001 Free Software Foundation, Inc.
+
+;; Copyright (C) 2001-2012 Free Software Foundation, Inc.
;; Author: Shenghuo Zhu <zsh@cs.rochester.edu>
;; This file is part of GNU Emacs.
-;; GNU Emacs is free software; you can redistribute it and/or modify
-;; it under the terms of the GNU General Public License as published
-;; by the Free Software Foundation; either version 2, or (at your
-;; option) any later version.
+;; GNU Emacs is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
-;; GNU Emacs is distributed in the hope that it will be useful, but
-;; WITHOUT ANY WARRANTY; without even the implied warranty of
-;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-;; General Public License for more details.
+;; GNU Emacs is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;; GNU General Public License for more details.
;; You should have received a copy of the GNU General Public License
-;; along with GNU Emacs; see the file COPYING. If not, write to the
-;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-;; Boston, MA 02111-1307, USA.
+;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
;;; Commentary:
;;; Code:
-(require 'mm-util)
-(require 'executable)
-
(eval-when-compile (require 'cl))
-(eval-and-compile
- (autoload 'url-insert-file-contents "url-handlers"))
+(require 'mm-util)
+(require 'gnus)
+
+(defvar url-current-object)
+(defvar url-package-name)
+(defvar url-package-version)
(defgroup mm-url nil
"A wrapper of url package and external url command for Gnus."
(defcustom mm-url-use-external (not
(condition-case nil
- (require 'url-handlers)
+ (require 'url)
(error nil)))
- "*If not-nil, use external grab program `mm-url-program'."
+ "*If non-nil, use external grab program `mm-url-program'."
+ :version "22.1"
:type 'boolean
:group 'mm-url)
(defvar mm-url-predefined-programs
- '((wget "wget" "-q" "-O" "-")
+ '((wget "wget" "--user-agent=mm-url" "-q" "-O" "-")
+ (w3m "w3m" "-dump_source")
(lynx "lynx" "-source")
- (curl "curl")))
+ (curl "curl" "--silent" "--user-agent" "mm-url" "--location")))
-(defcustom mm-url-program
+(defcustom mm-url-program
(cond
((executable-find "wget") 'wget)
+ ((executable-find "w3m") 'w3m)
((executable-find "lynx") 'lynx)
((executable-find "curl") 'curl)
(t "GET"))
- "The url grab program."
- :type '(choice
+ "The url grab program.
+Likely values are `wget', `w3m', `lynx' and `curl'."
+ :version "22.1"
+ :type '(choice
(symbol :tag "wget" wget)
+ (symbol :tag "w3m" w3m)
(symbol :tag "lynx" lynx)
(symbol :tag "curl" curl)
(string :tag "other"))
(defcustom mm-url-arguments nil
"The arguments for `mm-url-program'."
+ :version "22.1"
:type '(repeat string)
:group 'mm-url)
+\f
+;;; Internal variables
+
;; Stolen from w3.
(defvar mm-url-html-entities
'(
"A list of characters that are _NOT_ reserved in the URL spec.
This is taken from RFC 2396.")
+(defun mm-url-load-url ()
+ "Load `url-insert-file-contents'."
+ (unless (condition-case ()
+ (progn
+ (require 'url-handlers)
+ (require 'url-parse)
+ (require 'url-vars))
+ (error nil))
+ ;; w3-4.0pre0.46 or earlier version.
+ (require 'w3-vars)
+ (require 'url)))
+
+;;;###autoload
(defun mm-url-insert-file-contents (url)
+ "Insert file contents of URL.
+If `mm-url-use-external' is non-nil, use `mm-url-program'."
(if mm-url-use-external
- (mm-url-insert-file-contents-external url)
- (url-insert-file-contents url)))
+ (progn
+ (if (string-match "^file:/+" url)
+ (insert-file-contents (substring url (1- (match-end 0))))
+ (mm-url-insert-file-contents-external url))
+ (goto-char (point-min))
+ (if (fboundp 'url-generic-parse-url)
+ (setq url-current-object
+ (url-generic-parse-url url)))
+ (list url (buffer-size)))
+ (mm-url-load-url)
+ (let ((name buffer-file-name)
+ (url-request-extra-headers
+ ;; ISTM setting a Connection header was a workaround for
+ ;; older versions of url included with w3, but it does more
+ ;; harm than good with the one shipped with Emacs. --ansel
+ (if (not (and (boundp 'url-version)
+ (equal url-version "Emacs")))
+ (list (cons "Connection" "Close"))))
+ result)
+ (setq result (url-insert-file-contents url))
+ (save-excursion
+ (goto-char (point-min))
+ (while (re-search-forward "\r 1000\r ?" nil t)
+ (replace-match "")))
+ (setq buffer-file-name name)
+ (if (and (fboundp 'url-generic-parse-url)
+ (listp result))
+ (setq url-current-object (url-generic-parse-url
+ (car result))))
+ result)))
+;;;###autoload
(defun mm-url-insert-file-contents-external (url)
+ "Insert file contents of URL using `mm-url-program'."
(let (program args)
(if (symbolp mm-url-program)
(let ((item (cdr (assq mm-url-program mm-url-predefined-programs))))
args (append (cdr item) (list url))))
(setq program mm-url-program
args (append mm-url-arguments (list url))))
- (apply 'call-process program nil t nil args)))
+ (unless (eq 0 (apply 'call-process program nil t nil args))
+ (error "Couldn't fetch %s" url))))
+
+(defvar mm-url-timeout 30
+ "The number of seconds before timing out an URL fetch.")
+
+(defvar mm-url-retries 10
+ "The number of retries after timing out when fetching an URL.")
(defun mm-url-insert (url &optional follow-refresh)
"Insert the contents from an URL in the current buffer.
If FOLLOW-REFRESH is non-nil, redirect refresh url in META."
- (let ((name buffer-file-name))
- (if follow-refresh
- (save-restriction
- (narrow-to-region (point) (point))
- (mm-url-insert-file-contents url)
- (goto-char (point-min))
- (when (re-search-forward
- "<meta[ \t\r\n]*http-equiv=\"Refresh\"[^>]*URL=\\([^\"]+\\)\"" nil t)
- (let ((url (match-string 1)))
- (delete-region (point-min) (point-max))
- (mm-url-insert url t))))
- (mm-url-insert-file-contents url))
- (setq buffer-file-name name)))
+ (let ((times mm-url-retries)
+ (done nil)
+ (first t)
+ result)
+ (while (and (not (zerop (decf times)))
+ (not done))
+ (with-timeout (mm-url-timeout)
+ (unless first
+ (message "Trying again (%s)..." (- mm-url-retries times)))
+ (setq first nil)
+ (if follow-refresh
+ (save-restriction
+ (narrow-to-region (point) (point))
+ (mm-url-insert-file-contents url)
+ (goto-char (point-min))
+ (when (re-search-forward
+ "<meta[ \t\r\n]*http-equiv=\"Refresh\"[^>]*URL=\\([^\"]+\\)\"" nil t)
+ (let ((url (match-string 1)))
+ (delete-region (point-min) (point-max))
+ (setq result (mm-url-insert url t)))))
+ (setq result (mm-url-insert-file-contents url)))
+ (setq done t)))
+ result))
(defun mm-url-decode-entities ()
"Decode all HTML entities."
(goto-char (point-min))
- (while (re-search-forward "&\\(#[0-9]+\\|[a-z]+\\);" nil t)
- (let ((elem (if (eq (aref (match-string 1) 0) ?\#)
- (let ((c
- (string-to-number (substring
- (match-string 1) 1))))
- (if (mm-char-or-char-int-p c) c 32))
- (or (cdr (assq (intern (match-string 1))
- mm-url-html-entities))
- ?#))))
+ (while (re-search-forward "&\\(#[0-9]+\\|#x[0-9a-f]+\\|[a-z]+[0-9]*\\);"
+ nil t)
+ (let* ((entity (match-string 1))
+ (elem (if (eq (aref entity 0) ?\#)
+ (let ((c
+ ;; Hex number: ㈒
+ (if (eq (aref entity 1) ?x)
+ (string-to-number (substring entity 2)
+ 16)
+ ;; Decimal number: 
+ (string-to-number (substring entity 1)))))
+ (setq c (or (cdr (assq c mm-extra-numeric-entities))
+ (mm-ucs-to-char c)))
+ (if (mm-char-or-char-int-p c) c ?#))
+ (or (cdr (assq (intern entity)
+ mm-url-html-entities))
+ ?#))))
(unless (stringp elem)
(setq elem (char-to-string elem)))
(replace-match elem t t))))
+(defun mm-url-decode-entities-nbsp ()
+ "Decode all HTML entities and to a space."
+ (let ((mm-url-html-entities (cons '(nbsp . 32) mm-url-html-entities)))
+ (mm-url-decode-entities)))
+
(defun mm-url-decode-entities-string (string)
(with-temp-buffer
(insert string)
(mm-url-decode-entities)
- (buffer-substring (point-min) (point-max))))
+ (buffer-string)))
(defun mm-url-form-encode-xwfu (chunk)
"Escape characters in a string for application/x-www-form-urlencoded.
((= char ? ) "+")
((memq char mm-url-unreserved-chars) (char-to-string char))
(t (upcase (format "%%%02x" char)))))
- ;; Fixme: Should this actually be accepting multibyte? Is there a
- ;; better way in XEmacs?
- (if (featurep 'mule)
- (encode-coding-string chunk
- (if (fboundp 'find-coding-systems-string)
- (car (find-coding-systems-string chunk))
- buffer-file-coding-system))
- chunk)
+ (mm-encode-coding-string chunk
+ (if (fboundp 'find-coding-systems-string)
+ (car (find-coding-systems-string chunk))
+ buffer-file-coding-system))
""))
+(defun mm-url-encode-www-form-urlencoded (pairs)
+ "Return PAIRS encoded for forms."
+ (mapconcat
+ (lambda (data)
+ (concat (mm-url-form-encode-xwfu (car data)) "="
+ (mm-url-form-encode-xwfu (cdr data))))
+ pairs "&"))
+
+(autoload 'mml-compute-boundary "mml")
+
+(defun mm-url-remove-markup ()
+ "Remove all HTML markup, leaving just plain text."
+ (goto-char (point-min))
+ (while (search-forward "<!--" nil t)
+ (delete-region (match-beginning 0)
+ (or (search-forward "-->" nil t)
+ (point-max))))
+ (goto-char (point-min))
+ (while (re-search-forward "<[^>]+>" nil t)
+ (replace-match "" t t)))
+
(provide 'mm-url)
;;; mm-url.el ends here
projects / gnus / blobdiff