pop3.el (pop3-wait-for-messages): Use pop3-accept-process-output instead of nnheader...

[gnus] / lisp / pop3.el

diff --git a/lisp/pop3.el b/lisp/pop3.el
index f69c1d5..63ed800 100644 (file)
--- a/lisp/pop3.el
+++ b/lisp/pop3.el
@@ -1,7 +1,7 @@
 ;;; pop3.el --- Post Office Protocol (RFC 1460) interface
 
 ;;; pop3.el --- Post Office Protocol (RFC 1460) interface
 

-;; Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004
-;;        Free Software Foundation, Inc.
+;; Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
+;;   2004, 2005, 2006, 2007, 2008, 2009, 2010 Free Software Foundation, Inc.

 
 ;; Author: Richard L. Pieri <ratinox@peorth.gweep.net>
 ;; Maintainer: FSF
 
 ;; Author: Richard L. Pieri <ratinox@peorth.gweep.net>
 ;; Maintainer: FSF


@@ -9,10 +9,10 @@
 
 ;; This file is part of GNU Emacs.
 
 
 ;; This file is part of GNU Emacs.
 

-;; GNU Emacs is free software; you can redistribute it and/or modify
+;; GNU Emacs is free software: you can redistribute it and/or modify

 ;; it under the terms of the GNU General Public License as published by
 ;; it under the terms of the GNU General Public License as published by

-;; the Free Software Foundation; either version 2, or (at your option)
-;; any later version.
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.

 
 ;; GNU Emacs is distributed in the hope that it will be useful,
 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
 
 ;; GNU Emacs is distributed in the hope that it will be useful,
 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of


@@ -20,9 +20,7 @@
 ;; GNU General Public License for more details.
 
 ;; You should have received a copy of the GNU General Public License
 ;; GNU General Public License for more details.
 
 ;; You should have received a copy of the GNU General Public License

-;; along with GNU Emacs; see the file COPYING.  If not, write to the
-;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-;; Boston, MA 02111-1307, USA.
+;; along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.

 
 ;;; Commentary:
 
 
 ;;; Commentary:
 


@@ -35,10 +33,12 @@
 
 ;;; Code:
 
 
 ;;; Code:
 

+(eval-when-compile (require 'cl))

 (require 'mail-utils)
 (require 'mail-utils)

+(defvar parse-time-months)

 
 (defgroup pop3 nil
 
 (defgroup pop3 nil

-  "Post Office Protocol"
+  "Post Office Protocol."

   :group 'mail
   :group 'mail-source)
 
   :group 'mail
   :group 'mail-source)
 


@@ -46,26 +46,26 @@
                             (getenv "LOGNAME")
                             (getenv "USER"))
   "*POP3 maildrop."
                             (getenv "LOGNAME")
                             (getenv "USER"))
   "*POP3 maildrop."

-  :version "21.4" ;; Oort Gnus
+  :version "22.1" ;; Oort Gnus

   :type 'string
   :group 'pop3)
 
 (defcustom pop3-mailhost (or (getenv "MAILHOST") ;; nil -> mismatch
                             "pop3")
   "*POP3 mailhost."
   :type 'string
   :group 'pop3)
 
 (defcustom pop3-mailhost (or (getenv "MAILHOST") ;; nil -> mismatch
                             "pop3")
   "*POP3 mailhost."

-  :version "21.4" ;; Oort Gnus
+  :version "22.1" ;; Oort Gnus

   :type 'string
   :group 'pop3)
 
 (defcustom pop3-port 110
   "*POP3 port."
   :type 'string
   :group 'pop3)
 
 (defcustom pop3-port 110
   "*POP3 port."

-  :version "21.4" ;; Oort Gnus
+  :version "22.1" ;; Oort Gnus

   :type 'number
   :group 'pop3)
 
 (defcustom pop3-password-required t
   "*Non-nil if a password is required when connecting to POP server."
   :type 'number
   :group 'pop3)
 
 (defcustom pop3-password-required t
   "*Non-nil if a password is required when connecting to POP server."

-  :version "21.4" ;; Oort Gnus
+  :version "22.1" ;; Oort Gnus

   :type 'boolean
   :group 'pop3)
 
   :type 'boolean
   :group 'pop3)
 


@@ -75,16 +75,27 @@
 
 (defcustom pop3-authentication-scheme 'pass
   "*POP3 authentication scheme.
 
 (defcustom pop3-authentication-scheme 'pass
   "*POP3 authentication scheme.

-Defaults to 'pass, for the standard USER/PASS authentication.  Other valid
-values are 'apop."
-  :version "21.4" ;; Oort Gnus
-  :type '(choice (const :tag "USER/PASS" pass)
+Defaults to `pass', for the standard USER/PASS authentication.  The other
+valid value is 'apop'."
+  :type '(choice (const :tag "Normal user/password" pass)

                 (const :tag "APOP" apop))
                 (const :tag "APOP" apop))

+  :version "22.1" ;; Oort Gnus

   :group 'pop3)
 
 (defcustom pop3-leave-mail-on-server nil
   :group 'pop3)
 
 (defcustom pop3-leave-mail-on-server nil

-  "*Non-nil if the mail is to be left on the POP server after fetching."
-  :version "21.4" ;; Oort Gnus
+  "*Non-nil if the mail is to be left on the POP server after fetching.
+
+If `pop3-leave-mail-on-server' is non-nil the mail is to be left
+on the POP server after fetching.  Note that POP servers maintain
+no state information between sessions, so what the client
+believes is there and what is actually there may not match up.
+If they do not, then you may get duplicate mails or the whole
+thing can fall apart and leave you with a corrupt mailbox."
+  ;; We can't use the UILD support from XEmacs mail-lib or cvs.m17n.org:
+  ;; http://thread.gmane.org/v9lld8fml4.fsf@marauder.physik.uni-ulm.de
+  ;; http://thread.gmane.org/b9yy8hzy9ej.fsf@jpl.org
+  ;; Any volunteer to re-implement this?
+  :version "22.1" ;; Oort Gnus

   :type 'boolean
   :group 'pop3)
 
   :type 'boolean
   :group 'pop3)
 


@@ -95,40 +106,115 @@ Used for APOP authentication.")
 (defvar pop3-read-point nil)
 (defvar pop3-debug nil)
 
 (defvar pop3-read-point nil)
 (defvar pop3-debug nil)
 

-;; Borrowed from nnheader-accept-process-output in nnheader.el.
-(defvar pop3-read-timeout
-  (if (string-match "windows-nt\\|os/2\\|emx\\|cygwin"
-                   (symbol-name system-type))
-      ;; http://thread.gmane.org/v9655t3pjo.fsf@marauder.physik.uni-ulm.de
-      ;;
-      ;; IIRC, values lower than 1.0 didn't/don't work on Windows/DOS.
-      ;;
-      ;; There should probably be a runtime test to determine the timing
-      ;; resolution, or a primitive to report it.  I don't know off-hand
-      ;; what's possible.  Perhaps better, maybe the Windows/DOS primitive
-      ;; could round up non-zero timeouts to a minimum of 1.0?
-      1.0
-    0.1)
-  "How long pop3 should wait between checking for the end of output.
+;; Borrowed from nnheader-accept-process-output in nnheader.el.  See the
+;; comments there for explanations about the values.
+
+(eval-and-compile
+  (if (and (fboundp 'nnheader-accept-process-output)
+          (boundp 'nnheader-read-timeout))
+      (defalias 'pop3-accept-process-output 'nnheader-accept-process-output)
+    ;; Borrowed from `nnheader.el':
+    (defvar pop3-read-timeout
+      (if (string-match "windows-nt\\|os/2\\|cygwin"
+                       (symbol-name system-type))
+         1.0
+       0.01)
+      "How long pop3 should wait between checking for the end of output.

 Shorter values mean quicker response, but are more CPU intensive.")
 Shorter values mean quicker response, but are more CPU intensive.")

-
-;; Borrowed from nnheader-accept-process-output in nnheader.el.
-(defun pop3-accept-process-output (process)
-  (accept-process-output
-   process
-   (truncate pop3-read-timeout)
-   (truncate (* (- pop3-read-timeout
-                  (truncate pop3-read-timeout))
-               1000))))
-
-(defun pop3-movemail (&optional crashbox)
-  "Transfer contents of a maildrop to the specified CRASHBOX."
-  (or crashbox (setq crashbox (expand-file-name "~/.crashbox")))
+    (defun pop3-accept-process-output (process)
+      (accept-process-output
+       process
+       (truncate pop3-read-timeout)
+       (truncate (* (- pop3-read-timeout
+                      (truncate pop3-read-timeout))
+                   1000))))))
+
+;;;###autoload
+(defun pop3-movemail (file)
+  "Transfer contents of a maildrop to the specified FILE.
+Use streaming commands."

   (let* ((process (pop3-open-server pop3-mailhost pop3-port))
   (let* ((process (pop3-open-server pop3-mailhost pop3-port))

-        (crashbuf (get-buffer-create " *pop3-retr*"))
-        (n 1)
-        message-count
-        (pop3-password pop3-password))
+        message-count message-total-size)
+    (pop3-logon process)
+    (with-current-buffer (process-buffer process)
+      (let ((size (pop3-stat process)))
+       (setq message-count (car size)
+             message-total-size (cadr size)))
+      (when (plusp message-count)
+       (pop3-send-streaming-command
+        process "RETR" message-count message-total-size)
+       (pop3-write-to-file file)
+       (unless pop3-leave-mail-on-server
+         (pop3-send-streaming-command
+          process "DELE" message-count nil))))
+    (pop3-quit process)
+    t))
+
+(defun pop3-send-streaming-command (process command count total-size)
+  (erase-buffer)
+  (let ((i 1))
+    (while (>= count i)
+      (process-send-string process (format "%s %d\r\n" command i))
+      ;; Only do 100 messages at a time to avoid pipe stalls.
+      (when (zerop (% i 100))
+       (pop3-wait-for-messages process i total-size))
+      (incf i)))
+  (pop3-wait-for-messages process count total-size))
+
+(defun pop3-wait-for-messages (process count total-size)
+  (while (< (pop3-number-of-responses total-size) count)
+    (when total-size
+      (message "pop3 retrieved %dKB (%d%%)"
+              (truncate (/ (buffer-size) 1000))
+              (truncate (* (/ (* (buffer-size) 1.0)
+                              total-size) 100))))
+    (pop3-accept-process-output process)))
+
+(defun pop3-write-to-file (file)
+  (let ((pop-buffer (current-buffer))
+       (start (point-min))
+       beg end
+       temp-buffer)
+    (with-temp-buffer
+      (setq temp-buffer (current-buffer))
+      (with-current-buffer pop-buffer
+       (goto-char (point-min))
+       (while (re-search-forward "^\\+OK" nil t)
+         (forward-line 1)
+         (setq beg (point))
+         (when (re-search-forward "^\\.\r?\n" nil t)
+           (setq start (point))
+           (forward-line -1)
+           (setq end (point)))
+         (with-current-buffer temp-buffer
+           (goto-char (point-max))
+           (let ((hstart (point)))
+             (insert-buffer-substring pop-buffer beg end)
+             (pop3-clean-region hstart (point))
+             (goto-char (point-max))
+             (pop3-munge-message-separator hstart (point))
+             (goto-char (point-max))))))
+      (let ((coding-system-for-write 'binary))
+       (goto-char (point-min))
+       ;; Check whether something inserted a newline at the start and
+       ;; delete it.
+       (when (eolp)
+         (delete-char 1))
+       (write-region (point-min) (point-max) file nil 'nomesg)))))
+
+(defun pop3-number-of-responses (endp)
+  (let ((responses 0))
+    (save-excursion
+      (goto-char (point-min))
+      (while (or (and (re-search-forward "^\\+OK" nil t)
+                     (or (not endp)
+                         (re-search-forward "^\\.\r?\n" nil t)))
+                (re-search-forward "^-ERR " nil t))
+       (incf responses)))
+    responses))
+
+(defun pop3-logon (process)
+  (let ((pop3-password pop3-password))

     ;; for debugging only
     (if pop3-debug (switch-to-buffer (process-buffer process)))
     ;; query for password
     ;; for debugging only
     (if pop3-debug (switch-to-buffer (process-buffer process)))
     ;; query for password


@@ -140,31 +226,7 @@ Shorter values mean quicker response, but are more CPU intensive.")
          ((equal 'pass pop3-authentication-scheme)
           (pop3-user process pop3-maildrop)
           (pop3-pass process))
          ((equal 'pass pop3-authentication-scheme)
           (pop3-user process pop3-maildrop)
           (pop3-pass process))

-         (t (error "Invalid POP3 authentication scheme")))
-    (setq message-count (car (pop3-stat process)))
-    (unwind-protect
-       (while (<= n message-count)
-         (message "Retrieving message %d of %d from %s..."
-                  n message-count pop3-mailhost)
-         (pop3-retr process n crashbuf)
-         (save-excursion
-           (set-buffer crashbuf)
-           (let ((coding-system-for-write 'binary))
-             (write-region (point-min) (point-max) crashbox t 'nomesg))
-           (set-buffer (process-buffer process))
-           (while (> (buffer-size) 5000)
-             (goto-char (point-min))
-             (forward-line 50)
-             (delete-region (point-min) (point))))
-          (unless pop3-leave-mail-on-server
-            (pop3-dele process n))
-         (setq n (+ 1 n))
-         (if pop3-debug (sit-for 1) (sit-for 0.1))
-         )
-      (pop3-quit process))
-    (kill-buffer crashbuf)
-    )
-  t)
+         (t (error "Invalid POP3 authentication scheme")))))

 
 (defun pop3-get-message-count ()
   "Return the number of messages in the maildrop."
 
 (defun pop3-get-message-count ()
   "Return the number of messages in the maildrop."


@@ -187,6 +249,30 @@ Shorter values mean quicker response, but are more CPU intensive.")
     (pop3-quit process)
     message-count))
 
     (pop3-quit process)
     message-count))
 

+(autoload 'open-tls-stream "tls")
+(autoload 'starttls-open-stream "starttls")
+(autoload 'starttls-negotiate "starttls") ; avoid warning
+
+(defcustom pop3-stream-type nil
+  "*Transport security type for POP3 connexions.
+This may be either nil (plain connexion), `ssl' (use an
+SSL/TSL-secured stream) or `starttls' (use the starttls mechanism
+to turn on TLS security after opening the stream).  However, if
+this is nil, `ssl' is assumed for connexions to port
+995 (pop3s)."
+  :version "23.1" ;; No Gnus
+  :group 'pop3
+  :type '(choice (const :tag "Plain" nil)
+                (const :tag "SSL/TLS" ssl)
+                (const starttls)))
+
+(eval-and-compile
+  (if (fboundp 'set-process-query-on-exit-flag)
+      (defalias 'pop3-set-process-query-on-exit-flag
+       'set-process-query-on-exit-flag)
+    (defalias 'pop3-set-process-query-on-exit-flag
+      'process-kill-without-query)))
+

 (defun pop3-open-server (mailhost port)
   "Open TCP connection to MAILHOST on PORT.
 Returns the process associated with the connection."
 (defun pop3-open-server (mailhost port)
   "Open TCP connection to MAILHOST on PORT.
 Returns the process associated with the connection."


@@ -198,21 +284,60 @@ Returns the process associated with the connection."
                                             mailhost)))
       (erase-buffer)
       (setq pop3-read-point (point-min))
                                             mailhost)))
       (erase-buffer)
       (setq pop3-read-point (point-min))

-      (setq process (open-network-stream "POP" (current-buffer) mailhost port))
+      (setq process
+           (cond
+            ((or (eq pop3-stream-type 'ssl)
+                 (and (not pop3-stream-type) (member port '(995 "pop3s"))))
+             ;; gnutls-cli, openssl don't accept service names
+             (if (or (equal port "pop3s")
+                     (null port))
+                 (setq port 995))
+             (let ((process (open-tls-stream "POP" (current-buffer)
+                                             mailhost port)))
+               (when process
+                 ;; There's a load of info printed that needs deleting.
+                 (let ((again 't))
+                   ;; repeat until
+                   ;; - either we received the +OK line
+                   ;; - or accept-process-output timed out without getting
+                   ;;   anything
+                   (while (and again
+                               (setq again (memq (process-status process)
+                                                 '(open run))))
+                     (setq again (pop3-accept-process-output process))
+                     (goto-char (point-max))
+                     (forward-line -1)
+                     (cond ((looking-at "\\+OK")
+                            (setq again nil)
+                            (delete-region (point-min) (point)))
+                           ((not again)
+                            (pop3-quit process)
+                            (error "POP SSL connexion failed")))))
+                 process)))
+            ((eq pop3-stream-type 'starttls)
+             ;; gnutls-cli, openssl don't accept service names
+             (if (equal port "pop3")
+                 (setq port 110))
+             (let ((process (starttls-open-stream "POP" (current-buffer)
+                                                  mailhost (or port 110))))
+               (pop3-send-command process "STLS")
+               (let ((response (pop3-read-response process t)))
+                 (if (and response (string-match "+OK" response))
+                     (starttls-negotiate process)
+                   (pop3-quit process)
+                   (error "POP server doesn't support starttls")))
+               process))
+            (t
+             (open-network-stream "POP" (current-buffer) mailhost port))))

       (let ((response (pop3-read-response process t)))
        (setq pop3-timestamp
              (substring response (or (string-match "<" response) 0)
                         (+ 1 (or (string-match ">" response) -1)))))
       (let ((response (pop3-read-response process t)))
        (setq pop3-timestamp
              (substring response (or (string-match "<" response) 0)
                         (+ 1 (or (string-match ">" response) -1)))))

+      (pop3-set-process-query-on-exit-flag process nil)

       process)))
 
 ;; Support functions
 
       process)))
 
 ;; Support functions
 

-(defun pop3-process-filter (process output)
-  (save-excursion
-    (set-buffer (process-buffer process))
-    (goto-char (point-max))
-    (insert output)))
-

 (defun pop3-send-command (process command)
   (set-buffer (process-buffer process))
   (goto-char (point-max))
 (defun pop3-send-command (process command)
   (set-buffer (process-buffer process))
   (goto-char (point-max))


@@ -238,7 +363,7 @@ Return the response string if optional second argument is non-nil."
       (setq match-end (point))
       (goto-char pop3-read-point)
       (if (looking-at "-ERR")
       (setq match-end (point))
       (goto-char pop3-read-point)
       (if (looking-at "-ERR")

-         (error (buffer-substring (point) (- match-end 2)))
+         (error "%s" (buffer-substring (point) (- match-end 2)))

        (if (not (looking-at "+OK"))
            (progn (setq pop3-read-point match-end) nil)
          (setq pop3-read-point match-end)
        (if (not (looking-at "+OK"))
            (progn (setq pop3-read-point match-end) nil)
          (setq pop3-read-point match-end)


@@ -259,8 +384,6 @@ Return the response string if optional second argument is non-nil."
       (forward-char)))
   (set-marker end nil))
 
       (forward-char)))
   (set-marker end nil))
 

-(eval-when-compile (defvar parse-time-months))
-

 ;; Copied from message-make-date.
 (defun pop3-make-date (&optional now)
   "Make a valid date header.
 ;; Copied from message-make-date.
 (defun pop3-make-date (&optional now)
   "Make a valid date header.


@@ -305,6 +428,8 @@ If NOW, use that time instead."
            ;; Date: 08 Jul 1996 23:22:24 -0400
            ;; should be
            ;; Tue Jul 9 09:04:21 1996
            ;; Date: 08 Jul 1996 23:22:24 -0400
            ;; should be
            ;; Tue Jul 9 09:04:21 1996

+
+           ;; Fixme: This should use timezone on the date field contents.

            (setq date
                  (cond ((not date)
                         "Tue Jan 1 00:00:0 1900")
            (setq date
                  (cond ((not date)
                         "Tue Jan 1 00:00:0 1900")