X-Git-Url: http://cgit.sxemacs.org/?p=gnus;a=blobdiff_plain;f=lisp%2Fnnimap.el;h=c3c25cbf1942acc5dbf2a4fcfb7837240670cd16;hp=a5d371422d683d8d785c9d1d41f7040a96c27da6;hb=92bc187d1fcbe58bbd9c638b69a614c3bbad208e;hpb=e81c8cc1b52bca12903aed4e4afb2f41dfbf6cb4 diff --git a/lisp/nnimap.el b/lisp/nnimap.el index a5d371422..c3c25cbf1 100644 --- a/lisp/nnimap.el +++ b/lisp/nnimap.el @@ -37,6 +37,12 @@ (require 'gnus) (require 'nnoo) (require 'netrc) +(require 'utf7) +(require 'tls) +(require 'parse-time) + +(autoload 'auth-source-forget-user-or-password "auth-source") +(autoload 'auth-source-user-or-password "auth-source") (nnoo-declare nnimap) @@ -50,7 +56,7 @@ it will default to `imap'.") (defvoo nnimap-stream 'ssl "How nnimap will talk to the IMAP server. -Values are `ssl' and `network'.") +Values are `ssl', `network', `starttls' or `shell'.") (defvoo nnimap-shell-program (if (boundp 'imap-shell-program) (if (listp imap-shell-program) @@ -61,28 +67,41 @@ Values are `ssl' and `network'.") (defvoo nnimap-inbox nil "The mail box where incoming mail arrives and should be split out of.") -(defvoo nnimap-expunge-inbox nil - "If non-nil, expunge the inbox after fetching mail. -This is always done if the server supports UID EXPUNGE, but it's -not done by default on servers that doesn't support that command.") +(defvoo nnimap-split-methods nil + "How mail is split. +Uses the same syntax as nnmail-split-methods") + +(defvoo nnimap-split-fancy nil + "Uses the same syntax as nnmail-split-fancy.") + +(make-obsolete-variable 'nnimap-split-rule "see `nnimap-split-methods'" + "Emacs 24.1") (defvoo nnimap-authenticator nil "How nnimap authenticate itself to the server. Possible choices are nil (use default methods) or `anonymous'.") -(defvoo nnimap-fetch-partial-articles nil - "If non-nil, nnimap will fetch partial articles. -If t, nnimap will fetch only the first part. If a string, it -will fetch all parts that have types that match that string. A -likely value would be \"text/\" to automatically fetch all -textual parts.") +(defvoo nnimap-expunge t + "If non-nil, expunge articles after deleting them. +This is always done if the server supports UID EXPUNGE, but it's +not done by default on servers that doesn't support that command.") -(defvoo nnimap-expunge nil) +(defvoo nnimap-streaming t + "If non-nil, try to use streaming commands with IMAP servers. +Switching this off will make nnimap slower, but it helps with +some servers.") (defvoo nnimap-connection-alist nil) (defvoo nnimap-current-infos nil) +(defvoo nnimap-fetch-partial-articles nil + "If non-nil, Gnus will fetch partial articles. +If t, nnimap will fetch only the first part. If a string, it +will fetch all parts that have types that match that string. A +likely value would be \"text/\" to automatically fetch all +textual parts.") + (defvar nnimap-process nil) (defvar nnimap-status-string "") @@ -90,15 +109,19 @@ textual parts.") (defvar nnimap-split-download-body-default nil "Internal variable with default value for `nnimap-split-download-body'.") +(defvar nnimap-keepalive-timer nil) +(defvar nnimap-process-buffers nil) + (defstruct nnimap - group process commands capabilities select-result newlinep) + group process commands capabilities select-result newlinep server + last-command-time greeting) (defvar nnimap-object nil) (defvar nnimap-mark-alist - '((read "\\Seen") - (tick "\\Flagged") - (reply "\\Answered") + '((read "\\Seen" %Seen) + (tick "\\Flagged" %Flagged) + (reply "\\Answered" %Answered) (expire "gnus-expire") (dormant "gnus-dormant") (score "gnus-score") @@ -106,8 +129,6 @@ textual parts.") (download "gnus-download") (forward "gnus-forward"))) -(defvar nnimap-split-methods nil) - (defun nnimap-buffer () (nnimap-find-process-buffer nntp-server-buffer)) @@ -116,7 +137,6 @@ textual parts.") (erase-buffer) (when (nnimap-possibly-change-group group server) (with-current-buffer (nnimap-buffer) - (nnimap-send-command "SELECT %S" (utf7-encode group t)) (erase-buffer) (nnimap-wait-for-response (nnimap-send-command @@ -124,8 +144,7 @@ textual parts.") (nnimap-article-ranges (gnus-compress-sequence articles)) (format "(UID RFC822.SIZE BODYSTRUCTURE %s)" (format - (if (member "IMAP4REV1" - (nnimap-capabilities nnimap-object)) + (if (nnimap-ver4-p) "BODY.PEEK[HEADER.FIELDS %s]" "RFC822.HEADER.LINES %s") (append '(Subject From Date Message-Id @@ -135,19 +154,26 @@ textual parts.") (nnimap-transform-headers)) (insert-buffer-substring (nnimap-find-process-buffer (current-buffer)))) - t)) + 'headers)) (defun nnimap-transform-headers () (goto-char (point-min)) - (let (article bytes lines size) + (let (article bytes lines size string) (block nil (while (not (eobp)) (while (not (looking-at "^\\* [0-9]+ FETCH.*UID \\([0-9]+\\)")) (delete-region (point) (progn (forward-line 1) (point))) (when (eobp) (return))) - (setq article (match-string 1) - bytes (nnimap-get-length) + (setq article (match-string 1)) + ;; Unfold quoted {number} strings. + (while (re-search-forward "[^]] {\\([0-9]+\\)}\r\n" + (1+ (line-end-position)) t) + (setq size (string-to-number (match-string 1))) + (delete-region (+ (match-beginning 0) 2) (point)) + (setq string (delete-region (point) (+ (point) size))) + (insert (format "%S" string))) + (setq bytes (nnimap-get-length) lines nil) (beginning-of-line) (setq size @@ -157,7 +183,8 @@ textual parts.") (match-string 1))) (beginning-of-line) (when (search-forward "BODYSTRUCTURE" (line-end-position) t) - (let ((structure (ignore-errors (read (current-buffer))))) + (let ((structure (ignore-errors + (read (current-buffer))))) (while (and (consp structure) (not (stringp (car structure)))) (setq structure (car structure))) @@ -212,8 +239,10 @@ textual parts.") (buffer-disable-undo) (gnus-add-buffer) (set (make-local-variable 'after-change-functions) nil) - (set (make-local-variable 'nnimap-object) (make-nnimap)) + (set (make-local-variable 'nnimap-object) + (make-nnimap :server (nnoo-current-server 'nnimap))) (push (list buffer (current-buffer)) nnimap-connection-alist) + (push (current-buffer) nnimap-process-buffers) (current-buffer))) (defun nnimap-open-shell-stream (name buffer host port) @@ -226,7 +255,7 @@ textual parts.") ?s host ?p port))))) -(defun nnimap-credentials (address ports) +(defun nnimap-credentials (address ports &optional inhibit-create) (let (port credentials) ;; Request the credentials from all ports, but only query on the ;; last port if all the previous ones have failed. @@ -234,71 +263,155 @@ textual parts.") (setq port (pop ports))) (setq credentials (auth-source-user-or-password - '("login" "password") address port nil (null ports)))) + '("login" "password") address port nil + (if inhibit-create + nil + (null ports))))) credentials)) +(defun nnimap-keepalive () + (let ((now (current-time))) + (dolist (buffer nnimap-process-buffers) + (when (buffer-name buffer) + (with-current-buffer buffer + (when (and nnimap-object + (nnimap-last-command-time nnimap-object) + (> (time-to-seconds + (time-subtract + now + (nnimap-last-command-time nnimap-object))) + ;; More than five minutes since the last command. + (* 5 60))) + (nnimap-send-command "NOOP"))))))) + (defun nnimap-open-connection (buffer) - (with-current-buffer (nnimap-make-process-buffer buffer) - (let* ((coding-system-for-read 'binary) - (coding-system-for-write 'binary) - (ports - (cond - ((eq nnimap-stream 'network) - (open-network-stream - "*nnimap*" (current-buffer) nnimap-address - (or nnimap-server-port - (if (netrc-find-service-number "imap") - "imap" - "143"))) - '("143" "imap")) - ((eq nnimap-stream 'shell) - (nnimap-open-shell-stream - "*nnimap*" (current-buffer) nnimap-address - (or nnimap-server-port "imap")) - '("imap")) - ((eq nnimap-stream 'ssl) - (open-tls-stream - "*nnimap*" (current-buffer) nnimap-address - (or nnimap-server-port - (if (netrc-find-service-number "imaps") - "imaps" - "993"))) - '("143" "993" "imap" "imaps")))) - connection-result login-result credentials) - (setf (nnimap-process nnimap-object) - (get-buffer-process (current-buffer))) - (when (and (nnimap-process nnimap-object) - (memq (process-status (nnimap-process nnimap-object)) - '(open run))) - (gnus-set-process-query-on-exit-flag (nnimap-process nnimap-object) nil) - (when (setq connection-result (nnimap-wait-for-connection)) - (unless (equal connection-result "PREAUTH") - (if (not (setq credentials - (if (eq nnimap-authenticator 'anonymous) - (list "anonymous" - (message-make-address)) - (nnimap-credentials - nnimap-address - (if nnimap-server-port - (cons (format "%s" nnimap-server-port) ports) - ports))))) - (setq nnimap-object nil) - (setq login-result (nnimap-command "LOGIN %S %S" - (car credentials) - (cadr credentials))) - (unless (car login-result) - (delete-process (nnimap-process nnimap-object)) - (setq nnimap-object nil)))) - (when nnimap-object + (unless nnimap-keepalive-timer + (setq nnimap-keepalive-timer (run-at-time (* 60 15) (* 60 15) + 'nnimap-keepalive))) + (block nil + (with-current-buffer (nnimap-make-process-buffer buffer) + (let* ((coding-system-for-read 'binary) + (coding-system-for-write 'binary) + (port nil) + (ports + (cond + ((eq nnimap-stream 'network) + (open-network-stream + "*nnimap*" (current-buffer) nnimap-address + (setq port + (or nnimap-server-port + (if (netrc-find-service-number "imap") + "imap" + "143")))) + '("143" "imap")) + ((eq nnimap-stream 'shell) + (nnimap-open-shell-stream + "*nnimap*" (current-buffer) nnimap-address + (setq port (or nnimap-server-port "imap"))) + '("imap")) + ((eq nnimap-stream 'starttls) + (let ((tls-program (nnimap-extend-tls-programs))) + (open-tls-stream + "*nnimap*" (current-buffer) nnimap-address + (setq port (or nnimap-server-port "imap")) + 'starttls)) + '("imap")) + ((memq nnimap-stream '(ssl tls)) + (open-tls-stream + "*nnimap*" (current-buffer) nnimap-address + (setq port + (or nnimap-server-port + (if (netrc-find-service-number "imaps") + "imaps" + "993")))) + '("143" "993" "imap" "imaps")) + (t + (error "Unknown stream type: %s" nnimap-stream)))) + connection-result login-result credentials) + (setf (nnimap-process nnimap-object) + (get-buffer-process (current-buffer))) + (if (not (and (nnimap-process nnimap-object) + (memq (process-status (nnimap-process nnimap-object)) + '(open run)))) + (nnheader-report 'nnimap "Unable to contact %s:%s via %s" + nnimap-address port nnimap-stream) + (gnus-set-process-query-on-exit-flag (nnimap-process nnimap-object) nil) + (if (not (setq connection-result (nnimap-wait-for-connection))) + (nnheader-report 'nnimap + "%s" (buffer-substring + (point) (line-end-position))) + ;; Store the greeting (for debugging purposes). + (setf (nnimap-greeting nnimap-object) + (buffer-substring (line-beginning-position) + (line-end-position))) + ;; Store the capabilities. (setf (nnimap-capabilities nnimap-object) (mapcar #'upcase - (or (nnimap-find-parameter "CAPABILITY" (cdr login-result)) - (nnimap-find-parameter - "CAPABILITY" (cdr (nnimap-command "CAPABILITY")))))) - (when (member "QRESYNC" (nnimap-capabilities nnimap-object)) - (nnimap-command "ENABLE QRESYNC")) - t)))))) + (nnimap-find-parameter + "CAPABILITY" (cdr (nnimap-command "CAPABILITY"))))) + (when nnimap-server-port + (push (format "%s" nnimap-server-port) ports)) + ;; If this is a STARTTLS-capable server, then sever the + ;; connection and start a STARTTLS connection instead. + (when (and (eq nnimap-stream 'network) + (member "STARTTLS" (nnimap-capabilities nnimap-object))) + (let ((nnimap-stream 'starttls)) + (let ((tls-process + (nnimap-open-connection buffer))) + ;; If the STARTTLS connection was successful, we + ;; kill our first non-encrypted connection. If it + ;; wasn't successful, we just use our unencrypted + ;; connection. + (when (memq (process-status tls-process) '(open run)) + (delete-process (nnimap-process nnimap-object)) + (kill-buffer (current-buffer)) + (return tls-process))))) + (unless (equal connection-result "PREAUTH") + (if (not (setq credentials + (if (eq nnimap-authenticator 'anonymous) + (list "anonymous" + (message-make-address)) + (or + ;; First look for the credentials based + ;; on the virtual server name. + (nnimap-credentials + (nnoo-current-server 'nnimap) ports t) + ;; Then look them up based on the + ;; physical address. + (nnimap-credentials nnimap-address ports))))) + (setq nnimap-object nil) + (setq login-result (nnimap-command "LOGIN %S %S" + (car credentials) + (cadr credentials))) + (unless (car login-result) + ;; If the login failed, then forget the credentials + ;; that are now possibly cached. + (dolist (host (list (nnoo-current-server 'nnimap) + nnimap-address)) + (dolist (port ports) + (dolist (element '("login" "password")) + (auth-source-forget-user-or-password + element host port)))) + (delete-process (nnimap-process nnimap-object)) + (setq nnimap-object nil)))) + (when nnimap-object + (when (member "QRESYNC" (nnimap-capabilities nnimap-object)) + (nnimap-command "ENABLE QRESYNC")) + (nnimap-process nnimap-object)))))))) + +(defun nnimap-extend-tls-programs () + (let ((programs tls-program) + result) + (unless (consp programs) + (setq programs (list programs))) + (dolist (program programs) + (when (assoc (car (split-string program)) tls-starttls-switches) + (push (if (not (string-match "%s" program)) + (concat program " " "%s") + program) + result))) + (nreverse result))) (defun nnimap-find-parameter (parameter elems) (let (result) @@ -313,7 +426,10 @@ textual parts.") result)) (deffoo nnimap-close-server (&optional server) - t) + (when (nnoo-change-server 'nnimap server nil) + (ignore-errors + (delete-process (get-buffer-process (nnimap-buffer)))) + t)) (deffoo nnimap-request-close () t) @@ -330,7 +446,7 @@ textual parts.") (deffoo nnimap-request-article (article &optional group server to-buffer) (with-current-buffer nntp-server-buffer (let ((result (nnimap-possibly-change-group group server)) - parts) + parts structure) (when (stringp article) (setq article (nnimap-find-article-by-message-id group article))) (when (and result @@ -339,35 +455,125 @@ textual parts.") (with-current-buffer (nnimap-buffer) (erase-buffer) (when nnimap-fetch-partial-articles - (if (eq nnimap-fetch-partial-articles t) - (setq parts '(1)) - (nnimap-command "UID FETCH %d (BODYSTRUCTURE)" article) - (goto-char (point-min)) - (when (re-search-forward "FETCH.*BODYSTRUCTURE" nil t) - (let ((structure (ignore-errors (read (current-buffer))))) - (setq parts (nnimap-find-wanted-parts structure)))))) - (setq result - (nnimap-command - (if (member "IMAP4REV1" (nnimap-capabilities nnimap-object)) - "UID FETCH %d BODY.PEEK[]" - "UID FETCH %d RFC822.PEEK") - article)) - ;; Check that we really got an article. - (goto-char (point-min)) - (unless (looking-at "\\* [0-9]+ FETCH") - (setq result nil))) - (let ((buffer (nnimap-find-process-buffer (current-buffer)))) - (when (car result) - (with-current-buffer (or to-buffer nntp-server-buffer) - (insert-buffer-substring buffer) - (goto-char (point-min)) - (let ((bytes (nnimap-get-length))) - (delete-region (line-beginning-position) - (progn (forward-line 1) (point))) - (goto-char (+ (point) bytes)) - (delete-region (point) (point-max)) - (nnheader-ms-strip-cr)) - (cons group article)))))))) + (nnimap-command "UID FETCH %d (BODYSTRUCTURE)" article) + (goto-char (point-min)) + (when (re-search-forward "FETCH.*BODYSTRUCTURE" nil t) + (setq structure (ignore-errors + (let ((start (point))) + (forward-sexp 1) + (downcase-region start (point)) + (goto-char (point)) + (read (current-buffer)))) + parts (nnimap-find-wanted-parts structure)))) + (when (if parts + (nnimap-get-partial-article article parts structure) + (nnimap-get-whole-article article)) + (let ((buffer (current-buffer))) + (with-current-buffer (or to-buffer nntp-server-buffer) + (erase-buffer) + (insert-buffer-substring buffer) + (nnheader-ms-strip-cr) + (cons group article))))))))) + +(defun nnimap-get-whole-article (article) + (let ((result + (nnimap-command + (if (nnimap-ver4-p) + "UID FETCH %d BODY.PEEK[]" + "UID FETCH %d RFC822.PEEK") + article))) + ;; Check that we really got an article. + (goto-char (point-min)) + (unless (re-search-forward "\\* [0-9]+ FETCH" nil t) + (setq result nil)) + (when result + ;; Remove any data that may have arrived before the FETCH data. + (beginning-of-line) + (unless (bobp) + (delete-region (point-min) (point))) + (let ((bytes (nnimap-get-length))) + (delete-region (line-beginning-position) + (progn (forward-line 1) (point))) + (goto-char (+ (point) bytes)) + (delete-region (point) (point-max))) + t))) + +(defun nnimap-ver4-p () + (member "IMAP4REV1" (nnimap-capabilities nnimap-object))) + +(defun nnimap-get-partial-article (article parts structure) + (let ((result + (nnimap-command + "UID FETCH %d (%s %s)" + article + (if (nnimap-ver4-p) + "BODY.PEEK[HEADER]" + "RFC822.HEADER") + (if (nnimap-ver4-p) + (mapconcat (lambda (part) + (format "BODY.PEEK[%s]" part)) + parts " ") + (mapconcat (lambda (part) + (format "RFC822.PEEK[%s]" part)) + parts " "))))) + (when result + (nnimap-convert-partial-article structure)))) + +(defun nnimap-convert-partial-article (structure) + ;; First just skip past the headers. + (goto-char (point-min)) + (let ((bytes (nnimap-get-length)) + id parts) + ;; Delete "FETCH" line. + (delete-region (line-beginning-position) + (progn (forward-line 1) (point))) + (goto-char (+ (point) bytes)) + ;; Collect all the body parts. + (while (looking-at ".*BODY\\[\\([.0-9]+\\)\\]") + (setq id (match-string 1) + bytes (nnimap-get-length)) + (beginning-of-line) + (delete-region (point) (progn (forward-line 1) (point))) + (push (list id (buffer-substring (point) (+ (point) bytes))) + parts) + (delete-region (point) (+ (point) bytes))) + ;; Delete trailing junk. + (delete-region (point) (point-max)) + ;; Now insert all the parts again where they fit in the structure. + (nnimap-insert-partial-structure structure parts) + t)) + +(defun nnimap-insert-partial-structure (structure parts &optional subp) + (let (type boundary) + (let ((bstruc structure)) + (while (consp (car bstruc)) + (pop bstruc)) + (setq type (car bstruc)) + (setq bstruc (car (cdr bstruc))) + (when (and (stringp (car bstruc)) + (string= (downcase (car bstruc)) "boundary")) + (setq boundary (cadr bstruc)))) + (when subp + (insert (format "Content-type: multipart/%s; boundary=%S\n\n" + (downcase type) boundary))) + (while (not (stringp (car structure))) + (insert "\n--" boundary "\n") + (if (consp (caar structure)) + (nnimap-insert-partial-structure (pop structure) parts t) + (let ((bit (pop structure))) + (insert (format "Content-type: %s/%s" + (downcase (nth 0 bit)) + (downcase (nth 1 bit)))) + (if (member "CHARSET" (nth 2 bit)) + (insert (format + "; charset=%S\n" (cadr (member "CHARSET" (nth 2 bit))))) + (insert "\n")) + (insert (format "Content-transfer-encoding: %s\n" + (nth 5 bit))) + (insert "\n") + (when (assoc (nth 9 bit) parts) + (insert (cadr (assoc (nth 9 bit) parts))))))) + (insert "\n--" boundary "--\n"))) (defun nnimap-find-wanted-parts (structure) (message-flatten-list (nnimap-find-wanted-parts-1 structure ""))) @@ -383,13 +589,16 @@ textual parts.") (number-to-string num) (format "%s.%s" prefix num))) parts) - (let ((type (format "%s/%s" (nth 0 sub) (nth 1 sub)))) - (when (string-match nnimap-fetch-partial-articles type) - (push (if (string= prefix "") + (let ((type (format "%s/%s" (nth 0 sub) (nth 1 sub))) + (id (if (string= prefix "") (number-to-string num) - (format "%s.%s" prefix num)) - parts))) - (incf num)))) + (format "%s.%s" prefix num)))) + (setcar (nthcdr 9 sub) id) + (when (if (eq nnimap-fetch-partial-articles t) + (equal id "1") + (string-match nnimap-fetch-partial-articles type)) + (push id parts)))) + (incf num))) (nreverse parts))) (deffoo nnimap-request-group (group &optional server dont-check info) @@ -418,13 +627,12 @@ textual parts.") (when info (nnimap-update-infos marks (list info))) (goto-char (point-max)) - (cond - (marks - (setq high (nth 3 (car marks)) - low (nth 4 (car marks)))) - ((re-search-backward "UIDNEXT \\([0-9]+\\)" nil t) - (setq high (1- (string-to-number (match-string 1))) - low 1))))) + (let ((uidnext (nth 5 (car marks)))) + (setq high (or (if uidnext + (1- uidnext) + (nth 3 (car marks))) + 0) + low (or (nth 4 (car marks)) uidnext 1))))) (erase-buffer) (insert (format @@ -466,27 +674,27 @@ textual parts.") (deffoo nnimap-request-move-article (article group server accept-form &optional last internal-move-group) - (when (nnimap-possibly-change-group group server) - ;; If the move is internal (on the same server), just do it the easy - ;; way. - (let ((message-id (message-field-value "message-id"))) - (if internal-move-group - (let ((result - (with-current-buffer (nnimap-buffer) - (nnimap-command "UID COPY %d %S" - article - (utf7-encode internal-move-group t))))) - (when (car result) - (nnimap-delete-article article) - (cons internal-move-group - (nnimap-find-article-by-message-id - internal-move-group message-id)))) - (with-temp-buffer - (when (nnimap-request-article article group server (current-buffer)) - (let ((result (eval accept-form))) - (when result + (with-temp-buffer + (when (nnimap-request-article article group server (current-buffer)) + ;; If the move is internal (on the same server), just do it the easy + ;; way. + (let ((message-id (message-field-value "message-id"))) + (if internal-move-group + (let ((result + (with-current-buffer (nnimap-buffer) + (nnimap-command "UID COPY %d %S" + article + (utf7-encode internal-move-group t))))) + (when (car result) (nnimap-delete-article article) - result)))))))) + (cons internal-move-group + (nnimap-find-article-by-message-id + internal-move-group message-id)))) + ;; Move the article to a different method. + (let ((result (eval accept-form))) + (when result + (nnimap-delete-article article) + result))))))) (deffoo nnimap-request-expire-articles (articles group &optional server force) (cond @@ -496,12 +704,13 @@ textual parts.") articles) ((and force (eq nnmail-expiry-target 'delete)) - (unless (nnimap-delete-article articles) + (unless (nnimap-delete-article (gnus-compress-sequence articles)) (message "Article marked for deletion, but not expunged.")) nil) (t (let ((deletable-articles - (if force + (if (or force + (eq nnmail-expiry-wait 'immediate)) articles (gnus-sorted-intersection articles @@ -509,7 +718,7 @@ textual parts.") (if (null deletable-articles) articles (if (eq nnmail-expiry-target 'delete) - (nnimap-delete-article deletable-articles) + (nnimap-delete-article (gnus-compress-sequence deletable-articles)) (setq deletable-articles (nnimap-process-expiry-targets deletable-articles group server))) @@ -536,12 +745,11 @@ textual parts.") ;; Change back to the current group again. (nnimap-possibly-change-group group server) (setq deleted-articles (nreverse deleted-articles)) - (nnimap-delete-article deleted-articles) + (nnimap-delete-article (gnus-compress-sequence deleted-articles)) deleted-articles)) (defun nnimap-find-expired-articles (group) - (let ((cutoff (nnmail-expired-article-p - group nil nil nnml-inhibit-expiry))) + (let ((cutoff (nnmail-expired-article-p group nil nil))) (with-current-buffer (nnimap-buffer) (let ((result (nnimap-command @@ -558,16 +766,20 @@ textual parts.") (defun nnimap-find-article-by-message-id (group message-id) - (when (nnimap-possibly-change-group group nil) - (with-current-buffer (nnimap-buffer) - (let ((result - (nnimap-command "UID SEARCH HEADER Message-Id %S" message-id)) - article) - (when (car result) - ;; Select the last instance of the message in the group. - (and (setq article - (car (last (assoc "SEARCH" (cdr result))))) - (string-to-number article))))))) + (with-current-buffer (nnimap-buffer) + (erase-buffer) + (setf (nnimap-group nnimap-object) nil) + (nnimap-send-command "EXAMINE %S" (utf7-encode group t)) + (let ((sequence + (nnimap-send-command "UID SEARCH HEADER Message-Id %S" message-id)) + article result) + (setq result (nnimap-wait-for-response sequence)) + (when (and result + (car (setq result (nnimap-parse-response)))) + ;; Select the last instance of the message in the group. + (and (setq article + (car (last (assoc "SEARCH" (cdr result))))) + (string-to-number article)))))) (defun nnimap-delete-article (articles) (with-current-buffer (nnimap-buffer) @@ -580,13 +792,16 @@ textual parts.") t) (nnimap-expunge (nnimap-command "EXPUNGE") - t)))) + t) + (t (gnus-message 7 (concat "nnimap: nnimap-expunge is not set and the " + "server doesn't support UIDPLUS, so we won't " + "delete this article now")))))) (deffoo nnimap-request-scan (&optional group server) (when (and (nnimap-possibly-change-group nil server) - (equal group nnimap-inbox) nnimap-inbox nnimap-split-methods) + (message "nnimap %s splitting mail..." server) (nnimap-split-incoming-mail))) (defun nnimap-marks-to-flags (marks) @@ -600,6 +815,7 @@ textual parts.") (when (nnimap-possibly-change-group group server) (let (sequence) (with-current-buffer (nnimap-buffer) + (erase-buffer) ;; Just send all the STORE commands without waiting for ;; response. If they're successful, they're successful. (dolist (action actions) @@ -621,9 +837,10 @@ textual parts.") (deffoo nnimap-request-accept-article (group &optional server last) (when (nnimap-possibly-change-group nil server) (nnmail-check-syntax) - (let ((message (buffer-string)) - (message-id (message-field-value "message-id")) - sequence) + (let ((message-id (message-field-value "message-id")) + sequence message) + (nnimap-add-cr) + (setq message (buffer-string)) (with-current-buffer (nnimap-buffer) (setq sequence (nnimap-send-command "APPEND %S {%d}" (utf7-encode group t) @@ -634,7 +851,10 @@ textual parts.") "\n" "\r\n")) (let ((result (nnimap-get-response sequence))) - (when result + (if (not (car result)) + (progn + (message "%s" (nnheader-get-report-string 'nnimap)) + nil) (cons group (nnimap-find-article-by-message-id group message-id)))))))) @@ -664,6 +884,7 @@ textual parts.") sequences responses) (when groups (with-current-buffer (nnimap-buffer) + (setf (nnimap-group nnimap-object) nil) (dolist (group groups) (push (list (nnimap-send-command "EXAMINE %S" (utf7-encode group t)) group) @@ -697,6 +918,16 @@ textual parts.") (or highest exists))))))))) t)))) +(deffoo nnimap-request-newgroups (date &optional server) + (nnimap-possibly-change-group nil server) + (with-current-buffer nntp-server-buffer + (erase-buffer) + (dolist (group (with-current-buffer (nnimap-buffer) + (nnimap-get-groups))) + (unless (assoc group nnimap-current-infos) + ;; Insert dummy numbers here -- they don't matter. + (insert (format "%S 0 1 y\n" group)))))) + (deffoo nnimap-retrieve-group-data-early (server infos) (when (nnimap-possibly-change-group nil server) (with-current-buffer (nnimap-buffer) @@ -713,6 +944,7 @@ textual parts.") groups)) ;; Then request the data. (erase-buffer) + (setf (nnimap-group nnimap-object) nil) (dolist (elem groups) (if (and qresyncp (nth 2 elem)) @@ -734,7 +966,12 @@ textual parts.") (nnimap-send-command "UID FETCH %d:* FLAGS" start) start (car elem)) - sequences)))) + sequences))) + ;; Some servers apparently can't have many outstanding + ;; commands, so throttle them. + (when (and (not nnimap-streaming) + (car sequences)) + (nnimap-wait-for-response (caar sequences)))) sequences)))) (deffoo nnimap-finish-retrieve-group-infos (server infos sequences) @@ -742,26 +979,26 @@ textual parts.") (nnimap-possibly-change-group nil server)) (with-current-buffer (nnimap-buffer) ;; Wait for the final data to trickle in. - (nnimap-wait-for-response (cadar sequences)) - ;; Now we should have all the data we need, no matter whether - ;; we're QRESYNCING, fetching all the flags from scratch, or - ;; just fetching the last 100 flags per group. - (nnimap-update-infos (nnimap-flags-to-marks - (nnimap-parse-flags - (nreverse sequences))) - infos) - ;; Finally, just return something resembling an active file in - ;; the nntp buffer, so that the agent can save the info, too. - (with-current-buffer nntp-server-buffer - (erase-buffer) - (dolist (info infos) - (let* ((group (gnus-info-group info)) - (active (gnus-active group))) - (when active - (insert (format "%S %d %d y\n" - (gnus-group-real-name group) - (cdr active) - (car active)))))))))) + (when (nnimap-wait-for-response (cadar sequences) t) + ;; Now we should have all the data we need, no matter whether + ;; we're QRESYNCING, fetching all the flags from scratch, or + ;; just fetching the last 100 flags per group. + (nnimap-update-infos (nnimap-flags-to-marks + (nnimap-parse-flags + (nreverse sequences))) + infos) + ;; Finally, just return something resembling an active file in + ;; the nntp buffer, so that the agent can save the info, too. + (with-current-buffer nntp-server-buffer + (erase-buffer) + (dolist (info infos) + (let* ((group (gnus-info-group info)) + (active (gnus-active group))) + (when active + (insert (format "%S %d %d y\n" + (gnus-group-real-name group) + (cdr active) + (car active))))))))))) (defun nnimap-update-infos (flags infos) (dolist (info infos) @@ -769,28 +1006,46 @@ textual parts.") (nnimap-update-info info (cdr (assoc group flags)))))) (defun nnimap-update-info (info marks) - (when marks - (destructuring-bind (existing flags high low uidnext start-article) marks + (when (and marks + ;; Ignore groups with no UIDNEXT values. + (nth 4 marks)) + (destructuring-bind (existing flags high low uidnext start-article + permanent-flags) marks (let ((group (gnus-info-group info)) (completep (and start-article (= start-article 1)))) + (when uidnext + (setq high (1- uidnext))) ;; First set the active ranges based on high/low. (if (or completep (not (gnus-active group))) (gnus-set-active group - (if high - (cons low high) + (cond + ((and low high) + (cons low high)) + (uidnext ;; No articles in this group. - (cons (1- uidnext) uidnext))) - (setcdr (gnus-active group) high)) + (cons uidnext (1- uidnext))) + (start-article + (cons start-article (1- start-article))) + (t + ;; No articles and no uidnext. + nil))) + (gnus-set-active + group + (cons (car (gnus-active group)) + (or high (1- uidnext))))) + (when (and (not high) + uidnext) + (setq high (1- uidnext))) ;; Then update the list of read articles. (let* ((unread (gnus-compress-sequence (gnus-set-difference (gnus-set-difference existing - (cdr (assoc "\\Seen" flags))) - (cdr (assoc "\\Flagged" flags))))) + (cdr (assoc '%Seen flags))) + (cdr (assoc '%Flagged flags))))) (read (gnus-range-difference (cons start-article high) unread))) (when (> start-article 1) @@ -812,8 +1067,11 @@ textual parts.") (push (cons 'active (gnus-active group)) marks))) (dolist (type (cdr nnimap-mark-alist)) (let ((old-marks (assoc (car type) marks)) - (new-marks (gnus-compress-sequence - (cdr (assoc (cadr type) flags))))) + (new-marks + (gnus-compress-sequence + (cdr (or (assoc (caddr type) flags) ; %Flagged + (assoc (intern (cadr type) obarray) flags) + (assoc (cadr type) flags)))))) ; "\Flagged" (setq marks (delq old-marks marks)) (pop old-marks) (when (and old-marks @@ -835,12 +1093,13 @@ textual parts.") (push (list group info active) nnimap-current-infos)))) (defun nnimap-flags-to-marks (groups) - (let (data group totalp uidnext articles start-article mark) + (let (data group totalp uidnext articles start-article mark permanent-flags) (dolist (elem groups) (setq group (car elem) - uidnext (cadr elem) - start-article (caddr elem) - articles (cdddr elem)) + uidnext (nth 1 elem) + start-article (nth 2 elem) + permanent-flags (nth 3 elem) + articles (nthcdr 4 elem)) (let ((high (caar articles)) marks low existing) (dolist (article articles) @@ -850,36 +1109,49 @@ textual parts.") (setq mark (assoc flag marks)) (if (not mark) (push (list flag (car article)) marks) - (setcdr mark (cons (car article) (cdr mark))))) - (push (list group existing marks high low uidnext start-article) - data)))) + (setcdr mark (cons (car article) (cdr mark)))))) + (push (list group existing marks high low uidnext start-article + permanent-flags) + data))) data)) (defun nnimap-parse-flags (sequences) (goto-char (point-min)) - (let (start end articles groups uidnext elems) + ;; Change \Delete etc to %Delete, so that the reader can read it. + (subst-char-in-region (point-min) (point-max) + ?\\ ?% t) + (let (start end articles groups uidnext elems permanent-flags) (dolist (elem sequences) (destructuring-bind (group-sequence flag-sequence totalp group) elem + (setq start (point)) ;; The EXAMINE was successful. (when (and (search-forward (format "\n%d OK " group-sequence) nil t) (progn (forward-line 1) - (setq start (point)) - (if (re-search-backward "UIDNEXT \\([0-9]+\\)" - (or end (point-min)) t) - (setq uidnext (string-to-number (match-string 1))) - (setq uidnext nil)) - (goto-char start)) + (setq end (point)) + (goto-char start) + (setq permanent-flags + (and (search-forward "PERMANENTFLAGS " + (or end (point-min)) t) + (read (current-buffer)))) + (goto-char start) + (setq uidnext + (and (search-forward "UIDNEXT " + (or end (point-min)) t) + (read (current-buffer)))) + (goto-char end) + (forward-line -1)) ;; The UID FETCH FLAGS was successful. (search-forward (format "\n%d OK " flag-sequence) nil t)) - (setq end (point)) - (goto-char start) - (while (re-search-forward "^\\* [0-9]+ FETCH (\\(.*\\))" end t) - (setq elems (nnimap-parse-line (match-string 1))) - (push (cons (string-to-number (cadr (member "UID" elems))) - (cadr (member "FLAGS" elems))) + (setq start (point)) + (goto-char end) + (while (search-forward " FETCH " start t) + (setq elems (read (current-buffer))) + (push (cons (cadr (memq 'UID elems)) + (cadr (memq 'FLAGS elems))) articles)) - (push (nconc (list group uidnext totalp) articles) groups) + (push (nconc (list group uidnext totalp permanent-flags) articles) + groups) (setq articles nil)))) groups)) @@ -944,6 +1216,7 @@ textual parts.") (defun nnimap-command (&rest args) (erase-buffer) + (setf (nnimap-last-command-time nnimap-object) (current-time)) (let* ((sequence (apply #'nnimap-send-command args)) (response (nnimap-get-response sequence))) (if (equal (caar response) "OK") @@ -963,25 +1236,30 @@ textual parts.") (goto-char (point-min)) (while (and (memq (process-status process) '(open run)) - (not (re-search-forward "^\\* .*\n" nil t))) + (not (re-search-forward "^[*.] .*\n" nil t))) (nnheader-accept-process-output process) (goto-char (point-min))) (forward-line -1) - (and (looking-at "\\* \\([A-Z0-9]+\\)") + (and (looking-at "[*.] \\([A-Z0-9]+\\)") (match-string 1)))) (defun nnimap-wait-for-response (sequence &optional messagep) - (let ((process (get-buffer-process (current-buffer)))) + (let ((process (get-buffer-process (current-buffer))) + openp) (goto-char (point-max)) - (while (and (memq (process-status process) - '(open run)) - (not (re-search-backward (format "^%d .*\n" sequence) - (max (point-min) (- (point) 500)) - t))) + (while (and (setq openp (memq (process-status process) + '(open run))) + (not (re-search-backward + (format "^%d .*\n" sequence) + (if nnimap-streaming + (max (point-min) (- (point) 500)) + (point-min)) + t))) (when messagep - (message "Read %dKB" (/ (buffer-size) 1000))) + (message "nnimap read %dk" (/ (buffer-size) 1000))) (nnheader-accept-process-output process) - (goto-char (point-max))))) + (goto-char (point-max))) + openp)) (defun nnimap-parse-response () (let ((lines (split-string (nnimap-last-response-string) "\r\n" t)) @@ -1008,13 +1286,18 @@ textual parts.") (cond ((eql char ?\[) (split-string (buffer-substring - (1+ (point)) (1- (search-forward "]"))))) + (1+ (point)) + (1- (search-forward "]" (line-end-position) 'move))))) ((eql char ?\() (split-string (buffer-substring - (1+ (point)) (1- (search-forward ")"))))) + (1+ (point)) + (1- (search-forward ")" (line-end-position) 'move))))) ((eql char ?\") (forward-char 1) - (buffer-substring (point) (1- (search-forward "\"")))) + (buffer-substring + (point) + (1- (or (search-forward "\"" (line-end-position) 'move) + (point))))) (t (buffer-substring (point) (if (search-forward " " nil t) (1- (point)) @@ -1055,8 +1338,7 @@ textual parts.") (nnimap-article-ranges articles) (format "(UID %s%s)" (format - (if (member "IMAP4REV1" - (nnimap-capabilities nnimap-object)) + (if (nnimap-ver4-p) "BODY.PEEK[HEADER] BODY.PEEK" "RFC822.PEEK")) (if nnimap-split-download-body-default @@ -1067,7 +1349,11 @@ textual parts.") (defun nnimap-split-incoming-mail () (with-current-buffer (nnimap-buffer) (let ((nnimap-incoming-split-list nil) - (nnmail-split-methods nnimap-split-methods) + (nnmail-split-methods (if (eq nnimap-split-methods 'default) + nnmail-split-methods + nnimap-split-methods)) + (nnmail-split-fancy (or nnimap-split-fancy + nnmail-split-fancy)) (nnmail-inhibit-default-split-group t) (groups (nnimap-get-groups)) new-articles) @@ -1082,47 +1368,56 @@ textual parts.") (nnmail-split-incoming (current-buffer) #'nnimap-save-mail-spec nil nil - #'nnimap-dummy-active-number) + #'nnimap-dummy-active-number + #'nnimap-save-mail-spec) (when nnimap-incoming-split-list (let ((specs (nnimap-make-split-specs nnimap-incoming-split-list)) - sequences) + sequences junk-articles) ;; Create any groups that doesn't already exist on the ;; server first. (dolist (spec specs) - (unless (member (car spec) groups) + (when (and (not (member (car spec) groups)) + (not (eq (car spec) 'junk))) (nnimap-command "CREATE %S" (utf7-encode (car spec) t)))) ;; Then copy over all the messages. (erase-buffer) (dolist (spec specs) (let ((group (car spec)) (ranges (cdr spec))) - (push (list (nnimap-send-command "UID COPY %s %S" - (nnimap-article-ranges ranges) - (utf7-encode group t)) - ranges) - sequences))) + (if (eq group 'junk) + (setq junk-articles ranges) + (push (list (nnimap-send-command + "UID COPY %s %S" + (nnimap-article-ranges ranges) + (utf7-encode group t)) + ranges) + sequences)))) ;; Wait for the last COPY response... (when sequences (nnimap-wait-for-response (caar sequences)) ;; And then mark the successful copy actions as deleted, ;; and possibly expunge them. (nnimap-mark-and-expunge-incoming - (nnimap-parse-copied-articles sequences))))))))) + (nnimap-parse-copied-articles sequences))) + (nnimap-mark-and-expunge-incoming junk-articles))))))) (defun nnimap-mark-and-expunge-incoming (range) (when range (setq range (nnimap-article-ranges range)) - (nnimap-send-command - "UID STORE %s +FLAGS.SILENT (\\Deleted)" range) - (cond - ;; If the server supports it, we now delete the message we have - ;; just copied over. - ((member "UIDPLUS" (nnimap-capabilities nnimap-object)) - (nnimap-send-command "UID EXPUNGE %s" range)) - ;; If it doesn't support UID EXPUNGE, then we only expunge if the - ;; user has configured it. - (nnimap-expunge-inbox - (nnimap-send-command "EXPUNGE"))))) + (erase-buffer) + (let ((sequence + (nnimap-send-command + "UID STORE %s +FLAGS.SILENT (\\Deleted)" range))) + (cond + ;; If the server supports it, we now delete the message we have + ;; just copied over. + ((member "UIDPLUS" (nnimap-capabilities nnimap-object)) + (setq sequence (nnimap-send-command "UID EXPUNGE %s" range))) + ;; If it doesn't support UID EXPUNGE, then we only expunge if the + ;; user has configured it. + (nnimap-expunge + (setq sequence (nnimap-send-command "EXPUNGE")))) + (nnimap-wait-for-response sequence)))) (defun nnimap-parse-copied-articles (sequences) (let (sequence copied range) @@ -1137,8 +1432,8 @@ textual parts.") (let (new) (dolist (elem flags) (when (or (null (cdr elem)) - (and (not (member "\\Deleted" (cdr elem))) - (not (member "\\Seen" (cdr elem))))) + (and (not (memq '%Deleted (cdr elem))) + (not (memq '%Seen (cdr elem))))) (push (car elem) new))) (gnus-compress-sequence (nreverse new)))) @@ -1185,7 +1480,10 @@ textual parts.") (if (not (re-search-forward "X-nnimap-article: \\([0-9]+\\)" nil t)) (error "Invalid nnimap mail") (setq article (string-to-number (match-string 1)))) - (push (list article group-art) + (push (list article + (if (eq group-art 'junk) + (list (cons 'junk 1)) + group-art)) nnimap-incoming-split-list))) (provide 'nnimap)