From: Gemini Lasswell <gazally@runbox.com>
To: Michael Albinus <michael.albinus@gmx.de>
Cc: 28227@debbugs.gnu.org, npostavs@users.sourceforge.net
Subject: bug#28227: 26.0.50; Tramp tests are slow
Date: Mon, 04 Sep 2017 15:43:17 -0700 [thread overview]
Message-ID: <87a82ankze.fsf@runbox.com> (raw)
In-Reply-To: <877exili1a.fsf@detlef>
[-- Attachment #1: Type: text/plain, Size: 176 bytes --]
Michael Albinus writes:
> Looks good to me, thanks a lot! You could apply this to master.
Hi Michael,
Here's a new version of the patch which incorporates your suggestions:
[-- Attachment #2: 0001-Reduce-Tramp-s-memory-usage.patch --]
[-- Type: text/plain, Size: 12240 bytes --]
From 41f70cc2b89ac78f6b314cf140a4e7a9250c8eb2 Mon Sep 17 00:00:00 2001
From: Gemini Lasswell <gazally@runbox.com>
Date: Wed, 30 Aug 2017 07:11:41 -0700
Subject: [PATCH] Reduce Tramp's memory usage
Construct Tramp syntax strings and regular expressions once instead
of every time they are used, and store them in alists keyed by Tramp
syntax.
* tramp.el (tramp-build-remote-file-name-spec-regexp)
(tramp-build-file-name-structure): New functions.
(tramp-prefix-format-alist, tramp-prefix-regexp-alist)
(tramp-method-regexp-alist)
(tramp-postfix-method-format-alist)
(tramp-postfix-method-regexp-alist)
(tramp-prefix-ipv6-format-alist, tramp-prefix-ipv6-regexp-alist)
(tramp-postfix-ipv6-format-alist)
(tramp-postfix-ipv6-regexp-alist)
(tramp-postfix-host-format-alist)
(tramp-postfix-host-regexp-alist)
(tramp-remote-file-name-spec-regexp-alist)
(tramp-file-name-structure-alist): New constants.
(tramp-lookup-syntax): New function.
(tramp-prefix-format, tramp-prefix-regexp, tramp-method-regexp)
(tramp-postfix-method-format, tramp-postfix-method-regexp)
(tramp-prefix-ipv6-format, tramp-prefix-ipv6-regexp)
(tramp-postfix-ipv6-format, tramp-postfix-ipv6-regexp)
(tramp-postfix-host-format, tramp-postfix-host-regexp)
(tramp-remote-file-name-spec-regexp, tramp-file-name-structure):
Use it.
---
lisp/net/tramp.el | 172 ++++++++++++++++++++++++++++++++++++++++--------------
1 file changed, 127 insertions(+), 45 deletions(-)
diff --git a/lisp/net/tramp.el b/lisp/net/tramp.el
index 1a5cda7e20..adc59cb542 100644
--- a/lisp/net/tramp.el
+++ b/lisp/net/tramp.el
@@ -700,40 +700,69 @@ tramp-syntax-values
(setq values (mapcar 'last values)
values (mapcar 'car values))))
+(defun tramp-lookup-syntax (alist)
+ "Look up a syntax string in ALIST according to `tramp-compat-tramp-syntax.'
+Raise an error if `tramp-syntax' is invalid."
+ (or (cdr (assq (tramp-compat-tramp-syntax) alist))
+ (error "Wrong `tramp-syntax' %s" tramp-syntax)))
+
+(defconst tramp-prefix-format-alist
+ '((default . "/")
+ (simplified . "/")
+ (separate . "/["))
+ "Alist mapping Tramp syntax to strings beginning Tramp file names.")
+
(defun tramp-prefix-format ()
"String matching the very beginning of Tramp file names.
Used in `tramp-make-tramp-file-name'."
- (cond ((eq (tramp-compat-tramp-syntax) 'default) "/")
- ((eq (tramp-compat-tramp-syntax) 'simplified) "/")
- ((eq (tramp-compat-tramp-syntax) 'separate) "/[")
- (t (error "Wrong `tramp-syntax' %s" tramp-syntax))))
+ (tramp-lookup-syntax tramp-prefix-format-alist))
+
+(defconst tramp-prefix-regexp-alist
+ (mapcar (lambda (x)
+ (cons (car x) (concat "^" (regexp-quote (cdr x)))))
+ tramp-prefix-format-alist)
+ "Alist of regexps matching the beginnings of Tramp file names.
+Keyed by Tramp syntax. Derived from `tramp-prefix-format-alist'.")
(defun tramp-prefix-regexp ()
"Regexp matching the very beginning of Tramp file names.
Should always start with \"^\". Derived from `tramp-prefix-format'."
- (concat "^" (regexp-quote (tramp-prefix-format))))
+ (tramp-lookup-syntax tramp-prefix-regexp-alist))
+
+(defconst tramp-method-regexp-alist
+ '((default . "[a-zA-Z0-9-]+")
+ (simplified . "")
+ (separate . "[a-zA-Z0-9-]*"))
+ "Alist mapping Tramp syntax to regexps matching methods identifiers.")
(defun tramp-method-regexp ()
"Regexp matching methods identifiers.
The `ftp' syntax does not support methods."
- (cond ((eq (tramp-compat-tramp-syntax) 'default) "[a-zA-Z0-9-]+")
- ((eq (tramp-compat-tramp-syntax) 'simplified) "")
- ((eq (tramp-compat-tramp-syntax) 'separate) "[a-zA-Z0-9-]*")
- (t (error "Wrong `tramp-syntax' %s" tramp-syntax))))
+ (tramp-lookup-syntax tramp-method-regexp-alist))
+
+(defconst tramp-postfix-method-format-alist
+ '((default . ":")
+ (simplified . "")
+ (separate . "/"))
+ "Alist mapping Tramp syntax to the delimiter after the method.")
(defun tramp-postfix-method-format ()
"String matching delimiter between method and user or host names.
The `ftp' syntax does not support methods.
Used in `tramp-make-tramp-file-name'."
- (cond ((eq (tramp-compat-tramp-syntax) 'default) ":")
- ((eq (tramp-compat-tramp-syntax) 'simplified) "")
- ((eq (tramp-compat-tramp-syntax) 'separate) "/")
- (t (error "Wrong `tramp-syntax' %s" tramp-syntax))))
+ (tramp-lookup-syntax tramp-postfix-method-format-alist))
+
+(defconst tramp-postfix-method-regexp-alist
+ (mapcar (lambda (x)
+ (cons (car x) (regexp-quote (cdr x))))
+ tramp-postfix-method-format-alist)
+ "Alist mapping Tramp syntax to regexp matching delimiter after method.
+Derived from `tramp-postfix-method-format-alist'.")
(defun tramp-postfix-method-regexp ()
"Regexp matching delimiter between method and user or host names.
Derived from `tramp-postfix-method-format'."
- (regexp-quote (tramp-postfix-method-format)))
+ (tramp-lookup-syntax tramp-postfix-method-regexp-alist))
(defconst tramp-user-regexp "[^/|: \t]+"
"Regexp matching user names.")
@@ -769,18 +798,28 @@ tramp-postfix-user-regexp
(defconst tramp-host-regexp "[a-zA-Z0-9_.-]+"
"Regexp matching host names.")
+(defconst tramp-prefix-ipv6-format-alist
+ '((default . "[")
+ (simplified . "[")
+ (separate . ""))
+ "Alist mapping Tramp syntax to strings prefixing IPv6 addresses.")
+
(defun tramp-prefix-ipv6-format ()
"String matching left hand side of IPv6 addresses.
Used in `tramp-make-tramp-file-name'."
- (cond ((eq (tramp-compat-tramp-syntax) 'default) "[")
- ((eq (tramp-compat-tramp-syntax) 'simplified) "[")
- ((eq (tramp-compat-tramp-syntax) 'separate) "")
- (t (error "Wrong `tramp-syntax' %s" tramp-syntax))))
+ (tramp-lookup-syntax tramp-prefix-ipv6-format-alist))
+
+(defconst tramp-prefix-ipv6-regexp-alist
+ (mapcar (lambda (x)
+ (cons (car x) (regexp-quote (cdr x))))
+ tramp-prefix-ipv6-format-alist)
+ "Alist mapping Tramp syntax to regexp matching prefix of IPv6 addresses.
+Derived from `tramp-prefix-ipv6-format-alist'")
(defun tramp-prefix-ipv6-regexp ()
"Regexp matching left hand side of IPv6 addresses.
Derived from `tramp-prefix-ipv6-format'."
- (regexp-quote (tramp-prefix-ipv6-format)))
+ (tramp-lookup-syntax tramp-prefix-ipv6-regexp-alist))
;; The following regexp is a bit sloppy. But it shall serve our
;; purposes. It covers also IPv4 mapped IPv6 addresses, like in
@@ -789,18 +828,28 @@ tramp-ipv6-regexp
"\\(?:\\(?:[a-zA-Z0-9]+\\)?:\\)+[a-zA-Z0-9.]+"
"Regexp matching IPv6 addresses.")
+(defconst tramp-postfix-ipv6-format-alist
+ '((default . "]")
+ (simplified . "]")
+ (separate . ""))
+ "Alist mapping Tramp syntax to suffix for IPv6 addresses.")
+
(defun tramp-postfix-ipv6-format ()
"String matching right hand side of IPv6 addresses.
Used in `tramp-make-tramp-file-name'."
- (cond ((eq (tramp-compat-tramp-syntax) 'default) "]")
- ((eq (tramp-compat-tramp-syntax) 'simplified) "]")
- ((eq (tramp-compat-tramp-syntax) 'separate) "")
- (t (error "Wrong `tramp-syntax' %s" tramp-syntax))))
+ (tramp-lookup-syntax tramp-postfix-ipv6-format-alist))
+
+(defconst tramp-postfix-ipv6-regexp-alist
+ (mapcar (lambda (x)
+ (cons (car x) (regexp-quote (cdr x))))
+ tramp-postfix-ipv6-format-alist)
+ "Alist mapping Tramp syntax to regexps matching IPv6 suffixes.
+Derived from `tramp-postfix-ipv6-format-alist'.")
(defun tramp-postfix-ipv6-regexp ()
"Regexp matching right hand side of IPv6 addresses.
Derived from `tramp-postfix-ipv6-format'."
- (regexp-quote (tramp-postfix-ipv6-format)))
+ (tramp-lookup-syntax tramp-postfix-ipv6-format-alist))
(defconst tramp-prefix-port-format "#"
"String matching delimiter between host names and port numbers.")
@@ -827,18 +876,28 @@ tramp-postfix-hop-regexp
"Regexp matching delimiter after ad-hoc hop definitions.
Derived from `tramp-postfix-hop-format'.")
+(defconst tramp-postfix-host-format-alist
+ '((default . ":")
+ (simplified . ":")
+ (separate . "]"))
+ "Alist mapping Tramp syntax to strings between host and local names.")
+
(defun tramp-postfix-host-format ()
"String matching delimiter between host names and localnames.
Used in `tramp-make-tramp-file-name'."
- (cond ((eq (tramp-compat-tramp-syntax) 'default) ":")
- ((eq (tramp-compat-tramp-syntax) 'simplified) ":")
- ((eq (tramp-compat-tramp-syntax) 'separate) "]")
- (t (error "Wrong `tramp-syntax' %s" tramp-syntax))))
+ (tramp-lookup-syntax tramp-postfix-host-format-alist))
+
+(defconst tramp-postfix-host-regexp-alist
+ (mapcar (lambda (x)
+ (cons (car x) (regexp-quote (cdr x))))
+ tramp-postfix-host-format-alist)
+ "Alist mapping Tramp syntax to regexp matching name delimiters.
+Derived from `tramp-postfix-host-format-alist'.")
(defun tramp-postfix-host-regexp ()
"Regexp matching delimiter between host names and localnames.
Derived from `tramp-postfix-host-format'."
- (regexp-quote (tramp-postfix-host-format)))
+ (tramp-lookup-syntax tramp-postfix-host-regexp-alist))
(defconst tramp-localname-regexp ".*$"
"Regexp matching localnames.")
@@ -851,16 +910,46 @@ tramp-unknown-id-integer
;;; File name format:
+(defun tramp-build-remote-file-name-spec-regexp (syntax)
+ "Construct a regexp matching a Tramp file name for a Tramp SYNTAX."
+ (let ((tramp-syntax syntax))
+ (concat
+ "\\(" (tramp-method-regexp) "\\)" (tramp-postfix-method-regexp)
+ "\\(?:" "\\(" tramp-user-regexp "\\)" tramp-postfix-user-regexp "\\)?"
+ "\\(" "\\(?:" tramp-host-regexp "\\|"
+ (tramp-prefix-ipv6-regexp)
+ "\\(?:" tramp-ipv6-regexp "\\)?"
+ (tramp-postfix-ipv6-regexp) "\\)?"
+ "\\(?:" tramp-prefix-port-regexp tramp-port-regexp "\\)?" "\\)?")))
+
+(defconst tramp-remote-file-name-spec-regexp-alist
+ `((default . ,(tramp-build-remote-file-name-spec-regexp 'default))
+ (simplified . ,(tramp-build-remote-file-name-spec-regexp 'simplified))
+ (separate . ,(tramp-build-remote-file-name-spec-regexp 'separate)))
+ "Alist mapping Tramp syntax to regexps matching Tramp file names.")
+
(defun tramp-remote-file-name-spec-regexp ()
"Regular expression matching a Tramp file name between prefix and postfix."
- (concat
- "\\(" (tramp-method-regexp) "\\)" (tramp-postfix-method-regexp)
- "\\(?:" "\\(" tramp-user-regexp "\\)" tramp-postfix-user-regexp "\\)?"
- "\\(" "\\(?:" tramp-host-regexp "\\|"
- (tramp-prefix-ipv6-regexp)
- "\\(?:" tramp-ipv6-regexp "\\)?"
- (tramp-postfix-ipv6-regexp) "\\)?"
- "\\(?:" tramp-prefix-port-regexp tramp-port-regexp "\\)?" "\\)?"))
+ (tramp-lookup-syntax tramp-remote-file-name-spec-regexp-alist))
+
+(defun tramp-build-file-name-structure (syntax)
+ "Construct the Tramp file name structure for SYNTAX.
+See `tramp-file-name-structure'."
+ (let ((tramp-syntax syntax))
+ (list
+ (concat
+ (tramp-prefix-regexp)
+ "\\(" "\\(?:" (tramp-remote-file-name-spec-regexp)
+ tramp-postfix-hop-regexp "\\)+" "\\)?"
+ (tramp-remote-file-name-spec-regexp) (tramp-postfix-host-regexp)
+ "\\(" tramp-localname-regexp "\\)")
+ 5 6 7 8 1)))
+
+(defconst tramp-file-name-structure-alist
+ `((default . ,(tramp-build-file-name-structure 'default))
+ (simplified . ,(tramp-build-file-name-structure 'simplified))
+ (separate . ,(tramp-build-file-name-structure 'separate)))
+ "Alist mapping Tramp syntax to the file name structure for that syntax.")
(defun tramp-file-name-structure ()
"List of six elements (REGEXP METHOD USER HOST FILE HOP), detailing \
@@ -881,14 +970,7 @@ tramp-file-name-structure
means the opening parentheses are counted to identify the pair.
See also `tramp-file-name-regexp'."
- (list
- (concat
- (tramp-prefix-regexp)
- "\\(" "\\(?:" (tramp-remote-file-name-spec-regexp)
- tramp-postfix-hop-regexp "\\)+" "\\)?"
- (tramp-remote-file-name-spec-regexp) (tramp-postfix-host-regexp)
- "\\(" tramp-localname-regexp "\\)")
- 5 6 7 8 1))
+ (tramp-lookup-syntax tramp-file-name-structure-alist))
(defun tramp-file-name-regexp ()
"Regular expression matching file names handled by Tramp.
--
2.14.1
[-- Attachment #3: Type: text/plain, Size: 150 bytes --]
I submitted a request for write permissions this weekend, so I'll push
it after I hear back about that, if there's no other feedback.
Best,
Gemini
next prev parent reply other threads:[~2017-09-04 22:43 UTC|newest]
Thread overview: 9+ messages / expand[flat|nested] mbox.gz Atom feed top
2017-08-24 21:03 bug#28227: 26.0.50; Tramp tests are slow Gemini Lasswell
2017-08-25 8:04 ` Michael Albinus
[not found] ` <874lsok15a.fsf@runbox.com>
2017-08-31 2:26 ` npostavs
2017-08-31 14:46 ` Gemini Lasswell
2017-09-01 12:28 ` Michael Albinus
2017-09-04 22:43 ` Gemini Lasswell [this message]
2017-09-05 7:45 ` Michael Albinus
2017-09-09 16:27 ` Gemini Lasswell
2017-09-09 17:47 ` Michael Albinus
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=87a82ankze.fsf@runbox.com \
--to=gazally@runbox.com \
--cc=28227@debbugs.gnu.org \
--cc=michael.albinus@gmx.de \
--cc=npostavs@users.sourceforge.net \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this external index
https://git.savannah.gnu.org/cgit/emacs.git
https://git.savannah.gnu.org/cgit/emacs/org-mode.git
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.