Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Stefan Monnier]

Hi Vitalie,

For tentative branches where you don't want to have to follow all the
coding conventions (especially the convention about formatting and
content of commit messages), please use branch names of the form "scratch/*".


        Stefan


>>>>> "Vitalie" == Vitalie Spinu <spinuvit@gmail.com> writes:

> branch: widen-limits
> commit c331b6626a427fb89303fea75faebd8c39d343a8
> Author: Vitalie Spinu <spinuvit@gmail.com>
> Commit: Vitalie Spinu <spinuvit@gmail.com>

>     Implement buffer-widen-limits functionality
    
>      `widen` now respects restrictions imposed by new variable
>      `hard-widen-limits`
> ---
>  src/buffer.c  |   19 +++++++++++++++++--
>  src/buffer.h  |   17 +++++++++++++++++
>  src/editfns.c |   15 ++++++++++++++-
>  3 files changed, 48 insertions(+), 3 deletions(-)

> diff --git a/src/buffer.c b/src/buffer.c
> index f06d7e0..1b62d3a 100644
> --- a/src/buffer.c
> +++ b/src/buffer.c
> @@ -329,6 +329,11 @@ bset_scroll_up_aggressively (struct buffer *b, Lisp_Object val)
b-> scroll_up_aggressively_ = val;
>  }
>  static void
> +bset_widen_limits (struct buffer *b, Lisp_Object val)
> +{
> +  b->widen_limits_ = val;
> +}
> +static void
>  bset_selective_display (struct buffer *b, Lisp_Object val)
>  {
b-> selective_display_ = val;
> @@ -847,6 +852,7 @@ CLONE nil means the indirect buffer's state is reset to default values.  */)
>        bset_display_count (b, make_number (0));
>        bset_backed_up (b, Qnil);
>        bset_auto_save_file_name (b, Qnil);
> +      bset_widen_limits (b, b->base_buffer->widen_limits_);
>        set_buffer_internal_1 (b);
>        Fset (intern ("buffer-save-without-query"), Qnil);
>        Fset (intern ("buffer-file-number"), Qnil);
> @@ -961,6 +967,7 @@ reset_buffer_local_variables (struct buffer *b, bool permanent_too)
>       things that depend on the major mode.
>       default-major-mode is handled at a higher level.
>       We ignore it here.  */
> +  bset_widen_limits(b, Qnil);
>    bset_major_mode (b, Qfundamental_mode);
>    bset_keymap (b, Qnil);
>    bset_mode_name (b, QSFundamental);
> @@ -2167,7 +2174,7 @@ so the buffer is truly empty after this.  */)
>  {
>    Fwiden ();
 
> -  del_range (BEG, Z);
> +  del_range (BEGWL, ZWL);
 
current_buffer-> last_window_start = 1;
>    /* Prevent warnings, or suspension of auto saving, that would happen
> @@ -5037,6 +5044,7 @@ init_buffer_once (void)
>    bset_display_count (&buffer_local_flags, make_number (-1));
>    bset_display_time (&buffer_local_flags, make_number (-1));
>    bset_enable_multibyte_characters (&buffer_local_flags, make_number (-1));
> +  bset_widen_limits (&buffer_local_flags, make_number (-1));
 
>    /* These used to be stuck at 0 by default, but now that the all-zero value
>       means Qnil, we have to initialize them explicitly.  */
> @@ -5160,6 +5168,7 @@ init_buffer_once (void)
>    bset_cursor_type (&buffer_defaults, Qt);
>    bset_extra_line_spacing (&buffer_defaults, Qnil);
>    bset_cursor_in_non_selected_windows (&buffer_defaults, Qt);
> +  bset_widen_limits (&buffer_defaults, Qnil);
 
>    bset_enable_multibyte_characters (&buffer_defaults, Qt);
>    bset_buffer_file_coding_system (&buffer_defaults, Qnil);
> @@ -5367,7 +5376,6 @@ defvar_per_buffer (struct Lisp_Buffer_Objfwd *bo_fwd, const char *namestring,
>      emacs_abort ();
>  }
 
> -
>  /* Initialize the buffer routines.  */
>  void
>  syms_of_buffer (void)
> @@ -5796,6 +5804,13 @@ If you set this to -2, that means don't turn off auto-saving in this buffer
>  if its text size shrinks.   If you use `buffer-swap-text' on a buffer,
>  you probably should set this to -2 in that buffer.  */);
 
> +  DEFVAR_PER_BUFFER ("buffer-widen-limits", &BVAR (current_buffer, widen_limits),
> +                     Qnil,
> +                     doc: /* When non-nil `widen` will widen to these limits.
> +Must be a cons of the form (MIN . MAX) where MIN and MAX are integers
> +of hard widen limits in this buffer. This is an experimental variable
> +intended primarily for multi-mode engines.  */);
> +
>    DEFVAR_PER_BUFFER ("selective-display", &BVAR (current_buffer, selective_display),
>  		     Qnil,
>  		     doc: /* Non-nil enables selective display.
> diff --git a/src/buffer.h b/src/buffer.h
> index 87b7cee..4075bbf 100644
> --- a/src/buffer.h
> +++ b/src/buffer.h
> @@ -59,6 +59,10 @@ INLINE_HEADER_BEGIN
>  #define Z (current_buffer->text->z)
>  #define Z_BYTE (current_buffer->text->z_byte)
 
> +/* Positions that take into account widen limits.  */
> +#define BEGWL (BUF_BEGWL (current_buffer))
> +#define ZWL (BUF_ZWL(current_buffer))
> +
>  /* Macros for the addresses of places in the buffer.  */
 
>  /* Address of beginning of buffer.  */
> @@ -128,6 +132,15 @@ INLINE_HEADER_BEGIN
>      : NILP (BVAR (buf, begv_marker)) ? buf->begv_byte	\
>      : marker_byte_position (BVAR (buf, begv_marker)))
 
> +/* Hard positions in buffer. */
> +#define BUF_BEGWL(buf)  	                                \
> +  ((NILP (BVAR (buf, widen_limits))) ?  BUF_BEG (buf)    \
> +   : XINT( XCAR (BVAR (buf, widen_limits))))
> +
> +#define BUF_ZWL(buf)  	                                \
> +  ((NILP (BVAR (buf, widen_limits))) ?  BUF_Z (buf)      \
> +   : XINT( XCDR (BVAR (buf, widen_limits))))
> +
>  /* Position of point in buffer.  */
>  #define BUF_PT(buf)					\
>     (buf == current_buffer ? PT				\
> @@ -150,6 +163,7 @@ INLINE_HEADER_BEGIN
>      : NILP (BVAR (buf, zv_marker)) ? buf->zv_byte	\
>      : marker_byte_position (BVAR (buf, zv_marker)))
 
> +
>  /* Position of gap in buffer.  */
>  #define BUF_GPT(buf) ((buf)->text->gpt)
>  #define BUF_GPT_BYTE(buf) ((buf)->text->gpt_byte)
> @@ -748,6 +762,9 @@ struct buffer
>       See `cursor-type' for other values.  */
>    Lisp_Object cursor_in_non_selected_windows_;
 
> +  /* Cons of hard widen limits */
> +  Lisp_Object widen_limits_;
> +
>    /* No more Lisp_Object beyond this point.  Except undo_list,
>       which is handled specially in Fgarbage_collect.  */
 
> diff --git a/src/editfns.c b/src/editfns.c
> index 2ac0537..e5ab637 100644
> --- a/src/editfns.c
> +++ b/src/editfns.c
> @@ -3480,12 +3480,25 @@ DEFUN ("delete-and-extract-region", Fdelete_and_extract_region,
>      return empty_unibyte_string;
>    return del_range_1 (XINT (start), XINT (end), 1, 1);
>  }
> +
>  \f
>  DEFUN ("widen", Fwiden, Swiden, 0, 0, "",
>         doc: /* Remove restrictions (narrowing) from current buffer.
> -This allows the buffer's full text to be seen and edited.  */)
> +This allows the buffer's full text to be seen and edited.
> +If `buffer-widen-limits` is non-nil, widen only to those limits.  */)
>    (void)
>  {
> +
> +  if (!NILP (BVAR(current_buffer, widen_limits)))
> +    {
> +      Lisp_Object hl = BVAR(current_buffer,  widen_limits);
> +      CHECK_CONS(hl);
> +      CHECK_NUMBER(XCAR(hl));
> +      CHECK_NUMBER(XCDR(hl));
> +      Fnarrow_to_region(XCAR(hl), XCDR(hl));
> +      return Qnil;
> +    }
> +
>    if (BEG != BEGV || Z != ZV)
current_buffer-> clip_changed = 1;
>    BEGV = BEG;

> _______________________________________________
> Emacs-diffs mailing list
> Emacs-diffs@gnu.org
> https://lists.gnu.org/mailman/listinfo/emacs-diffs

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Vitalie Spinu]

Sure. Fixed. This convention is not mentioned in CONTRIBUTE btw.

  Vitalie

>> On Tue, Mar 22 2016 08:08, Stefan Monnier wrote:

> Hi Vitalie,

> For tentative branches where you don't want to have to follow all the
> coding conventions (especially the convention about formatting and
> content of commit messages), please use branch names of the form "scratch/*".

>         Stefan

>>>>>> "Vitalie" == Vitalie Spinu <spinuvit@gmail.com> writes:

>> branch: widen-limits
>> commit c331b6626a427fb89303fea75faebd8c39d343a8
>> Author: Vitalie Spinu <spinuvit@gmail.com>
>> Commit: Vitalie Spinu <spinuvit@gmail.com>

>>     Implement buffer-widen-limits functionality

>>      `widen` now respects restrictions imposed by new variable
>>      `hard-widen-limits`
>> ---
>>  src/buffer.c  |   19 +++++++++++++++++--
>>  src/buffer.h  |   17 +++++++++++++++++
>>  src/editfns.c |   15 ++++++++++++++-
>>  3 files changed, 48 insertions(+), 3 deletions(-)

>> diff --git a/src/buffer.c b/src/buffer.c
>> index f06d7e0..1b62d3a 100644
>> --- a/src/buffer.c
>> +++ b/src/buffer.c
>> @@ -329,6 +329,11 @@ bset_scroll_up_aggressively (struct buffer *b, Lisp_Object val)
> b-> scroll_up_aggressively_ = val;
>>  }
>>  static void
>> +bset_widen_limits (struct buffer *b, Lisp_Object val)
>> +{
>> +  b->widen_limits_ = val;
>> +}
>> +static void
>>  bset_selective_display (struct buffer *b, Lisp_Object val)
>>  {
> b-> selective_display_ = val;
>> @@ -847,6 +852,7 @@ CLONE nil means the indirect buffer's state is reset to default values.  */)
>>        bset_display_count (b, make_number (0));
>>        bset_backed_up (b, Qnil);
>>        bset_auto_save_file_name (b, Qnil);
>> +      bset_widen_limits (b, b->base_buffer->widen_limits_);
>>        set_buffer_internal_1 (b);
>>        Fset (intern ("buffer-save-without-query"), Qnil);
>>        Fset (intern ("buffer-file-number"), Qnil);
>> @@ -961,6 +967,7 @@ reset_buffer_local_variables (struct buffer *b, bool permanent_too)
>>       things that depend on the major mode.
>>       default-major-mode is handled at a higher level.
>>       We ignore it here.  */
>> +  bset_widen_limits(b, Qnil);
>>    bset_major_mode (b, Qfundamental_mode);
>>    bset_keymap (b, Qnil);
>>    bset_mode_name (b, QSFundamental);
>> @@ -2167,7 +2174,7 @@ so the buffer is truly empty after this.  */)
>>  {
>>    Fwiden ();

>> -  del_range (BEG, Z);
>> +  del_range (BEGWL, ZWL);

> current_buffer-> last_window_start = 1;
>>    /* Prevent warnings, or suspension of auto saving, that would happen
>> @@ -5037,6 +5044,7 @@ init_buffer_once (void)
>>    bset_display_count (&buffer_local_flags, make_number (-1));
>>    bset_display_time (&buffer_local_flags, make_number (-1));
>>    bset_enable_multibyte_characters (&buffer_local_flags, make_number (-1));
>> +  bset_widen_limits (&buffer_local_flags, make_number (-1));

>>    /* These used to be stuck at 0 by default, but now that the all-zero value
>>       means Qnil, we have to initialize them explicitly.  */
>> @@ -5160,6 +5168,7 @@ init_buffer_once (void)
>>    bset_cursor_type (&buffer_defaults, Qt);
>>    bset_extra_line_spacing (&buffer_defaults, Qnil);
>>    bset_cursor_in_non_selected_windows (&buffer_defaults, Qt);
>> +  bset_widen_limits (&buffer_defaults, Qnil);

>>    bset_enable_multibyte_characters (&buffer_defaults, Qt);
>>    bset_buffer_file_coding_system (&buffer_defaults, Qnil);
>> @@ -5367,7 +5376,6 @@ defvar_per_buffer (struct Lisp_Buffer_Objfwd *bo_fwd, const char *namestring,
>>      emacs_abort ();
>>  }

>> -
>>  /* Initialize the buffer routines.  */
>>  void
>>  syms_of_buffer (void)
>> @@ -5796,6 +5804,13 @@ If you set this to -2, that means don't turn off auto-saving in this buffer
>>  if its text size shrinks.   If you use `buffer-swap-text' on a buffer,
>>  you probably should set this to -2 in that buffer.  */);

>> +  DEFVAR_PER_BUFFER ("buffer-widen-limits", &BVAR (current_buffer, widen_limits),
>> +                     Qnil,
>> +                     doc: /* When non-nil `widen` will widen to these limits.
>> +Must be a cons of the form (MIN . MAX) where MIN and MAX are integers
>> +of hard widen limits in this buffer. This is an experimental variable
>> +intended primarily for multi-mode engines.  */);
>> +
>>    DEFVAR_PER_BUFFER ("selective-display", &BVAR (current_buffer, selective_display),
>>  		     Qnil,
>>  		     doc: /* Non-nil enables selective display.
>> diff --git a/src/buffer.h b/src/buffer.h
>> index 87b7cee..4075bbf 100644
>> --- a/src/buffer.h
>> +++ b/src/buffer.h
>> @@ -59,6 +59,10 @@ INLINE_HEADER_BEGIN
>>  #define Z (current_buffer->text->z)
>>  #define Z_BYTE (current_buffer->text->z_byte)

>> +/* Positions that take into account widen limits.  */
>> +#define BEGWL (BUF_BEGWL (current_buffer))
>> +#define ZWL (BUF_ZWL(current_buffer))
>> +
>>  /* Macros for the addresses of places in the buffer.  */

>>  /* Address of beginning of buffer.  */
>> @@ -128,6 +132,15 @@ INLINE_HEADER_BEGIN
>>      : NILP (BVAR (buf, begv_marker)) ? buf->begv_byte	\
>>      : marker_byte_position (BVAR (buf, begv_marker)))

>> +/* Hard positions in buffer. */
>> +#define BUF_BEGWL(buf)  	                                \
>> +  ((NILP (BVAR (buf, widen_limits))) ?  BUF_BEG (buf)    \
>> +   : XINT( XCAR (BVAR (buf, widen_limits))))
>> +
>> +#define BUF_ZWL(buf)  	                                \
>> +  ((NILP (BVAR (buf, widen_limits))) ?  BUF_Z (buf)      \
>> +   : XINT( XCDR (BVAR (buf, widen_limits))))
>> +
>>  /* Position of point in buffer.  */
>>  #define BUF_PT(buf)					\
>>     (buf == current_buffer ? PT				\
>> @@ -150,6 +163,7 @@ INLINE_HEADER_BEGIN
>>      : NILP (BVAR (buf, zv_marker)) ? buf->zv_byte	\
>>      : marker_byte_position (BVAR (buf, zv_marker)))

>> +
>>  /* Position of gap in buffer.  */
>>  #define BUF_GPT(buf) ((buf)->text->gpt)
>>  #define BUF_GPT_BYTE(buf) ((buf)->text->gpt_byte)
>> @@ -748,6 +762,9 @@ struct buffer
>>       See `cursor-type' for other values.  */
>>    Lisp_Object cursor_in_non_selected_windows_;

>> +  /* Cons of hard widen limits */
>> +  Lisp_Object widen_limits_;
>> +
>>    /* No more Lisp_Object beyond this point.  Except undo_list,
>>       which is handled specially in Fgarbage_collect.  */

>> diff --git a/src/editfns.c b/src/editfns.c
>> index 2ac0537..e5ab637 100644
>> --- a/src/editfns.c
>> +++ b/src/editfns.c
>> @@ -3480,12 +3480,25 @@ DEFUN ("delete-and-extract-region", Fdelete_and_extract_region,
>>      return empty_unibyte_string;
>>    return del_range_1 (XINT (start), XINT (end), 1, 1);
>>  }
>> +
>>  \f
>>  DEFUN ("widen", Fwiden, Swiden, 0, 0, "",
>>         doc: /* Remove restrictions (narrowing) from current buffer.
>> -This allows the buffer's full text to be seen and edited.  */)
>> +This allows the buffer's full text to be seen and edited.
>> +If `buffer-widen-limits` is non-nil, widen only to those limits.  */)
>>    (void)
>>  {
>> +
>> +  if (!NILP (BVAR(current_buffer, widen_limits)))
>> +    {
>> +      Lisp_Object hl = BVAR(current_buffer,  widen_limits);
>> +      CHECK_CONS(hl);
>> +      CHECK_NUMBER(XCAR(hl));
>> +      CHECK_NUMBER(XCDR(hl));
>> +      Fnarrow_to_region(XCAR(hl), XCDR(hl));
>> +      return Qnil;
>> +    }
>> +
>>    if (BEG != BEGV || Z != ZV)
> current_buffer-> clip_changed = 1;
>>    BEGV = BEG;

>> _______________________________________________
>> Emacs-diffs mailing list
>> Emacs-diffs@gnu.org
>> https://lists.gnu.org/mailman/listinfo/emacs-diffs

RE: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Drew Adams]

> >>  DEFUN ("widen", Fwiden, Swiden, 0, 0, "",
> >>         doc: /* Remove restrictions (narrowing) from current buffer.
> >> -This allows the buffer's full text to be seen and edited.  */)
> >> +This allows the buffer's full text to be seen and edited.
> >> +If `buffer-widen-limits` is non-nil, widen only to those limits.  */)

Why do you need this?  Why don't you just _narrow_ to those limits,
instead of making `widen' do it?

Why does "widening" need a separate set of limits?  That's what we have
narrowing for.  Any restriction of the buffer is a narrowing (which
concept is defined relative to the full buffer), regardless of whether
the previous state was a narrower narrowing (restriction).

Cf. the first line of that doc string, which seems to be contradicted
by the possibility that `widen' can put you in a different restriction.

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Vitalie Spinu]

>> On Tue, Mar 22 2016 12:56, Drew Adams wrote:

> Why does "widening" need a separate set of limits?  

Because in multi-modes most of critical operations such as syntax parsing,
syntax-propertize, font-locking and indentations inside submodes occurs in
narrowed regions. That is, sub-mode is placed in a bubble. The problem is that
that buble is easy to escape with widening. These extra limits are intended to
make that escape impossible (at least till the sub-mode start using those hard
limits itself).

> Cf. the first line of that doc string, which seems to be contradicted

Good catch!

Thanks,

  Vitalie
  

RE: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Drew Adams]

> > Why does "widening" need a separate set of limits?
> 
> Because in multi-modes most of critical operations such as syntax parsing,
> syntax-propertize, font-locking and indentations inside submodes occurs in
> narrowed regions. That is, sub-mode is placed in a bubble. The problem is
> that that buble is easy to escape with widening. These extra limits are
> intended to make that escape impossible (at least till the sub-mode start
> using those hard limits itself).

Thanks for the explanation.  I think I see what you are trying to do.

I don't see why changing `widen' would be the only, or necessarily
the best, way to meet that need (which is essentially to make a
sub-mode treat given bounds as if they were the buffer limits).

But you answered my question.  Thank you.

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Andreas Röhler]

On 23.03.2016 01:44, Drew Adams wrote:
>>> Why does "widening" need a separate set of limits?
>> Because in multi-modes most of critical operations such as syntax parsing,
>> syntax-propertize, font-locking and indentations inside submodes occurs in
>> narrowed regions. That is, sub-mode is placed in a bubble. The problem is
>> that that buble is easy to escape with widening. These extra limits are
>> intended to make that escape impossible (at least till the sub-mode start
>> using those hard limits itself).
> Thanks for the explanation.  I think I see what you are trying to do.
>
> I don't see why changing `widen' would be the only, or necessarily
> the best, way to meet that need (which is essentially to make a
> sub-mode treat given bounds as if they were the buffer limits).
>
> But you answered my question.  Thank you.
>

Reads as a classical fix at the wrong place - not the first one in Emacs.
Instead of introducing extra limits --and then inner- and outer-extras 
of that extra etc.-- preventing unwanted widen instead seems the way to go.

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Vitalie Spinu]

>> On Wed, Mar 23 2016 08:16, Andreas Röhler wrote:

> preventing unwanted widen instead seems the way to go.

That's precisely what extra limit do. Prevent unwanted widen. How do you propose
to implement that?

I see 4 ways to go about it:

  1) Add an extra prog-widen and teach all modes out there to use it in contexts
     like syntax-parsing, indentation, font-lock and who know what else. A half
     backed version of this in already part of emacs.

  2) Have low level restrictions directly in `widen` and a macro
     `with-widen-limit` that multi-modes can use. This is the current patch.

  3) Have two types of narrowing (soft and hard). This is harder to implement
     but has the benefit that it can be used in non-transient situations like
     Info mode.

  4) Bring widen to elisp and allow minor modes (and Info mode) advice widen in
     whatever way they see.

I think (1) is a bad idea. (4) is simplest and very general. (3) might be useful
but it's hard. (2) is implemented to get rid of (1).

I proposed (4) very early in the thread, but didn't hear much support for
it. There are only three trivial usages of Fwiden in C code. Bringing `narrow`
to elisp is equally easy.

  Vitalie

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Andreas Röhler]

On 23.03.2016 12:58, Vitalie Spinu wrote:
>>> On Wed, Mar 23 2016 08:16, Andreas Röhler wrote:
>> preventing unwanted widen instead seems the way to go.
> That's precisely what extra limit do. Prevent unwanted widen. How do you propose
> to implement that?
>
> I see 4 ways to go about it:
>
>    1) Add an extra prog-widen and teach all modes out there to use it in contexts
>       like syntax-parsing, indentation, font-lock and who know what else. A half
>       backed version of this in already part of emacs.
>
>    2) Have low level restrictions directly in `widen` and a macro
>       `with-widen-limit` that multi-modes can use. This is the current patch.
>
>    3) Have two types of narrowing (soft and hard). This is harder to implement
>       but has the benefit that it can be used in non-transient situations like
>       Info mode.
>
>    4) Bring widen to elisp and allow minor modes (and Info mode) advice widen in
>       whatever way they see.
>
> I think (1) is a bad idea. (4) is simplest and very general. (3) might be useful
> but it's hard. (2) is implemented to get rid of (1).
>
> I proposed (4) very early in the thread, but didn't hear much support for
> it. There are only three trivial usages of Fwiden in C code. Bringing `narrow`
> to elisp is equally easy.
>
>    Vitalie

Hi Vitali,

may you point me to spot/bug where widen goes wrong?

Thanks,

Andreas

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Vitalie Spinu]

>> On Wed, Mar 23 2016 14:02, Andreas Röhler wrote:

> may you point me to spot/bug where widen goes wrong?

Widen doesn't go wrong in itself. It what you do when you widen. In multi-modes
which use narrowing to create a micro-universe for inner modes, inner mode might
widen then compute some stuff on code from other language regions. This leads to
errors of all kind.

For example when multi-mode advices font-lock-default-fontify-region it cannot
control what individual functions in font-lock-keywords are doing. In case of
syntax-propertize-function it's a complete black box. The function can decide to
do whatever there.

Luckily if major-modes doesn't use widen or parse-partial-sexp directly it all
seem to work quite well with proper advice of relevant functions.

  Vitalie

RE: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Drew Adams]

> > preventing unwanted widen instead seems the way to go.
> 
> That's precisely what extra limit do. Prevent unwanted widen. How do you
> propose
> to implement that?
> 
> I see 4 ways to go about it:
> 
>   1) Add an extra prog-widen and teach all modes out there to use it in
> contexts like syntax-parsing, indentation, font-lock and who know what else. A
> half backed version of this in already part of emacs.
> 
>   2) Have low level restrictions directly in `widen` and a macro
>      `with-widen-limit` that multi-modes can use. This is the current patch.
> 
>   3) Have two types of narrowing (soft and hard). This is harder to
> implement but has the benefit that it can be used in non-transient situations
> like Info mode.
> 
>   4) Bring widen to elisp and allow minor modes (and Info mode) advice widen
> in whatever way they see.
> 
> I think (1) is a bad idea. (4) is simplest and very general. (3) might be
> useful but it's hard. (2) is implemented to get rid of (1).
> 
> I proposed (4) very early in the thread, but didn't hear much support for
> it. There are only three trivial usages of Fwiden in C code. Bringing
> `narrow` to elisp is equally easy.

At least now you're backing up to talk about multiple possible
solutions, instead of digging in deeper in your discussion of a
single implementation (or is it two implementations that you and
Stefan are considering?).

I asked for a clear statement (at least a summary, but preferably a
mini-spec) of the _problem_ you are trying to solve.  My request was
ignored.  So be it.  Continue on, the two of you ... but don't be
surprised if people wonder about your solution after a while.

As for solutions (for what I'm only guessing might be the problem),
I suppose there are additional possibilities, including perhaps a
`with-buffer-limits' macro or perhaps even adapting `point-min' and
`point-max' (e.g. by let-binding global variables that they respond
to - e.g., variables `point-min' and `point-max').

Dunno.  It's hard to guess at a solution without understanding the
problem.  All I've gathered so far is that you want certain zones
of the buffer to act as if the major mode is this or that, and in
those zones you want some code (which code? all code?) to treat
`point-min' and `point-max' as the zone limits, i.e., to act as if
there is nothing outside of the zone limits.

Just what that means in detail is not clear (not specified).
Something about font-locking ... and some other blah.  Not clear.

No, I don't need to understand; I'm just one user, and I probably
have nothing useful to offer as a suggestion.  But my guess is
that if more people here had an idea of what problem you are
trying to solve then you might get some useful input.  Just a guess.

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Eli Zaretskii]

> From: Vitalie Spinu <spinuvit@gmail.com>
> Date: Wed, 23 Mar 2016 15:17:03 +0100
> Cc: emacs-devel@gnu.org
> 
> Widen doesn't go wrong in itself. It what you do when you widen. In multi-modes
> which use narrowing to create a micro-universe for inner modes, inner mode might
> widen then compute some stuff on code from other language regions. This leads to
> errors of all kind.
> 
> For example when multi-mode advices font-lock-default-fontify-region it cannot
> control what individual functions in font-lock-keywords are doing. In case of
> syntax-propertize-function it's a complete black box. The function can decide to
> do whatever there.
> 
> Luckily if major-modes doesn't use widen or parse-partial-sexp directly it all
> seem to work quite well with proper advice of relevant functions.

Isn't prog-widen the solution to those issues?

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Andreas Röhler]

On 23.03.2016 15:17, Vitalie Spinu wrote:
>
>
>>> On Wed, Mar 23 2016 14:02, Andreas Röhler wrote:
>> may you point me to spot/bug where widen goes wrong?
> Widen doesn't go wrong in itself. It what you do when you widen. In multi-modes
> which use narrowing to create a micro-universe for inner modes, inner mode might
> widen then compute some stuff on code from other language regions. This leads to
> errors of all kind.
>
> For example when multi-mode advices font-lock-default-fontify-region it cannot
> control what individual functions in font-lock-keywords are doing. In case of
> syntax-propertize-function it's a complete black box. The function can decide to
> do whatever there.
>
> Luckily if major-modes doesn't use widen or parse-partial-sexp directly it all
> seem to work quite well with proper advice of relevant functions.
>
>    Vitalie
>

Okay, thanks.

Still think it's best to continue with a real-case scenario, discussing 
possible solutions.

Andreas

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Andreas Röhler]

On 23.03.2016 16:34, Eli Zaretskii wrote:
>> From: Vitalie Spinu <spinuvit@gmail.com>
>> Date: Wed, 23 Mar 2016 15:17:03 +0100
>> Cc: emacs-devel@gnu.org
>>
>> Widen doesn't go wrong in itself. It what you do when you widen. In multi-modes
>> which use narrowing to create a micro-universe for inner modes, inner mode might
>> widen then compute some stuff on code from other language regions. This leads to
>> errors of all kind.
>>
>> For example when multi-mode advices font-lock-default-fontify-region it cannot
>> control what individual functions in font-lock-keywords are doing. In case of
>> syntax-propertize-function it's a complete black box. The function can decide to
>> do whatever there.
>>
>> Luckily if major-modes doesn't use widen or parse-partial-sexp directly it all
>> seem to work quite well with proper advice of relevant functions.
> Isn't prog-widen the solution to those issues?

Hi Eli,

doku of prog-widen says

"This variable enables the major mode of the
main language to use the indentation engine of the sub-mode"

This also doesn't sound right. A mode should not need to fiddle with 
other modes. Seems it's time to introduce a true multi-mode, and abandon 
the notion of a buffers major-mode when multi-mode is on - even if the 
symbol of a than regional-mode is still called major-mode for legacy 
reasons.

A multi-mode could keep a register, an index about the modes to employ 
according to position.

Best,

Andreas

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Eli Zaretskii]

> Cc: emacs-devel@gnu.org
> From: Andreas Röhler <andreas.roehler@online.de>
> Date: Wed, 23 Mar 2016 18:24:38 +0100
> 
> > Isn't prog-widen the solution to those issues?
> 
> Hi Eli,
> 
> doku of prog-widen says
> 
> "This variable enables the major mode of the
> main language to use the indentation engine of the sub-mode"
> 
> This also doesn't sound right.

Please read the description in the ELisp manual instead.

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Andreas Röhler]

On 23.03.2016 18:55, Eli Zaretskii wrote:
>> Cc: emacs-devel@gnu.org
>> From: Andreas Röhler <andreas.roehler@online.de>
>> Date: Wed, 23 Mar 2016 18:24:38 +0100
>>
>>> Isn't prog-widen the solution to those issues?
>> Hi Eli,
>>
>> doku of prog-widen says
>>
>> "This variable enables the major mode of the
>> main language to use the indentation engine of the sub-mode"
>>
>> This also doesn't sound right.
> Please read the description in the ELisp manual instead.

So let's read this (prog-widen):

"    ... to remove any restrictions
      imposed by the mode’s indentation engine and restore the
      restrictions recorded in ‘prog-indentation-context’.  This prevents
      the indentation engine of a sub-mode from inadvertently operating
      on text outside of the chunk it was supposed to indent, and
      preserves the restriction imposed by the superior mode.  When no
      superior mode is in effect, this function just calls ‘widen’.
"

Don't see in which way this should be better. It lays the burden of 
dealing with the mode in place into prog-mode. IMO wrong place, wrong 
direction.

Expect prog-mode do deliver very basic things common to all programming 
modes. Not dealing with and fixing special needs there.

Modes must meet the specific languages. Prog-mode must not be specific 
and not provide tools for storing things like indentation-context. Let 
the modes indent, fontify and jump around like they want - not thwarting 
their settings seems all needed here.

A vision for multiple major modes [was: Re: [Emacs-diffs] widen-limits c331b66:]

[Alan Mackenzie]

Hello, Vitalie.

On Wed, Mar 23, 2016 at 12:58:27PM +0100, Vitalie Spinu wrote:

> >> On Wed, Mar 23 2016 08:16, Andreas Röhler wrote:

> > preventing unwanted widen instead seems the way to go.

> That's precisely what extra limit do. Prevent unwanted widen. How do you propose
> to implement that?

> I see 4 ways to go about it:

>   1) Add an extra prog-widen and teach all modes out there to use it in contexts
>      like syntax-parsing, indentation, font-lock and who know what else. A half
>      backed version of this in already part of emacs.

>   2) Have low level restrictions directly in `widen` and a macro
>      `with-widen-limit` that multi-modes can use. This is the current patch.

>   3) Have two types of narrowing (soft and hard). This is harder to implement
>      but has the benefit that it can be used in non-transient situations like
>      Info mode.

>   4) Bring widen to elisp and allow minor modes (and Info mode) advice widen in
>      whatever way they see.

> I think (1) is a bad idea. (4) is simplest and very general. (3) might be useful
> but it's hard. (2) is implemented to get rid of (1).

> I proposed (4) very early in the thread, but didn't hear much support for
> it. There are only three trivial usages of Fwiden in C code. Bringing `narrow`
> to elisp is equally easy.

All these options strike me as artificial, ad hoc, and ugly.  I would go
for option number (5) - to transcend the "unwanted widen" problem - to
enhance Emacs such that users and Lisp hackers can freely narrow and
widen _without_ upsetting the @dfn{super mode} (the multiple mode
handling mode).

What is a major mode?  It is a collection of local variable settings, a
syntax table, an abbreviation table, a mode specific key map, font lock
mode settings, an indentation engine, Imenu settings, and one or two
other things.

Let us then have all these things in our super mode, such that their
current values are according to where point is - if we have an AWK
script embedded in a shell script, when point is in the AWK bit, the
mode line should say "AWK", the C-c C-? bindings from CC Mode should be
in force, the font locking should be AWK's, etc.

For this we will need a new type of local variable, an "island-local" or
"span-local" variable, or whatever you want to call it.  Values of these
variables will vary according to where point is.

To transcend the "unwanted widen" problem, there will be a very special
variable `restrict-to-island' or `restrict-to-span', or .....  When
bound to non-nil (by the super mode), this instructs certain primitives
to confine their attention to the individual island/span (or possibly a
chain of them).  There will be no restrictions on `widen' or
`parse-partial-sexp', because there won't need to be.
`parse-partial-sexp' would simply skip over "foreign spans" looking for
the delimiter marking the beginning of the interesting span.  Regexp
searching would likewise restrict its attentions, as would several other
facilities.

Although the above vision implies a lot of development work, there is
nothing there which is beyond our abilities to implement readily.  It
would give us a true multi major mode capability, yet the impact on
individual major modes would be minimal.

>   Vitalie

-- 
Alan Mackenzie (Nuremberg, Germany).

RE: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Drew Adams]

> >> doku of prog-widen says
> >>
> >> "This variable enables the major mode of the
> >> main language to use the indentation engine of the sub-mode"
> >>
> >> This also doesn't sound right.
> > Please read the description in the ELisp manual instead.
> 
> So let's read this (prog-widen):
> 
> "    ... to remove any restrictions
>       imposed by the mode's indentation engine and restore the
>       restrictions recorded in 'prog-indentation-context'.  This prevents
>       the indentation engine of a sub-mode from inadvertently operating
>       on text outside of the chunk it was supposed to indent, and
>       preserves the restriction imposed by the superior mode.  When no
>       superior mode is in effect, this function just calls 'widen'.
> "
> 
> Don't see in which way this should be better. It lays the burden of
> dealing with the mode in place into prog-mode. IMO wrong place, wrong
> direction.
> 
> Expect prog-mode do deliver very basic things common to all programming
> modes. Not dealing with and fixing special needs there.
> 
> Modes must meet the specific languages. Prog-mode must not be specific
> and not provide tools for storing things like indentation-context. Let
> the modes indent, fontify and jump around like they want - not thwarting
> their settings seems all needed here.

Worse yet -

What's a "sub-mode"?  The term is introduced nowhere in either the Emacs
manual or the Elisp manual.  It's not in the Elisp manual index.

It is used only in the node mentioned, and with no definition or
explanation.

That's a bug, IMO.  One might assume that it just means any major mode
that inherits from (i.e., derived from) `prog-mode' (since the term is
used only in the context of `prog-mode').  But one should not have to
assume.

And the text about indentation and the use of sub-mode for only a given
chunk of text in a buffer is incomprehensible without some explanation.
Normally, a major mode, no matter whether it is derived from some other
major mode, has its effect on the entire buffer.

It seems there is some non-negligible functionality/behavior/feature
that has not been documented.  Not up to Emacs standards, it appears.

It looks like someone perhaps implemented something and just tossed
some minimal doc into the manual, in the form of doc-string-like text
for a couple functions.  Insufficient.

Please explain what this is all about.

Re: A vision for multiple major modes [was: Re: [Emacs-diffs] widen-limits c331b66:]

[Vitalie Spinu]

> To transcend the "unwanted widen" problem, there will be a very special
> variable `restrict-to-island' or `restrict-to-span', 

A second type of narrowing. That is what Stefan was insisting upon and that's
what I will provide next patch for.

> Although the above vision implies a lot of development work, there is
> nothing there which is beyond our abilities to implement readily.  It
> would give us a true multi major mode capability, yet the impact on
> individual major modes would be minimal.

A lot of development work is already happening in various generic multi-mode
engines. It's hard, but feasible and stuff mostly works without changing any of
the existing code. Making parse-partial-sexp understand islands won't give
much. You can already do that well enough by advising syntax-ppss.

  Vitalie

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Vitalie Spinu]

>> On Wed, Mar 23 2016 14:57, Drew Adams wrote:

> It looks like someone perhaps implemented something and just tossed
> some minimal doc into the manual, in the form of doc-string-like text
> for a couple functions.

The issue of prog-widen and prog-indentation-context was discussed extensively
in the thread that led to this diff.

  https://lists.gnu.org/archive/html/emacs-devel/2016-03/threads.html#00859

Too long to be read from scratch I suppose.  My takeaway is that those features
are raw and simplistic and bring more complications than they solve. Hopefully
there will be a common consensus to put those on hold for now.

  Vitalie

Re: A vision for multiple major modes [was: Re: [Emacs-diffs] widen-limits c331b66:]

[Dmitry Gutov]

Hi Alan,

On 03/23/2016 11:16 PM, Alan Mackenzie wrote:

> All these options strike me as artificial, ad hoc, and ugly.  I would go
> for option number (5) - to transcend the "unwanted widen" problem - to
> enhance Emacs such that users and Lisp hackers can freely narrow and
> widen _without_ upsetting the @dfn{super mode} (the multiple mode
> handling mode).

I disagree about ugly. Co-opting narrowing to do something useful for 
once is a pretty neat, and minimal, approach.

> Let us then have all these things in our super mode, such that their
> current values are according to where point is - if we have an AWK
> script embedded in a shell script, when point is in the AWK bit, the
> mode line should say "AWK", the C-c C-? bindings from CC Mode should be
> in force, the font locking should be AWK's, etc.

Each multi-mode package implements something like this already. It 
doesn't work well e.g. because font-lock rules of each particular 
language, indentation code, etc, are free to widen.

> For this we will need a new type of local variable, an "island-local" or
> "span-local" variable, or whatever you want to call it.  Values of these
> variables will vary according to where point is.

That part is already doable (and done), for most practical purposes.

> To transcend the "unwanted widen" problem, there will be a very special
> variable `restrict-to-island' or `restrict-to-span', or .....  When
> bound to non-nil (by the super mode), this instructs certain primitives
> to confine their attention to the individual island/span (or possibly a
> chain of them).  There will be no restrictions on `widen' or
> `parse-partial-sexp', because there won't need to be.
> `parse-partial-sexp' would simply skip over "foreign spans" looking for
> the delimiter marking the beginning of the interesting span.  Regexp
> searching would likewise restrict its attentions, as would several other
> facilities.

You'll have to present the total list of facilities, decide how the 
islands would be applied, and other issues will likely come up from 
unexpected places.

For instance, you said that there could be island-local variables. Can I 
put some cache into one? Earlier, you suggested that the islands would 
be applied via text properties. What happens to all island-local 
variables when someone, somewhere, changes an island's boundary (maybe 
adds, maybe removes, maybe moves it)? On the one hand, we'd probably 
want to retain some variables, in order not to rerun the major mode 
functions over and over again. On the other hand, if we were to put e.g. 
syntax-ppss-last into an island-local variable (and it's a logical 
continuation of this idea), after island boundaries change it should 
what... become unbound? Nil? Or carefully managed by the mult-mode 
package, which happens already.

Next, at which points exactly would Emacs look at the island boundaries 
and change the island-local variables to the values set in the current 
island? Probably not after each point movement. In post-command-hook? 
That's also already done.

> Although the above vision implies a lot of development work, there is
> nothing there which is beyond our abilities to implement readily.  It
> would give us a true multi major mode capability, yet the impact on
> individual major modes would be minimal.

I'm sure this is eventually doable. But this proposal looks rather 
similar to what Lennart Borgman has been asking, in multi-mode related 
discussions, on several occasions separated by years. Also in broad 
strokes (probably a bit broader that these). Nobody has been both 
capable and invested enough into the issue of multi-mode buffers to even 
start working on it, AFAIK.

On the other hand, using narrowing for multi-mode purpose is a familiar 
ground already, and the changes in Emacs core required to do so are 
minimal. And most of the code written for Emacs has been taught to 
respect narrowing bounds (even if only by the virtue of always using 
(point-min) instead of 1), so we can utilize that.

Another (probably minor, it's hard to tell now) disadvantage, is if the 
multi-mode package sets narrowing bounds itself, it will decide which 
islands are visible from the current island, dynamically, so to speak. 
Maybe just the current one. Or it can copy just a couple of islands from 
the same mode to a temp buffer, call the indentation function there, and 
use the result. Doing that using an islands framework limits it to a 
predefined set of semantics (e.g. all Ruby islands see all other Ruby 
islands).

That's not to say that being able to make parse-partial-sexp to skip 
over certain intervals wouldn't be valuable. But you can do that, sort 
of, already, by applying existing text properties to those intervals 
(like beginning-of-comment/end-of-comment, or just "whitespace" over the 
whole of it), and then removing them at the end of an operation. But the 
end benefits might not be high enough to justify the necessary work and 
the increase in complexity in internals.

RE: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Drew Adams]

> > It looks like someone perhaps implemented something and just tossed
> > some minimal doc into the manual, in the form of doc-string-like text
> > for a couple functions.
> 
> The issue of prog-widen and prog-indentation-context was discussed
> extensively in the thread that led to this diff.
>   https://lists.gnu.org/archive/html/emacs-devel/2016-03/threads.html#00859

Maybe so.  My comment was about the inadequate doc, which does not define
or in any way explain "sub-mode", its relation to chunks of text within
a "super-mode" etc.

This stuff needs to be defined.  Can sub-modes be nested (sub-sub-mode etc.)?
Etc.  There is nothing clear about this stuff, based on the doc.  And my
comment was limited to the doc - it was not a comment about any mail thread.

> Too long to be read from scratch I suppose.  My takeaway is that those
> features are raw and simplistic and bring more complications than they 
> solve. Hopefully there will be a common consensus to put those on hold
> for now.

I have no opinion on what should be done, other than that whatever is
intended to be added as a feature should be presented and discussed
here first (IMO).  I asked for a mini-spec of the requirements, but
have seen nothing - no reasonable description of the whole problem
the feature is intended to address.

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Vitalie Spinu]

>> On Wed, Mar 23 2016 18:14, Andreas Röhler wrote:

> Still think it's best to continue with a real-case scenario, discussing
> possible solutions.

Here is a non multi-mode real case:

 1) goto arbitrary Info page
 2) narrow to a region
 3) widen

You will get the whole info manual. Hard narrowing will allow widening to
original page in step (3).


  Vitalie
   

RE: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Drew Adams]

> > Still think it's best to continue with a real-case scenario, discussing
> > possible solutions.
> 
> Here is a non multi-mode real case:
> 
>  1) goto arbitrary Info page
>  2) narrow to a region
>  3) widen
> 
> You will get the whole info manual.

No.  You will get the full current Info file, which is what you
asked for (with `C-x n w').

> Hard narrowing will allow widening to original page in step (3).

What "original page"?  Perhaps you mean the current Info node?

What do you mean by "allow"?  Do you mean that `C-x w' will no
longer widen to the full Info file?  That would be a regression.

You should use another function if you want to return to the
current Info node in Info mode.  It is sufficient to hit `g'
and give the current node name.

No need for any new-fangled hard-widening feature for this
use case.  And no need to break the use of widening to get
to the full Info file.

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Vitalie Spinu]

>> On Wed, Mar 23 2016 17:37, Drew Adams wrote:

> No.  You will get the full current Info file, which is what you
> asked for (with `C-x n w').

I asked for original view (page, node call it as you want). Info gave me a
buffer, I narrowed, then widened and got surprised. The user need not be exposed
to internal implementation of Info.

>> Hard narrowing will allow widening to original page in step (3).

> What do you mean by "allow"?  Do you mean that `C-x w' will no
> longer widen to the full Info file?  

"allow" means that it would be possible to use it for that purpose. It's not up
to me to decide if hard narrowing will be used in info or how it will be used.

  Vitalie

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Eli Zaretskii]

> Date: Wed, 23 Mar 2016 14:57:49 -0700 (PDT)
> From: Drew Adams <drew.adams@oracle.com>
> Cc: spinuvit@gmail.com, emacs-devel@gnu.org
> 
> And the text about indentation and the use of sub-mode for only a given
> chunk of text in a buffer is incomprehensible without some explanation.
> Normally, a major mode, no matter whether it is derived from some other
> major mode, has its effect on the entire buffer.
> 
> It seems there is some non-negligible functionality/behavior/feature
> that has not been documented.  Not up to Emacs standards, it appears.
> 
> It looks like someone perhaps implemented something and just tossed
> some minimal doc into the manual, in the form of doc-string-like text
> for a couple functions.  Insufficient.

"Compliments" such as these for something that caused me a
considerable effort to produce where no documentation by the code
developers existed in the first place is a very efficient method of
convincing me never to make such efforts again.  Not for this
community, anyway.

Thanks a lot!

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Eli Zaretskii]

> From: Vitalie Spinu <spinuvit@gmail.com>
> Cc: Andreas Röhler <andreas.roehler@online.de>,  Eli
>  Zaretskii <eliz@gnu.org>,  emacs-devel@gnu.org
> Date: Wed, 23 Mar 2016 23:13:58 +0100
> 
> The issue of prog-widen and prog-indentation-context was discussed extensively
> in the thread that led to this diff.
> 
>   https://lists.gnu.org/archive/html/emacs-devel/2016-03/threads.html#00859
> 
> Too long to be read from scratch I suppose.  My takeaway is that those features
> are raw and simplistic and bring more complications than they solve. Hopefully
> there will be a common consensus to put those on hold for now.

If that's the case, how come these features were admitted into our
codebase?  Should we rip them out, as long as it's not too late?

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Andreas Röhler]

On 24.03.2016 01:03, Vitalie Spinu wrote:
>
>>> On Wed, Mar 23 2016 18:14, Andreas Röhler wrote:
>> Still think it's best to continue with a real-case scenario, discussing
>> possible solutions.
> Here is a non multi-mode real case:
>
>   1) goto arbitrary Info page
>   2) narrow to a region
>   3) widen
>
> You will get the whole info manual. Hard narrowing will allow widening to
> original page in step (3).
>
>
>    Vitalie
>     
Don't see why info couldn't care for that - for example by indexing 
previous narrowing and re-narrow following widen if appropriate.

May think of nested narrows that way, which wouldn't require changes of 
Emacs core.

But info seems a special case anyway, while a true multi-mode really 
being at stake. Instead of diving into info now, suggest to continue 
with a multi-mode issue.

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Dmitry Gutov]

On 03/24/2016 05:38 AM, Eli Zaretskii wrote:

 >> [prog-indentation-context]

> Should we rip them out, as long as it's not too late?

Yes. If nobody beats me to it, I'll revert it in emacs-25 this weekend. 
The yet-unfinished discussion aside, it doesn't look ready for 25.1 
either way.

I'm sorry you had to work on documenting it; hopefully we can reuse the 
pieces of that text, or at least refer to it when documenting the 
eventual alternative approach.

RE: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Drew Adams]

> > No.  You will get the full current Info file, which is what you
> > asked for (with `C-x n w').
> 
> I asked for original view (page, node call it as you want). Info
> gave me a buffer, I narrowed, then widened and got surprised.

One could argue that the surprise is your fault. ;-)  You asked
to widen to the full buffer, and that's what you got.  Presumably,
if you ask for that then you want the whole buffer (whatever that
might be), and in this context presumably you know what that whole
buffer is (an Info file).  If not, you shouldn't be using `C-x n w'.

> The user need not be exposed to internal implementation of Info.

Need not?  A user of `C-x n w' should have some expectation that
what s?he sees is not the full buffer, and s?he should not have
a false preconception about what the full buffer might be like.

On the other hand, if we take your view about surprise here
then the issue is more general: `C-x n w' could be surprising
in any context where Emacs narrows a buffer by default.

Taking that into account, it should perhaps be the case that
`C-x n w', just like `C-x n n', should have property `disabled'
set to non-nil by default.  That should remove your surprise.

(And it's not just an "internal implementation", IMO.  Not every
Emacs users needs to know that Info uses files and display of a
node narrows to part of a file, of course.  But that information
is available to users, and they can make use of it.)

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Dmitry Gutov]

On 03/24/2016 03:53 PM, Drew Adams wrote:

> One could argue that the surprise is your fault. ;-)  You asked
> to widen to the full buffer, and that's what you got.  Presumably,
> if you ask for that then you want the whole buffer (whatever that
> might be), and in this context presumably you know what that whole
> buffer is (an Info file).  If not, you shouldn't be using `C-x n w'.

Just how, then, should a user undo the result of narrow-to-region, which 
is a user-level operation?

Even if it's one that's disabled by default.

RE: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Drew Adams]

> > One could argue that the surprise is your fault. ;-)  You asked
> > to widen to the full buffer, and that's what you got.  Presumably,
> > if you ask for that then you want the whole buffer (whatever that
> > might be), and in this context presumably you know what that whole
> > buffer is (an Info file).  If not, you shouldn't be using `C-x n w'.
> 
> Just how, then, should a user undo the result of narrow-to-region, which
> is a user-level operation?  Even if it's one that's disabled by default.

Good question.  Depends what you mean by "undo" it.  Depends
what the user wants.

In vanilla Emacs, so far, `C-x n w' undoes a buffer restriction,
restoring the full buffer.  That's clear.

But if you mean something other than that as "undo" then you need
to specify just what that other is.

FWIW, my library `zones.el' has this very question at heart, and
it provides some other useful meanings of "undo" for narrowing.
I have no problem with vanilla Emacs adding additional ones.

What I question is co-opting `widen' to redefine it so that it
performs only one particular sort of new kind of "undo" for
narrowing.

Instead, please consider any forms of "undo" for narrowing, other
than widening to the full buffer, to be changes to a different
narrowing.  As long as the result is a buffer restriction (a
narrowing), such a change is still narrowing.

FWIW, zones.el provides for multiple narrowings, and "undoing"
from one to another.

The first title in the doc for this is "Problem: Narrowing is
Fine-Grained, But Widening Is Not".  So I think I understand
the use of having other kinds of undo, besides just `C-x n w'.

(https://www.emacswiki.org/emacs/MultipleNarrowings)

The Emacs world is a wider place than what you have in mind
for one form of widening, whatever that might be.  Your form
can line up with any number of others - they are all narrowings
(buffer restrictions), and I doubt that any one of them should
be privileged to be considered "the hard" widening.

RE: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Drew Adams]

> > And the text about indentation and the use of sub-mode for only a given
> > chunk of text in a buffer is incomprehensible without some explanation.
> > Normally, a major mode, no matter whether it is derived from some other
> > major mode, has its effect on the entire buffer.
> >
> > It seems there is some non-negligible functionality/behavior/feature
> > that has not been documented.  Not up to Emacs standards, it appears.
> >
> > It looks like someone perhaps implemented something and just tossed
> > some minimal doc into the manual, in the form of doc-string-like text
> > for a couple functions.  Insufficient.
> 
> "Compliments" such as these for something that caused me a
> considerable effort to produce where no documentation by the code
> developers existed in the first place is a very efficient method of
> convincing me never to make such efforts again.  Not for this
> community, anyway.  Thanks a lot!

Please consider it instead as a statement that "the code
developers" for this feature (whatever it might be) should
themselves have documented it.  Which is also what I think
you suggest.

My plaint wrt what the user sees now (incomplete doc) is
essentially the same as your plaint that the code developers
did not provide adequate doc.  I don't at all complain about
your effort to improve a complete lack by providing something
helpful.

You improved things by adding some explanation.  I made a
(minor) effort to improve things after that, by saying that,
IMO, the doc is still not sufficient.

My guess is that those responsible for adding this feature
are the best placed to provide the technical content (if not
necessarily the wording) for telling users what it is.

I don't know what that content is - dunno what the feature is.
As a user learning about this question for the first time, I
searched the manuals for "sub-mode" and similar expressions
to learn more, and came up empty-handed.  I reported that.

Perhaps you too have only a partial feel for this feature.
I have no gripe whatsoever with what you've done to help us
understand this, Eli.  Quite the contrary.

Had you not documented this at all, I would not have noticed
it and would not have posed questions about it.  You need not
be defensive about my saying the doc is incomplete here.
You can be proud that you tried to make it more complete.

Thank you for that.  In fact, I thank Emacs everyday for your
unflagging concern about the doc and your efforts to maintain
and improve it.

This concern with doc is an Emacs tradition that dates from
Day One and for which RMS was always a prominent advocate and
participant.  Today, you remain the main stalwart in this
endeavor.  Thank goodness you are still at it.

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Stefan Monnier]

> What I question is co-opting `widen' to redefine it so that it
> performs only one particular sort of new kind of "undo" for
> narrowing.

The important part is that we need a command that makes the buffer "as
wide as is meaningful".  Usually that's the whole buffer, but in
info-mode it's not.


        Stefan

RE: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Drew Adams]

> > What I question is co-opting `widen' to redefine it so that it
> > performs only one particular sort of new kind of "undo" for
> > narrowing.
> 
> The important part is that we need a command that makes the buffer "as
> wide as is meaningful".  Usually that's the whole buffer, but in
> info-mode it's not.

The important part is to define one or more "meaningful" behaviors
for any given context (such as Info).

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Stefan Monnier]

> The important part is to define one or more "meaningful" behaviors
> for any given context (such as Info).

Indeed, there can be several "meaningful" choices.

Currently, narrowing doesn't say anything about what is being narrowed
or why, so we can't distinguish between different cases, which means
that it's impossible for something like font-lock or syntax-ppss (which
can require additional contextual info in order to correctly process
the narrowed pat of the buffer) to know how much to widen.


        Stefan

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Eli Zaretskii]

> Cc: andreas.roehler@online.de, drew.adams@oracle.com, emacs-devel@gnu.org
> From: Dmitry Gutov <dgutov@yandex.ru>
> Date: Thu, 24 Mar 2016 14:24:37 +0200
> 
> On 03/24/2016 05:38 AM, Eli Zaretskii wrote:
> 
>  >> [prog-indentation-context]
> 
> > Should we rip them out, as long as it's not too late?
> 
> Yes. If nobody beats me to it, I'll revert it in emacs-25 this weekend. 
> The yet-unfinished discussion aside, it doesn't look ready for 25.1 
> either way.

If you want to leave it in the code on master, don't forget to puts
some telltale phrase in the commit log.

> I'm sorry you had to work on documenting it; hopefully we can reuse the 
> pieces of that text, or at least refer to it when documenting the 
> eventual alternative approach.

Stuff happens.  If we learn from this experience for the future, at
least the effort wouldn't be completely wasted.

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Eli Zaretskii]

> Date: Thu, 24 Mar 2016 07:41:50 -0700 (PDT)
> From: Drew Adams <drew.adams@oracle.com>
> Cc: andreas.roehler@online.de, spinuvit@gmail.com, emacs-devel@gnu.org
> 
> > > And the text about indentation and the use of sub-mode for only a given
> > > chunk of text in a buffer is incomprehensible without some explanation.
> > > Normally, a major mode, no matter whether it is derived from some other
> > > major mode, has its effect on the entire buffer.
> > >
> > > It seems there is some non-negligible functionality/behavior/feature
> > > that has not been documented.  Not up to Emacs standards, it appears.
> > >
> > > It looks like someone perhaps implemented something and just tossed
> > > some minimal doc into the manual, in the form of doc-string-like text
> > > for a couple functions.  Insufficient.
> > 
> > "Compliments" such as these for something that caused me a
> > considerable effort to produce where no documentation by the code
> > developers existed in the first place is a very efficient method of
> > convincing me never to make such efforts again.  Not for this
> > community, anyway.  Thanks a lot!
> 
> Please consider it instead as a statement that "the code
> developers" for this feature (whatever it might be) should
> themselves have documented it.  Which is also what I think
> you suggest.
> 
> My plaint wrt what the user sees now (incomplete doc) is
> essentially the same as your plaint that the code developers
> did not provide adequate doc.  I don't at all complain about
> your effort to improve a complete lack by providing something
> helpful.

The documentation is complete.  I have just re-read it to be sure, and
I see no flaws there.  If you still think there are gaps, and if the
current text is unclear to you, that's a clear sign that the use cases
in which these features are supposed to be used are something you
never bumped into, and therefore you don't have a clear picture of the
issues.  Which is not a catastrophe, but it does mean that you are not
the best person to criticize this part of the documentation.

> As a user learning about this question for the first time, I
> searched the manuals for "sub-mode" and similar expressions
> to learn more, and came up empty-handed.  I reported that.

It is clear to anyone who understand the situations which need these
features.  Examples of such situations are part of the text in the
manual.  The fact is that this discussion thread references "sub-mode"
or "submode" several times, and people who used that terminology
evidently have to difficulty understanding what it is.

RE: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Drew Adams]

> > > > And the text about indentation and the use of sub-mode for only a
> given
> > > > chunk of text in a buffer is incomprehensible without some
> explanation.
> > > > Normally, a major mode, no matter whether it is derived from some
> other
> > > > major mode, has its effect on the entire buffer.
> > > >
> > > > It seems there is some non-negligible functionality/behavior/feature
> > > > that has not been documented.  Not up to Emacs standards, it appears.
> > > >
> > > > It looks like someone perhaps implemented something and just tossed
> > > > some minimal doc into the manual, in the form of doc-string-like text
> > > > for a couple functions.  Insufficient.
> > >
> > > "Compliments" such as these for something that caused me a
> > > considerable effort to produce where no documentation by the code
> > > developers existed in the first place is a very efficient method of
> > > convincing me never to make such efforts again.  Not for this
> > > community, anyway.  Thanks a lot!
> >
> > Please consider it instead as a statement that "the code
> > developers" for this feature (whatever it might be) should
> > themselves have documented it.  Which is also what I think
> > you suggest.
> >
> > My plaint wrt what the user sees now (incomplete doc) is
> > essentially the same as your plaint that the code developers
> > did not provide adequate doc.  I don't at all complain about
> > your effort to improve a complete lack by providing something
> > helpful.
> 
> The documentation is complete.  I have just re-read it to be sure, and
> I see no flaws there.  If you still think there are gaps, and if the
> current text is unclear to you, that's a clear sign that the use cases
> in which these features are supposed to be used are something you
> never bumped into, and therefore you don't have a clear picture of the
> issues.  Which is not a catastrophe, but it does mean that you are not
> the best person to criticize this part of the documentation.
> 
> > As a user learning about this question for the first time, I
> > searched the manuals for "sub-mode" and similar expressions
> > to learn more, and came up empty-handed.  I reported that.
> 
> It is clear to anyone who understand the situations which need these
> features.  Examples of such situations are part of the text in the
> manual.  The fact is that this discussion thread references "sub-mode"
> or "submode" several times, and people who used that terminology
> evidently have to difficulty understanding what it is.
                 ^^
                 no (?)

Excuse me, but that's a crock.  You are essentially saying that it is
OK to refer to a "florksit" over and over, with no explanation of what
such a critter is, as long as some developers who implemented the code
that uses and provides for florsits understand what it is.  This is
the Elisp manual - for all users of Elisp.

Imagine if we did that for other concepts: keymap, syntax table, or
whatever.  Instead, we introduce the concept with some care, so that
readers of doc about functions that make use of keymaps etc. know what
that doc is talking about.

As I said earlier, one might guess that a "sub-mode" is a mode that
derives from another mode.  OK.  But one will not guess what that might
mean for zones of text in one or another sub-mode, what it might mean
for indentation there, or whether sub-modes can nest or otherwise
overlap.  And so on.

None of this is clear to a reader of this doc, and it should be, IMO.
What's a "sub-mode" in this context?  Answer that clearly in the doc
and Bob will be our uncle.

Re: A vision for multiple major modes [was: Re: [Emacs-diffs] widen-limits c331b66:]

[Alan Mackenzie]

Hello, Vitalie.

On Wed, Mar 23, 2016 at 10:58:44PM +0100, Vitalie Spinu wrote:

> > To transcend the "unwanted widen" problem, there will be a very special
> > variable `restrict-to-island' or `restrict-to-span', 

> A second type of narrowing. That is what Stefan was insisting upon and that's
> what I will provide next patch for.

A type of narrowing is not a good way of thinking about it.

> > Although the above vision implies a lot of development work, there is
> > nothing there which is beyond our abilities to implement readily.  It
> > would give us a true multi major mode capability, yet the impact on
> > individual major modes would be minimal.

> A lot of development work is already happening in various generic multi-mode
> engines. It's hard, but feasible and stuff mostly works without changing any of
> the existing code.

It "mostly" works, sort of, from what I can gather in threads like this
one.  My impression is that with better support in the Emacs core, it
could work fully, without unlovely artifices.

But nobody else has so far shown much interest, so it seems it won't
happen.

> Making parse-partial-sexp understand islands won't give much.

That was not what my post, to which you are replying, was about.  It was
far more ambitious than that.

> You can already do that well enough by advising syntax-ppss.

I doubt that very much.  syntax-ppss is just one of many ways of using
parse-partial-sexp.  But I'd love to see the code.  Has it been
committed, and if so, into which branch?

>   Vitalie

-- 
Alan Mackenzie (Nuremberg, Germany).

Re: A vision for multiple major modes [was: Re: [Emacs-diffs] widen-limits c331b66:]

[Alan Mackenzie]

Hello, Dmitry.

On Thu, Mar 24, 2016 at 12:34:58AM +0200, Dmitry Gutov wrote:
> On 03/23/2016 11:16 PM, Alan Mackenzie wrote:

> > Let us then have all these things in our super mode, such that their
> > current values are according to where point is - if we have an AWK
> > script embedded in a shell script, when point is in the AWK bit, the
> > mode line should say "AWK", the C-c C-? bindings from CC Mode should be
> > in force, the font locking should be AWK's, etc.

> Each multi-mode package implements something like this already. It 
> doesn't work well e.g. because font-lock rules of each particular 
> language, indentation code, etc, are free to widen.

With islands it should work well, regardless of whether any major mode
widens or narrows.  That's because primitives (such as regexp search)
would constrain themselves to the island.

> > For this we will need a new type of local variable, an "island-local" or
> > "span-local" variable, or whatever you want to call it.  Values of these
> > variables will vary according to where point is.

> That part is already doable (and done), for most practical purposes.

Except you said above that it doesn't work very well.

> > To transcend the "unwanted widen" problem, there will be a very special
> > variable `restrict-to-island' or `restrict-to-span', or .....  When
> > bound to non-nil (by the super mode), this instructs certain primitives
> > to confine their attention to the individual island/span (or possibly a
> > chain of them).  There will be no restrictions on `widen' or
> > `parse-partial-sexp', because there won't need to be.
> > `parse-partial-sexp' would simply skip over "foreign spans" looking for
> > the delimiter marking the beginning of the interesting span.  Regexp
> > searching would likewise restrict its attentions, as would several other
> > facilities.

> You'll have to present the total list of facilities, decide how the 
> islands would be applied, and other issues will likely come up from 
> unexpected places.

That will require more investigation.  But syntax based searching and
regexp based searching will certainly be amongst them.

> For instance, you said that there could be island-local variables. Can I 
> put some cache into one?

I can't see why not.

> Earlier, you suggested that the islands would be applied via text
> properties. What happens to all island-local variables when someone,
> somewhere, changes an island's boundary (maybe adds, maybe removes,
> maybe moves it)?

The island-local variables would stay with the island, so that when
somebody inserts or removes text the right thing would be done.  If
somebody deletes the island, those variables would disappear (just as
buffer local ones do when a buffer is deleted).

> On the one hand, we'd probably want to retain some variables, in order
> not to rerun the major mode functions over and over again. On the
> other hand, if we were to put e.g.  syntax-ppss-last into an
> island-local variable (and it's a logical continuation of this idea),
> after island boundaries change it should what... become unbound? Nil?

That's for the application to decide.  The island local binding would
countinue to exist for as long as the island exists, and the application
would be free to use it.

> Or carefully managed by the mult-mode package, which happens already.

> Next, at which points exactly would Emacs look at the island boundaries 
> and change the island-local variables to the values set in the current 
> island? Probably not after each point movement. In post-command-hook? 
> That's also already done.

It wouldn't use any high level facility like post-command-hook.  The
mechanism would be more like a buffer local variable, entirely handled
by the C level, so that such a binding would simply exist.

> > Although the above vision implies a lot of development work, there is
> > nothing there which is beyond our abilities to implement readily.  It
> > would give us a true multi major mode capability, yet the impact on
> > individual major modes would be minimal.

I'm worried by this.  Already narrowing/widening which is currently
simple, straightforward, and rigorously defined, is looking like
becoming complicated.  Major modes are going to be constrained in what
they can do.  I get the impression that major modes are going to be
restricted in how they can use parse-partial-sexp.  These are serious
matters, but I haven't seen any widespread debate on emacs-devel about
them.

> I'm sure this is eventually doable. But this proposal looks rather 
> similar to what Lennart Borgman has been asking, in multi-mode related 
> discussions, on several occasions separated by years. Also in broad 
> strokes (probably a bit broader that these). Nobody has been both 
> capable and invested enough into the issue of multi-mode buffers to even 
> start working on it, AFAIK.

> On the other hand, using narrowing for multi-mode purpose is a familiar 
> ground already, and the changes in Emacs core required to do so are 
> minimal. And most of the code written for Emacs has been taught to 
> respect narrowing bounds (even if only by the virtue of always using 
> (point-min) instead of 1), so we can utilize that.

What is "(narrow-to-region 1 (point-max))" going to become?  It seems
there will be a need for "the bounds of the island", which will impose
complexities in major mode code.

> Another (probably minor, it's hard to tell now) disadvantage, is if the 
> multi-mode package sets narrowing bounds itself, it will decide which 
> islands are visible from the current island, dynamically, so to speak.

How will this be done when narrowing is the tool used?  It you narrow to
an island, none of the other islands in a chain are visible.  If you
narrow to the smallest region which spans those islands, you leave a lot
of stuff visible which shouldn't be.

> Maybe just the current one. Or it can copy just a couple of islands from 
> the same mode to a temp buffer, call the indentation function there, and 
> use the result. Doing that using an islands framework limits it to a 
> predefined set of semantics (e.g. all Ruby islands see all other Ruby 
> islands).

I don't see why.  There's no reason why islands couldn't be linked to
and unlinked from eachother freely at runtime.

> That's not to say that being able to make parse-partial-sexp to skip 
> over certain intervals wouldn't be valuable. But you can do that, sort 
> of, already, by applying existing text properties to those intervals 
> (like beginning-of-comment/end-of-comment, or just "whitespace" over the 
> whole of it), and then removing them at the end of an operation.

Not really.  If you "whitespace" a region out, you've got to remember
the text properties that were there originally, and restore them
afterwards.  There's no easy way to do that.  The comment-fence text
properties won't work either, because, in the general case, there will
already be comment-fence properties in your region which will foul
things up.  I'm worried that the multi-mode projects will suborn these
text properties, making them unavailable for major modes to use.

> But the end benefits might not be high enough to justify the necessary
> work and the increase in complexity in internals.

They might not.  They might.  Basically, nobody else really seams
interested in my idea, so it doesn't look like it will happen.

-- 
Alan Mackenzie (Nuremberg, Germany).

Removing prog-indentation-context (was: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality)

[Stefan Monnier]

>> >> [prog-indentation-context]
>> > Should we rip them out, as long as it's not too late?
>> Yes. If nobody beats me to it, I'll revert it in emacs-25 this weekend. 
>> The yet-unfinished discussion aside, it doesn't look ready for 25.1 
>> either way.

FWIW, I think it's a mistake to remove it.  We need to move forward on
multi-mode support, and even if prog-indentation-context is not "the
right way" (I doubt there is such a thing anyway), adapting some major
modes to it will most likely not be wasted time, because it will most
likely make it easier to adapt those modes to whichever other approach
we may switch to in some future.

More importantly, I don't think we'll ever agree on what should be done
in this respect because we'll only know what works and what doesn't
*after* we install it and make it "the official way".

I think any attempt at adding support for multi-mode is inevitably
flawed (unless it is designed from scratch and doesn't intend to try and
work with existing modes) and it's easy to see those flaws.  So we can
only make progress if we (somewhat arbitrarily) pick one candidate and
run with it.  The "run with it" part will hopefully help align the
various major modes, thus making it easier for a second candidate to
make further progress, and so on and so forth.

At least, that was the reason why I decided to go with
prog-indentation-context.  It was not because I thought it was The Right
Way, the final word on the matter.


        Stefan

Re: A vision for multiple major modes [was: Re: [Emacs-diffs] widen-limits c331b66:]

[Vitalie Spinu]

>> On Thu, Mar 24 2016 18:38, Alan Mackenzie wrote:

>> For instance, you said that there could be island-local variables. Can I 
>> put some cache into one?

> I can't see why not.

Not sure why would you need island local variables but sub-mode local variables
are very meaningful. For instance, polymode gives each submode its own indirect
buffer. So all those critical variables that you mentioned are already sub-mode
local and independent between sub-modes. Polymode never bothers with thing like
syntax-tables, font-lock keywords and most of other buffer local vars.

> The island-local variables would stay with the island, so that when somebody
> inserts or removes text the right thing would be done.  If somebody deletes
> the island, those variables would disappear (just as buffer local ones do when
> a buffer is deleted).

Are islands of same mode connected in any way? If you delete one char of a chunk
head, the island disappears. You type it back and the islands re-appears. Is
this a new island? Or is it the same old one? Are local variables from old
island preserved or not?

This stuff is stuffy. If you are so excited about your general proposal, then
why not give it a try and implement a proof of concept - a new multi-mode
engine?

> What is "(narrow-to-region 1 (point-max))" going to become?  

Out-of-range error. Unless a mode is already smart enough to use (point-min) it
will have to learn that the hard way.

> I'm worried that the multi-mode projects will suborn these
> text properties, making them unavailable for major modes to use.

Not if there will be special "whitespace-begin/end" syntax class designed for
multitudes only. But that work won't be complete without tackling regexp
search. Your idea of "whitespacing" the regexp search might be handy but it's
probably a lot of complex work with long list of problems which no-one has ever
seen before.

For instance a lot of searches are user level and must be on the whole
buffer. So you will need to differentiate between sub-mode search/replace and
user search/replace. Then you need to define the semantics of matching or
replacing on the boundaries. What if a mode wants to replace a regexp that
starts in one chunk and ends in another? More generally, you will need to decide
if the mode is allowed to manipulate regions that start in one of its chunks and
end in another of its chunks. And if so with what semantics. Etc. etc.

Restricting a sub-mode to it's own bubble within a single span is a much simpler
implementation and is quite well understood by now. In so many years no-one came
across with a better idea than widen/narrow. AFAIK this situation is unlikely to
change in the foreseeable future.

  Vitalie

Re: A vision for multiple major modes [was: Re: [Emacs-diffs] widen-limits c331b66:]

[Vitalie Spinu]

>> On Thu, Mar 24 2016 17:44, Alan Mackenzie wrote:

> I doubt that very much.  syntax-ppss is just one of many ways of using
> parse-partial-sexp.  

Most common one (or the only one) that might need across chunk parsing.  Other
uses might be behaving correctly within a narrowed span.

The only adjustment of parse-partial-sexp that might be necessary with respect
to hard narrowing is to start parsing from (max FROM hard-point-min). But that
remains to be seen with concrete use cases.

> But I'd love to see the code.  Has it been committed, and if so, into which
> branch?

I have never needed that with polymode, so I don't have an example (this might
change in the near future though). The implementation would flow as: parse one
span with syntax-ppss, then go to next span, install syntax-ppss-last at the
beggining of it and call syntax-ppss at the next desired point etc.

  Vitalie

  

Re: A vision for multiple major modes [was: Re: [Emacs-diffs] widen-limits c331b66:]

[Dmitry Gutov]

On 03/24/2016 08:38 PM, Alan Mackenzie wrote:

>> Each multi-mode package implements something like this already. It
>> doesn't work well e.g. because font-lock rules of each particular
>> language, indentation code, etc, are free to widen.
>
> With islands it should work well, regardless of whether any major mode
> widens or narrows.  That's because primitives (such as regexp search)
> would constrain themselves to the island.

It might work well. Without so much as a proof of concept, there's no 
way to tell, really.

>>> For this we will need a new type of local variable, an "island-local" or
>>> "span-local" variable, or whatever you want to call it.  Values of these
>>> variables will vary according to where point is.
>
>> That part is already doable (and done), for most practical purposes.
>
> Except you said above that it doesn't work very well.

Submode-local, and chunk-local, variables work fine. But you don't get 
functioning facilities just by having variables.

>> You'll have to present the total list of facilities, decide how the
>> islands would be applied, and other issues will likely come up from
>> unexpected places.
>
> That will require more investigation.  But syntax based searching and
> regexp based searching will certainly be amongst them.

What about looking-at? char-after? syntax-after? forward-char? 
goto-char? Speaking of the last one, will the buffer positions, as 
visible to the submode code, be renumerated, or will they have gaps?

The above are just some prominent functions that are frequently used in 
indentation code.

> The island-local variables would stay with the island, so that when
> somebody inserts or removes text the right thing would be done.  If
> somebody deletes the island, those variables would disappear (just as
> buffer local ones do when a buffer is deleted).

Depending on your answer to the previous question, even simply inserting 
text inside one of the preceding islands might make syntax-ppss-cache 
out of date in the current island.

>> On the one hand, we'd probably want to retain some variables, in order
>> not to rerun the major mode functions over and over again. On the
>> other hand, if we were to put e.g.  syntax-ppss-last into an
>> island-local variable (and it's a logical continuation of this idea),
>> after island boundaries change it should what... become unbound? Nil?
>
> That's for the application to decide.  The island local binding would
> countinue to exist for as long as the island exists, and the application
> would be free to use it.

Something would have to create and maintain the island identities, as 
users add and remove text, including island delimiters, it's not like 
Emacs could do that automagically. My point is, implementing significant 
part in Elisp is unavoidable anyway.

>> Next, at which points exactly would Emacs look at the island boundaries
>> and change the island-local variables to the values set in the current
>> island? Probably not after each point movement. In post-command-hook?
>> That's also already done.
>
> It wouldn't use any high level facility like post-command-hook.  The
> mechanism would be more like a buffer local variable, entirely handled
> by the C level, so that such a binding would simply exist.

You didn't answer my question, though. When will the bindings apply?

>>> Although the above vision implies a lot of development work, there is
>>> nothing there which is beyond our abilities to implement readily.  It
>>> would give us a true multi major mode capability, yet the impact on
>>> individual major modes would be minimal.
>
> I'm worried by this.  Already narrowing/widening which is currently
> simple, straightforward, and rigorously defined, is looking like
> becoming complicated.  Major modes are going to be constrained in what
> they can do.

A lot of code is already constrained. That's why we most often use 
(point-min) instead of 1. The only unconstrained code we currently have 
is the one that starts with (widen).

> I get the impression that major modes are going to be
> restricted in how they can use parse-partial-sexp.  These are serious
> matters, but I haven't seen any widespread debate on emacs-devel about
> them.

Most parse-partial-sexp uses don't call `widen' in advance.

>> On the other hand, using narrowing for multi-mode purpose is a familiar
>> ground already, and the changes in Emacs core required to do so are
>> minimal. And most of the code written for Emacs has been taught to
>> respect narrowing bounds (even if only by the virtue of always using
>> (point-min) instead of 1), so we can utilize that.
>
> What is "(narrow-to-region 1 (point-max))" going to become?  It seems
> there will be a need for "the bounds of the island", which will impose
> complexities in major mode code.

Replacing '1' with (point-min) everywhere sounds like a minor change all 
modes can bear.

I don't know if we'd need the island-beginnig/island-end accessors in 
your proposal.

> How will this be done when narrowing is the tool used?  It you narrow to
> an island, none of the other islands in a chain are visible.  If you
> narrow to the smallest region which spans those islands, you leave a lot
> of stuff visible which shouldn't be.

a) You can assign whitespace syntax to that stuff (which should cover 
many use cases).

b) You can use a temporary buffer.

>> Doing that using an islands framework limits it to a
>> predefined set of semantics (e.g. all Ruby islands see all other Ruby
>> islands).
>
> I don't see why.  There's no reason why islands couldn't be linked to
> and unlinked from eachother freely at runtime.

Very well.

>> That's not to say that being able to make parse-partial-sexp to skip
>> over certain intervals wouldn't be valuable. But you can do that, sort
>> of, already, by applying existing text properties to those intervals
>> (like beginning-of-comment/end-of-comment, or just "whitespace" over the
>> whole of it), and then removing them at the end of an operation.
>
> Not really.  If you "whitespace" a region out, you've got to remember
> the text properties that were there originally, and restore them
> afterwards.  There's no easy way to do that.

I think it's doable using char-property-alias-alist. The multi-mode will 
define its own property as an alias of `syntax-table', and then put it 
on and remove it from text at will.

> The comment-fence text
> properties won't work either, because, in the general case, there will
> already be comment-fence properties in your region which will foul
> things up.

Yeah, that's a plausible problem. So at least your proposal could pivot 
into something like new kind of high-priority, generic comment fences. 
Or maybe just being able to assign priorities to the "generic comment" 
syntax-table values.

> I'm worried that the multi-mode projects will suborn these
> text properties, making them unavailable for major modes to use.

See the "alias" suggestion.

>> But the end benefits might not be high enough to justify the necessary
>> work and the increase in complexity in internals.
>
> They might not.  They might.  Basically, nobody else really seams
> interested in my idea, so it doesn't look like it will happen.

It sounds somewhat attractive to me, but we should understand more 
clearly the goals we'll reach with it, as well as the semantics of the 
new feature.

In order to deliver on the promise of seamless-ness, I think the islands 
should be able to believe that every one of them starts with position 1, 
and that they all have no gaps. I.e. every primitive would have to 
behave that way, at least as long as the relevant global variable is 
non-nil.

Re: Removing prog-indentation-context

[Dmitry Gutov]

On 03/24/2016 08:55 PM, Stefan Monnier wrote:
> FWIW, I think it's a mistake to remove it.  We need to move forward on
> multi-mode support, and even if prog-indentation-context is not "the
> right way" (I doubt there is such a thing anyway),

Are you against removing it in Emacs 25.1 in particular (but retaining 
it, for now, on master)? We don't include support for it even in the 
built-in major modes, save one. And python-mode doesn't support the more 
controversial third element. The built-in antlr-mode doesn't use it 
either, even though it was supposed to be the main beneficiary.

It's silly to expect third-party authors to adopt it when we haven't 
done so ourselves.

By that measure, prog-indentation-context has failed, at least for 
inclusion in the upcoming release.

> adapting some major
> modes to it will most likely not be wasted time, because it will most
> likely make it easier to adapt those modes to whichever other approach
> we may switch to in some future.

Respectfully, I more disagree than agree. If we put aside the 
PREVIOUS-CHUNKS element, we have two elements left:

- (START . END). Yes, making modes use `prog-widen' instead of `widen' 
is a good change, but it's a trivial search-replace. There's not much 
benefit in doing it in advance, or waiting for third-party code to catch 
up, etc. The proposed alternative: hard widen limits. If it's adopted, 
it'll make using prog-widen simply unnecessary.

- FIRST-COLUMN. The proposed alternative: prog-indentation-function that 
returns a column number. If we choose this one, it's likely to result in 
a rather natural code transformation in modes adopting it. It's also a 
different one that what we'd make to use FIRST-COLUMN, at least if we 
take smie and js as examples.

> More importantly, I don't think we'll ever agree on what should be done
> in this respect because we'll only know what works and what doesn't
> *after* we install it and make it "the official way".

Why? The advancement prog-indentation-context offers is rather minimal 
for new multi-mode packages to crop up overnight. And even if they 
would, they could just as well test their changes using the master 
branch (we do have a certain fraction of users continually building from 
it, even if they don't participate in the development).

On the other hand, you have maintainers of the two active multi-mode 
packages (polymode more than mmm-mode) right here, in this discussion. 
And we're both capable of building Emacs from master, as well as 
implementing features that depend on it. That must be true for 
Christopher as well.

> The "run with it" part will hopefully help align the
> various major modes, thus making it easier for a second candidate to
> make further progress, and so on and so forth.

Yes, well, it didn't, so far.

And there are better ways to evaluate an API proposal, such as asking 
for patches that add support for all of its parts, in advance.

If the demand for multi-mode support is less than we'd hope (resulting 
in fewer or slower patches), it can well incubate on a feature branch.

In the meantime, the basic needs of the web development crowd are being 
served by web-mode (which is not an actual multi-mode, and would likely 
not benefit from features discussed here). So it's not like a lot of 
people's is at a standstill because of our indecision.

> At least, that was the reason why I decided to go with
> prog-indentation-context.  It was not because I thought it was The Right
> Way, the final word on the matter.

Emacs's development cycles are long, and backward compatibility promise 
is significant. I don't want to get into situation with multiple blessed 
solutions, all inferior in some way, all with spotty support in existing 
code.

Re: Removing prog-indentation-context

[Dmitry Gutov]

Sorry,

On 03/25/2016 02:53 AM, Dmitry Gutov wrote:

> In the meantime, the basic needs of the web development crowd are being
> served by web-mode (which is not an actual multi-mode, and would likely
> not benefit from features discussed here). So it's not like a lot of
> people's  is at a standstill because of our indecision.
            ^ work

Re: Removing prog-indentation-context

[Stefan Monnier]

>> FWIW, I think it's a mistake to remove it.  We need to move forward on
>> multi-mode support, and even if prog-indentation-context is not "the
>> right way" (I doubt there is such a thing anyway),
> Are you against removing it in Emacs 25.1 in particular (but retaining it,
> for now, on master)?

Yes.


        Stefan

Re: Removing prog-indentation-context

[Dmitry Gutov]

Hi John,

Could you please rule on this disagreement?

The main arguments for and against are in the two parent messages.

On 03/25/2016 04:09 AM, Stefan Monnier wrote:

>> Are you against removing it in Emacs 25.1 in particular (but retaining it,
>> for now, on master)?
>
> Yes.

Re: Removing prog-indentation-context

[Vitalie Spinu]

I think the third element of prog-indentation-context is unlikely to be used by
major-modes anyways. Firstly due to inherent complexity of the concept, secondly
due to undefined, multi-mode specific, semantics of what to do with those
spans/chunks.

In any case, multi-mode engines can ignore prog-indentation-context. My bet is
that they will always do so, at least because there is no reliable way to
identify modes which use it and modes which don't.

  Vitalie

>> On Fri, Mar 25 2016 02:53, Dmitry Gutov wrote:

> On 03/24/2016 08:55 PM, Stefan Monnier wrote:
>> FWIW, I think it's a mistake to remove it.  We need to move forward on
>> multi-mode support, and even if prog-indentation-context is not "the
>> right way" (I doubt there is such a thing anyway),

> Are you against removing it in Emacs 25.1 in particular (but retaining it, for
> now, on master)? We don't include support for it even in the built-in major
> modes, save one. And python-mode doesn't support the more controversial third
> element. The built-in antlr-mode doesn't use it either, even though it was
> supposed to be the main beneficiary.

> It's silly to expect third-party authors to adopt it when we haven't done so
> ourselves.

> By that measure, prog-indentation-context has failed, at least for inclusion in
> the upcoming release.

>> adapting some major
>> modes to it will most likely not be wasted time, because it will most
>> likely make it easier to adapt those modes to whichever other approach
>> we may switch to in some future.

> Respectfully, I more disagree than agree. If we put aside the PREVIOUS-CHUNKS
> element, we have two elements left:

> - (START . END). Yes, making modes use `prog-widen' instead of `widen' is a good
> change, but it's a trivial search-replace. There's not much benefit in doing it
> in advance, or waiting for third-party code to catch up, etc. The proposed
> alternative: hard widen limits. If it's adopted, it'll make using prog-widen
> simply unnecessary.

> - FIRST-COLUMN. The proposed alternative: prog-indentation-function that returns
> a column number. If we choose this one, it's likely to result in a rather
> natural code transformation in modes adopting it. It's also a different one that
> what we'd make to use FIRST-COLUMN, at least if we take smie and js as examples.

>> More importantly, I don't think we'll ever agree on what should be done
>> in this respect because we'll only know what works and what doesn't
>> *after* we install it and make it "the official way".

> Why? The advancement prog-indentation-context offers is rather minimal for new
> multi-mode packages to crop up overnight. And even if they would, they could
> just as well test their changes using the master branch (we do have a certain
> fraction of users continually building from it, even if they don't participate
> in the development).

> On the other hand, you have maintainers of the two active multi-mode packages
> (polymode more than mmm-mode) right here, in this discussion. And we're both
> capable of building Emacs from master, as well as implementing features that
> depend on it. That must be true for Christopher as well.

>> The "run with it" part will hopefully help align the
>> various major modes, thus making it easier for a second candidate to
>> make further progress, and so on and so forth.

> Yes, well, it didn't, so far.

> And there are better ways to evaluate an API proposal, such as asking for
> patches that add support for all of its parts, in advance.

> If the demand for multi-mode support is less than we'd hope (resulting in fewer
> or slower patches), it can well incubate on a feature branch.

> In the meantime, the basic needs of the web development crowd are being served
> by web-mode (which is not an actual multi-mode, and would likely not benefit
> from features discussed here). So it's not like a lot of people's is at a
> standstill because of our indecision.

>> At least, that was the reason why I decided to go with
>> prog-indentation-context.  It was not because I thought it was The Right
>> Way, the final word on the matter.

> Emacs's development cycles are long, and backward compatibility promise is
> significant. I don't want to get into situation with multiple blessed solutions,
> all inferior in some way, all with spotty support in existing code.

Re: A vision for multiple major modes [was: Re: [Emacs-diffs] widen-limits c331b66:]

[Phillip Lord]

Alan Mackenzie <acm@muc.de> writes:

>> I proposed (4) very early in the thread, but didn't hear much support for
>> it. There are only three trivial usages of Fwiden in C code. Bringing `narrow`
>> to elisp is equally easy.
>
> All these options strike me as artificial, ad hoc, and ugly.  I would go
> for option number (5) - to transcend the "unwanted widen" problem - to
> enhance Emacs such that users and Lisp hackers can freely narrow and
> widen _without_ upsetting the @dfn{super mode} (the multiple mode
> handling mode).
>
> What is a major mode?  It is a collection of local variable settings, a
> syntax table, an abbreviation table, a mode specific key map, font lock
> mode settings, an indentation engine, Imenu settings, and one or two
> other things.

And a visualisation of all of the above. It's also an assumption that
it's unique in a buffer. And, it's not really the above, because you can
change any of these values in an individual buffer, even if it stays in
the same mode. So, more a major mode is a mechanism for setting all of
the above.

> Although the above vision implies a lot of development work, there is
> nothing there which is beyond our abilities to implement readily.  It
> would give us a true multi major mode capability, yet the impact on
> individual major modes would be minimal.

We already have a mechanism to seperate and make instances of all the
things that you speak out above -- it's called a new buffer; the most
sensible way, then, is to have multiple modes is, surely, to have
multiple buffers.

Phil

Re: Removing prog-indentation-context

[John Wiegley]

[-- Attachment #1: Type: text/plain, Size: 563 bytes --]

>>>>> Dmitry Gutov <dgutov@yandex.ru> writes:

> Could you please rule on this disagreement?

Please remove it from Emacs 25.1.

I strongly dislike "just putting it out there", and hoping some crowd-sourced
opinion machine will let us know at some point how the code should improve. If
it's not ready, it does not belong in our release. We can wait until we have a
better answer. There is no rush.

-- 
John Wiegley                  GPG fingerprint = 4710 CF98 AF9B 327B B80F
http://newartisans.com                          60E1 46C4 BD1A 7AC1 4BA2

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 629 bytes --]

Re: A vision for multiple major modes [was: Re: [Emacs-diffs] widen-limits c331b66:]

[Alan Mackenzie]

Hello, Dmitry.

On Fri, Mar 25, 2016 at 02:11:34AM +0200, Dmitry Gutov wrote:
> On 03/24/2016 08:38 PM, Alan Mackenzie wrote:

> > With islands it should work well, regardless of whether any major mode
> > widens or narrows.  That's because primitives (such as regexp search)
> > would constrain themselves to the island.

> It might work well. Without so much as a proof of concept, there's no 
> way to tell, really.

I can't really see a proof of concept happening.  Either the thing is
implemented or it's not.  There's hardly a half way point.

> Submode-local, and chunk-local, variables work fine. But you don't get 
> functioning facilities just by having variables.

No.  But you certainly don't get them without variables.  ;-)

> >> You'll have to present the total list of facilities, decide how the
> >> islands would be applied, and other issues will likely come up from
> >> unexpected places.

> > That will require more investigation.  But syntax based searching and
> > regexp based searching will certainly be amongst them.

> What about
looking-at? Probably.
char-after? No.
syntax-after? No. (This works only on a single char, hence must be in a
              single island, it can't span two.)
forward-char? Maybe.  In fact, there's a good argument for not moving
              forward when `forward-char' hits an island boundary.
goto-char? No.

(All assuming `restrict-to-island' is bound to non-nil.)

> Speaking of the last one, will the buffer positions, as visible to the
> submode code, be renumerated, or will they have gaps?

It would retain the global buffer position, for consistency with the rest
of Emacs.  For example, in narrowed buffers, the position relative to
point-min is never used.

> > The island-local variables would stay with the island, so that when
> > somebody inserts or removes text the right thing would be done.  If
> > somebody deletes the island, those variables would disappear (just as
> > buffer local ones do when a buffer is deleted).

> Depending on your answer to the previous question, even simply inserting 
> text inside one of the preceding islands might make syntax-ppss-cache 
> out of date in the current island.

That could probably be solved by making positions in that cache relative
to the beginning of the island, and things like that.  I think you'd want
to make `syntax-propertize--done' island local, too.

> >> On the one hand, we'd probably want to retain some variables, in order
> >> not to rerun the major mode functions over and over again. On the
> >> other hand, if we were to put e.g.  syntax-ppss-last into an
> >> island-local variable (and it's a logical continuation of this idea),
> >> after island boundaries change it should what... become unbound? Nil?

> > That's for the application to decide.  The island local binding would
> > countinue to exist for as long as the island exists, and the application
> > would be free to use it.

> Something would have to create and maintain the island identities, as 
> users add and remove text, including island delimiters, it's not like 
> Emacs could do that automagically. My point is, implementing significant 
> part in Elisp is unavoidable anyway.

The job of the super mode would be to maintain the islands, including
creating them, and deleting them when the user deletes text which
characterizes an island.  This would of course be mainly in Lisp, yes.

> >> Next, at which points exactly would Emacs look at the island boundaries
> >> and change the island-local variables to the values set in the current
> >> island? Probably not after each point movement. In post-command-hook?
> >> That's also already done.

> > It wouldn't use any high level facility like post-command-hook.  The
> > mechanism would be more like a buffer local variable, entirely handled
> > by the C level, so that such a binding would simply exist.

> You didn't answer my question, though. When will the bindings apply?

Sorry, an island local binding would be current when point is within that
island.  Or, perhaps there could be some variable which would be set to a
position to do this job.

> > What is "(narrow-to-region 1 (point-max))" going to become?  It seems
> > there will be a need for "the bounds of the island", which will impose
> > complexities in major mode code.

> Replacing '1' with (point-min) everywhere sounds like a minor change all 
> modes can bear.

> I don't know if we'd need the island-beginnig/island-end accessors in 
> your proposal.

I think we would.  (narrow-to-region (point-min) (point-max)) is a null
operation.  If the major mode has narrowed, and needs to set point-min to
"1", it will need to know the island-beginning, somehow.

> >> That's not to say that being able to make parse-partial-sexp to skip
> >> over certain intervals wouldn't be valuable. But you can do that, sort
> >> of, already, by applying existing text properties to those intervals
> >> (like beginning-of-comment/end-of-comment, or just "whitespace" over the
> >> whole of it), and then removing them at the end of an operation.

> > Not really.  If you "whitespace" a region out, you've got to remember
> > the text properties that were there originally, and restore them
> > afterwards.  There's no easy way to do that.

> I think it's doable using char-property-alias-alist. The multi-mode will 
> define its own property as an alias of `syntax-table', and then put it 
> on and remove it from text at will.

I don't think that would work at the moment.  syntax-table text
properties would take priority over syntax-table-1 properties.  Something
similar to char-property-alias-alist, but giving syntax-table-1 priority,
would need to be implemented.

> >> But the end benefits might not be high enough to justify the necessary
> >> work and the increase in complexity in internals.

> > They might not.  They might.  Basically, nobody else really seams
> > interested in my idea, so it doesn't look like it will happen.

> It sounds somewhat attractive to me, but we should understand more 
> clearly the goals we'll reach with it, as well as the semantics of the 
> new feature.

It would relieve super modes of the tedious necessity to maintain island
local variables using functions in post-command-hook, and things like
that.  It would not be necessary to use widening and narrowing to
restrict indentation/fontification code to an island.

> In order to deliver on the promise of seamless-ness, I think the islands 
> should be able to believe that every one of them starts with position 1, 
> and that they all have no gaps. I.e. every primitive would have to 
> behave that way, at least as long as the relevant global variable is 
> non-nil.

I don't think having islands start at position 1 is a good idea.  But
having the relevant primitives skip the gaps between chained islands, and
stop at an island boundary when not chained, when `restrict-to-island' is
non-nil, would be a central idea.

-- 
Alan Mackenzie (Nuremberg, Germany).

Re: A vision for multiple major modes [was: Re: [Emacs-diffs] widen-limits c331b66:]

[Dmitry Gutov]

On 03/27/2016 03:09 PM, Alan Mackenzie wrote:

> I can't really see a proof of concept happening.  Either the thing is
> implemented or it's not.  There's hardly a half way point.

"Full implementation" would work, too. But it would still be considered 
a prototype. Point is, we can't really decide whether we'll end up using 
it, before trying it for a while.

>>> That will require more investigation.  But syntax based searching and
>>> regexp based searching will certainly be amongst them.

I don't think that's enough. E.g. (goto-char (match-end n)) is a common 
idiom for syntax highlighting and indentation code.

>> [island-affected primitives]
> looking-at? Probably.
> char-after? No.
> syntax-after? No. (This works only on a single char, hence must be in a
>               single island, it can't span two.)
> forward-char? Maybe.  In fact, there's a good argument for not moving
>               forward when `forward-char' hits an island boundary.

Being confined to the current island is not what I had in mind. If 
that's the idea of how such primitives would work, I don't really see a 
lot of benefit over using narrowing around all those calls.

>> Speaking of the last one, will the buffer positions, as visible to the
>> submode code, be renumerated, or will they have gaps?
>
> It would retain the global buffer position, for consistency with the rest
> of Emacs.  For example, in narrowed buffers, the position relative to
> point-min is never used.

True. I'm more worried about gaps. But they could be treated like 
whitespace, I suppose.

>> Depending on your answer to the previous question, even simply inserting
>> text inside one of the preceding islands might make syntax-ppss-cache
>> out of date in the current island.
>
> That could probably be solved by making positions in that cache relative
> to the beginning of the island, and things like that.  I think you'd want
> to make `syntax-propertize--done' island local, too.

Right. So then, some yet-unknown body of code has to become 
island-aware, and the improvement is not that seamless anymore.

>>>> Next, at which points exactly would Emacs look at the island boundaries
>>>> and change the island-local variables to the values set in the current
>>>> island? Probably not after each point movement. In post-command-hook?
>>>> That's also already done.
>
>>> It wouldn't use any high level facility like post-command-hook.  The
>>> mechanism would be more like a buffer local variable, entirely handled
>>> by the C level, so that such a binding would simply exist.
>
>> You didn't answer my question, though. When will the bindings apply?
>
> Sorry, an island local binding would be current when point is within that
> island.  Or, perhaps there could be some variable which would be set to a
> position to do this job.

What if var has an island-local binding? And my function has this:

(setq var 1)
(goto-char xx) ; where xx is in a different island

will its value change mid-program? What if xx is in the same island? 
Will the value change then?

I don't think either should be true. Then, the "adequate" model would 
amount to changing them in post-command-hook anyway.

>> I don't know if we'd need the island-beginnig/island-end accessors in
>> your proposal.
>
> I think we would.  (narrow-to-region (point-min) (point-max)) is a null
> operation.  If the major mode has narrowed, and needs to set point-min to
> "1", it will need to know the island-beginning, somehow.

What if it just calls widen?

>>> If you "whitespace" a region out, you've got to remember
>>> the text properties that were there originally, and restore them
>>> afterwards.  There's no easy way to do that.
>
>> I think it's doable using char-property-alias-alist. The multi-mode will
>> define its own property as an alias of `syntax-table', and then put it
>> on and remove it from text at will.
>
> I don't think that would work at the moment.  syntax-table text
> properties would take priority over syntax-table-1 properties.  Something
> similar to char-property-alias-alist, but giving syntax-table-1 priority,
> would need to be implemented.

You're probably right.

> It would relieve super modes of the tedious necessity to maintain island
> local variables using functions in post-command-hook, and things like
> that.

They'd still have to maintain the island boundaries. And the lists of 
variables to set island-locally, or mode-locally, or chunk-locally. That 
constitutes the majority of the related code already.

> It would not be necessary to use widening and narrowing to
> restrict indentation/fontification code to an island.

So making it seem, to the Lisp code, like the multiple related islands 
don't have anything (for some definition of "anything") between them is 
not a priority? That would be the high point of the "islands" idea from 
my perspective. But having only some primitives affected would likely 
break this use case.

Here's an example of ERB code Vitalie brought up recently:

<% if foo %>
   <%= bar(boo) %>
<% end %>

The parts of the buffer between %'s would be Ruby islands. The 
suggestion was to use the value returned by the indentation function in 
the second island (second line) to indent the "real" buffer contents.

But: neither of the islands contains a newline. If they are combined in 
the most straightforward way, the Ruby line will be 'if foo bar(boo)', 
and its indentation would be not what we're looking for. I think the 
current ways to look for a newline,

   (progn (skip-syntax-backward " ")
          (eq (char-before ?\n))

or

   (looking-at "\\s-$")

will fail to work. Also note that neither of the two islands in this 
example contains the newline in question.

To sum up, hard examples are hard. And yet, the new feature should make 
some use cases possible that were impossible before, not just allow for 
easier island-local variables.

> I don't think having islands start at position 1 is a good idea.  But
> having the relevant primitives skip the gaps between chained islands, and
> stop at an island boundary when not chained, when `restrict-to-island' is
> non-nil, would be a central idea.

The "not chained" idea would require some explanation.

Re: Removing prog-indentation-context

[Dmitry Gutov]

On 03/27/2016 12:29 AM, John Wiegley wrote:

>> Could you please rule on this disagreement?
>
> Please remove it from Emacs 25.1.

Thanks. I'll pushed a removal commit to a branch, and will merge it soon.

TBH, removing it altogether is rather sad, given that the first two 
elements constitute a reasonable solution. Even so, we have a better 
solution in sight for at least one of them, and I don't see a good 
migration path from here to there, in the next Emacs version.

Re: [Emacs-diffs] widen-limits c331b66: Implement buffer-widen-limits functionality

[Dmitry Gutov]

On 03/24/2016 05:56 PM, Eli Zaretskii wrote:

> If you want to leave it in the code on master, don't forget to puts
> some telltale phrase in the commit log.

Right, thanks.

Re: A vision for multiple major modes [was: Re: [Emacs-diffs] widen-limits c331b66:]

[Filipp Gunbin]

On 24/03/2016 18:38 +0000, Alan Mackenzie wrote:

> They might not.  They might.  Basically, nobody else really seams
> interested in my idea, so it doesn't look like it will happen.

Alan, your proposed solution seems very nice to me.

Probably the islands could be nested in some way (maybe like bidi rtl
and ltr text).

For example, if we have HTML with embedded PHP, the whole buffer would
be in HTML major mode; top-level PHP code spans will be islands nested
beneath top level (nesting level 1) and there could be HTML lines nested
inside the PHP ones (say, when PHP cycles over some collection and
prints HTML snippet for each element of it) - that would be nesting
level 2.  That will make usual narrowing more helpful and context-aware.

However, the major mode can decide to put all PHP on the level 1
nesting, so the whole buffer would be HTML (nesting 0) interspersed with
PHP (nesting 1).

Filipp

Re: Removing prog-indentation-context

[Dmitry Gutov]

On 03/25/2016 05:45 PM, Vitalie Spinu wrote:

> I think the third element of prog-indentation-context is unlikely to be used by
> major-modes anyways. Firstly due to inherent complexity of the concept, secondly
> due to undefined, multi-mode specific, semantics of what to do with those
> spans/chunks.

I concur.

> In any case, multi-mode engines can ignore prog-indentation-context. My bet is
> that they will always do so, at least because there is no reliable way to
> identify modes which use it and modes which don't.

prog-indent-function would indeed be a better alternative to 
prog-first-column, not least because of being able to easily identify 
modes that support it.

prog-widen, though, would be a decent fallback, if none of the 
hard-widen or "islands" discussions reaches a satisfying conclusion.

In that case, we'd be saddled with (font-lock|syntax-ppss)-dont-widen, 
but since there's nothing a multi-mode can do when a submode widens (is 
there?), it would just behave the same and hope that all major modes 
either don't use widen at all, or switch to prog-widen.

Re: Removing prog-indentation-context

[Stefan Monnier]

> In that case, we'd be saddled with (font-lock|syntax-ppss)-dont-widen, but

We really should fix this.  `font-lock-dont-widen' is fundamentally
broken (it's usually the major mode which tells font-lock not to widen
because the major mode has narrowed in a special way, but then the user
can re-narrow on top of that and this narrowing *should* be lifted
during font-lock, but font-lock-dont-widen prevents it).


        Stefan

Re: Removing prog-indentation-context

[Dmitry Gutov]

On 03/29/2016 01:08 AM, Stefan Monnier wrote:
>> In that case, we'd be saddled with (font-lock|syntax-ppss)-dont-widen, but
>
> We really should fix this.  `font-lock-dont-widen' is fundamentally
> broken (it's usually the major mode which tells font-lock not to widen
> because the major mode has narrowed in a special way, but then the user
> can re-narrow on top of that and this narrowing *should* be lifted
> during font-lock, but font-lock-dont-widen prevents it).

Wouldn't it also be fixed if font-lock and syntax-ppss used prog-widen 
instead of widen?

Re: Removing prog-indentation-context

[Stefan Monnier]

>>> In that case, we'd be saddled with (font-lock|syntax-ppss)-dont-widen, but
>> We really should fix this.  `font-lock-dont-widen' is fundamentally
>> broken (it's usually the major mode which tells font-lock not to widen
>> because the major mode has narrowed in a special way, but then the user
>> can re-narrow on top of that and this narrowing *should* be lifted
>> during font-lock, but font-lock-dont-widen prevents it).
> Wouldn't it also be fixed if font-lock and syntax-ppss used prog-widen
> instead of widen?

Probably, yes.


        Stefan

Re: A vision for multiple major modes [was: Re: [Emacs-diffs] widen-limits c331b66:]

[Alan Mackenzie]

Hello, Dmitry.

On Mon, Mar 28, 2016 at 01:59:21AM +0300, Dmitry Gutov wrote:
> On 03/27/2016 03:09 PM, Alan Mackenzie wrote:

> > I can't really see a proof of concept happening.  Either the thing is
> > implemented or it's not.  There's hardly a half way point.

> "Full implementation" would work, too. But it would still be considered 
> a prototype. Point is, we can't really decide whether we'll end up using 
> it, before trying it for a while.

Of course.

> >>> That will require more investigation.  But syntax based searching and
> >>> regexp based searching will certainly be amongst them.

> I don't think that's enough. E.g. (goto-char (match-end n)) is a common 
> idiom for syntax highlighting and indentation code.

Yes, but that match is going to be in the same island, or at the very
least, in the chain of islands containing the current one.

> >> [island-affected primitives]
> > looking-at? Probably.
> > char-after? No.
> > syntax-after? No. (This works only on a single char, hence must be in a
> >               single island, it can't span two.)
> > forward-char? Maybe.  In fact, there's a good argument for not moving
> >               forward when `forward-char' hits an island boundary.

> Being confined to the current island is not what I had in mind. If 
> that's the idea of how such primitives would work, I don't really see a 
> lot of benefit over using narrowing around all those calls.

Are you thinking more that `forward-char' should move from the end of an
island to the beginning of the next island in a chain?

> >> Speaking of the last one, will the buffer positions, as visible to the
> >> submode code, be renumerated, or will they have gaps?

> > It would retain the global buffer position, for consistency with the rest
> > of Emacs.  For example, in narrowed buffers, the position relative to
> > point-min is never used.

> True. I'm more worried about gaps. But they could be treated like 
> whitespace, I suppose.

For example, by `forward-char'?  What primitives were you thinking of,
here?

> >> Depending on your answer to the previous question, even simply inserting
> >> text inside one of the preceding islands might make syntax-ppss-cache
> >> out of date in the current island.

> > That could probably be solved by making positions in that cache relative
> > to the beginning of the island, and things like that.  I think you'd want
> > to make `syntax-propertize--done' island local, too.

> Right. So then, some yet-unknown body of code has to become 
> island-aware, and the improvement is not that seamless anymore.

The way I see it, the super modes would have to be totally aware of the
island mechanism, but the major modes would be largely, it not totally,
unaware of it.  `syntax-ppss' seems more part of the super mode
mechanism.

> >>> It wouldn't use any high level facility like post-command-hook.  The
> >>> mechanism would be more like a buffer local variable, entirely handled
> >>> by the C level, so that such a binding would simply exist.

> >> You didn't answer my question, though. When will the bindings apply?

> > Sorry, an island local binding would be current when point is within that
> > island.  Or, perhaps there could be some variable which would be set to a
> > position to do this job.

> What if var has an island-local binding? And my function has this:

> (setq var 1)
> (goto-char xx) ; where xx is in a different island

> will its value change mid-program? What if xx is in the same island? 
> Will the value change then?

A different island local binding would become current, so yes its value
would thereby change.  Much like if var had a buffer local binding, and
you do (set-buffer "foo"), the value of var you'd just set would no
longer be in the current binding.  if xx is in the same island, the
binding just setq'd would remain the same.

> I don't think either should be true. Then, the "adequate" model would 
> amount to changing them in post-command-hook anyway.

What sort of variables are you thinking about here?  [ Some time later:]
Could it be that it might be better to have "island chain local"
variables rather than merely "island local" ones?  So that the three ruby
lines in your example below would be three islands in a chain, and would
thus all share a set of "island chain local" bindings rather than each
line having its own binding?

> >> I don't know if we'd need the island-beginning/island-end accessors in
> >> your proposal.

> > I think we would.  (narrow-to-region (point-min) (point-max)) is a null
> > operation.  If the major mode has narrowed, and needs to set point-min to
> > "1", it will need to know the island-beginning, somehow.

> What if it just calls widen?

That also sets point-max, which would specifically not be wanted.

> They [ super modes ]'d still have to maintain the island boundaries.
> And the lists of variables to set island-locally, or mode-locally, or
> chunk-locally. That constitutes the majority of the related code
> already.

Yes.  How could it be otherwise?  But with sharp tools, the work could be
easier.

> > It would not be necessary to use widening and narrowing to
> > restrict indentation/fontification code to an island.

> So making it seem, to the Lisp code, like the multiple related islands 
> don't have anything (for some definition of "anything") between them is 
> not a priority?

By "related", I think you mean the same thing I meaan by "chained" - the
three ruby mode lines enclosed in <%...%> in your example below, would be
chained together by the super mode.

By "anything between them", I take it you mean certain primitives (still
to be pinned down precisely) would make it look like the successive
islands in a chain followed on from eachother without "anything between
them".

We could make it a priority.

> That would be the high point of the "islands" idea from my perspective.
> But having only some primitives affected would likely break this use
> case.

> Here's an example of ERB code Vitalie brought up recently:

> <% if foo %>
>    <%= bar(boo) %>
> <% end %>

> The parts of the buffer between %'s would be Ruby islands. The 
> suggestion was to use the value returned by the indentation function in 
> the second island (second line) to indent the "real" buffer contents.

The question seems to be, are we indenting in the super mode or in each
island?  The answer seems to be you'd want to indent in the super mode
here, I think.

> But: neither of the islands contains a newline. If they are combined in 
> the most straightforward way, the Ruby line will be 'if foo bar(boo)', 
> and its indentation would be not what we're looking for. I think the 
> current ways to look for a newline,

>    (progn (skip-syntax-backward " ")
>           (eq (char-before ?\n))

> or

>    (looking-at "\\s-$")

> will fail to work. Also note that neither of the two islands in this 
> example contains the newline in question.

Two?  Don't you mean three, here?  Is this a minor mistake in your
counting or am I missing something important because I don't grok ruby?

> To sum up, hard examples are hard. And yet, the new feature should make 
> some use cases possible that were impossible before, not just allow for 
> easier island-local variables.

> > I don't think having islands start at position 1 is a good idea.  But
> > having the relevant primitives skip the gaps between chained islands, and
> > stop at an island boundary when not chained, when `restrict-to-island' is
> > non-nil, would be a central idea.

> The "not chained" idea would require some explanation.

For example, islands in a shell script which were individual AWK scripts
would not be chained together.  But the three (?two) lines of ruby in the
example above would be chained together.  I think we need to work this
out more precisely.

For example, in the three ruby line scenario above, is there any variable
you can think of for which it would be advantageous to have a binding
for each island rather than a shared binding for the set of three (?two)?

I'm kind of envisioning successive `forward-char's moving from the end
of "<% if foo %>" to the beginning of "<%= bar(boo) %>", or even from
the end of "if foo" to the beginning of "bar(boo)".  How does this sound?
Or am I talking total rubbish?  (It's getting late).

-- 
Alan Mackenzie (Nuremberg, Germany).

Re: A vision for multiple major modes [was: Re: [Emacs-diffs] widen-limits c331b66:]

[Dmitry Gutov]

On 03/29/2016 03:07 AM, Alan Mackenzie wrote:

>> E.g. (goto-char (match-end n)) is a common
>> idiom for syntax highlighting and indentation code.
>
> Yes, but that match is going to be in the same island, or at the very
> least, in the chain of islands containing the current one.

Would looking-at skip over intermediate islands? Basically, I wonder if

  (= (+ (match-beginning 0) (length (match-string 0))
     (match-end 0))

is going to always hold in that new world.

> Are you thinking more that `forward-char' should move from the end of an
> island to the beginning of the next island in a chain?

As one option, yes. Let's call it option A, and it entails renumerating 
buffer positions to avoid gaps.

>> True. I'm more worried about gaps. But they could be treated like
>> whitespace, I suppose.
>
> For example, by `forward-char'?  What primitives were you thinking of,
> here?

This would be option B. forward-char doesn't care if the characters are 
whitespace or not. I don't know if e.g. search-forward would see the 
contents of the intermediate islands as a bunch of actual space 
characters (it's something to consider).

Most importantly, the contents of intermediate islands would match 
"\\s-", and parse-partial-sexp would similarly skip over them.

>> Right. So then, some yet-unknown body of code has to become
>> island-aware, and the improvement is not that seamless anymore.
>
> The way I see it, the super modes would have to be totally aware of the
> island mechanism, but the major modes would be largely, it not totally,
> unaware of it.  `syntax-ppss' seems more part of the super mode
> mechanism.

At the very least, that does sound somewhat complicated.

>> What if var has an island-local binding? And my function has this:
>
>> (setq var 1)
>> (goto-char xx) ; where xx is in a different island
>
>> will its value change mid-program? What if xx is in the same island?
>> Will the value change then?
>
> A different island local binding would become current, so yes its value
> would thereby change.  Much like if var had a buffer local binding, and
> you do (set-buffer "foo"), the value of var you'd just set would no
> longer be in the current binding.  if xx is in the same island, the
> binding just setq'd would remain the same.

That might be something to look out for. Changing a buffer is an 
operation with big consequences, we know that already, but using 
goto-char didn't have too many implications until now (aside from 
point-entered hooks, I guess, which are being phased out).

>> I don't think either should be true. Then, the "adequate" model would
>> amount to changing them in post-command-hook anyway.
>
> What sort of variables are you thinking about here?  [ Some time later:]
> Could it be that it might be better to have "island chain local"
> variables rather than merely "island local" ones?

mmm-mode has both kinds.

> So that the three ruby
> lines in your example below would be three islands in a chain, and would
> thus all share a set of "island chain local" bindings rather than each
> line having its own binding?

Yes, probably. But that doesn't solve the above concern.

>> So making it seem, to the Lisp code, like the multiple related islands
>> don't have anything (for some definition of "anything") between them is
>> not a priority?
>
> By "related", I think you mean the same thing I meaan by "chained" - the
> three ruby mode lines enclosed in <%...%> in your example below, would be
> chained together by the super mode.

Yes, we can call them chained. As long as it doesn't imply anything 
about any presence of islands of other types between them.

>> Here's an example of ERB code Vitalie brought up recently:
>
>> <% if foo %>
>>    <%= bar(boo) %>
>> <% end %>
>
>> The parts of the buffer between %'s would be Ruby islands. The
>> suggestion was to use the value returned by the indentation function in
>> the second island (second line) to indent the "real" buffer contents.
>
> The question seems to be, are we indenting in the super mode or in each
> island?  The answer seems to be you'd want to indent in the super mode
> here, I think.

Indentation in the super mode would look like this:

   <% if foo %>
   <%= bar(boo) %>
   <% end %>

which is not what we'd really want. But yes, a good solution would take 
html-mode's indentation logic as a base, and modify it with the 
indentation offsets provided by the islands.

>> But: neither of the islands contains a newline. If they are combined in
>> the most straightforward way, the Ruby line will be 'if foo bar(boo)',
>> and its indentation would be not what we're looking for. I think the
>> current ways to look for a newline,
>
>>    (progn (skip-syntax-backward " ")
>>           (eq (char-before ?\n))
>
>> or
>
>>    (looking-at "\\s-$")
>
>> will fail to work. Also note that neither of the two islands in this
>> example contains the newline in question.
>
> Two?  Don't you mean three, here?  Is this a minor mistake in your
> counting or am I missing something important because I don't grok ruby?

The mistake would be mine, but really, I'm just talking about the first 
two. We're discussing the indentation of the second line. The third one 
is just here to make the example look more complete.

>> The "not chained" idea would require some explanation.
>
> For example, islands in a shell script which were individual AWK scripts
> would not be chained together.  But the three (?two) lines of ruby in the
> example above would be chained together.  I think we need to work this
> out more precisely.

Makes sense.

> For example, in the three ruby line scenario above, is there any variable
> you can think of for which it would be advantageous to have a binding
> for each island rather than a shared binding for the set of three (?two)?

Not off the top of my head. Maybe there's none.

> I'm kind of envisioning successive `forward-char's moving from the end
> of "<% if foo %>" to the beginning of "<%= bar(boo) %>", or even from
> the end of "if foo" to the beginning of "bar(boo)".  How does this sound?

Sounds like what I've calling option A above. Basically, my vague 
concerns are:

- Performance (checking islands boundaries in each primitive must incur 
overhead, even when there are no islands).
- Code complexity: new code paths that might be exercised not very often 
in the future. Hence, they could be prone to breakage. A dedicated test 
suite would help with that, though.

Also, think back to the problem of the absence of newlines in the Ruby 
islands in the above example. The indentation engine needs them, and it 
might be harder to make newlines appear both inside and outside of the 
Ruby islands (the html-mode indentation code needs them too, I'd wager). 
This is where option B has another advantage, aside from (probably) 
being easier to implement.

Re: A vision for multiple major modes [was: Re: [Emacs-diffs] widen-limits c331b66:]

[Alan Mackenzie]

Hello, Dmitry.

On Fri, Apr 01, 2016 at 04:15:18AM +0300, Dmitry Gutov wrote:
> On 03/29/2016 03:07 AM, Alan Mackenzie wrote:

> >> E.g. (goto-char (match-end n)) is a common
> >> idiom for syntax highlighting and indentation code.

> > Yes, but that match is going to be in the same island, or at the very
> > least, in the chain of islands containing the current one.

> Would looking-at skip over intermediate islands?

Yes, somehow.  That sort of looking-at is going to contain something
which matches whitespace in its regexp.  An intermediate island would
count as whitespace, one way or another.

> Basically, I wonder if

>   (= (+ (match-beginning 0) (length (match-string 0))
>      (match-end 0))

> is going to always hold in that new world.

Yes, certainly.  But (match-string 0) would include the island inside it.

> > Are you thinking more that `forward-char' should move from the end of an
> > island to the beginning of the next island in a chain?

> As one option, yes. Let's call it option A, and it entails renumerating 
> buffer positions to avoid gaps.

I'm solidly against using alternative buffer positions.  The unforeseen
side effects we'd have to cope with would be excessive.  The machinery
we'd have to set up to convert from "real" offsets to "no gaps" offsets
would be horrendous.  There's no reason forward-char shouldn't jump to
the next island in a chain without renumbering buffer positions.
(Assuming `restrict-to-island' has been bound to non-nil by the super
mode.)

> >> True. I'm more worried about gaps. But they could be treated like
> >> whitespace, I suppose.

> > For example, by `forward-char'?  What primitives were you thinking of,
> > here?

> This would be option B. forward-char doesn't care if the characters are 
> whitespace or not. I don't know if e.g. search-forward would see the 
> contents of the intermediate islands as a bunch of actual space 
> characters (it's something to consider).

When the key variable `restrict-to-island' is bound to non-nil, things
like search-forward would skip over islands.  Otherwise it would see the
insides of islands.  The latter would be what a user doing C-s would
normally want to happen.

> Most importantly, the contents of intermediate islands would match 
> "\\s-", and parse-partial-sexp would similarly skip over them.

Yes, with `restrict-to-island' non-nil.  parse-partial-sexp would always
save its state on encountering the start of an island, and restore it at
the end of the island.  For the island itself, it would set the state to
that at the end of the previous island in the chain, or to "nil".

One of the main points of islands is to isolate them syntactically from
their surroundings - a bit like a "super comment".

> >> Right. So then, some yet-unknown body of code has to become
> >> island-aware, and the improvement is not that seamless anymore.

> > The way I see it, the super modes would have to be totally aware of the
> > island mechanism, but the major modes would be largely, it not totally,
> > unaware of it.  `syntax-ppss' seems more part of the super mode
> > mechanism.

> At the very least, that does sound somewhat complicated.

I don't agree.  The essence of being a super mode is to create, organise
and coordinate the islands the buffer contains.

> >> What if var has an island-local binding? And my function has this:

> >> (setq var 1)
> >> (goto-char xx) ; where xx is in a different island

> >> will its value change mid-program? What if xx is in the same island?
> >> Will the value change then?

> > A different island local binding would become current, so yes its value
> > would thereby change.  Much like if var had a buffer local binding, and
> > you do (set-buffer "foo"), the value of var you'd just set would no
> > longer be in the current binding.  if xx is in the same island, the
> > binding just setq'd would remain the same.

> That might be something to look out for. Changing a buffer is an 
> operation with big consequences, we know that already, but using 
> goto-char didn't have too many implications until now (aside from 
> point-entered hooks, I guess, which are being phased out).

Most (all?) goto-chars in a major mode are going to be within an island
(or chain of islands).  Only the super mode is going to have to be aware
of this effect.

I think thinking of it as "changing the value of a variable" is a clumsy
way to regard it.  "Accessing the correct binding for the current buffer
position" is more how I would describe it.

> >> I don't think either should be true. Then, the "adequate" model would
> >> amount to changing them in post-command-hook anyway.

> > What sort of variables are you thinking about here?  [ Some time later:]
> > Could it be that it might be better to have "island chain local"
> > variables rather than merely "island local" ones?

> mmm-mode has both kinds.

Maybe, but having two extra kinds of locals is more than twice as
complicated as only having one.

> > So that the three ruby
> > lines in your example below would be three islands in a chain, and would
> > thus all share a set of "island chain local" bindings rather than each
> > line having its own binding?

> Yes, probably. But that doesn't solve the above concern.

I think "the above concern" was about the lack of newlines in the three
ruby lines of the example, and the difficulties this would cause
indentation.  One thing we could do is to make an island syntactically
equivalent to a newline for the enclosing code.  Maybe.

> >> So making it seem, to the Lisp code, like the multiple related islands
> >> don't have anything (for some definition of "anything") between them is
> >> not a priority?

> > By "related", I think you mean the same thing I meaan by "chained" - the
> > three ruby mode lines enclosed in <%...%> in your example below, would be
> > chained together by the super mode.

> Yes, we can call them chained. As long as it doesn't imply anything 
> about any presence of islands of other types between them.

> >> Here's an example of ERB code Vitalie brought up recently:

> >> <% if foo %>
> >>    <%= bar(boo) %>
> >> <% end %>

> >> The parts of the buffer between %'s would be Ruby islands. The
> >> suggestion was to use the value returned by the indentation function in
> >> the second island (second line) to indent the "real" buffer contents.

> > The question seems to be, are we indenting in the super mode or in each
> > island?  The answer seems to be you'd want to indent in the super mode
> > here, I think.

> Indentation in the super mode would look like this:

>    <% if foo %>
>    <%= bar(boo) %>
>    <% end %>

> which is not what we'd really want.

But we'd surely be happy enough with

<% if foo %>
<%     = bar(boo) %>
<% end %>

, if that is the way the user typed in the code.

> But yes, a good solution would take html-mode's indentation logic as a
> base, and modify it with the indentation offsets provided by the
> islands.

As a matter of interest, is it really likely that users will type in
embedded ruby code with <%...%> delimiters around each line, rather than
one pair around, say, a function or an entire program?  Is this ruby
snippet normal coding, or was it brought up to illustrate an awkward
situation we might have to handle?

> >> But: neither of the islands contains a newline. If they are combined in
> >> the most straightforward way, the Ruby line will be 'if foo bar(boo)',
> >> and its indentation would be not what we're looking for. I think the
> >> current ways to look for a newline,

> >>    (progn (skip-syntax-backward " ")
> >>           (eq (char-before ?\n))

> >> or

> >>    (looking-at "\\s-$")

> >> will fail to work. Also note that neither of the two islands in this
> >> example contains the newline in question.

> > For example, in the three ruby line scenario above, is there any variable
> > you can think of for which it would be advantageous to have a binding
> > for each island rather than a shared binding for the set of three (?two)?

> Not off the top of my head. Maybe there's none.

I would guess there aren't any.

> > I'm kind of envisioning successive `forward-char's moving from the end
> > of "<% if foo %>" to the beginning of "<%= bar(boo) %>", or even from
> > the end of "if foo" to the beginning of "bar(boo)".  How does this sound?

> Sounds like what I've calling option A above. Basically, my vague 
> concerns are:

> - Performance (checking islands boundaries in each primitive must incur 
> overhead, even when there are no islands).

There would clearly be some overhead.  This would be negligible for the
most common case of `restrict-to-island' being nil.

> - Code complexity: new code paths that might be exercised not very often 
> in the future. Hence, they could be prone to breakage. A dedicated test 
> suite would help with that, though.

If the abstraction we're talking about is sound, then the code complexity
will be manageable.  People cope with buffer local variables, people cope
with the regexp engine searching for syntactic things.

> Also, think back to the problem of the absence of newlines in the Ruby 
> islands in the above example. The indentation engine needs them, and it 
> might be harder to make newlines appear both inside and outside of the 
> Ruby islands (the html-mode indentation code needs them too, I'd wager). 
> This is where option B has another advantage, aside from (probably) 
> being easier to implement.

-- 
Alan Mackenzie (Nuremberg, Germany).

Re: A vision for multiple major modes [was: Re: [Emacs-diffs] widen-limits c331b66:]

[Dmitry Gutov]

Hi Alan,

On 04/05/2016 07:29 PM, Alan Mackenzie wrote:

>> As one option, yes. Let's call it option A, and it entails renumerating
>> buffer positions to avoid gaps.
>
> I'm solidly against using alternative buffer positions.  The unforeseen
> side effects we'd have to cope with would be excessive.  The machinery
> we'd have to set up to convert from "real" offsets to "no gaps" offsets
> would be horrendous.

That's my expectation, too.

> There's no reason forward-char shouldn't jump to
> the next island in a chain without renumbering buffer positions.
> (Assuming `restrict-to-island' has been bound to non-nil by the super
> mode.)

s/shouldn't/couldn't; I'm not sure it should, actually. forward-char 
doesn't normally skip over syntactic whitespace.

> When the key variable `restrict-to-island' is bound to non-nil, things
> like search-forward would skip over islands.  Otherwise it would see the
> insides of islands.  The latter would be what a user doing C-s would
> normally want to happen.

Maybe seeing inside islands is not that terrible. Good code should check 
syntax-ppss anyway, even if it finds a match inside a foreign island.

> parse-partial-sexp would always
> save its state on encountering the start of an island, and restore it at
> the end of the island.

Sure, OK. Or it could just treat the foreign islands as comments, and 
avoid creating a stack of states. Whichever way is easier to implement.

> For the island itself, it would set the state to
> that at the end of the previous island in the chain, or to "nil".

Are you thinking of syntax-ppss instead? The return value of 
parse-partial-sexp must has to depend on where is the start position.

Although if it uses a stack of states internally, it will be easier for 
syntax-ppss to work with (syntax-ppss would be able to reuse cache 
points within islands foreign to the current one).

> I think thinking of it as "changing the value of a variable" is a clumsy
> way to regard it.  "Accessing the correct binding for the current buffer
> position" is more how I would describe it.

Any way we call it now will essentially be a post factum rationalization 
if a programmer is surprised by the behavior. I'm describing the 
behavior. Maybe it's not too bad, IDK.

>>> What sort of variables are you thinking about here?  [ Some time later:]
>>> Could it be that it might be better to have "island chain local"
>>> variables rather than merely "island local" ones?
>
>> mmm-mode has both kinds.
>
> Maybe, but having two extra kinds of locals is more than twice as
> complicated as only having one.

"island chain local" variables are used a lot more often than the 
"island local" ones, in mmm-mode.

> I think "the above concern" was about the lack of newlines in the three
> ruby lines of the example, and the difficulties this would cause
> indentation.  One thing we could do is to make an island syntactically
> equivalent to a newline for the enclosing code.  Maybe.

Yes, it doesn't sounds like a natural solution. Adding a newline in the 
middle of a line might change the meaning of an expression, or at least 
throw off some heuristics we're using in indentation code.

> But we'd surely be happy enough with
>
> <% if foo %>
> <%     = bar(boo) %>
> <% end %>

Not really, no. That's just not how ERB is traditionally formatted. And 
see below (note how the HTML tags, which belong to the superior mode, 
are indented an extra level when inside <% if ... %>...<% end %>).

EJS and JSP are also formatted similarly.

> As a matter of interest, is it really likely that users will type in
> embedded ruby code with <%...%> delimiters around each line, rather than
> one pair around, say, a function or an entire program?

The latter would be an exception. Here's a real-life ERB example:

https://github.com/swanson/stringer/blob/master/app/views/archive.erb

>> - Code complexity: new code paths that might be exercised not very often
>> in the future. Hence, they could be prone to breakage. A dedicated test
>> suite would help with that, though.
>
> If the abstraction we're talking about is sound, then the code complexity
> will be manageable.  People cope with buffer local variables, people cope
> with the regexp engine searching for syntactic things.

Yes, a sound abstraction is key.

Re: A vision for multiple major modes [was: Re: [Emacs-diffs] widen-limits c331b66:]

[Alan Mackenzie]

Hello, Dmitry.

Sorry this reply has taken so long.  I've been preoccupied with things
in real life.

On Wed, Apr 06, 2016 at 01:52:23AM +0300, Dmitry Gutov wrote:
> Hi Alan,

> On 04/05/2016 07:29 PM, Alan Mackenzie wrote:

> > There's no reason forward-char shouldn't jump to
> > the next island in a chain without renumbering buffer positions.
> > (Assuming `restrict-to-island' has been bound to non-nil by the super
> > mode.)

I've now renamed `restrict-to-island' to `in-islands', because it's
shorter to write and read, and also with the "s", it's more accurate.

> s/shouldn't/couldn't; I'm not sure it should, actually. forward-char 
> doesn't normally skip over syntactic whitespace.

I think I agree with you here.  All the searching and skipping commands
will have to be considered.  For example, under what circumstances
should skip-syntax-forward skip the gap between two chained islands?
I'm coming round to the idea that both a "foreign" island and a gap in a
chain should be treated as a single unit of whitespace, and should match
"\\s-" - and possibly "[[:space:]]".

> > When the key variable `restrict-to-island' is bound to non-nil, things
> > like search-forward would skip over islands.  Otherwise it would see the
> > insides of islands.  The latter would be what a user doing C-s would
> > normally want to happen.

> Maybe seeing inside islands is not that terrible. Good code should check 
> syntax-ppss anyway, even if it finds a match inside a foreign island.

I'm thinking of major modes searching for somethingi, rather than users'
searching.

> > parse-partial-sexp would always
> > save its state on encountering the start of an island, and restore it at
> > the end of the island.

> Sure, OK. Or it could just treat the foreign islands as comments, and 
> avoid creating a stack of states. Whichever way is easier to implement.

No, it can't treat a foreign island as a comment, since its end point
might be inside that island.

[ .... ]

> Although if it uses a stack of states internally, it will be easier for 
> syntax-ppss to work with (syntax-ppss would be able to reuse cache 
> points within islands foreign to the current one).

I think it must use a stack of states.

> > I think thinking of it as "changing the value of a variable" is a clumsy
> > way to regard it.  "Accessing the correct binding for the current buffer
> > position" is more how I would describe it.

> Any way we call it now will essentially be a post factum rationalization 
> if a programmer is surprised by the behavior. I'm describing the 
> behavior. Maybe it's not too bad, IDK.

Most programmers will be working on major modes, or things like them, so
the bindings will stay the same.  I'm hoping there won't be surprises
there.

[ .... ]

> > I think "the above concern" was about the lack of newlines in the three
> > ruby lines of the example, and the difficulties this would cause
> > indentation.  One thing we could do is to make an island syntactically
> > equivalent to a newline for the enclosing code.  Maybe.

> Yes, it doesn't sounds like a natural solution. Adding a newline in the 
> middle of a line might change the meaning of an expression, or at least 
> throw off some heuristics we're using in indentation code.

I now think this is a tangent to the discussion.  Whatever solution is
settled on, it will probably be a part of a particular super mode (? ERB
Mode??) rather than part of the island mechanism.  I now don't think
that that newline idea was good.

[ .... ]

> > As a matter of interest, is it really likely that users will type in
> > embedded ruby code with <%...%> delimiters around each line, rather than
> > one pair around, say, a function or an entire program?

> The latter would be an exception. Here's a real-life ERB example:

> https://github.com/swanson/stringer/blob/master/app/views/archive.erb

Thanks.  I've downloaded it.

> >> - Code complexity: new code paths that might be exercised not very often
> >> in the future. Hence, they could be prone to breakage. A dedicated test
> >> suite would help with that, though.
> >
> > If the abstraction we're talking about is sound, then the code complexity
> > will be manageable.  People cope with buffer local variables, people cope
> > with the regexp engine searching for syntactic things.

> Yes, a sound abstraction is key.

I'm writing a more detailed outline of the island idea.  I hope to post
it on emacs-devel in the next day or two.

There are two things I'm currently unhappy about - (i) That buffer local
variables (as they are now) will need to be rigidly classified as either
ones which would be island chain local, or ones which would be entire
buffer variables.  This applies particularly to variables defined and
used in the C code; (ii) It's looking like major modes will explicitly
have to bind `in-islands' to non-nil when their commands are executed.

-- 
Alan Mackenzie (Nuremberg, Germany).