unofficial mirror of bug-gnu-emacs@gnu.org 
 help / color / mirror / code / Atom feed
* bug#59468: 29.0.50; c-ts-mode cannot fontify after macros are encountered
       [not found] <87k03n4v0r.fsf.ref@yahoo.com>
@ 2022-11-22  1:50 ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
  2022-11-22 20:11   ` Yuan Fu
                     ` (2 more replies)
  0 siblings, 3 replies; 16+ messages in thread
From: Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors @ 2022-11-22  1:50 UTC (permalink / raw)
  To: 59468

Insert the following code in a buffer, and enable c-ts-mode.

#define CheckExtension(name)				\
  if (!name)						\
    {							\
      if (display)					\
	eglTerminate (display);				\
      fprintf (stderr, "Missing: egl%s\n", #name + 1);	\
      return False;					\
    }

#define CheckExtensionGl(name)				\
  if (!name)						\
    {							\
      /* If the context remains current, then nothing	\
	 will get released upon eglTerminate.  */	\
      eglMakeCurrent (display, EGL_NO_SURFACE,		\
		      EGL_NO_SURFACE, EGL_NO_CONTEXT);	\
      eglTerminate (display);				\
      fprintf (stderr, "Missing: gl%s\n", #name + 1);	\
      return False;					\
    }

#define LoadProc(name, ext, extname)			\
  if (HaveEglExtension (extname))			\
    I##name						\
      = (void *) eglGetProcAddress ("egl" #name ext)

#define LoadProcGl(name, ext, extname)			\
  if (HaveGlExtension (extname))			\
    I##name						\
      = (void *) eglGetProcAddress ("gl" #name ext)

#define CheckGlExtension(name)			\
  if (!HaveGlExtension (name))			\
    {						\
      fprintf (stderr, "Missing %s\n", name);	\
						\
      eglMakeCurrent (display, EGL_NO_SURFACE,	\
		      EGL_NO_SURFACE,		\
		      EGL_NO_CONTEXT);		\
      eglTerminate (display);			\
      return False;				\
    }

There will be no fontification of types or string constants inside the
macro bodies.  CC Mode works fine.

Now, insert the following code below the macro definition:

static Visual *
FindVisual (VisualID visual, int *depth)
{
  XVisualInfo vinfo, *visuals;
  Visual *value;
  int nvisuals;
  const char *override;
}

visual, depth, visuals, value and override are not fontified as
identifiers.  Line feed characters below the macros are also fontfied as
"errors".

Within the following code:

static BufferActivityRecord *
FindBufferActivityRecord (PictureBuffer *buffer, PictureTarget *target)
{
  BufferActivityRecord *record;

  /* Look through the global activity list for a record matching
     the given values.  */
  record = all_activity.global_next;
  while (record != &all_activity)
    {
      if (record->buffer == buffer
	  && record->target == target)
	return record;

      record = record->global_next;
    }

  return NULL;
}

"record" is fontified as an identifier, but not within the condition in
the while loop.

Within the following code:

static void
compare_single_row_8bpc4 (unsigned char *data, unsigned char *xdata,
			  size_t size,
			  struct image_difference_statistics *statistics)
{
  unsigned char channel_a, channel_b;
  int diff;
  size_t i;

  for (i = 0; i < size; ++i)
    {
      channel_a = data[i];
      channel_b = xdata[i];

      diff = (int) channel_b - (int) channel_a;
      statistics->min_diff = MIN (statistics->min_diff,
				  diff);
      statistics->max_diff = MAX (statistics->max_diff,
				  diff);
    }
}

the "i" in "size_t i" is fontified as an identifier the first time, but
not in future appearances in the function.

Within src/xterm.c, every appearance of ATOM_REFS_INIT is fontified in
the error face.  Also, inside `xm_setup_dnd_targets',

  xm_targets_table_header header;
  xm_targets_table_rec **recs UNINIT;
  xm_byte_order byteorder;

"recs" is fontified in the error face.

	      recs = xmalloc (sizeof *recs);
	      recs[0] = xmalloc (FLEXSIZEOF (struct xm_targets_table_rec,
					     targets, ntargets * 4));

	      recs[0]->n_targets = ntargets;

"struct" is fontified in the error face.  Within
x_sync_note_frame_times:

#ifdef FRAME_DEBUG
  uint_fast64_t last_frame_ms = output->last_frame_time / 1000;
  fprintf (stderr,
	   "Drawing the last frame took: %"PRIuFAST64" ms (%"PRIuFAST64")\n",
	   last_frame_ms, time);
#endif

"PRIuFAST64" is fontified in the error face.  Within
x_dnd_begin_drag_and_drop:

	  if (x_dnd_movement_frame
	      /* FIXME: how come this can end up with movement frames
		 from other displays on GTK builds?  */
	      && (FRAME_X_DISPLAY (x_dnd_movement_frame)
		  == FRAME_X_DISPLAY (f))
	      /* If both those variables are false, then F is no
		 longer protected from deletion by Lisp code.  This
		 can only happen during the final iteration of the DND
		 event loop.  */
	      && (x_dnd_in_progress || x_dnd_waiting_for_finish))
	    {
	      XSETFRAME (frame_object, x_dnd_movement_frame);

is fontified incorrectly: the "if" is not fontified at all, and the last
three lines show up in the error faces.

In addition:

Lisp_Object
x_dnd_begin_drag_and_drop (struct frame *f, Time time, Atom xaction,
			   Lisp_Object return_frame, Atom *ask_action_list,
			   const char **ask_action_names, size_t n_ask_actions,
			   bool allow_current_frame, Atom *target_atoms,
			   int ntargets, Lisp_Object selection_target_list,
			   bool follow_tooltip)

"Lisp_Object" and "x_dnd_begin_drag_and_drop" are not fontified at all.

Here:

/* Select for input extension events used by scroll bars.  This will
   result in the corresponding core events not being generated for
   SCROLL_BAR.  */

MAYBE_UNUSED static void
xi_select_scroll_bar_events (struct x_display_info *dpyinfo,
			     Window scroll_bar)

"void" is fontified in the error face.

Here, in handle_one_xevent:

#ifdef HAVE_XINPUT2
      if (event->type != GenericEvent)
#endif
	any = x_any_window_to_frame (dpyinfo, event->xany.window);
#ifdef HAVE_XINPUT2
      else
	any = NULL;
#endif

the "else" is fontified as a type.

static int
handle_one_xevent (struct x_display_info *dpyinfo,
#ifndef HAVE_XINPUT2
		   const XEvent *event,
#else
		   XEvent *event,
#endif
		   int *finish, struct input_event *hold_quit)

the "const XEvent *" is fontified in the error face.

Here:

    case GraphicsExpose:	/* This occurs when an XCopyArea's
                                   source area was obscured or not
                                   available.  */
      f = x_window_to_frame (dpyinfo, event->xgraphicsexpose.drawable);
      if (f)
        {
          expose_frame (f, event->xgraphicsexpose.x,
                        event->xgraphicsexpose.y,
                        event->xgraphicsexpose.width,
                        event->xgraphicsexpose.height);
#ifndef USE_TOOLKIT_SCROLL_BARS
	  x_scroll_bar_handle_exposure (f, (XEvent *) event);
#endif
#ifdef USE_GTK
	  x_clear_under_internal_border (f);
#endif
#ifdef HAVE_XDBE
	  show_back_buffer (f);
#endif
        }
#ifdef USE_X_TOOLKIT
      else
        goto OTHER;
#endif /* USE_X_TOOLKIT */
      break;

"goto OTHER" is fontified in the error face, and "else" as a type.

Here:

#ifdef HAVE_XINPUT2
	      if (event->xkey.time == pending_keystroke_time)
		{
		  source = xi_device_from_id (dpyinfo,
					      dpyinfo->pending_keystroke_source);

		  if (source)
		    inev.ie.device = source->name;
		}
#endif

"inev" is fontified in the error face.

Here:

#ifdef USE_GTK
      /* See comment in EnterNotify above */
      else if (dpyinfo->last_mouse_glyph_frame)
        x_note_mouse_movement (dpyinfo->last_mouse_glyph_frame,
			       &event->xmotion, Qnil);
#endif

"else" and "dpyinfo" are fontified as types.

#ifdef USE_MOTIF
	    Widget widget;

	    widget = XtWindowToWidget (dpyinfo->display,
				       event->xbutton.window);

	    if (widget && XmIsCascadeButton (widget)
		&& XtIsSensitive (widget))
	      {
#endif
		if (!f->output_data.x->saved_menu_event)
		  f->output_data.x->saved_menu_event = xmalloc (sizeof *event);
		*f->output_data.x->saved_menu_event = *event;
		inev.ie.kind = MENU_BAR_ACTIVATE_EVENT;
		XSETFRAME (inev.ie.frame_or_window, f);
		*finish = X_EVENT_DROP;
#ifdef USE_MOTIF
	      }
#endif

here, the last closing brace is fontified in the error face.

Here:

static void NO_INLINE
x_error_quitter (Display *display, XErrorEvent *event)
{
  char buf[256], buf1[400 + INT_STRLEN_BOUND (int)
		      + INT_STRLEN_BOUND (unsigned long)];

"NO_INLINE" and "unsigned long" are fontified in the error face.

CC mode fontifies all of the examples above fine, and they are not even
pre-standard C.  Would someone please fix c-ts-mode to fontify them
correctly as well?

In GNU Emacs 29.0.50 (build 238, x86_64-pc-linux-gnu) of 2022-11-22
 built on trinity
Repository revision: aeadba1418d8fc18f17b4ae415cde35e9e272e7a
Repository branch: master
Windowing system distributor 'The X.Org Foundation', version 11.0.12101099
System Description: Fedora Linux 37 (Workstation Edition)

Configured using:
 'configure --with-x --with-x-toolkit=no --without-cairo
 --with-dumping=unexec --cache-file=/tmp/ccache'

Configured features:
ACL DBUS FREETYPE GIF GLIB GMP GNUTLS GSETTINGS HARFBUZZ JPEG JSON LCMS2
LIBSELINUX LIBSYSTEMD LIBXML2 MODULES NOTIFY INOTIFY OLDXMENU PNG RSVG
SECCOMP SOUND SQLITE3 THREADS TIFF TREE_SITTER UNEXEC WEBP X11 XDBE XFT
XIM XINPUT2 XPM ZLIB

Important settings:
  value of $LANG: en_GB.utf8
  value of $XMODIFIERS: @im=ibus
  locale-coding-system: utf-8-unix

Major mode: C/*l

Minor modes in effect:
  tooltip-mode: t
  global-eldoc-mode: t
  show-paren-mode: t
  electric-indent-mode: t
  mouse-wheel-mode: t
  tool-bar-mode: t
  menu-bar-mode: t
  file-name-shadow-mode: t
  global-font-lock-mode: t
  font-lock-mode: t
  blink-cursor-mode: t
  line-number-mode: t
  indent-tabs-mode: t
  transient-mark-mode: t
  auto-composition-mode: t
  auto-encryption-mode: t
  auto-compression-mode: t
  abbrev-mode: t

Load-path shadows:
None found.

Features:
(shadow sort emacsbug mail-extr message sendmail mailcap yank-media puny
rfc822 mml mml-sec epa derived epg rfc6068 epg-config gnus-util
time-date mm-decode mm-bodies mm-encode mail-parse rfc2231 rfc2047
rfc2045 mm-util ietf-drums mail-prsvr mailabbrev mail-utils gmm-utils
mailheader reporter find-dired ffap url-parse auth-source eieio
eieio-core cl-macs password-cache json subr-x map byte-opt gv bytecomp
byte-compile url-vars thingatpt files-x grep compile
text-property-search comint ansi-osc ansi-color ring c-ts-mode rx
treesit cl-seq vc-dispatcher vc-svn cc-mode cc-fonts cc-guess cc-menus
cc-cmds cc-styles cc-align cc-engine cc-vars cc-defs misearch
multi-isearch dired-aux cl-loaddefs cl-lib dired dired-loaddefs rmc
iso-transl tooltip cconv eldoc paren electric uniquify ediff-hook
vc-hooks lisp-float-type elisp-mode mwheel term/x-win x-win
term/common-win x-dnd tool-bar dnd fontset image regexp-opt fringe
tabulated-list replace newcomment text-mode lisp-mode prog-mode register
page tab-bar menu-bar rfn-eshadow isearch easymenu timer select
scroll-bar mouse jit-lock font-lock syntax font-core term/tty-colors
frame minibuffer nadvice seq simple cl-generic indonesian philippine
cham georgian utf-8-lang misc-lang vietnamese tibetan thai tai-viet lao
korean japanese eucjp-ms cp51932 hebrew greek romanian slovak czech
european ethiopic indian cyrillic chinese composite emoji-zwj charscript
charprop case-table epa-hook jka-cmpr-hook help abbrev obarray oclosure
cl-preloaded button loaddefs theme-loaddefs faces cus-face macroexp
files window text-properties overlay sha1 md5 base64 format env
code-pages mule custom widget keymap hashtable-print-readable backquote
threads dbusbind inotify lcms2 dynamic-setting system-font-setting
font-render-setting xinput2 x multi-tty make-network-process emacs)

Memory information:
((conses 16 171735 10469)
 (symbols 48 26041 14)
 (strings 32 45784 1739)
 (string-bytes 1 1373049)
 (vectors 16 26115)
 (vector-slots 8 653341 15564)
 (floats 8 75 179)
 (intervals 56 3663 0)
 (buffers 992 18))





^ permalink raw reply	[flat|nested] 16+ messages in thread

* bug#59468: 29.0.50; c-ts-mode cannot fontify after macros are  encountered
  2022-11-22  1:50 ` bug#59468: 29.0.50; c-ts-mode cannot fontify after macros are encountered Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
@ 2022-11-22 20:11   ` Yuan Fu
  2022-11-23  0:41     ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
  2022-11-23  3:30     ` Eli Zaretskii
  2022-11-23  1:27   ` Yuan Fu via Bug reports for GNU Emacs, the Swiss army knife of text editors
  2022-11-24  0:03   ` Yuan Fu
  2 siblings, 2 replies; 16+ messages in thread
From: Yuan Fu @ 2022-11-22 20:11 UTC (permalink / raw)
  To: Po Lu; +Cc: 59468


Po Lu <luangruo@yahoo.com> writes:

> Insert the following code in a buffer, and enable c-ts-mode.
>
> #define CheckExtension(name)				\
>   if (!name)						\
>     {							\
>       if (display)					\
> 	eglTerminate (display);				\
>       fprintf (stderr, "Missing: egl%s\n", #name + 1);	\
>       return False;					\
>     }
>
> #define CheckExtensionGl(name)				\
>   if (!name)						\
>     {							\
>       /* If the context remains current, then nothing	\
> 	 will get released upon eglTerminate.  */	\
>       eglMakeCurrent (display, EGL_NO_SURFACE,		\
> 		      EGL_NO_SURFACE, EGL_NO_CONTEXT);	\
>       eglTerminate (display);				\
>       fprintf (stderr, "Missing: gl%s\n", #name + 1);	\
>       return False;					\
>     }
>
> #define LoadProc(name, ext, extname)			\
>   if (HaveEglExtension (extname))			\
>     I##name						\
>       = (void *) eglGetProcAddress ("egl" #name ext)
>
> #define LoadProcGl(name, ext, extname)			\
>   if (HaveGlExtension (extname))			\
>     I##name						\
>       = (void *) eglGetProcAddress ("gl" #name ext)
>
> #define CheckGlExtension(name)			\
>   if (!HaveGlExtension (name))			\
>     {						\
>       fprintf (stderr, "Missing %s\n", name);	\
> 						\
>       eglMakeCurrent (display, EGL_NO_SURFACE,	\
> 		      EGL_NO_SURFACE,		\
> 		      EGL_NO_CONTEXT);		\
>       eglTerminate (display);			\
>       return False;				\
>     }
>
> There will be no fontification of types or string constants inside the
> macro bodies.  CC Mode works fine.
>
> Now, insert the following code below the macro definition:
>
> static Visual *
> FindVisual (VisualID visual, int *depth)
> {
>   XVisualInfo vinfo, *visuals;
>   Visual *value;
>   int nvisuals;
>   const char *override;
> }
>
> visual, depth, visuals, value and override are not fontified as
> identifiers.  Line feed characters below the macros are also fontfied as
> "errors".
>
> Within the following code:
>
> static BufferActivityRecord *
> FindBufferActivityRecord (PictureBuffer *buffer, PictureTarget *target)
> {
>   BufferActivityRecord *record;
>
>   /* Look through the global activity list for a record matching
>      the given values.  */
>   record = all_activity.global_next;
>   while (record != &all_activity)
>     {
>       if (record->buffer == buffer
> 	  && record->target == target)
> 	return record;
>
>       record = record->global_next;
>     }
>
>   return NULL;
> }
>
> "record" is fontified as an identifier, but not within the condition in
> the while loop.
>
> Within the following code:
>
> static void
> compare_single_row_8bpc4 (unsigned char *data, unsigned char *xdata,
> 			  size_t size,
> 			  struct image_difference_statistics *statistics)
> {
>   unsigned char channel_a, channel_b;
>   int diff;
>   size_t i;
>
>   for (i = 0; i < size; ++i)
>     {
>       channel_a = data[i];
>       channel_b = xdata[i];
>
>       diff = (int) channel_b - (int) channel_a;
>       statistics->min_diff = MIN (statistics->min_diff,
> 				  diff);
>       statistics->max_diff = MAX (statistics->max_diff,
> 				  diff);
>     }
> }
>
> the "i" in "size_t i" is fontified as an identifier the first time, but
> not in future appearances in the function.
>
> Within src/xterm.c, every appearance of ATOM_REFS_INIT is fontified in
> the error face.  Also, inside `xm_setup_dnd_targets',
>
>   xm_targets_table_header header;
>   xm_targets_table_rec **recs UNINIT;
>   xm_byte_order byteorder;
>
> "recs" is fontified in the error face.
>
> 	      recs = xmalloc (sizeof *recs);
> 	      recs[0] = xmalloc (FLEXSIZEOF (struct xm_targets_table_rec,
> 					     targets, ntargets * 4));
>
> 	      recs[0]->n_targets = ntargets;
>
> "struct" is fontified in the error face.  Within
> x_sync_note_frame_times:
>
> #ifdef FRAME_DEBUG
>   uint_fast64_t last_frame_ms = output->last_frame_time / 1000;
>   fprintf (stderr,
> 	   "Drawing the last frame took: %"PRIuFAST64" ms (%"PRIuFAST64")\n",
> 	   last_frame_ms, time);
> #endif
>
> "PRIuFAST64" is fontified in the error face.  Within
> x_dnd_begin_drag_and_drop:
>
> 	  if (x_dnd_movement_frame
> 	      /* FIXME: how come this can end up with movement frames
> 		 from other displays on GTK builds?  */
> 	      && (FRAME_X_DISPLAY (x_dnd_movement_frame)
> 		  == FRAME_X_DISPLAY (f))
> 	      /* If both those variables are false, then F is no
> 		 longer protected from deletion by Lisp code.  This
> 		 can only happen during the final iteration of the DND
> 		 event loop.  */
> 	      && (x_dnd_in_progress || x_dnd_waiting_for_finish))
> 	    {
> 	      XSETFRAME (frame_object, x_dnd_movement_frame);
>
> is fontified incorrectly: the "if" is not fontified at all, and the last
> three lines show up in the error faces.
>
> In addition:
>
> Lisp_Object
> x_dnd_begin_drag_and_drop (struct frame *f, Time time, Atom xaction,
> 			   Lisp_Object return_frame, Atom *ask_action_list,
> 			   const char **ask_action_names, size_t n_ask_actions,
> 			   bool allow_current_frame, Atom *target_atoms,
> 			   int ntargets, Lisp_Object selection_target_list,
> 			   bool follow_tooltip)
>
> "Lisp_Object" and "x_dnd_begin_drag_and_drop" are not fontified at all.
>
> Here:
>
> /* Select for input extension events used by scroll bars.  This will
>    result in the corresponding core events not being generated for
>    SCROLL_BAR.  */
>
> MAYBE_UNUSED static void
> xi_select_scroll_bar_events (struct x_display_info *dpyinfo,
> 			     Window scroll_bar)
>
> "void" is fontified in the error face.
>
> Here, in handle_one_xevent:
>
> #ifdef HAVE_XINPUT2
>       if (event->type != GenericEvent)
> #endif
> 	any = x_any_window_to_frame (dpyinfo, event->xany.window);
> #ifdef HAVE_XINPUT2
>       else
> 	any = NULL;
> #endif
>
> the "else" is fontified as a type.
>
> static int
> handle_one_xevent (struct x_display_info *dpyinfo,
> #ifndef HAVE_XINPUT2
> 		   const XEvent *event,
> #else
> 		   XEvent *event,
> #endif
> 		   int *finish, struct input_event *hold_quit)
>
> the "const XEvent *" is fontified in the error face.
>
> Here:
>
>     case GraphicsExpose:	/* This occurs when an XCopyArea's
>                                    source area was obscured or not
>                                    available.  */
>       f = x_window_to_frame (dpyinfo, event->xgraphicsexpose.drawable);
>       if (f)
>         {
>           expose_frame (f, event->xgraphicsexpose.x,
>                         event->xgraphicsexpose.y,
>                         event->xgraphicsexpose.width,
>                         event->xgraphicsexpose.height);
> #ifndef USE_TOOLKIT_SCROLL_BARS
> 	  x_scroll_bar_handle_exposure (f, (XEvent *) event);
> #endif
> #ifdef USE_GTK
> 	  x_clear_under_internal_border (f);
> #endif
> #ifdef HAVE_XDBE
> 	  show_back_buffer (f);
> #endif
>         }
> #ifdef USE_X_TOOLKIT
>       else
>         goto OTHER;
> #endif /* USE_X_TOOLKIT */
>       break;
>
> "goto OTHER" is fontified in the error face, and "else" as a type.
>
> Here:
>
> #ifdef HAVE_XINPUT2
> 	      if (event->xkey.time == pending_keystroke_time)
> 		{
> 		  source = xi_device_from_id (dpyinfo,
> 					      dpyinfo->pending_keystroke_source);
>
> 		  if (source)
> 		    inev.ie.device = source->name;
> 		}
> #endif
>
> "inev" is fontified in the error face.
>
> Here:
>
> #ifdef USE_GTK
>       /* See comment in EnterNotify above */
>       else if (dpyinfo->last_mouse_glyph_frame)
>         x_note_mouse_movement (dpyinfo->last_mouse_glyph_frame,
> 			       &event->xmotion, Qnil);
> #endif
>
> "else" and "dpyinfo" are fontified as types.
>
> #ifdef USE_MOTIF
> 	    Widget widget;
>
> 	    widget = XtWindowToWidget (dpyinfo->display,
> 				       event->xbutton.window);
>
> 	    if (widget && XmIsCascadeButton (widget)
> 		&& XtIsSensitive (widget))
> 	      {
> #endif
> 		if (!f->output_data.x->saved_menu_event)
> 		  f->output_data.x->saved_menu_event = xmalloc (sizeof *event);
> 		*f->output_data.x->saved_menu_event = *event;
> 		inev.ie.kind = MENU_BAR_ACTIVATE_EVENT;
> 		XSETFRAME (inev.ie.frame_or_window, f);
> 		*finish = X_EVENT_DROP;
> #ifdef USE_MOTIF
> 	      }
> #endif
>
> here, the last closing brace is fontified in the error face.
>
> Here:
>
> static void NO_INLINE
> x_error_quitter (Display *display, XErrorEvent *event)
> {
>   char buf[256], buf1[400 + INT_STRLEN_BOUND (int)
> 		      + INT_STRLEN_BOUND (unsigned long)];
>
> "NO_INLINE" and "unsigned long" are fontified in the error face.
>
> CC mode fontifies all of the examples above fine, and they are not even
> pre-standard C.  Would someone please fix c-ts-mode to fontify them
> correctly as well?
>

Thanks for these reports! Buuuuut as we’ve seen, tree-sitter really
doesn’t handle macros that well. I’ll see what I can do when I find the
time, but there really isn’t any good solutions to this (right now, that
I can think of). I’m not even sure the author of tree-sitter-c would
accept changes that try to parse macros: he has made it clear that the
purpose of tree-sitter-c is to support post-processed C and leave macros
and preprocessor to error-recovery.

Yuan





^ permalink raw reply	[flat|nested] 16+ messages in thread

* bug#59468: 29.0.50; c-ts-mode cannot fontify after macros are encountered
  2022-11-22 20:11   ` Yuan Fu
@ 2022-11-23  0:41     ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
  2022-11-23  3:30     ` Eli Zaretskii
  1 sibling, 0 replies; 16+ messages in thread
From: Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors @ 2022-11-23  0:41 UTC (permalink / raw)
  To: Yuan Fu; +Cc: 59468

Yuan Fu <casouri@gmail.com> writes:

> Thanks for these reports! Buuuuut as we’ve seen, tree-sitter really
> doesn’t handle macros that well. I’ll see what I can do when I find the
> time, but there really isn’t any good solutions to this (right now, that
> I can think of). I’m not even sure the author of tree-sitter-c would
> accept changes that try to parse macros: he has made it clear that the
> purpose of tree-sitter-c is to support post-processed C and leave macros
> and preprocessor to error-recovery.

Thanks, and too bad.  However, one cannot help but wonder how he is
supposed to write only "post-processed" C.  I think that attitude of the
tree-sitter-c developer is extremely nasty.





^ permalink raw reply	[flat|nested] 16+ messages in thread

* bug#59468: 29.0.50; c-ts-mode cannot fontify after macros are  encountered
  2022-11-22  1:50 ` bug#59468: 29.0.50; c-ts-mode cannot fontify after macros are encountered Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
  2022-11-22 20:11   ` Yuan Fu
@ 2022-11-23  1:27   ` Yuan Fu via Bug reports for GNU Emacs, the Swiss army knife of text editors
  2022-11-24  0:03   ` Yuan Fu
  2 siblings, 0 replies; 16+ messages in thread
From: Yuan Fu via Bug reports for GNU Emacs, the Swiss army knife of text editors @ 2022-11-23  1:27 UTC (permalink / raw)
  To: Po Lu; +Cc: 59468


Po Lu <luangruo@yahoo.com> writes:

> Yuan Fu <casouri@gmail.com> writes:
>
>> Thanks for these reports! Buuuuut as we’ve seen, tree-sitter really
>> doesn’t handle macros that well. I’ll see what I can do when I find the
>> time, but there really isn’t any good solutions to this (right now, that
>> I can think of). I’m not even sure the author of tree-sitter-c would
>
> Thanks, and too bad.  However, one cannot help but wonder how he is
> supposed to write only "post-processed" C.

I think the idea is that most macros are defined like functions, so they
are just parsed as function invocations. Other things can be handled by
tree-sitter’s error recovery to some degree. Not saying this is correct,
but that’s probably the idea.


>> accept changes that try to parse macros: he has made it clear that the
>> purpose of tree-sitter-c is to support post-processed C and leave macros
>> and preprocessor to error-recovery.
>
> I think that attitude of the
> tree-sitter-c developer is extremely nasty.

For the record, because I don’t want to put words into tree-sitter’s
author’s mouth, I came up with this observation from [1], and I could be
wrong. Also, I think it’s more about the limitation of parser-base
approach than about policy.

[1]: https://github.com/tree-sitter/tree-sitter-c/issues/7

Yuan





^ permalink raw reply	[flat|nested] 16+ messages in thread

* bug#59468: 29.0.50; c-ts-mode cannot fontify after macros are  encountered
  2022-11-22 20:11   ` Yuan Fu
  2022-11-23  0:41     ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
@ 2022-11-23  3:30     ` Eli Zaretskii
  2022-11-23  4:13       ` Yuan Fu
  1 sibling, 1 reply; 16+ messages in thread
From: Eli Zaretskii @ 2022-11-23  3:30 UTC (permalink / raw)
  To: Yuan Fu; +Cc: luangruo, 59468

> Cc: 59468@debbugs.gnu.org
> From: Yuan Fu <casouri@gmail.com>
> Date: Tue, 22 Nov 2022 12:11:17 -0800
> 
> I’m not even sure the author of tree-sitter-c would accept changes that
> try to parse macros: he has made it clear that the purpose of
> tree-sitter-c is to support post-processed C and leave macros and
> preprocessor to error-recovery.

What do you mean by "error-recovery"?

Is it possible to do something on our end for processing the error nodes,
which would remedy the issue with macros?  Or maybe some other way of
handling the macros by ourselves?





^ permalink raw reply	[flat|nested] 16+ messages in thread

* bug#59468: 29.0.50; c-ts-mode cannot fontify after macros are  encountered
  2022-11-23  3:30     ` Eli Zaretskii
@ 2022-11-23  4:13       ` Yuan Fu
  2022-11-23  7:36         ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
  2022-11-23 12:43         ` Eli Zaretskii
  0 siblings, 2 replies; 16+ messages in thread
From: Yuan Fu @ 2022-11-23  4:13 UTC (permalink / raw)
  To: Eli Zaretskii; +Cc: luangruo, 59468



> On Nov 22, 2022, at 7:30 PM, Eli Zaretskii <eliz@gnu.org> wrote:
> 
>> Cc: 59468@debbugs.gnu.org
>> From: Yuan Fu <casouri@gmail.com>
>> Date: Tue, 22 Nov 2022 12:11:17 -0800
>> 
>> I’m not even sure the author of tree-sitter-c would accept changes that
>> try to parse macros: he has made it clear that the purpose of
>> tree-sitter-c is to support post-processed C and leave macros and
>> preprocessor to error-recovery.
> 
> What do you mean by "error-recovery"?

Basically if tree-sitter encounters something that doesn’t comply to the grammar, it tries to skip a number of tokens, and parse the rest of the file normally. This way any error is localized and the rest of the file can still be parsed. It could also insert “missing nodes” to the tree (eg, a missing parentheses) if that works better than skipping tokens. Those tokens that are skipped are put under an ERROR node, to indicated that they are skipped and normally shouldn’t be there in well-formed text.

> Is it possible to do something on our end for processing the error nodes,
> which would remedy the issue with macros?  Or maybe some other way of
> handling the macros by ourselves?

For each of the ERROR nodes, we can look at the skipped tokens inside, and look around with some regex searching. That would take some non-trivial effort, I imagine.

Yuan




^ permalink raw reply	[flat|nested] 16+ messages in thread

* bug#59468: 29.0.50; c-ts-mode cannot fontify after macros are encountered
  2022-11-23  4:13       ` Yuan Fu
@ 2022-11-23  7:36         ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
  2022-11-23 12:53           ` Eli Zaretskii
  2022-11-23 12:43         ` Eli Zaretskii
  1 sibling, 1 reply; 16+ messages in thread
From: Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors @ 2022-11-23  7:36 UTC (permalink / raw)
  To: Yuan Fu; +Cc: Eli Zaretskii, 59468

Yuan Fu <casouri@gmail.com> writes:

> For each of the ERROR nodes, we can look at the skipped tokens inside,
> and look around with some regex searching.
                            ^^^^^^^^^^^^^^^

I thought that is the sort of thing we aim to avoid by using
tree-sitter.





^ permalink raw reply	[flat|nested] 16+ messages in thread

* bug#59468: 29.0.50; c-ts-mode cannot fontify after macros are  encountered
  2022-11-23  4:13       ` Yuan Fu
  2022-11-23  7:36         ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
@ 2022-11-23 12:43         ` Eli Zaretskii
  1 sibling, 0 replies; 16+ messages in thread
From: Eli Zaretskii @ 2022-11-23 12:43 UTC (permalink / raw)
  To: Yuan Fu; +Cc: luangruo, 59468

> From: Yuan Fu <casouri@gmail.com>
> Date: Tue, 22 Nov 2022 20:13:49 -0800
> Cc: luangruo@yahoo.com,
>  59468@debbugs.gnu.org
> 
> > Is it possible to do something on our end for processing the error nodes,
> > which would remedy the issue with macros?  Or maybe some other way of
> > handling the macros by ourselves?
> 
> For each of the ERROR nodes, we can look at the skipped tokens inside, and look around with some regex searching. That would take some non-trivial effort, I imagine.

If nothing better comes up, maybe this is what we should do.  For
fontifications, perhaps using some small subset of CC Mode for level-2
fontifications should be relatively easy?





^ permalink raw reply	[flat|nested] 16+ messages in thread

* bug#59468: 29.0.50; c-ts-mode cannot fontify after macros are encountered
  2022-11-23  7:36         ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
@ 2022-11-23 12:53           ` Eli Zaretskii
  0 siblings, 0 replies; 16+ messages in thread
From: Eli Zaretskii @ 2022-11-23 12:53 UTC (permalink / raw)
  To: Po Lu; +Cc: casouri, 59468

> From: Po Lu <luangruo@yahoo.com>
> Cc: Eli Zaretskii <eliz@gnu.org>,  59468@debbugs.gnu.org
> Date: Wed, 23 Nov 2022 15:36:21 +0800
> 
> Yuan Fu <casouri@gmail.com> writes:
> 
> > For each of the ERROR nodes, we can look at the skipped tokens inside,
> > and look around with some regex searching.
>                             ^^^^^^^^^^^^^^^
> 
> I thought that is the sort of thing we aim to avoid by using
> tree-sitter.

Would you rather do nothing about these error nodes?

If they happen too frequently, then it would be a real problem for the
language(s) where that happens.  But if they happen relatively infrequently,
then falling back on regexps could be an okay solution, IMO.





^ permalink raw reply	[flat|nested] 16+ messages in thread

* bug#59468: 29.0.50; c-ts-mode cannot fontify after macros are  encountered
  2022-11-22  1:50 ` bug#59468: 29.0.50; c-ts-mode cannot fontify after macros are encountered Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
  2022-11-22 20:11   ` Yuan Fu
  2022-11-23  1:27   ` Yuan Fu via Bug reports for GNU Emacs, the Swiss army knife of text editors
@ 2022-11-24  0:03   ` Yuan Fu
  2022-11-24  0:36     ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
  2022-11-26  2:39     ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
  2 siblings, 2 replies; 16+ messages in thread
From: Yuan Fu @ 2022-11-24  0:03 UTC (permalink / raw)
  To: Po Lu; +Cc: 59468


Po Lu <luangruo@yahoo.com> writes:

Ok, I had a look at each of these cases, comments:

> Insert the following code in a buffer, and enable c-ts-mode.
>
> #define CheckExtension(name)				\
>   if (!name)						\
>     {							\
>       if (display)					\
> 	eglTerminate (display);				\
>       fprintf (stderr, "Missing: egl%s\n", #name + 1);	\
>       return False;					\
>     }
>
> #define CheckExtensionGl(name)				\
>   if (!name)						\
>     {							\
>       /* If the context remains current, then nothing	\
> 	 will get released upon eglTerminate.  */	\
>       eglMakeCurrent (display, EGL_NO_SURFACE,		\
> 		      EGL_NO_SURFACE, EGL_NO_CONTEXT);	\
>       eglTerminate (display);				\
>       fprintf (stderr, "Missing: gl%s\n", #name + 1);	\
>       return False;					\
>     }
>
> #define LoadProc(name, ext, extname)			\
>   if (HaveEglExtension (extname))			\
>     I##name						\
>       = (void *) eglGetProcAddress ("egl" #name ext)
>
> #define LoadProcGl(name, ext, extname)			\
>   if (HaveGlExtension (extname))			\
>     I##name						\
>       = (void *) eglGetProcAddress ("gl" #name ext)
>
> #define CheckGlExtension(name)			\
>   if (!HaveGlExtension (name))			\
>     {						\
>       fprintf (stderr, "Missing %s\n", name);	\
> 						\
>       eglMakeCurrent (display, EGL_NO_SURFACE,	\
> 		      EGL_NO_SURFACE,		\
> 		      EGL_NO_CONTEXT);		\
>       eglTerminate (display);			\
>       return False;				\
>     }
>
> There will be no fontification of types or string constants inside the
> macro bodies.  CC Mode works fine.

Tree-sitter right now doesn’t parse the content of macro bodies. I think
it should be reasonable for tree-sitter-c to parse them. In the meantime
I’ll see if I can trick tree-sitter to parse it as normal C source.


> Now, insert the following code below the macro definition:
>
> static Visual *
> FindVisual (VisualID visual, int *depth)
> {
>   XVisualInfo vinfo, *visuals;
>   Visual *value;
>   int nvisuals;
>   const char *override;
> }
>
> visual, depth, visuals, value and override are not fontified as
> identifiers.  Line feed characters below the macros are also fontfied as
> "errors".

That should fixed by a recent change.

> Within the following code:
>
> static BufferActivityRecord *
> FindBufferActivityRecord (PictureBuffer *buffer, PictureTarget *target)
> {
>   BufferActivityRecord *record;
>
>   /* Look through the global activity list for a record matching
>      the given values.  */
>   record = all_activity.global_next;
>   while (record != &all_activity)
>     {
>       if (record->buffer == buffer
> 	  && record->target == target)
> 	return record;
>
>       record = record->global_next;
>     }
>
>   return NULL;
> }
>
> "record" is fontified as an identifier, but not within the condition in
> the while loop.

That’s expected, since the font-lock rule is to fontify LHS of
assignments (the first record), but not all appearances of identifiers.
Now we have font-lock rules that fontifies all occurrences of functions
and variables. I personally don’t find them useful, but if someone wants
it it’s there.

>
> Within the following code:
>
> static void
> compare_single_row_8bpc4 (unsigned char *data, unsigned char *xdata,
> 			  size_t size,
> 			  struct image_difference_statistics *statistics)
> {
>   unsigned char channel_a, channel_b;
>   int diff;
>   size_t i;
>
>   for (i = 0; i < size; ++i)
>     {
>       channel_a = data[i];
>       channel_b = xdata[i];
>
>       diff = (int) channel_b - (int) channel_a;
>       statistics->min_diff = MIN (statistics->min_diff,
> 				  diff);
>       statistics->max_diff = MAX (statistics->max_diff,
> 				  diff);
>     }
> }
>
> the "i" in "size_t i" is fontified as an identifier the first time, but
> not in future appearances in the function.

In the first appearance, i is a declaration, so it’s fontified. Like the
previous case, not all i are declaration or assignment, so they are not
all fontified. And now you can enable the "variable" feture to fontify
all of them.

>
> Within src/xterm.c, every appearance of ATOM_REFS_INIT is fontified in
> the error face.  Also, inside `xm_setup_dnd_targets',
>
>   xm_targets_table_header header;
>   xm_targets_table_rec **recs UNINIT;
>   xm_byte_order byteorder;

I think tree-sitter isn’t expecting another identifier (UNINIT) after
recs. Here if we don’t enable highlighting errors recs would be
highlighted fine, as an identifier.

>
> "recs" is fontified in the error face.
>
> 	      recs = xmalloc (sizeof *recs);
> 	      recs[0] = xmalloc (FLEXSIZEOF (struct xm_targets_table_rec,
> 					     targets, ntargets * 4));
>
> 	      recs[0]->n_targets = ntargets;
>
> "struct" is fontified in the error face.  Within
> x_sync_note_frame_times:

Tree-sitter thinks FLEXSIZEOF is a function call and struct
xm_targets_table_rec is an argument. We can add a heuristic here:
(ERROR (identifier)) is either type or struct/enum keyword. I should be
able to fix that.

> #ifdef FRAME_DEBUG
>   uint_fast64_t last_frame_ms = output->last_frame_time / 1000;
>   fprintf (stderr,
> 	   "Drawing the last frame took: %"PRIuFAST64" ms (%"PRIuFAST64")\n",
> 	   last_frame_ms, time);
> #endif
>
> "PRIuFAST64" is fontified in the error face.

Is this valid C? If it it, we should report to tree-sitter-c. If not, we
can add a heuristic.

Within
> x_dnd_begin_drag_and_drop:
>
> 	  if (x_dnd_movement_frame
> 	      /* FIXME: how come this can end up with movement frames
> 		 from other displays on GTK builds?  */
> 	      && (FRAME_X_DISPLAY (x_dnd_movement_frame)
> 		  == FRAME_X_DISPLAY (f))
> 	      /* If both those variables are false, then F is no
> 		 longer protected from deletion by Lisp code.  This
> 		 can only happen during the final iteration of the DND
> 		 event loop.  */
> 	      && (x_dnd_in_progress || x_dnd_waiting_for_finish))
> 	    {
> 	      XSETFRAME (frame_object, x_dnd_movement_frame);
>
> is fontified incorrectly: the "if" is not fontified at all, and the last
> three lines show up in the error faces.

It fontifies fine here. I guess it’s fixes by some changes I made,
although I don’t recall making changes related to if statements.

> In addition:
>
> Lisp_Object
> x_dnd_begin_drag_and_drop (struct frame *f, Time time, Atom xaction,
> 			   Lisp_Object return_frame, Atom *ask_action_list,
> 			   const char **ask_action_names, size_t n_ask_actions,
> 			   bool allow_current_frame, Atom *target_atoms,
> 			   int ntargets, Lisp_Object selection_target_list,
> 			   bool follow_tooltip)
>
> "Lisp_Object" and "x_dnd_begin_drag_and_drop" are not fontified at all.

They are fontified fine here.

> Here:
>
> /* Select for input extension events used by scroll bars.  This will
>    result in the corresponding core events not being generated for
>    SCROLL_BAR.  */
>
> MAYBE_UNUSED static void
> xi_select_scroll_bar_events (struct x_display_info *dpyinfo,
> 			     Window scroll_bar)
>
> "void" is fontified in the error face.

I’ll try to come up with a heuristic.

> Here, in handle_one_xevent:
>
> #ifdef HAVE_XINPUT2
>       if (event->type != GenericEvent)
> #endif
> 	any = x_any_window_to_frame (dpyinfo, event->xany.window);
> #ifdef HAVE_XINPUT2
>       else
> 	any = NULL;
> #endif
>
> the "else" is fontified as a type.

The "else" is separated from the "if" in the parse tree by the
preprocessor directive, I’ll report this to tree-sitter-c and see if
they can do anything.

>
> static int
> handle_one_xevent (struct x_display_info *dpyinfo,
> #ifndef HAVE_XINPUT2
> 		   const XEvent *event,
> #else
> 		   XEvent *event,
> #endif
> 		   int *finish, struct input_event *hold_quit)
>
> the "const XEvent *" is fontified in the error face.

Similar to the previous case. There could be better ways to handle
ifdef’s in tree-sitter-c, I don’t know if they are easy to implement.


> Here:
>
>     case GraphicsExpose:	/* This occurs when an XCopyArea's
>                                    source area was obscured or not
>                                    available.  */
>       f = x_window_to_frame (dpyinfo, event->xgraphicsexpose.drawable);
>       if (f)
>         {
>           expose_frame (f, event->xgraphicsexpose.x,
>                         event->xgraphicsexpose.y,
>                         event->xgraphicsexpose.width,
>                         event->xgraphicsexpose.height);
> #ifndef USE_TOOLKIT_SCROLL_BARS
> 	  x_scroll_bar_handle_exposure (f, (XEvent *) event);
> #endif
> #ifdef USE_GTK
> 	  x_clear_under_internal_border (f);
> #endif
> #ifdef HAVE_XDBE
> 	  show_back_buffer (f);
> #endif
>         }
> #ifdef USE_X_TOOLKIT
>       else
>         goto OTHER;
> #endif /* USE_X_TOOLKIT */
>       break;
>
> "goto OTHER" is fontified in the error face, and "else" as a type.

Same as above.

> Here:
>
> #ifdef HAVE_XINPUT2
> 	      if (event->xkey.time == pending_keystroke_time)
> 		{
> 		  source = xi_device_from_id (dpyinfo,
> 					      dpyinfo->pending_keystroke_source);
>
> 		  if (source)
> 		    inev.ie.device = source->name;
> 		}
> #endif
>
> "inev" is fontified in the error face.

Fontifies fine here, should be fixed.

> Here:
>
> #ifdef USE_GTK
>       /* See comment in EnterNotify above */
>       else if (dpyinfo->last_mouse_glyph_frame)
>         x_note_mouse_movement (dpyinfo->last_mouse_glyph_frame,
> 			       &event->xmotion, Qnil);
> #endif
>
> "else" and "dpyinfo" are fontified as types.

Same.

> #ifdef USE_MOTIF
> 	    Widget widget;
>
> 	    widget = XtWindowToWidget (dpyinfo->display,
> 				       event->xbutton.window);
>
> 	    if (widget && XmIsCascadeButton (widget)
> 		&& XtIsSensitive (widget))
> 	      {
> #endif
> 		if (!f->output_data.x->saved_menu_event)
> 		  f->output_data.x->saved_menu_event = xmalloc (sizeof *event);
> 		*f->output_data.x->saved_menu_event = *event;
> 		inev.ie.kind = MENU_BAR_ACTIVATE_EVENT;
> 		XSETFRAME (inev.ie.frame_or_window, f);
> 		*finish = X_EVENT_DROP;
> #ifdef USE_MOTIF
> 	      }
> #endif
>
> here, the last closing brace is fontified in the error face.

Still, blame #ifdef. I don’t have good ideas for these.

> Here:
>
> static void NO_INLINE
> x_error_quitter (Display *display, XErrorEvent *event)
> {
>   char buf[256], buf1[400 + INT_STRLEN_BOUND (int)
> 		      + INT_STRLEN_BOUND (unsigned long)];
>
> "NO_INLINE" and "unsigned long" are fontified in the error face.

I’ll try heuristics.

Yuan





^ permalink raw reply	[flat|nested] 16+ messages in thread

* bug#59468: 29.0.50; c-ts-mode cannot fontify after macros are encountered
  2022-11-24  0:03   ` Yuan Fu
@ 2022-11-24  0:36     ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
  2022-11-24  9:19       ` Eli Zaretskii
  2022-11-26  2:39     ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
  1 sibling, 1 reply; 16+ messages in thread
From: Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors @ 2022-11-24  0:36 UTC (permalink / raw)
  To: Yuan Fu; +Cc: 59468

Yuan Fu <casouri@gmail.com> writes:

> Po Lu <luangruo@yahoo.com> writes:
>
> Ok, I had a look at each of these cases, comments:
>
>> Insert the following code in a buffer, and enable c-ts-mode.
>>
>> #define CheckExtension(name)				\
>>   if (!name)						\
>>     {							\
>>       if (display)					\
>> 	eglTerminate (display);				\
>>       fprintf (stderr, "Missing: egl%s\n", #name + 1);	\
>>       return False;					\
>>     }
>>
>> #define CheckExtensionGl(name)				\
>>   if (!name)						\
>>     {							\
>>       /* If the context remains current, then nothing	\
>> 	 will get released upon eglTerminate.  */	\
>>       eglMakeCurrent (display, EGL_NO_SURFACE,		\
>> 		      EGL_NO_SURFACE, EGL_NO_CONTEXT);	\
>>       eglTerminate (display);				\
>>       fprintf (stderr, "Missing: gl%s\n", #name + 1);	\
>>       return False;					\
>>     }
>>
>> #define LoadProc(name, ext, extname)			\
>>   if (HaveEglExtension (extname))			\
>>     I##name						\
>>       = (void *) eglGetProcAddress ("egl" #name ext)
>>
>> #define LoadProcGl(name, ext, extname)			\
>>   if (HaveGlExtension (extname))			\
>>     I##name						\
>>       = (void *) eglGetProcAddress ("gl" #name ext)
>>
>> #define CheckGlExtension(name)			\
>>   if (!HaveGlExtension (name))			\
>>     {						\
>>       fprintf (stderr, "Missing %s\n", name);	\
>> 						\
>>       eglMakeCurrent (display, EGL_NO_SURFACE,	\
>> 		      EGL_NO_SURFACE,		\
>> 		      EGL_NO_CONTEXT);		\
>>       eglTerminate (display);			\
>>       return False;				\
>>     }
>>
>> There will be no fontification of types or string constants inside the
>> macro bodies.  CC Mode works fine.
>
> Tree-sitter right now doesn’t parse the content of macro bodies. I think
> it should be reasonable for tree-sitter-c to parse them. In the meantime
> I’ll see if I can trick tree-sitter to parse it as normal C source.
>
>
>> Now, insert the following code below the macro definition:
>>
>> static Visual *
>> FindVisual (VisualID visual, int *depth)
>> {
>>   XVisualInfo vinfo, *visuals;
>>   Visual *value;
>>   int nvisuals;
>>   const char *override;
>> }
>>
>> visual, depth, visuals, value and override are not fontified as
>> identifiers.  Line feed characters below the macros are also fontfied as
>> "errors".
>
> That should fixed by a recent change.
>
>> Within the following code:
>>
>> static BufferActivityRecord *
>> FindBufferActivityRecord (PictureBuffer *buffer, PictureTarget *target)
>> {
>>   BufferActivityRecord *record;
>>
>>   /* Look through the global activity list for a record matching
>>      the given values.  */
>>   record = all_activity.global_next;
>>   while (record != &all_activity)
>>     {
>>       if (record->buffer == buffer
>> 	  && record->target == target)
>> 	return record;
>>
>>       record = record->global_next;
>>     }
>>
>>   return NULL;
>> }
>>
>> "record" is fontified as an identifier, but not within the condition in
>> the while loop.
>
> That’s expected, since the font-lock rule is to fontify LHS of
> assignments (the first record), but not all appearances of identifiers.
> Now we have font-lock rules that fontifies all occurrences of functions
> and variables. I personally don’t find them useful, but if someone wants
> it it’s there.
>
>>
>> Within the following code:
>>
>> static void
>> compare_single_row_8bpc4 (unsigned char *data, unsigned char *xdata,
>> 			  size_t size,
>> 			  struct image_difference_statistics *statistics)
>> {
>>   unsigned char channel_a, channel_b;
>>   int diff;
>>   size_t i;
>>
>>   for (i = 0; i < size; ++i)
>>     {
>>       channel_a = data[i];
>>       channel_b = xdata[i];
>>
>>       diff = (int) channel_b - (int) channel_a;
>>       statistics->min_diff = MIN (statistics->min_diff,
>> 				  diff);
>>       statistics->max_diff = MAX (statistics->max_diff,
>> 				  diff);
>>     }
>> }
>>
>> the "i" in "size_t i" is fontified as an identifier the first time, but
>> not in future appearances in the function.
>
> In the first appearance, i is a declaration, so it’s fontified. Like the
> previous case, not all i are declaration or assignment, so they are not
> all fontified. And now you can enable the "variable" feture to fontify
> all of them.
>
>>
>> Within src/xterm.c, every appearance of ATOM_REFS_INIT is fontified in
>> the error face.  Also, inside `xm_setup_dnd_targets',
>>
>>   xm_targets_table_header header;
>>   xm_targets_table_rec **recs UNINIT;
>>   xm_byte_order byteorder;
>
> I think tree-sitter isn’t expecting another identifier (UNINIT) after
> recs. Here if we don’t enable highlighting errors recs would be
> highlighted fine, as an identifier.
>
>>
>> "recs" is fontified in the error face.
>>
>> 	      recs = xmalloc (sizeof *recs);
>> 	      recs[0] = xmalloc (FLEXSIZEOF (struct xm_targets_table_rec,
>> 					     targets, ntargets * 4));
>>
>> 	      recs[0]->n_targets = ntargets;
>>
>> "struct" is fontified in the error face.  Within
>> x_sync_note_frame_times:
>
> Tree-sitter thinks FLEXSIZEOF is a function call and struct
> xm_targets_table_rec is an argument. We can add a heuristic here:
> (ERROR (identifier)) is either type or struct/enum keyword. I should be
> able to fix that.
>
>> #ifdef FRAME_DEBUG
>>   uint_fast64_t last_frame_ms = output->last_frame_time / 1000;
>>   fprintf (stderr,
>> 	   "Drawing the last frame took: %"PRIuFAST64" ms (%"PRIuFAST64")\n",
>> 	   last_frame_ms, time);
>> #endif
>>
>> "PRIuFAST64" is fontified in the error face.

> Is this valid C? If it it, we should report to tree-sitter-c. If not, we
> can add a heuristic.

This is valid C.  PRIuFAST64 is a standard macro containing the format
specifier for a uint_fast64_t.

> It fontifies fine here. I guess it’s fixes by some changes I made,
> although I don’t recall making changes related to if statements.

Maybe you scrolled up the file instead of down?  That sometimes matters.

> The "else" is separated from the "if" in the parse tree by the
> preprocessor directive, I’ll report this to tree-sitter-c and see if
> they can do anything.
>
>>
>> static int
>> handle_one_xevent (struct x_display_info *dpyinfo,
>> #ifndef HAVE_XINPUT2
>> 		   const XEvent *event,
>> #else
>> 		   XEvent *event,
>> #endif
>> 		   int *finish, struct input_event *hold_quit)
>>
>> the "const XEvent *" is fontified in the error face.
>
> Similar to the previous case. There could be better ways to handle
> ifdef’s in tree-sitter-c, I don’t know if they are easy to implement.
>
>
>> Here:
>>
>>     case GraphicsExpose:	/* This occurs when an XCopyArea's
>>                                    source area was obscured or not
>>                                    available.  */
>>       f = x_window_to_frame (dpyinfo, event->xgraphicsexpose.drawable);
>>       if (f)
>>         {
>>           expose_frame (f, event->xgraphicsexpose.x,
>>                         event->xgraphicsexpose.y,
>>                         event->xgraphicsexpose.width,
>>                         event->xgraphicsexpose.height);
>> #ifndef USE_TOOLKIT_SCROLL_BARS
>> 	  x_scroll_bar_handle_exposure (f, (XEvent *) event);
>> #endif
>> #ifdef USE_GTK
>> 	  x_clear_under_internal_border (f);
>> #endif
>> #ifdef HAVE_XDBE
>> 	  show_back_buffer (f);
>> #endif
>>         }
>> #ifdef USE_X_TOOLKIT
>>       else
>>         goto OTHER;
>> #endif /* USE_X_TOOLKIT */
>>       break;
>>
>> "goto OTHER" is fontified in the error face, and "else" as a type.
>
> Same as above.
>
>> Here:
>>
>> #ifdef HAVE_XINPUT2
>> 	      if (event->xkey.time == pending_keystroke_time)
>> 		{
>> 		  source = xi_device_from_id (dpyinfo,
>> 					      dpyinfo->pending_keystroke_source);
>>
>> 		  if (source)
>> 		    inev.ie.device = source->name;
>> 		}
>> #endif
>>
>> "inev" is fontified in the error face.
>
> Fontifies fine here, should be fixed.
>
>> Here:
>>
>> #ifdef USE_GTK
>>       /* See comment in EnterNotify above */
>>       else if (dpyinfo->last_mouse_glyph_frame)
>>         x_note_mouse_movement (dpyinfo->last_mouse_glyph_frame,
>> 			       &event->xmotion, Qnil);
>> #endif
>>
>> "else" and "dpyinfo" are fontified as types.
>
> Same.
>
>> #ifdef USE_MOTIF
>> 	    Widget widget;
>>
>> 	    widget = XtWindowToWidget (dpyinfo->display,
>> 				       event->xbutton.window);
>>
>> 	    if (widget && XmIsCascadeButton (widget)
>> 		&& XtIsSensitive (widget))
>> 	      {
>> #endif
>> 		if (!f->output_data.x->saved_menu_event)
>> 		  f->output_data.x->saved_menu_event = xmalloc (sizeof *event);
>> 		*f->output_data.x->saved_menu_event = *event;
>> 		inev.ie.kind = MENU_BAR_ACTIVATE_EVENT;
>> 		XSETFRAME (inev.ie.frame_or_window, f);
>> 		*finish = X_EVENT_DROP;
>> #ifdef USE_MOTIF
>> 	      }
>> #endif
>>
>> here, the last closing brace is fontified in the error face.
>
> Still, blame #ifdef. I don’t have good ideas for these.

Can't #ifdef simply not be passed to tree-sitter, if it is so bad at
understanding them?





^ permalink raw reply	[flat|nested] 16+ messages in thread

* bug#59468: 29.0.50; c-ts-mode cannot fontify after macros are encountered
  2022-11-24  0:36     ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
@ 2022-11-24  9:19       ` Eli Zaretskii
  2022-11-24 10:36         ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
  0 siblings, 1 reply; 16+ messages in thread
From: Eli Zaretskii @ 2022-11-24  9:19 UTC (permalink / raw)
  To: Po Lu; +Cc: casouri, 59468

> Cc: 59468@debbugs.gnu.org
> Date: Thu, 24 Nov 2022 08:36:13 +0800
> From:  Po Lu via "Bug reports for GNU Emacs,
>  the Swiss army knife of text editors" <bug-gnu-emacs@gnu.org>
> 
> >> #ifdef FRAME_DEBUG
> >>   uint_fast64_t last_frame_ms = output->last_frame_time / 1000;
> >>   fprintf (stderr,
> >> 	   "Drawing the last frame took: %"PRIuFAST64" ms (%"PRIuFAST64")\n",
> >> 	   last_frame_ms, time);
> >> #endif
> >>
> >> "PRIuFAST64" is fontified in the error face.
> 
> > Is this valid C? If it it, we should report to tree-sitter-c. If not, we
> > can add a heuristic.
> 
> This is valid C.  PRIuFAST64 is a standard macro containing the format
> specifier for a uint_fast64_t.
> 
> > It fontifies fine here. I guess it’s fixes by some changes I made,
> > although I don’t recall making changes related to if statements.
> 
> Maybe you scrolled up the file instead of down?  That sometimes matters.

I don't see an error, either.  I did scroll from above.





^ permalink raw reply	[flat|nested] 16+ messages in thread

* bug#59468: 29.0.50; c-ts-mode cannot fontify after macros are encountered
  2022-11-24  9:19       ` Eli Zaretskii
@ 2022-11-24 10:36         ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
  2022-11-24 10:55           ` Eli Zaretskii
  0 siblings, 1 reply; 16+ messages in thread
From: Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors @ 2022-11-24 10:36 UTC (permalink / raw)
  To: Eli Zaretskii; +Cc: casouri, 59468

Eli Zaretskii <eliz@gnu.org> writes:

>> Cc: 59468@debbugs.gnu.org
>> Date: Thu, 24 Nov 2022 08:36:13 +0800
>> From:  Po Lu via "Bug reports for GNU Emacs,
>>  the Swiss army knife of text editors" <bug-gnu-emacs@gnu.org>
>> 
>> >> #ifdef FRAME_DEBUG
>> >>   uint_fast64_t last_frame_ms = output->last_frame_time / 1000;
>> >>   fprintf (stderr,
>> >> 	   "Drawing the last frame took: %"PRIuFAST64" ms (%"PRIuFAST64")\n",
>> >> 	   last_frame_ms, time);
>> >> #endif
>> >>
>> >> "PRIuFAST64" is fontified in the error face.
>> 
>> > Is this valid C? If it it, we should report to tree-sitter-c. If not, we
>> > can add a heuristic.
>> 
>> This is valid C.  PRIuFAST64 is a standard macro containing the format
>> specifier for a uint_fast64_t.
>> 
>> > It fontifies fine here. I guess it’s fixes by some changes I made,
>> > although I don’t recall making changes related to if statements.
>> 
>> Maybe you scrolled up the file instead of down?  That sometimes matters.
>
> I don't see an error, either.  I did scroll from above.

Try going to the end and scrolling up using M-v.  That makes the
error(s) show up for me.





^ permalink raw reply	[flat|nested] 16+ messages in thread

* bug#59468: 29.0.50; c-ts-mode cannot fontify after macros are encountered
  2022-11-24 10:36         ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
@ 2022-11-24 10:55           ` Eli Zaretskii
  2022-11-24 12:12             ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
  0 siblings, 1 reply; 16+ messages in thread
From: Eli Zaretskii @ 2022-11-24 10:55 UTC (permalink / raw)
  To: Po Lu; +Cc: casouri, 59468

> From: Po Lu <luangruo@yahoo.com>
> Cc: casouri@gmail.com,  59468@debbugs.gnu.org
> Date: Thu, 24 Nov 2022 18:36:06 +0800
> 
> Eli Zaretskii <eliz@gnu.org> writes:
> 
> >> Cc: 59468@debbugs.gnu.org
> >> Date: Thu, 24 Nov 2022 08:36:13 +0800
> >> From:  Po Lu via "Bug reports for GNU Emacs,
> >>  the Swiss army knife of text editors" <bug-gnu-emacs@gnu.org>
> >> 
> >> >> #ifdef FRAME_DEBUG
> >> >>   uint_fast64_t last_frame_ms = output->last_frame_time / 1000;
> >> >>   fprintf (stderr,
> >> >> 	   "Drawing the last frame took: %"PRIuFAST64" ms (%"PRIuFAST64")\n",
> >> >> 	   last_frame_ms, time);
> >> >> #endif
> >> >>
> >> >> "PRIuFAST64" is fontified in the error face.
> >> 
> >> > Is this valid C? If it it, we should report to tree-sitter-c. If not, we
> >> > can add a heuristic.
> >> 
> >> This is valid C.  PRIuFAST64 is a standard macro containing the format
> >> specifier for a uint_fast64_t.
> >> 
> >> > It fontifies fine here. I guess it’s fixes by some changes I made,
> >> > although I don’t recall making changes related to if statements.
> >> 
> >> Maybe you scrolled up the file instead of down?  That sometimes matters.
> >
> > I don't see an error, either.  I did scroll from above.
> 
> Try going to the end and scrolling up using M-v.  That makes the
> error(s) show up for me.

Still no cigar.

Are you using the latest master?





^ permalink raw reply	[flat|nested] 16+ messages in thread

* bug#59468: 29.0.50; c-ts-mode cannot fontify after macros are encountered
  2022-11-24 10:55           ` Eli Zaretskii
@ 2022-11-24 12:12             ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
  0 siblings, 0 replies; 16+ messages in thread
From: Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors @ 2022-11-24 12:12 UTC (permalink / raw)
  To: Eli Zaretskii; +Cc: casouri, 59468

Eli Zaretskii <eliz@gnu.org> writes:

> Still no cigar.
>
> Are you using the latest master?

Yes.  However, that specific problem (with
x_dnd_begin_drag_and_drop) went away after I deleted
treesit.elc.





^ permalink raw reply	[flat|nested] 16+ messages in thread

* bug#59468: 29.0.50; c-ts-mode cannot fontify after macros are encountered
  2022-11-24  0:03   ` Yuan Fu
  2022-11-24  0:36     ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
@ 2022-11-26  2:39     ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
  1 sibling, 0 replies; 16+ messages in thread
From: Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors @ 2022-11-26  2:39 UTC (permalink / raw)
  To: Yuan Fu; +Cc: 59468

[-- Attachment #1: Type: text/plain, Size: 1008 bytes --]

Yuan Fu <casouri@gmail.com> writes:

> Within
>> x_dnd_begin_drag_and_drop:
>>
>> 	  if (x_dnd_movement_frame
>> 	      /* FIXME: how come this can end up with movement frames
>> 		 from other displays on GTK builds?  */
>> 	      && (FRAME_X_DISPLAY (x_dnd_movement_frame)
>> 		  == FRAME_X_DISPLAY (f))
>> 	      /* If both those variables are false, then F is no
>> 		 longer protected from deletion by Lisp code.  This
>> 		 can only happen during the final iteration of the DND
>> 		 event loop.  */
>> 	      && (x_dnd_in_progress || x_dnd_waiting_for_finish))
>> 	    {
>> 	      XSETFRAME (frame_object, x_dnd_movement_frame);
>>
>> is fontified incorrectly: the "if" is not fontified at all, and the last
>> three lines show up in the error faces.
>
> It fontifies fine here. I guess it’s fixes by some changes I made,
> although I don’t recall making changes related to if statements.

Ok, so with a fresh build from master it turns out that this still isn't
fixed.


[-- Attachment #2: Screenshot from 2022-11-26 10-31-22.png --]
[-- Type: image/png, Size: 28146 bytes --]

[-- Attachment #3: Type: text/plain, Size: 715 bytes --]


>> In addition:
>>
>> Lisp_Object
>> x_dnd_begin_drag_and_drop (struct frame *f, Time time, Atom xaction,
>> 			   Lisp_Object return_frame, Atom *ask_action_list,
>> 			   const char **ask_action_names, size_t n_ask_actions,
>> 			   bool allow_current_frame, Atom *target_atoms,
>> 			   int ntargets, Lisp_Object selection_target_list,
>> 			   bool follow_tooltip)
>>
>> "Lisp_Object" and "x_dnd_begin_drag_and_drop" are not fontified at all.
>
> They are fontified fine here.

This appeared to be fixed, but it turns out that it was not actually
fixed.  "Lisp_Object" is fontified as an identifier, despite it being a
type, and "x_dnd_begin_drag_and_drop" is fontified as an identifier, not
a function name.


[-- Attachment #4: Screenshot from 2022-11-26 10-33-09.png --]
[-- Type: image/png, Size: 31781 bytes --]

[-- Attachment #5: Type: text/plain, Size: 75 bytes --]


Also, the end of x_dnd_begin_drag_and_drop is now fontified as an error:


[-- Attachment #6: Screenshot from 2022-11-26 10-37-33.png --]
[-- Type: image/png, Size: 20518 bytes --]

[-- Attachment #7: Type: text/plain, Size: 846 bytes --]


>> Here:
>>
>> /* Select for input extension events used by scroll bars.  This will
>>    result in the corresponding core events not being generated for
>>    SCROLL_BAR.  */
>>
>> MAYBE_UNUSED static void
>> xi_select_scroll_bar_events (struct x_display_info *dpyinfo,
>> 			     Window scroll_bar)
>>
>> "void" is fontified in the error face.
>
> I’ll try to come up with a heuristic.

Now, MAYBE_UNUSED is fontified as a type, and void as an identifier.  :(

>> #ifdef USE_GTK
>>       /* See comment in EnterNotify above */
>>       else if (dpyinfo->last_mouse_glyph_frame)
>>         x_note_mouse_movement (dpyinfo->last_mouse_glyph_frame,
>> 			       &event->xmotion, Qnil);
>> #endif
>>
>> "else" and "dpyinfo" are fontified as types.
>
> Same.

This seemed to work for a while, but it's broken again.


[-- Attachment #8: Screenshot from 2022-11-26 10-34-25.png --]
[-- Type: image/png, Size: 25178 bytes --]

[-- Attachment #9: Type: text/plain, Size: 194 bytes --]


Also, here's a new problem:

  char buf[256], buf1[400 + INT_STRLEN_BOUND (int)
		      + INT_STRLEN_BOUND (unsigned long)];

"int" and "unsigned long" are fontified as identifiers, not types!

^ permalink raw reply	[flat|nested] 16+ messages in thread

end of thread, other threads:[~2022-11-26  2:39 UTC | newest]

Thread overview: 16+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
     [not found] <87k03n4v0r.fsf.ref@yahoo.com>
2022-11-22  1:50 ` bug#59468: 29.0.50; c-ts-mode cannot fontify after macros are encountered Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
2022-11-22 20:11   ` Yuan Fu
2022-11-23  0:41     ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
2022-11-23  3:30     ` Eli Zaretskii
2022-11-23  4:13       ` Yuan Fu
2022-11-23  7:36         ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
2022-11-23 12:53           ` Eli Zaretskii
2022-11-23 12:43         ` Eli Zaretskii
2022-11-23  1:27   ` Yuan Fu via Bug reports for GNU Emacs, the Swiss army knife of text editors
2022-11-24  0:03   ` Yuan Fu
2022-11-24  0:36     ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
2022-11-24  9:19       ` Eli Zaretskii
2022-11-24 10:36         ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
2022-11-24 10:55           ` Eli Zaretskii
2022-11-24 12:12             ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors
2022-11-26  2:39     ` Po Lu via Bug reports for GNU Emacs, the Swiss army knife of text editors

Code repositories for project(s) associated with this public inbox

	https://git.savannah.gnu.org/cgit/emacs.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).