Re: Identifying the face between STRETCH and right fringe.

[Robert Pluim <rpluim@gmail.com>]

Eli Zaretskii <eliz@gnu.org> writes:

>> From: Robert Pluim <rpluim@gmail.com>
>> Cc: emacs-devel@gnu.org
>> Date: Tue, 27 Nov 2018 14:55:23 +0100
>> 
>> >> When we get here, face->background == 1, and FRAME_BACKGROUND_PIXEL ==
>> >> 0xfffeffff
>> >> 
>> >> Looking through nsfns.m, the problem becomes obvious: the NS port uses
>> >> indices into a color table to specify the background colour of faces,
>> >> and FRAME_BACKGROUND_PIXEL is an RGBA value.
>> >
>> > Why does NS use indices here, and not RGBA values?
>> 
>> Probably because NS doesnʼt really use the RGBA values directly at
>> all, but uses the indices all the time.
>
> So where does the RGBA value in FRAME_BACKGROUND_PIXEL come from?
>
>> > If no better/cleaner idea emerges, how about having an NS-specific
>> > code here that computed the it->face's background RGBA by indexing
>> > into the color table, before comparing that with
>> > FRAME_BACKGROUND_PIXEL?
>> 
>> Thatʼs doable.
>
> Thanks.
>
> Note that there are a couple more of such comparisons, so perhaps a
> macro is in order, with different implementations for NS and the rest.

I love those rare moments when I implement something one way, and then
Eli tells me to do it that way :-)

In theory there are a couple of other fields in the face structure
that might need the same type of treatment, but I didnʼt see any code
comparing them to RGBA pixel values.

I suck at naming things, comments welcome.

diff --git i/src/dispextern.h w/src/dispextern.h
index 579665c2ff..776d14080e 100644
--- i/src/dispextern.h
+++ w/src/dispextern.h
@@ -74,10 +74,13 @@ typedef HDC XImagePtr_or_DC;
 
 #ifdef HAVE_NS
 #include "nsgui.h"
+#define FACE_COLOR_TO_PIXEL(face_color, frame) ns_color_index_to_rgba(face_color, frame)
 /* Following typedef needed to accommodate the MSDOS port, believe it or not.  */
 typedef struct ns_display_info Display_Info;
 typedef Pixmap XImagePtr;
 typedef XImagePtr XImagePtr_or_DC;
+#else
+#define FACE_COLOR_TO_PIXEL(face_color, frame) face_color
 #endif
 
 #ifdef HAVE_WINDOW_SYSTEM
diff --git i/src/nsgui.h w/src/nsgui.h
index 4e7d7d35da..0c29eed5e4 100644
--- i/src/nsgui.h
+++ w/src/nsgui.h
@@ -73,6 +73,8 @@ typedef unichar XChar2b;
 #define XCHAR2B_BYTE2(chp) \
   (*(chp) & 0x00ff)
 
+/* For xdisp.c */
+extern unsigned long ns_color_index_to_rgba(int idx, struct frame *f);
 
 /* XXX: xfaces requires these structures, but the question is are we
         forced to use them?  */
diff --git i/src/nsterm.m w/src/nsterm.m
index bcc23ffeaf..cacfc57fd7 100644
--- i/src/nsterm.m
+++ w/src/nsterm.m
@@ -2356,6 +2356,22 @@ so some key presses (TAB) are swallowed by the system.  */
   return 1;
 }
 
+/* Convert an index into the color table into an RGBA value.  Used in
+   xdisp.c:extend_face_to_end_of_line when comparing faces and frame
+   color values.  */
+
+unsigned long
+ns_color_index_to_rgba(int idx, struct frame *f)
+{
+  NSColor *col;
+  col = ns_lookup_indexed_color (idx, f);
+
+  EmacsCGFloat r, g, b, a;
+  [col getRed: &r green: &g blue: &b alpha: &a];
+
+  return ARGB_TO_ULONG((int)(a*255),
+                       (int)(r*255), (int)(g*255), (int)(b*255));
+}
 
 void
 ns_query_color(void *col, XColor *color_def, int setPixel)
diff --git i/src/xdisp.c w/src/xdisp.c
index fa7691cdd0..e9048afb86 100644
--- i/src/xdisp.c
+++ w/src/xdisp.c
@@ -20287,7 +20287,7 @@ extend_face_to_end_of_line (struct it *it)
   if (FRAME_WINDOW_P (f)
       && MATRIX_ROW_DISPLAYS_TEXT_P (it->glyph_row)
       && face->box == FACE_NO_BOX
-      && face->background == FRAME_BACKGROUND_PIXEL (f)
+      && FACE_COLOR_TO_PIXEL(face->background, f) == FRAME_BACKGROUND_PIXEL (f)
 #ifdef HAVE_WINDOW_SYSTEM
       && !face->stipple
 #endif
@@ -20432,7 +20432,7 @@ extend_face_to_end_of_line (struct it *it)
 	  && (it->glyph_row->used[LEFT_MARGIN_AREA]
 	      < WINDOW_LEFT_MARGIN_WIDTH (it->w))
 	  && !it->glyph_row->mode_line_p
-	  && default_face->background != FRAME_BACKGROUND_PIXEL (f))
+	  && FACE_COLOR_TO_PIXEL(face->background, f) != FRAME_BACKGROUND_PIXEL (f))
 	{
 	  struct glyph *g = it->glyph_row->glyphs[LEFT_MARGIN_AREA];
 	  struct glyph *e = g + it->glyph_row->used[LEFT_MARGIN_AREA];
@@ -20473,7 +20473,7 @@ extend_face_to_end_of_line (struct it *it)
 	  && (it->glyph_row->used[RIGHT_MARGIN_AREA]
 	      < WINDOW_RIGHT_MARGIN_WIDTH (it->w))
 	  && !it->glyph_row->mode_line_p
-	  && default_face->background != FRAME_BACKGROUND_PIXEL (f))
+	  && FACE_COLOR_TO_PIXEL(face->background, f) != FRAME_BACKGROUND_PIXEL (f))
 	{
 	  struct glyph *g = it->glyph_row->glyphs[RIGHT_MARGIN_AREA];
 	  struct glyph *e = g + it->glyph_row->used[RIGHT_MARGIN_AREA];