Re: Process output truncation when using UTF-8

[Kenichi Handa <handa@m17n.org>]

In article <874r3f1ocb.fsf@zamazal.org>, Milan Zamazal <pdm@zamazal.org> writes:
> When I start a process, set its input and output encoding
> to utf-8 and send a long string to it through
> process-send-string, usually some final part of the sent
> string is missing on the recipient's side.

Thank you for the report.  I've just installed the attached
change in RC and HEAD.   It should be applicable also to
Emacs 21.3.

---
Ken'ichi HANDA
handa@m17n.org

2003-05-30  Kenichi Handa  <handa@m17n.org>

	* coding.c (ccl_coding_driver): Set ccl->eight_bit_control
	properly before calling ccl_driver.

	* ccl.h (struct ccl_program) <eight_bit_control: Comment fixed.

	* ccl.c (CCL_WRITE_CHAR): Increment extra_bytes only when it is
	nonzero.
	(ccl_driver): Initialize extra_bytes to ccl->eight_bit_control.
	(setup_ccl_program): Initialize ccl->eight_bit_control to zero.

Index: coding.c
===================================================================
RCS file: /cvsroot/emacs/emacs/src/coding.c,v
retrieving revision 1.222.4.16
retrieving revision 1.222.4.17
diff -u -c -r1.222.4.16 -r1.222.4.17
cvs server: conflicting specifications of output style
*** coding.c	7 Mar 2003 04:37:21 -0000	1.222.4.16
--- coding.c	30 May 2003 08:12:19 -0000	1.222.4.17
***************
*** 4477,4483 ****
--- 4477,4486 ----
        if (ccl->eol_type ==CODING_EOL_UNDECIDED)
  	ccl->eol_type = CODING_EOL_LF;
        ccl->cr_consumed = coding->spec.ccl.cr_carryover;
+       ccl->eight_bit_control = coding->dst_multibyte;
      }
+   else
+     ccl->eight_bit_control = 1;
    ccl->multibyte = coding->src_multibyte;
    if (coding->spec.ccl.eight_bit_carryover[0] != 0)
      {
Index: ccl.h
===================================================================
RCS file: /cvsroot/emacs/emacs/src/ccl.h,v
retrieving revision 1.16
retrieving revision 1.16.12.1
diff -u -c -r1.16 -r1.16.12.1
cvs server: conflicting specifications of output style
*** ccl.h	27 Feb 2001 03:29:08 -0000	1.16
--- ccl.h	30 May 2003 08:12:38 -0000	1.16.12.1
***************
*** 65,72 ****
  				   system.  */
    int suppress_error;		/* If nonzero, don't insert error
  				   message in the output.  */
!   int eight_bit_control;	/* Set to nonzero if CCL_WRITE_CHAR
! 				   writes eight-bit-control char.  */
  };
  
  /* This data type is used for the spec field of the structure
--- 65,75 ----
  				   system.  */
    int suppress_error;		/* If nonzero, don't insert error
  				   message in the output.  */
!   int eight_bit_control;	/* If nonzero, ccl_driver counts all
! 				   eight-bit-control bytes written by
! 				   CCL_WRITE_CHAR.  After execution,
! 				   if no such byte is written, set
! 				   this value to zero.  */
  };
  
  /* This data type is used for the spec field of the structure
Index: ccl.c
===================================================================
RCS file: /cvsroot/emacs/emacs/src/ccl.c,v
retrieving revision 1.71
retrieving revision 1.71.12.1
diff -u -c -r1.71 -r1.71.12.1
cvs server: conflicting specifications of output style
*** ccl.c	17 May 2001 09:09:14 -0000	1.71
--- ccl.c	30 May 2003 08:13:01 -0000	1.71.12.1
***************
*** 717,723 ****
  	if (bytes == 1)							\
  	  {								\
  	    *dst++ = (ch);						\
! 	    if ((ch) >= 0x80 && (ch) < 0xA0)				\
  	      /* We may have to convert this eight-bit char to		\
  		 multibyte form later.  */				\
  	      extra_bytes++;						\
--- 717,723 ----
  	if (bytes == 1)							\
  	  {								\
  	    *dst++ = (ch);						\
! 	    if (extra_bytes && (ch) >= 0x80 && (ch) < 0xA0)		\
  	      /* We may have to convert this eight-bit char to		\
  		 multibyte form later.  */				\
  	      extra_bytes++;						\
***************
*** 731,736 ****
--- 731,737 ----
        CCL_SUSPEND (CCL_STAT_SUSPEND_BY_DST);				\
    } while (0)
  
+ 
  /* Encode one character CH to multibyte form and write to the current
     output buffer.  The output bytes always forms a valid multibyte
     sequence.  */
***************
*** 874,880 ****
       each of them will be converted to multibyte form of 2-byte
       sequence.  For that conversion, we remember how many more bytes
       we must keep in DESTINATION in this variable.  */
!   int extra_bytes = 0;
  
    if (ic >= ccl->eof_ic)
      ic = CCL_HEADER_MAIN;
--- 875,881 ----
       each of them will be converted to multibyte form of 2-byte
       sequence.  For that conversion, we remember how many more bytes
       we must keep in DESTINATION in this variable.  */
!   int extra_bytes = ccl->eight_bit_control;
  
    if (ic >= ccl->eof_ic)
      ic = CCL_HEADER_MAIN;
***************
*** 1849,1855 ****
    ccl->ic = ic;
    ccl->stack_idx = stack_idx;
    ccl->prog = ccl_prog;
!   ccl->eight_bit_control = (extra_bytes > 0);
    if (consumed)
      *consumed = src - source;
    return (dst ? dst - destination : 0);
--- 1850,1856 ----
    ccl->ic = ic;
    ccl->stack_idx = stack_idx;
    ccl->prog = ccl_prog;
!   ccl->eight_bit_control = (extra_bytes > 1);
    if (consumed)
      *consumed = src - source;
    return (dst ? dst - destination : 0);
***************
*** 2004,2009 ****
--- 2005,2011 ----
    ccl->stack_idx = 0;
    ccl->eol_type = CODING_EOL_LF;
    ccl->suppress_error = 0;
+   ccl->eight_bit_control = 0;
    return 0;
  }