GitHub

Original file line numberDiff line numberDiff line change

@@ -166,6 +166,7 @@ mrb_value mrb_str_aref(mrb_state *mrb, mrb_value str, mrb_value idx, mrb_value l

166166

uint32_t mrb_byte_hash(const uint8_t*, mrb_int);

167167

uint32_t mrb_byte_hash_step(const uint8_t*, mrb_int, uint32_t);

168168
169+

mrb_int mrb_utf8_to_buf(char *buf, uint32_t cp);

169170

#ifdef MRB_UTF8_STRING

170171

mrb_int mrb_utf8len(const char *str, const char *end);

171172

mrb_int mrb_utf8_strlen(const char *str, mrb_int byte_len);

Original file line numberDiff line numberDiff line change

@@ -5375,30 +5375,8 @@ tokadd(parser_state *p, int32_t c)

53755375

len = 1;

53765376

}

53775377

else {

5378-

/* Unicode character */

5379-

c = -c;

5380-

if (c < 0x80) {

5381-

utf8[0] = (char)c;

5382-

len = 1;

5383-

}

5384-

else if (c < 0x800) {

5385-

utf8[0] = (char)(0xC0 | (c >> 6));

5386-

utf8[1] = (char)(0x80 | (c & 0x3F));

5387-

len = 2;

5388-

}

5389-

else if (c < 0x10000) {

5390-

utf8[0] = (char)(0xE0 | (c >> 12) );

5391-

utf8[1] = (char)(0x80 | ((c >> 6) & 0x3F));

5392-

utf8[2] = (char)(0x80 | ( c & 0x3F));

5393-

len = 3;

5394-

}

5395-

else {

5396-

utf8[0] = (char)(0xF0 | (c >> 18) );

5397-

utf8[1] = (char)(0x80 | ((c >> 12) & 0x3F));

5398-

utf8[2] = (char)(0x80 | ((c >> 6) & 0x3F));

5399-

utf8[3] = (char)(0x80 | ( c & 0x3F));

5400-

len = 4;

5401-

}

5378+

/* Unicode character (negative c indicates codepoint) */

5379+

len = (int)mrb_utf8_to_buf(utf8, (uint32_t)(-c));

54025380

}

54035381

if (p->tidx+len >= p->tsiz) {

54045382

if (p->tsiz >= MRB_PARSER_TOKBUF_MAX) {

Original file line numberDiff line numberDiff line change

@@ -12613,30 +12613,8 @@ tokadd(parser_state *p, int32_t c)

1261312613

len = 1;

1261412614

}

1261512615

else {

12616-

/* Unicode character */

12617-

c = -c;

12618-

if (c < 0x80) {

12619-

utf8[0] = (char)c;

12620-

len = 1;

12621-

}

12622-

else if (c < 0x800) {

12623-

utf8[0] = (char)(0xC0 | (c >> 6));

12624-

utf8[1] = (char)(0x80 | (c & 0x3F));

12625-

len = 2;

12626-

}

12627-

else if (c < 0x10000) {

12628-

utf8[0] = (char)(0xE0 | (c >> 12) );

12629-

utf8[1] = (char)(0x80 | ((c >> 6) & 0x3F));

12630-

utf8[2] = (char)(0x80 | ( c & 0x3F));

12631-

len = 3;

12632-

}

12633-

else {

12634-

utf8[0] = (char)(0xF0 | (c >> 18) );

12635-

utf8[1] = (char)(0x80 | ((c >> 12) & 0x3F));

12636-

utf8[2] = (char)(0x80 | ((c >> 6) & 0x3F));

12637-

utf8[3] = (char)(0x80 | ( c & 0x3F));

12638-

len = 4;

12639-

}

12616+

/* Unicode character (negative c indicates codepoint) */

12617+

len = (int)mrb_utf8_to_buf(utf8, (uint32_t)(-c));

1264012618

}

1264112619

if (p->tidx+len >= p->tsiz) {

1264212620

if (p->tsiz >= MRB_PARSER_TOKBUF_MAX) {

Original file line numberDiff line numberDiff line change

@@ -1124,24 +1124,7 @@ io_putc(mrb_state *mrb, mrb_value io)

11241124

if (len == 0) return c;

11251125
11261126

#ifdef MRB_UTF8_STRING

1127-

/* Determine UTF-8 character length from first byte */

1128-

unsigned char first = (unsigned char)ptr[0];

1129-

if (first < 0x80) {

1130-

write_len = 1; /* ASCII */

1131-

}

1132-

else if ((first & 0xE0) == 0xC0) {

1133-

write_len = 2; /* 2-byte UTF-8 */

1134-

}

1135-

else if ((first & 0xF0) == 0xE0) {

1136-

write_len = 3; /* 3-byte UTF-8 */

1137-

}

1138-

else if ((first & 0xF8) == 0xF0) {

1139-

write_len = 4; /* 4-byte UTF-8 */

1140-

}

1141-

else {

1142-

write_len = 1; /* Invalid UTF-8, write single byte */

1143-

}

1144-

if (write_len > len) write_len = len;

1127+

write_len = mrb_utf8len(ptr, ptr + len);

11451128

#else

11461129

write_len = 1; /* Non-UTF8: write single byte */

11471130

#endif

Original file line numberDiff line numberDiff line change

@@ -10,6 +10,7 @@

1010

#include <mruby/string.h>

1111

#include <mruby/variable.h>

1212

#include <mruby/endian.h>

13+

#include <mruby/internal.h>

1314

#include <mruby/presym.h>

1415
1516

#include <ctype.h>

@@ -772,36 +773,11 @@ static int

772773

pack_utf8(mrb_state *mrb, mrb_value o, mrb_value str, mrb_int sidx, int count, unsigned int flags)

773774

{

774775

char utf8[4];

775-

int len = 0;

776-

uint32_t c = 0;

777-
778-

c = (uint32_t)mrb_integer(o);

779-
780-

/* Unicode character */

781-

/* from mruby-compiler gem */

782-

if (c < 0x80) {

783-

utf8[0] = (char)c;

784-

len = 1;

785-

}

786-

else if (c < 0x800) {

787-

utf8[0] = (char)(0xC0 | (c >> 6));

788-

utf8[1] = (char)(0x80 | (c & 0x3F));

789-

len = 2;

790-

}

791-

else if (c < 0x10000) {

792-

utf8[0] = (char)(0xE0 | (c >> 12) );

793-

utf8[1] = (char)(0x80 | ((c >> 6) & 0x3F));

794-

utf8[2] = (char)(0x80 | ( c & 0x3F));

795-

len = 3;

796-

}

797-

else if (c < 0x200000) {

798-

utf8[0] = (char)(0xF0 | (c >> 18) );

799-

utf8[1] = (char)(0x80 | ((c >> 12) & 0x3F));

800-

utf8[2] = (char)(0x80 | ((c >> 6) & 0x3F));

801-

utf8[3] = (char)(0x80 | ( c & 0x3F));

802-

len = 4;

803-

}

804-

else {

776+

int len;

777+

uint32_t c = (uint32_t)mrb_integer(o);

778+
779+

len = (int)mrb_utf8_to_buf(utf8, c);

780+

if (len == 0) {

805781

mrb_raise(mrb, E_RANGE_ERROR, "pack(U): value out of range");

806782

}

807783
Original file line numberDiff line numberDiff line change

@@ -520,28 +520,8 @@ mrb_str_format(mrb_state *mrb, mrb_int argc, const mrb_value *argv, mrb_value fm

520520

/* Integer: encode directly to stack buffer (no allocation) */

521521

mrb_int code = mrb_integer(val);

522522

#ifdef MRB_UTF8_STRING

523-

if (code < 0x80) {

524-

cbuf[0] = (char)code;

525-

clen = 1;

526-

}

527-

else if (code < 0x800) {

528-

cbuf[0] = (char)(0xC0 | (code >> 6));

529-

cbuf[1] = (char)(0x80 | (code & 0x3F));

530-

clen = 2;

531-

}

532-

else if (code < 0x10000) {

533-

cbuf[0] = (char)(0xE0 | (code >> 12));

534-

cbuf[1] = (char)(0x80 | ((code >> 6) & 0x3F));

535-

cbuf[2] = (char)(0x80 | (code & 0x3F));

536-

clen = 3;

537-

}

538-

else {

539-

cbuf[0] = (char)(0xF0 | (code >> 18));

540-

cbuf[1] = (char)(0x80 | ((code >> 12) & 0x3F));

541-

cbuf[2] = (char)(0x80 | ((code >> 6) & 0x3F));

542-

cbuf[3] = (char)(0x80 | (code & 0x3F));

543-

clen = 4;

544-

}

523+

clen = (int)mrb_utf8_to_buf(cbuf, (uint32_t)code);

524+

if (clen == 0) clen = 1; /* invalid codepoint: write single byte */

545525

#else

546526

cbuf[0] = (char)(code & 0xff);

547527

clen = 1;

Original file line numberDiff line numberDiff line change

@@ -51,36 +51,15 @@ int_chr_utf8(mrb_state *mrb, mrb_value num)

5151

char utf8[4];

5252

mrb_int len;

5353

mrb_value str;

54-

uint32_t sb_flag = 0;

5554
5655

if (cp < 0 || 0x10FFFF < cp) {

5756

mrb_raisef(mrb, E_RANGE_ERROR, "%v out of char range", num);

5857

}

59-

if (cp < 0x80) {

60-

utf8[0] = (char)cp;

61-

len = 1;

62-

sb_flag = MRB_STR_SINGLE_BYTE;

63-

}

64-

else if (cp < 0x800) {

65-

utf8[0] = (char)(0xC0 | (cp >> 6));

66-

utf8[1] = (char)(0x80 | (cp & 0x3F));

67-

len = 2;

68-

}

69-

else if (cp < 0x10000) {

70-

utf8[0] = (char)(0xE0 | (cp >> 12));

71-

utf8[1] = (char)(0x80 | ((cp >> 6) & 0x3F));

72-

utf8[2] = (char)(0x80 | ( cp & 0x3F));

73-

len = 3;

74-

}

75-

else {

76-

utf8[0] = (char)(0xF0 | (cp >> 18));

77-

utf8[1] = (char)(0x80 | ((cp >> 12) & 0x3F));

78-

utf8[2] = (char)(0x80 | ((cp >> 6) & 0x3F));

79-

utf8[3] = (char)(0x80 | ( cp & 0x3F));

80-

len = 4;

81-

}

58+

len = mrb_utf8_to_buf(utf8, (uint32_t)cp);

8259

str = mrb_str_new(mrb, utf8, len);

83-

mrb_str_ptr(str)->flags |= sb_flag;

60+

if (len == 1) {

61+

RSTR_SET_ASCII_FLAG(mrb_str_ptr(str));

62+

}

8463

return str;

8564

}

8665

#endif

Original file line numberDiff line numberDiff line change

@@ -316,6 +316,39 @@ mrb_gc_free_str(mrb_state *mrb, struct RString *str)

316316

#define MASK01 0x01010101ul

317317

#endif

318318
319+

/*

320+

* Encode a Unicode codepoint to UTF-8 bytes.

321+

* buf must have at least 4 bytes of space.

322+

* Returns the number of bytes written (1-4), or 0 for invalid codepoint.

323+

*/

324+

mrb_int

325+

mrb_utf8_to_buf(char *buf, uint32_t cp)

326+

{

327+

if (cp < 0x80) {

328+

buf[0] = (char)cp;

329+

return 1;

330+

}

331+

else if (cp < 0x800) {

332+

buf[0] = (char)(0xC0 | (cp >> 6));

333+

buf[1] = (char)(0x80 | (cp & 0x3F));

334+

return 2;

335+

}

336+

else if (cp < 0x10000) {

337+

buf[0] = (char)(0xE0 | (cp >> 12));

338+

buf[1] = (char)(0x80 | ((cp >> 6) & 0x3F));

339+

buf[2] = (char)(0x80 | (cp & 0x3F));

340+

return 3;

341+

}

342+

else if (cp <= 0x10FFFF) {

343+

buf[0] = (char)(0xF0 | (cp >> 18));

344+

buf[1] = (char)(0x80 | ((cp >> 12) & 0x3F));

345+

buf[2] = (char)(0x80 | ((cp >> 6) & 0x3F));

346+

buf[3] = (char)(0x80 | (cp & 0x3F));

347+

return 4;

348+

}

349+

return 0; /* invalid codepoint */

350+

}

351+
319352

#ifdef MRB_UTF8_STRING

320353
321354

#define NOASCII(c) ((c) & 0x80)

Read the original on github.com ↗