12 #include "ruby/internal/config.h"
17 #include "internal/array.h"
18 #include "internal/inits.h"
19 #include "internal/object.h"
20 #include "internal/string.h"
21 #include "internal/transcode.h"
24 #include "transcode_data.h"
27 #define ENABLE_ECONV_NEWLINE_OPTION 1
30 static VALUE rb_eUndefinedConversionError;
31 static VALUE rb_eInvalidByteSequenceError;
32 static VALUE rb_eConverterNotFoundError;
34 VALUE rb_cEncodingConverter;
36 static ID id_destination_encoding;
37 static ID id_destination_encoding_name;
38 static ID id_error_bytes;
39 static ID id_error_char;
40 static ID id_incomplete_input;
41 static ID id_readagain_bytes;
42 static ID id_source_encoding;
43 static ID id_source_encoding_name;
45 static VALUE sym_invalid, sym_undef, sym_replace, sym_fallback;
46 static VALUE sym_xml, sym_text, sym_attr;
47 static VALUE sym_universal_newline;
48 static VALUE sym_crlf_newline;
49 static VALUE sym_cr_newline;
50 #ifdef ENABLE_ECONV_NEWLINE_OPTION
51 static VALUE sym_newline, sym_universal, sym_crlf, sym_cr, sym_lf;
53 static VALUE sym_partial_input;
55 static VALUE sym_invalid_byte_sequence;
56 static VALUE sym_undefined_conversion;
57 static VALUE sym_destination_buffer_full;
58 static VALUE sym_source_buffer_empty;
59 static VALUE sym_finished;
60 static VALUE sym_after_output;
61 static VALUE sym_incomplete_input;
63 static unsigned char *
64 allocate_converted_string(
const char *sname,
const char *dname,
65 const unsigned char *str,
size_t len,
66 unsigned char *caller_dst_buf,
size_t caller_dst_bufsize,
77 unsigned int next_table;
79 unsigned char next_byte;
80 unsigned int output_index;
82 ssize_t recognized_len;
83 ssize_t readagain_len;
98 char ary[
sizeof(double) >
sizeof(
void*) ?
sizeof(double) :
sizeof(
void*)];
99 double dummy_for_alignment;
102 #define TRANSCODING_READBUF(tc) \
103 ((tc)->transcoder->max_input <= (int)sizeof((tc)->readbuf.ary) ? \
104 (tc)->readbuf.ary : \
106 #define TRANSCODING_WRITEBUF(tc) \
107 ((tc)->transcoder->max_output <= (int)sizeof((tc)->writebuf.ary) ? \
108 (tc)->writebuf.ary : \
110 #define TRANSCODING_WRITEBUF_SIZE(tc) \
111 ((tc)->transcoder->max_output <= (int)sizeof((tc)->writebuf.ary) ? \
112 sizeof((tc)->writebuf.ary) : \
113 (size_t)(tc)->transcoder->max_output)
114 #define TRANSCODING_STATE_EMBED_MAX ((int)sizeof(union rb_transcoding_state_t))
115 #define TRANSCODING_STATE(tc) \
116 ((tc)->transcoder->state_size <= (int)sizeof((tc)->state) ? \
122 unsigned char *out_buf_start;
123 unsigned char *out_data_start;
124 unsigned char *out_data_end;
125 unsigned char *out_buf_end;
133 const char *source_encoding_name;
134 const char *destination_encoding_name;
136 const unsigned char *replacement_str;
137 size_t replacement_len;
138 const char *replacement_enc;
140 unsigned char *in_buf_start;
141 unsigned char *in_data_start;
142 unsigned char *in_data_end;
143 unsigned char *in_buf_end;
145 int replacement_allocated;
155 const char *source_encoding;
156 const char *destination_encoding;
157 const unsigned char *error_bytes_start;
158 size_t error_bytes_len;
159 size_t readagain_len;
172 #define DECORATOR_P(sname, dname) (*(sname) == '\0')
184 make_transcoder_entry(
const char *sname,
const char *dname)
189 if (!st_lookup(transcoder_table, (st_data_t)sname, &val)) {
190 val = (st_data_t)st_init_strcasetable();
191 st_add_direct(transcoder_table, (st_data_t)sname, val);
194 if (!st_lookup(table2, (st_data_t)dname, &val)) {
196 entry->sname = sname;
197 entry->dname = dname;
199 entry->transcoder = NULL;
200 val = (st_data_t)entry;
201 st_add_direct(table2, (st_data_t)dname, val);
207 get_transcoder_entry(
const char *sname,
const char *dname)
212 if (!st_lookup(transcoder_table, (st_data_t)sname, &val)) {
216 if (!st_lookup(table2, (st_data_t)dname, &val)) {
225 const char *
const sname =
tr->src_encoding;
226 const char *
const dname =
tr->dst_encoding;
230 entry = make_transcoder_entry(sname, dname);
231 if (entry->transcoder) {
236 entry->transcoder =
tr;
240 declare_transcoder(
const char *sname,
const char *dname,
const char *lib)
244 entry = make_transcoder_entry(sname, dname);
248 static const char transcoder_lib_prefix[] =
"enc/trans/";
251 rb_declare_transcoder(
const char *enc1,
const char *enc2,
const char *lib)
256 declare_transcoder(enc1, enc2, lib);
259 #define encoding_equal(enc1, enc2) (STRCASECMP((enc1), (enc2)) == 0)
270 const char *base_enc;
274 transcode_search_path_i(st_data_t key, st_data_t val, st_data_t arg)
276 const char *dname = (
const char *)key;
280 if (st_lookup(bfs->visited, (st_data_t)dname, &val)) {
287 *bfs->queue_last_ptr = q;
288 bfs->queue_last_ptr = &q->next;
290 st_add_direct(bfs->visited, (st_data_t)dname, (st_data_t)bfs->base_enc);
295 transcode_search_path(
const char *sname,
const char *dname,
296 void (*callback)(
const char *sname,
const char *dname,
int depth,
void *arg),
306 if (encoding_equal(sname, dname))
312 bfs.queue_last_ptr = &q->next;
315 bfs.visited = st_init_strcasetable();
316 st_add_direct(bfs.visited, (st_data_t)sname, (st_data_t)NULL);
322 bfs.queue_last_ptr = &bfs.queue;
324 if (!st_lookup(transcoder_table, (st_data_t)q->enc, &val)) {
330 if (st_lookup(table2, (st_data_t)dname, &val)) {
331 st_add_direct(bfs.visited, (st_data_t)dname, (st_data_t)q->enc);
337 bfs.base_enc = q->enc;
338 st_foreach(table2, transcode_search_path_i, (st_data_t)&bfs);
353 const char *enc = dname;
357 st_lookup(bfs.visited, (st_data_t)enc, &val);
361 enc = (
const char *)val;
366 st_lookup(bfs.visited, (st_data_t)enc, &val);
369 callback((
const char *)val, enc, --depth, arg);
370 enc = (
const char *)val;
374 st_free_table(bfs.visited);
379 int rb_require_internal_silent(
VALUE fname);
384 if (entry->transcoder)
385 return entry->transcoder;
388 const char *
const lib = entry->lib;
389 const size_t len = strlen(lib);
390 const size_t total_len =
sizeof(transcoder_lib_prefix) - 1 + len;
394 memcpy(path, transcoder_lib_prefix,
sizeof(transcoder_lib_prefix) - 1);
395 memcpy(path +
sizeof(transcoder_lib_prefix) - 1, lib, len);
398 rb_require_internal_silent(fn);
401 if (entry->transcoder)
402 return entry->transcoder;
408 get_replacement_character(
const char *encname,
size_t *len_ret,
const char **repl_encname_ptr)
410 if (encoding_equal(encname,
"UTF-8")) {
412 *repl_encname_ptr =
"UTF-8";
413 return "\xEF\xBF\xBD";
417 *repl_encname_ptr =
"US-ASCII";
426 static const unsigned char *
428 const unsigned char *in_start,
429 const unsigned char *inchar_start,
430 const unsigned char *in_p,
431 size_t *char_len_ptr)
433 const unsigned char *ptr;
434 if (inchar_start - in_start < tc->recognized_len) {
435 MEMCPY(TRANSCODING_READBUF(tc) + tc->recognized_len,
436 inchar_start,
unsigned char, in_p - inchar_start);
437 ptr = TRANSCODING_READBUF(tc);
440 ptr = inchar_start - tc->recognized_len;
442 *char_len_ptr = tc->recognized_len + (in_p - inchar_start);
447 transcode_restartable0(
const unsigned char **in_pos,
unsigned char **out_pos,
448 const unsigned char *in_stop,
unsigned char *out_stop,
453 int unitlen =
tr->input_unit_length;
454 ssize_t readagain_len = 0;
456 const unsigned char *inchar_start;
457 const unsigned char *in_p;
459 unsigned char *out_p;
461 in_p = inchar_start = *in_pos;
465 #define SUSPEND(ret, num) \
467 tc->resume_position = (num); \
468 if (0 < in_p - inchar_start) \
469 MEMMOVE(TRANSCODING_READBUF(tc)+tc->recognized_len, \
470 inchar_start, unsigned char, in_p - inchar_start); \
473 tc->recognized_len += in_p - inchar_start; \
474 if (readagain_len) { \
475 tc->recognized_len -= readagain_len; \
476 tc->readagain_len = readagain_len; \
479 resume_label ## num:; \
481 #define SUSPEND_OBUF(num) \
483 while (out_stop - out_p < 1) { SUSPEND(econv_destination_buffer_full, num); } \
486 #define SUSPEND_AFTER_OUTPUT(num) \
487 if ((opt & ECONV_AFTER_OUTPUT) && *out_pos != out_p) { \
488 SUSPEND(econv_after_output, num); \
491 #define next_table (tc->next_table)
492 #define next_info (tc->next_info)
493 #define next_byte (tc->next_byte)
494 #define writebuf_len (tc->writebuf_len)
495 #define writebuf_off (tc->writebuf_off)
497 switch (tc->resume_position) {
499 case 1:
goto resume_label1;
500 case 2:
goto resume_label2;
501 case 3:
goto resume_label3;
502 case 4:
goto resume_label4;
503 case 5:
goto resume_label5;
504 case 6:
goto resume_label6;
505 case 7:
goto resume_label7;
506 case 8:
goto resume_label8;
507 case 9:
goto resume_label9;
508 case 10:
goto resume_label10;
509 case 11:
goto resume_label11;
510 case 12:
goto resume_label12;
511 case 13:
goto resume_label13;
512 case 14:
goto resume_label14;
513 case 15:
goto resume_label15;
514 case 16:
goto resume_label16;
515 case 17:
goto resume_label17;
516 case 18:
goto resume_label18;
517 case 19:
goto resume_label19;
518 case 20:
goto resume_label20;
519 case 21:
goto resume_label21;
520 case 22:
goto resume_label22;
521 case 23:
goto resume_label23;
522 case 24:
goto resume_label24;
523 case 25:
goto resume_label25;
524 case 26:
goto resume_label26;
525 case 27:
goto resume_label27;
526 case 28:
goto resume_label28;
527 case 29:
goto resume_label29;
528 case 30:
goto resume_label30;
529 case 31:
goto resume_label31;
530 case 32:
goto resume_label32;
531 case 33:
goto resume_label33;
532 case 34:
goto resume_label34;
537 tc->recognized_len = 0;
538 next_table =
tr->conv_tree_start;
540 SUSPEND_AFTER_OUTPUT(24);
542 if (in_stop <= in_p) {
549 #define BYTE_ADDR(index) (tr->byte_array + (index))
550 #define WORD_ADDR(index) (tr->word_array + INFO2WORDINDEX(index))
551 #define BL_BASE BYTE_ADDR(BYTE_LOOKUP_BASE(WORD_ADDR(next_table)))
552 #define BL_INFO WORD_ADDR(BYTE_LOOKUP_INFO(WORD_ADDR(next_table)))
553 #define BL_MIN_BYTE (BL_BASE[0])
554 #define BL_MAX_BYTE (BL_BASE[1])
555 #define BL_OFFSET(byte) (BL_BASE[2+(byte)-BL_MIN_BYTE])
556 #define BL_ACTION(byte) (BL_INFO[BL_OFFSET((byte))])
558 next_byte = (
unsigned char)*in_p++;
560 if (next_byte < BL_MIN_BYTE || BL_MAX_BYTE < next_byte)
563 next_info = (
VALUE)BL_ACTION(next_byte);
566 switch (next_info & 0x1F) {
569 const unsigned char *p = inchar_start;
572 TRANSCODING_WRITEBUF(tc)[writebuf_off++] = (
unsigned char)*p++;
574 writebuf_len = writebuf_off;
576 while (writebuf_off < writebuf_len) {
578 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
582 case 0x00:
case 0x04:
case 0x08:
case 0x0C:
583 case 0x10:
case 0x14:
case 0x18:
case 0x1C:
584 SUSPEND_AFTER_OUTPUT(25);
585 while (in_p >= in_stop) {
590 next_byte = (
unsigned char)*in_p++;
591 next_table = (
unsigned int)next_info;
596 SUSPEND_OBUF(9); *out_p++ = getBT1(next_info);
599 SUSPEND_OBUF(10); *out_p++ = getBT1(next_info);
600 SUSPEND_OBUF(21); *out_p++ = getBT2(next_info);
603 SUSPEND_OBUF(11); *out_p++ = getBT1(next_info);
604 SUSPEND_OBUF(15); *out_p++ = getBT2(next_info);
605 SUSPEND_OBUF(16); *out_p++ = getBT3(next_info);
608 SUSPEND_OBUF(12); *out_p++ = getBT0(next_info);
609 SUSPEND_OBUF(17); *out_p++ = getBT1(next_info);
610 SUSPEND_OBUF(18); *out_p++ = getBT2(next_info);
611 SUSPEND_OBUF(19); *out_p++ = getBT3(next_info);
614 SUSPEND_OBUF(29); *out_p++ = getGB4bt0(next_info);
615 SUSPEND_OBUF(30); *out_p++ = getGB4bt1(next_info);
616 SUSPEND_OBUF(31); *out_p++ = getGB4bt2(next_info);
617 SUSPEND_OBUF(32); *out_p++ = getGB4bt3(next_info);
620 tc->output_index = 0;
621 while (tc->output_index < STR1_LENGTH(BYTE_ADDR(STR1_BYTEINDEX(next_info)))) {
622 SUSPEND_OBUF(28); *out_p++ = BYTE_ADDR(STR1_BYTEINDEX(next_info))[1+tc->output_index];
627 next_info = (
VALUE)(*
tr->func_ii)(TRANSCODING_STATE(tc), next_info);
631 const unsigned char *char_start;
633 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
634 next_info = (
VALUE)(*
tr->func_si)(TRANSCODING_STATE(tc), char_start, (size_t)char_len);
639 if (
tr->max_output <= out_stop - out_p)
640 out_p +=
tr->func_io(TRANSCODING_STATE(tc),
641 next_info, out_p, out_stop - out_p);
643 writebuf_len =
tr->func_io(TRANSCODING_STATE(tc),
645 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
647 while (writebuf_off < writebuf_len) {
649 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
655 const unsigned char *char_start;
658 if (
tr->max_output <= out_stop - out_p) {
659 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
660 out_p +=
tr->func_so(TRANSCODING_STATE(tc),
661 char_start, (
size_t)char_len,
662 out_p, out_stop - out_p);
665 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
666 writebuf_len =
tr->func_so(TRANSCODING_STATE(tc),
667 char_start, (
size_t)char_len,
668 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
670 while (writebuf_off < writebuf_len) {
672 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
679 const unsigned char *char_start;
682 if (
tr->max_output <= out_stop - out_p) {
683 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
684 out_p +=
tr->func_sio(TRANSCODING_STATE(tc),
685 char_start, (
size_t)char_len, next_info,
686 out_p, out_stop - out_p);
689 char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
690 writebuf_len =
tr->func_sio(TRANSCODING_STATE(tc),
691 char_start, (
size_t)char_len, next_info,
692 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
694 while (writebuf_off < writebuf_len) {
696 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
702 if (tc->recognized_len + (in_p - inchar_start) <= unitlen) {
703 if (tc->recognized_len + (in_p - inchar_start) < unitlen)
704 SUSPEND_AFTER_OUTPUT(26);
705 while ((opt &
ECONV_PARTIAL_INPUT) && tc->recognized_len + (in_stop - inchar_start) < unitlen) {
709 if (tc->recognized_len + (in_stop - inchar_start) <= unitlen) {
713 in_p = inchar_start + (unitlen - tc->recognized_len);
719 invalid_len = tc->recognized_len + (in_p - inchar_start);
720 discard_len = ((invalid_len - 1) / unitlen) * unitlen;
721 readagain_len = invalid_len - discard_len;
745 if (
tr->finish_func) {
747 if (
tr->max_output <= out_stop - out_p) {
748 out_p +=
tr->finish_func(TRANSCODING_STATE(tc),
749 out_p, out_stop - out_p);
752 writebuf_len =
tr->finish_func(TRANSCODING_STATE(tc),
753 TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
755 while (writebuf_off < writebuf_len) {
757 *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
772 transcode_restartable(
const unsigned char **in_pos,
unsigned char **out_pos,
773 const unsigned char *in_stop,
unsigned char *out_stop,
777 if (tc->readagain_len) {
778 unsigned char *readagain_buf =
ALLOCA_N(
unsigned char, tc->readagain_len);
779 const unsigned char *readagain_pos = readagain_buf;
780 const unsigned char *readagain_stop = readagain_buf + tc->readagain_len;
783 MEMCPY(readagain_buf, TRANSCODING_READBUF(tc) + tc->recognized_len,
784 unsigned char, tc->readagain_len);
785 tc->readagain_len = 0;
786 res = transcode_restartable0(&readagain_pos, out_pos, readagain_stop, out_stop, tc, opt|
ECONV_PARTIAL_INPUT);
788 MEMCPY(TRANSCODING_READBUF(tc) + tc->recognized_len + tc->readagain_len,
789 readagain_pos,
unsigned char, readagain_stop - readagain_pos);
790 tc->readagain_len += readagain_stop - readagain_pos;
794 return transcode_restartable0(in_pos, out_pos, in_stop, out_stop, tc, opt);
805 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size)
807 if (
tr->state_init_func) {
808 (
tr->state_init_func)(TRANSCODING_STATE(tc));
810 tc->resume_position = 0;
811 tc->recognized_len = 0;
812 tc->readagain_len = 0;
813 tc->writebuf_len = 0;
814 tc->writebuf_off = 0;
815 if ((
int)
sizeof(tc->readbuf.ary) <
tr->max_input) {
816 tc->readbuf.ptr =
xmalloc(
tr->max_input);
818 if ((
int)
sizeof(tc->writebuf.ary) <
tr->max_output) {
819 tc->writebuf.ptr =
xmalloc(
tr->max_output);
826 const unsigned char **input_ptr,
const unsigned char *input_stop,
827 unsigned char **output_ptr,
unsigned char *output_stop,
830 return transcode_restartable(
831 input_ptr, output_ptr,
832 input_stop, output_stop,
840 if (
tr->state_fini_func) {
841 (
tr->state_fini_func)(TRANSCODING_STATE(tc));
843 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size)
844 xfree(tc->state.ptr);
845 if ((
int)
sizeof(tc->readbuf.ary) <
tr->max_input)
846 xfree(tc->readbuf.ptr);
847 if ((
int)
sizeof(tc->writebuf.ary) <
tr->max_output)
848 xfree(tc->writebuf.ptr);
858 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size) {
859 size +=
tr->state_size;
861 if ((
int)
sizeof(tc->readbuf.ary) <
tr->max_input) {
862 size +=
tr->max_input;
864 if ((
int)
sizeof(tc->writebuf.ary) <
tr->max_output) {
865 size +=
tr->max_output;
871 rb_econv_alloc(
int n_hint)
880 ec->source_encoding_name = NULL;
881 ec->destination_encoding_name = NULL;
883 ec->replacement_str = NULL;
884 ec->replacement_len = 0;
885 ec->replacement_enc = NULL;
886 ec->replacement_allocated = 0;
887 ec->in_buf_start = NULL;
888 ec->in_data_start = NULL;
889 ec->in_data_end = NULL;
890 ec->in_buf_end = NULL;
891 ec->num_allocated = n_hint;
894 ec->num_finished = 0;
897 ec->last_error.error_tc = NULL;
898 ec->last_error.source_encoding = NULL;
899 ec->last_error.destination_encoding = NULL;
900 ec->last_error.error_bytes_start = NULL;
901 ec->last_error.error_bytes_len = 0;
902 ec->last_error.readagain_len = 0;
903 ec->source_encoding = NULL;
904 ec->destination_encoding = NULL;
915 if (ec->num_trans == ec->num_allocated) {
916 n = ec->num_allocated * 2;
918 ec->num_allocated = n;
925 ec->elems[i].tc = rb_transcoding_open_by_transcoder(
tr, 0);
926 ec->elems[i].out_buf_start = p;
927 ec->elems[i].out_buf_end = p + bufsize;
928 ec->elems[i].out_data_start = p;
929 ec->elems[i].out_data_end = p;
934 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding))
935 for (j = ec->num_trans-1; i <= j; j--) {
938 if (!DECORATOR_P(tr2->src_encoding, tr2->dst_encoding)) {
953 for (i = 0; i < n; i++) {
955 tr = load_transcoder_entry(entries[i]);
960 ec = rb_econv_alloc(n);
962 for (i = 0; i < n; i++) {
964 ret = rb_econv_add_transcoder_at(ec,
tr, ec->num_trans);
980 trans_open_i(
const char *sname,
const char *dname,
int depth,
void *arg)
984 if (!toarg->entries) {
987 toarg->entries[depth] = get_transcoder_entry(sname, dname);
991 rb_econv_open0(
const char *sname,
const char *dname,
int ecflags)
1002 if (*sname ==
'\0' && *dname ==
'\0') {
1009 toarg.entries = NULL;
1010 toarg.num_additional = 0;
1011 num_trans = transcode_search_path(sname, dname, trans_open_i, (
void *)&toarg);
1012 entries = toarg.entries;
1013 if (num_trans < 0) {
1019 ec = rb_econv_open_by_transcoder_entries(num_trans, entries);
1024 ec->flags = ecflags;
1025 ec->source_encoding_name = sname;
1026 ec->destination_encoding_name = dname;
1031 #define MAX_ECFLAGS_DECORATORS 32
1034 decorator_names(
int ecflags,
const char **decorators_ret)
1055 decorators_ret[num_decorators++] =
"xml_text_escape";
1057 decorators_ret[num_decorators++] =
"xml_attr_content_escape";
1059 decorators_ret[num_decorators++] =
"xml_attr_quote";
1062 decorators_ret[num_decorators++] =
"crlf_newline";
1064 decorators_ret[num_decorators++] =
"cr_newline";
1066 decorators_ret[num_decorators++] =
"universal_newline";
1068 return num_decorators;
1076 const char *decorators[MAX_ECFLAGS_DECORATORS];
1079 num_decorators = decorator_names(ecflags, decorators);
1080 if (num_decorators == -1)
1087 for (i = 0; i < num_decorators; i++)
1100 const unsigned char **input_ptr,
const unsigned char *input_stop,
1101 unsigned char **output_ptr,
unsigned char *output_stop,
1108 const unsigned char **ipp, *is, *iold;
1109 unsigned char **opp, *os, *oold;
1115 for (i = start; i < ec->num_trans; i++) {
1124 ipp = (
const unsigned char **)&prev_te->out_data_start;
1125 is = prev_te->out_data_end;
1128 if (i == ec->num_trans-1) {
1133 if (te->out_buf_start != te->out_data_start) {
1134 ssize_t len = te->out_data_end - te->out_data_start;
1135 ssize_t off = te->out_data_start - te->out_buf_start;
1136 MEMMOVE(te->out_buf_start, te->out_data_start,
unsigned char, len);
1137 te->out_data_start = te->out_buf_start;
1138 te->out_data_end -= off;
1140 opp = &te->out_data_end;
1141 os = te->out_buf_end;
1145 if (ec->num_finished != i)
1155 te->last_result = res = rb_transcoding_convert(te->tc, ipp, is, opp, os, f);
1156 if (iold != *ipp || oold != *opp)
1171 ec->num_finished = i+1;
1181 const unsigned char **input_ptr,
const unsigned char *input_stop,
1182 unsigned char **output_ptr,
unsigned char *output_stop,
1184 int *result_position_ptr)
1187 int needreport_index;
1190 unsigned char empty_buf;
1191 unsigned char *empty_ptr = &empty_buf;
1194 input_ptr = (
const unsigned char **)&empty_ptr;
1195 input_stop = empty_ptr;
1199 output_ptr = &empty_ptr;
1200 output_stop = empty_ptr;
1206 for (i = ec->num_trans-1; 0 <= i; i--) {
1207 switch (ec->elems[i].last_result) {
1214 goto found_needreport;
1221 rb_bug(
"unexpected transcode last result");
1231 res = rb_trans_conv(ec, NULL, NULL, output_ptr, output_stop,
1233 result_position_ptr);
1245 needreport_index = trans_sweep(ec, input_ptr, input_stop, output_ptr, output_stop, flags, sweep_start);
1246 sweep_start = needreport_index + 1;
1247 }
while (needreport_index != -1 && needreport_index != ec->num_trans-1);
1249 for (i = ec->num_trans-1; 0 <= i; i--) {
1258 if (result_position_ptr)
1259 *result_position_ptr = i;
1263 if (result_position_ptr)
1264 *result_position_ptr = -1;
1270 const unsigned char **input_ptr,
const unsigned char *input_stop,
1271 unsigned char **output_ptr,
unsigned char *output_stop,
1275 int result_position;
1278 memset(&ec->last_error, 0,
sizeof(ec->last_error));
1280 if (ec->num_trans == 0) {
1282 if (ec->in_buf_start && ec->in_data_start != ec->in_data_end) {
1283 if (output_stop - *output_ptr < ec->in_data_end - ec->in_data_start) {
1284 len = output_stop - *output_ptr;
1285 memcpy(*output_ptr, ec->in_data_start, len);
1286 *output_ptr = output_stop;
1287 ec->in_data_start += len;
1291 len = ec->in_data_end - ec->in_data_start;
1292 memcpy(*output_ptr, ec->in_data_start, len);
1294 ec->in_data_start = ec->in_data_end = ec->in_buf_start;
1300 if (output_stop - *output_ptr < input_stop - *input_ptr) {
1301 len = output_stop - *output_ptr;
1304 len = input_stop - *input_ptr;
1307 *(*output_ptr)++ = *(*input_ptr)++;
1311 memcpy(*output_ptr, *input_ptr, len);
1314 if (*input_ptr != input_stop)
1323 if (ec->elems[ec->num_trans-1].out_data_start) {
1324 unsigned char *data_start = ec->elems[ec->num_trans-1].out_data_start;
1325 unsigned char *data_end = ec->elems[ec->num_trans-1].out_data_end;
1326 if (data_start != data_end) {
1328 if (output_stop - *output_ptr < data_end - data_start) {
1329 len = output_stop - *output_ptr;
1330 memcpy(*output_ptr, data_start, len);
1331 *output_ptr = output_stop;
1332 ec->elems[ec->num_trans-1].out_data_start += len;
1336 len = data_end - data_start;
1337 memcpy(*output_ptr, data_start, len);
1339 ec->elems[ec->num_trans-1].out_data_start =
1340 ec->elems[ec->num_trans-1].out_data_end =
1341 ec->elems[ec->num_trans-1].out_buf_start;
1346 if (ec->in_buf_start &&
1347 ec->in_data_start != ec->in_data_end) {
1348 res = rb_trans_conv(ec, (
const unsigned char **)&ec->in_data_start, ec->in_data_end, output_ptr, output_stop,
1356 *input_ptr != input_stop) {
1357 input_stop = *input_ptr;
1358 res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1363 ec->num_trans == 1) {
1364 res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1369 res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1374 ec->last_error.result = res;
1379 ec->last_error.error_tc = error_tc;
1380 ec->last_error.source_encoding = error_tc->transcoder->src_encoding;
1381 ec->last_error.destination_encoding = error_tc->transcoder->dst_encoding;
1382 ec->last_error.error_bytes_start = TRANSCODING_READBUF(error_tc);
1383 ec->last_error.error_bytes_len = error_tc->recognized_len;
1384 ec->last_error.readagain_len = error_tc->readagain_len;
1390 static int output_replacement_character(
rb_econv_t *ec);
1396 unsigned char utfbuf[1024];
1397 const unsigned char *utf;
1399 int utf_allocated = 0;
1400 char charef_buf[16];
1401 const unsigned char *p;
1403 if (encoding_equal(ec->last_error.source_encoding,
"UTF-32BE")) {
1404 utf = ec->last_error.error_bytes_start;
1405 utf_len = ec->last_error.error_bytes_len;
1408 utf = allocate_converted_string(ec->last_error.source_encoding,
"UTF-32BE",
1409 ec->last_error.error_bytes_start, ec->last_error.error_bytes_len,
1410 utfbuf,
sizeof(utfbuf),
1414 if (utf != utfbuf && utf != ec->last_error.error_bytes_start)
1418 if (utf_len % 4 != 0)
1422 while (4 <= utf_len) {
1428 snprintf(charef_buf,
sizeof(charef_buf),
"&#x%X;", u);
1450 const unsigned char **input_ptr,
const unsigned char *input_stop,
1451 unsigned char **output_ptr,
unsigned char *output_stop,
1456 unsigned char empty_buf;
1457 unsigned char *empty_ptr = &empty_buf;
1462 input_ptr = (
const unsigned char **)&empty_ptr;
1463 input_stop = empty_ptr;
1467 output_ptr = &empty_ptr;
1468 output_stop = empty_ptr;
1472 ret = rb_econv_convert0(ec, input_ptr, input_stop, output_ptr, output_stop, flags);
1480 if (output_replacement_character(ec) == 0)
1491 if (output_replacement_character(ec) == 0)
1496 if (output_hex_charref(ec) == 0)
1514 tr = tc->transcoder;
1516 if (
tr->asciicompat_type == asciicompat_encoder)
1517 return tr->src_encoding;
1518 return tr->dst_encoding;
1521 static unsigned char *
1522 allocate_converted_string(
const char *sname,
const char *dname,
1523 const unsigned char *str,
size_t len,
1524 unsigned char *caller_dst_buf,
size_t caller_dst_bufsize,
1525 size_t *dst_len_ptr)
1527 unsigned char *dst_str;
1534 const unsigned char *sp;
1538 dst_bufsize = caller_dst_bufsize;
1548 dst_str = caller_dst_buf;
1550 dst_str =
xmalloc(dst_bufsize);
1553 dp = dst_str+dst_len;
1555 dst_len = dp - dst_str;
1557 if (SIZE_MAX/2 < dst_bufsize) {
1561 if (dst_str == caller_dst_buf) {
1564 memcpy(tmp, dst_str, dst_bufsize/2);
1568 dst_str =
xrealloc(dst_str, dst_bufsize);
1570 dp = dst_str+dst_len;
1572 dst_len = dp - dst_str;
1578 *dst_len_ptr = dst_len;
1582 if (dst_str != caller_dst_buf)
1591 const unsigned char *str,
size_t len,
const char *str_encoding)
1594 unsigned char insert_buf[4096];
1595 const unsigned char *insert_str = NULL;
1598 int last_trans_index;
1601 unsigned char **buf_start_p;
1602 unsigned char **data_start_p;
1603 unsigned char **data_end_p;
1604 unsigned char **buf_end_p;
1613 if (encoding_equal(insert_encoding, str_encoding)) {
1618 insert_str = allocate_converted_string(str_encoding, insert_encoding,
1619 str, len, insert_buf,
sizeof(insert_buf), &insert_len);
1620 if (insert_str == NULL)
1626 last_trans_index = ec->num_trans-1;
1627 if (ec->num_trans == 0) {
1629 buf_start_p = &ec->in_buf_start;
1630 data_start_p = &ec->in_data_start;
1631 data_end_p = &ec->in_data_end;
1632 buf_end_p = &ec->in_buf_end;
1634 else if (ec->elems[last_trans_index].tc->transcoder->asciicompat_type == asciicompat_encoder) {
1635 tc = ec->elems[last_trans_index].tc;
1636 need += tc->readagain_len;
1637 if (need < insert_len)
1639 if (last_trans_index == 0) {
1640 buf_start_p = &ec->in_buf_start;
1641 data_start_p = &ec->in_data_start;
1642 data_end_p = &ec->in_data_end;
1643 buf_end_p = &ec->in_buf_end;
1647 buf_start_p = &ee->out_buf_start;
1648 data_start_p = &ee->out_data_start;
1649 data_end_p = &ee->out_data_end;
1650 buf_end_p = &ee->out_buf_end;
1655 buf_start_p = &ee->out_buf_start;
1656 data_start_p = &ee->out_data_start;
1657 data_end_p = &ee->out_data_end;
1658 buf_end_p = &ee->out_buf_end;
1659 tc = ec->elems[last_trans_index].tc;
1662 if (*buf_start_p == NULL) {
1663 unsigned char *buf =
xmalloc(need);
1665 *data_start_p = buf;
1667 *buf_end_p = buf+need;
1669 else if ((
size_t)(*buf_end_p - *data_end_p) < need) {
1670 MEMMOVE(*buf_start_p, *data_start_p,
unsigned char, *data_end_p - *data_start_p);
1671 *data_end_p = *buf_start_p + (*data_end_p - *data_start_p);
1672 *data_start_p = *buf_start_p;
1673 if ((
size_t)(*buf_end_p - *data_end_p) < need) {
1675 size_t s = (*data_end_p - *buf_start_p) + need;
1679 *data_start_p = buf;
1680 *data_end_p = buf + (*data_end_p - *buf_start_p);
1682 *buf_end_p = buf + s;
1686 memcpy(*data_end_p, insert_str, insert_len);
1687 *data_end_p += insert_len;
1688 if (tc && tc->transcoder->asciicompat_type == asciicompat_encoder) {
1689 memcpy(*data_end_p, TRANSCODING_READBUF(tc)+tc->recognized_len, tc->readagain_len);
1690 *data_end_p += tc->readagain_len;
1691 tc->readagain_len = 0;
1694 if (insert_str != str && insert_str != insert_buf)
1695 xfree((
void*)insert_str);
1699 if (insert_str != str && insert_str != insert_buf)
1700 xfree((
void*)insert_str);
1709 if (ec->replacement_allocated) {
1710 xfree((
void *)ec->replacement_str);
1712 for (i = 0; i < ec->num_trans; i++) {
1713 rb_transcoding_close(ec->elems[i].tc);
1714 if (ec->elems[i].out_buf_start)
1715 xfree(ec->elems[i].out_buf_start);
1717 xfree(ec->in_buf_start);
1728 if (ec->replacement_allocated) {
1729 size += ec->replacement_len;
1731 for (i = 0; i < ec->num_trans; i++) {
1732 size += rb_transcoding_memsize(ec->elems[i].tc);
1734 if (ec->elems[i].out_buf_start) {
1735 size += ec->elems[i].out_buf_end - ec->elems[i].out_buf_start;
1738 size += ec->in_buf_end - ec->in_buf_start;
1747 if (ec->num_trans == 0)
1749 #if SIZEOF_SIZE_T > SIZEOF_INT
1750 if (ec->elems[0].tc->readagain_len > INT_MAX)
return INT_MAX;
1752 return (
int)ec->elems[0].tc->readagain_len;
1759 if (ec->num_trans == 0 || n == 0)
1761 tc = ec->elems[0].tc;
1762 memcpy(p, TRANSCODING_READBUF(tc) + tc->recognized_len + tc->readagain_len - n, n);
1763 tc->readagain_len -= n;
1767 const char *ascii_compat_name;
1768 const char *ascii_incompat_name;
1772 asciicompat_encoding_i(st_data_t key, st_data_t val, st_data_t arg)
1778 if (DECORATOR_P(entry->sname, entry->dname))
1780 tr = load_transcoder_entry(entry);
1781 if (
tr &&
tr->asciicompat_type == asciicompat_decoder) {
1782 data->ascii_compat_name =
tr->dst_encoding;
1795 if (!st_lookup(transcoder_table, (st_data_t)ascii_incompat_name, &v))
1806 if (table2->num_entries != 1)
1809 data.ascii_incompat_name = ascii_incompat_name;
1810 data.ascii_compat_name = NULL;
1811 st_foreach(table2, asciicompat_encoding_i, (st_data_t)&data);
1812 return data.ascii_compat_name;
1824 unsigned const char *sp, *se;
1825 unsigned char *ds, *dp, *de;
1844 max_output = ec->last_tc->transcoder->max_output;
1852 unsigned long new_capa = (
unsigned long)dlen + len + max_output;
1853 if (LONG_MAX < new_capa)
1857 sp = (
const unsigned char *)ss;
1863 switch (coderange) {
1866 cr = (int)coderange;
1875 len -= (
const char *)sp - ss;
1876 ss = (
const char *)sp;
1912 rb_econv_add_converter(
rb_econv_t *ec,
const char *sname,
const char *dname,
int n)
1917 if (ec->started != 0)
1920 entry = get_transcoder_entry(sname, dname);
1924 tr = load_transcoder_entry(entry);
1927 return rb_econv_add_transcoder_at(ec,
tr, n);
1931 rb_econv_decorate_at(
rb_econv_t *ec,
const char *decorator_name,
int n)
1933 return rb_econv_add_converter(ec,
"", decorator_name, n);
1941 if (ec->num_trans == 0)
1942 return rb_econv_decorate_at(ec, decorator_name, 0);
1944 tr = ec->elems[0].tc->transcoder;
1946 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding) &&
1947 tr->asciicompat_type == asciicompat_decoder)
1948 return rb_econv_decorate_at(ec, decorator_name, 1);
1950 return rb_econv_decorate_at(ec, decorator_name, 0);
1958 if (ec->num_trans == 0)
1959 return rb_econv_decorate_at(ec, decorator_name, 0);
1961 tr = ec->elems[ec->num_trans-1].tc->transcoder;
1963 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding) &&
1964 tr->asciicompat_type == asciicompat_encoder)
1965 return rb_econv_decorate_at(ec, decorator_name, ec->num_trans-1);
1967 return rb_econv_decorate_at(ec, decorator_name, ec->num_trans);
1973 const char *dname = 0;
1977 dname =
"universal_newline";
1980 dname =
"crlf_newline";
1983 dname =
"cr_newline";
1988 const rb_transcoder *transcoder = get_transcoder_entry(
"", dname)->transcoder;
1989 int num_trans = ec->num_trans;
1992 for (i=0; i < num_trans; i++) {
1993 if (transcoder == ec->elems[i].tc->transcoder) {
1994 rb_transcoding_close(ec->elems[i].tc);
1995 xfree(ec->elems[i].out_buf_start);
1999 ec->elems[j++] = ec->elems[i];
2007 econv_description(
const char *sname,
const char *dname,
int ecflags,
VALUE mesg)
2009 int has_description = 0;
2014 if (*sname !=
'\0' || *dname !=
'\0') {
2017 else if (*dname ==
'\0')
2021 has_description = 1;
2028 const char *pre =
"";
2029 if (has_description)
2055 has_description = 1;
2057 if (!has_description) {
2069 econv_description(sname, dname, ecflags, mesg);
2071 exc =
rb_exc_new3(rb_eConverterNotFoundError, mesg);
2081 const char *err = (
const char *)ec->last_error.error_bytes_start;
2082 size_t error_len = ec->last_error.error_bytes_len;
2085 size_t readagain_len = ec->last_error.readagain_len;
2091 ec->last_error.source_encoding);
2093 else if (readagain_len) {
2094 bytes2 =
rb_str_new(err+error_len, readagain_len);
2099 ec->last_error.source_encoding);
2104 ec->last_error.source_encoding);
2107 exc =
rb_exc_new3(rb_eInvalidByteSequenceError, mesg);
2114 VALUE bytes =
rb_str_new((
const char *)ec->last_error.error_bytes_start,
2115 ec->last_error.error_bytes_len);
2118 if (strcmp(ec->last_error.source_encoding,
"UTF-8") == 0) {
2120 const char *start, *end;
2122 start = (
const char *)ec->last_error.error_bytes_start;
2123 end = start + ec->last_error.error_bytes_len;
2133 if (strcmp(ec->last_error.source_encoding,
2134 ec->source_encoding_name) == 0 &&
2135 strcmp(ec->last_error.destination_encoding,
2136 ec->destination_encoding_name) == 0) {
2139 ec->last_error.source_encoding,
2140 ec->last_error.destination_encoding);
2144 mesg =
rb_sprintf(
"%s to %s in conversion from %s",
2146 ec->last_error.destination_encoding,
2147 ec->source_encoding_name);
2148 for (i = 0; i < ec->num_trans; i++) {
2150 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding))
2152 ec->elems[i].tc->transcoder->dst_encoding);
2155 exc =
rb_exc_new3(rb_eUndefinedConversionError, mesg);
2179 unsigned char *(*resize_destination)(
VALUE,
size_t,
size_t),
2181 unsigned char **out_start_ptr,
2182 unsigned char **out_pos,
2183 unsigned char **out_stop_ptr)
2185 size_t len = (*out_pos - *out_start_ptr);
2186 size_t new_len = (len + max_output) * 2;
2187 *out_start_ptr = resize_destination(destination, len, new_len);
2188 *out_pos = *out_start_ptr + len;
2189 *out_stop_ptr = *out_start_ptr + new_len;
2197 const unsigned char *replacement;
2198 const char *repl_enc;
2199 const char *ins_enc;
2202 if (ec->replacement_str)
2209 tr = tc->transcoder;
2211 replacement = (
const unsigned char *)get_replacement_character(ins_enc, &len, &repl_enc);
2214 replacement = (
unsigned char *)
"?";
2219 ec->replacement_str = replacement;
2220 ec->replacement_len = len;
2221 ec->replacement_enc = repl_enc;
2222 ec->replacement_allocated = 0;
2228 const unsigned char *str,
size_t len,
const char *encname)
2230 unsigned char *str2;
2232 const char *encname2;
2236 if (!*encname2 || encoding_equal(encname, encname2)) {
2238 MEMCPY(str2, str,
unsigned char, len);
2243 str2 = allocate_converted_string(encname, encname2, str, len, NULL, 0, &len2);
2248 if (ec->replacement_allocated) {
2249 xfree((
void *)ec->replacement_str);
2251 ec->replacement_allocated = 1;
2252 ec->replacement_str = str2;
2253 ec->replacement_len = len2;
2254 ec->replacement_enc = encname2;
2263 if (make_replacement(ec) == -1)
2274 #define hash_fallback rb_hash_aref
2295 transcode_loop(
const unsigned char **in_pos,
unsigned char **out_pos,
2296 const unsigned char *in_stop,
unsigned char *out_stop,
2298 unsigned char *(*resize_destination)(
VALUE,
size_t,
size_t),
2299 const char *src_encoding,
2300 const char *dst_encoding,
2307 unsigned char *out_start = *out_pos;
2320 fallback_func = hash_fallback;
2323 fallback_func = proc_fallback;
2326 fallback_func = method_fallback;
2329 fallback_func = aref_fallback;
2332 last_tc = ec->last_tc;
2333 max_output = last_tc ? last_tc->transcoder->max_output : 1;
2340 (
const char *)ec->last_error.error_bytes_start,
2341 ec->last_error.error_bytes_len,
2343 rep = (*fallback_func)(fallback, rep);
2348 if ((
int)ret == -1) {
2358 exc = make_econv_exception(ec);
2364 more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop);
2374 transcode_loop(
const unsigned char **in_pos,
unsigned char **out_pos,
2375 const unsigned char *in_stop,
unsigned char *out_stop,
2377 unsigned char *(*resize_destination)(
VALUE,
size_t,
size_t),
2378 const char *src_encoding,
2379 const char *dst_encoding,
2386 unsigned char *out_start = *out_pos;
2387 const unsigned char *ptr;
2395 last_tc = ec->last_tc;
2396 max_output = last_tc ? last_tc->transcoder->max_output : 1;
2401 unsigned char input_byte;
2402 const unsigned char *p = &input_byte;
2405 if (ptr < in_stop) {
2416 if (&input_byte != p)
2417 ptr += p - &input_byte;
2422 exc = make_econv_exception(ec);
2428 more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop);
2449 static unsigned char *
2450 str_transcoding_resize(
VALUE destination,
size_t len,
size_t new_len)
2457 econv_opts(
VALUE opt,
int ecflags)
2460 int newlineflag = 0;
2465 else if (v==sym_replace) {
2475 else if (v==sym_replace) {
2492 else if (v==sym_attr) {
2503 #ifdef ENABLE_ECONV_NEWLINE_OPTION
2508 if (v == sym_universal) {
2511 else if (v == sym_crlf) {
2514 else if (v == sym_cr) {
2517 else if (v == sym_lf) {
2535 newlineflag |= !
NIL_P(v);
2540 newlineflag |= !
NIL_P(v);
2545 newlineflag |= !
NIL_P(v);
2547 switch (newlineflag) {
2550 ecflags |= setflags;
2554 rb_warning(
":newline option precedes other newline options");
2568 if (
NIL_P(opthash)) {
2572 ecflags = econv_opts(opthash, ecflags);
2600 if (!
NIL_P(newhash))
2619 if (
NIL_P(opthash)) {
2624 rb_bug(
"rb_econv_open_opts called with invalid opthash");
2628 ec =
rb_econv_open(source_encoding, destination_encoding, ecflags);
2632 if (!
NIL_P(replacement)) {
2678 const char *sname, *dname;
2679 int sencidx, dencidx;
2681 dencidx = enc_arg(arg1, &dname, &denc);
2689 sencidx = enc_arg(arg2, &sname, &senc);
2700 str_transcode0(
int argc,
VALUE *argv,
VALUE *
self,
int ecflags,
VALUE ecopts)
2706 unsigned char *buf, *bp, *sp;
2707 const unsigned char *fromp;
2709 const char *sname, *dname;
2711 int explicitly_invalid_replace = TRUE;
2718 if (!ecflags)
return -1;
2722 explicitly_invalid_replace = FALSE;
2729 arg2 = argc<=1 ?
Qnil : argv[1];
2730 dencidx = str_transcode_enc_args(str, &arg1, &arg2, &sname, &senc, &dname, &denc);
2736 if (senc && senc == denc) {
2739 if (!
NIL_P(ecopts)) {
2742 dest = rb_enc_str_scrub(senc, str, rep);
2743 if (
NIL_P(dest)) dest = str;
2747 return NIL_P(arg2) ? -1 : dencidx;
2754 if (encoding_equal(sname, dname)) {
2755 return NIL_P(arg2) ? -1 : dencidx;
2765 if (encoding_equal(sname, dname)) {
2777 transcode_loop(&fromp, &bp, (sp+slen), (bp+blen), dest, str_transcoding_resize, sname, dname, ecflags, ecopts);
2778 if (fromp != sp+slen) {
2797 str_transcode(
int argc,
VALUE *argv,
VALUE *
self)
2803 argc =
rb_scan_args(argc, argv,
"02:", NULL, NULL, &opt);
2807 return str_transcode0(argc, argv,
self, ecflags, ecopts);
2811 str_encode_associate(
VALUE str,
int encidx)
2843 str_encode_bang(
int argc,
VALUE *argv,
VALUE str)
2851 encidx = str_transcode(argc, argv, &newstr);
2853 if (encidx < 0)
return str;
2854 if (newstr == str) {
2859 return str_encode_associate(str, encidx);
2924 int encidx = str_transcode(argc, argv, &newstr);
2925 return encoded_dup(newstr, str, encidx);
2934 int encidx = str_transcode0(argc, argv, &newstr, ecflags, ecopts);
2935 return encoded_dup(newstr, str, encidx);
2939 encoded_dup(
VALUE newstr,
VALUE str,
int encidx)
2942 if (newstr == str) {
2950 return str_encode_associate(newstr, encidx);
2959 econv_free(
void *ptr)
2966 econv_memsize(
const void *ptr)
2973 {0, econv_free, econv_memsize,},
2974 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
2978 econv_s_allocate(
VALUE klass)
2984 make_dummy_encoding(
const char *name)
2994 make_encoding(
const char *name)
2999 enc = make_dummy_encoding(name);
3004 make_encobj(
const char *name)
3028 econv_s_asciicompat_encoding(
VALUE klass,
VALUE arg)
3030 const char *arg_name, *result_name;
3033 enc_arg(&arg, &arg_name, &arg_enc);
3037 if (result_name == NULL)
3040 result_enc = make_encoding(result_name);
3046 econv_args(
int argc,
VALUE *argv,
3048 const char **sname_p,
const char **dname_p,
3053 VALUE opt, flags_v, ecopts;
3055 const char *sname, *dname;
3059 argc =
rb_scan_args(argc, argv,
"21:", snamev_p, dnamev_p, &flags_v, &opt);
3061 if (!
NIL_P(flags_v)) {
3063 rb_error_arity(argc + 1, 2, 3);
3068 else if (!
NIL_P(opt)) {
3101 *ecflags_p = ecflags;
3106 decorate_convpath(
VALUE convpath,
int ecflags)
3109 const char *decorators[MAX_ECFLAGS_DECORATORS];
3113 num_decorators = decorator_names(ecflags, decorators);
3114 if (num_decorators == -1)
3127 if (!DECORATOR_P(
tr->src_encoding,
tr->dst_encoding) &&
3128 tr->asciicompat_type == asciicompat_encoder) {
3130 rb_ary_store(convpath, len + num_decorators - 1, pair);
3134 rb_ary_store(convpath, len + num_decorators - 1, pair);
3138 for (i = 0; i < num_decorators; i++)
3145 search_convpath_i(
const char *sname,
const char *dname,
int depth,
void *arg)
3150 if (
NIL_P(*ary_p)) {
3154 if (DECORATOR_P(sname, dname)) {
3158 v =
rb_assoc_new(make_encobj(sname), make_encobj(dname));
3189 econv_s_search_convpath(
int argc,
VALUE *argv,
VALUE klass)
3191 VALUE snamev, dnamev;
3192 const char *sname, *dname;
3198 econv_args(argc, argv, &snamev, &dnamev, &sname, &dname, &senc, &denc, &ecflags, &ecopts);
3201 transcode_search_path(sname, dname, search_convpath_i, &convpath);
3203 if (
NIL_P(convpath)) {
3210 if (decorate_convpath(convpath, ecflags) == -1) {
3229 transcode_search_path(from_encoding, to_encoding, search_convpath_i,
3231 return RTEST(convpath);
3241 rb_econv_init_by_convpath_i(
const char *sname,
const char *dname,
int depth,
void *arg)
3249 ret = rb_econv_add_converter(a->ec, sname, dname, a->index);
3256 rb_econv_init_by_convpath(
VALUE self,
VALUE convpath,
3257 const char **sname_p,
const char **dname_p,
3265 const char *sname, *dname;
3271 VALUE snamev, dnamev;
3278 enc_arg(&snamev, &sname, &senc);
3280 enc_arg(&dnamev, &dname, &denc);
3286 if (DECORATOR_P(sname, dname)) {
3287 ret = rb_econv_add_converter(ec, sname, dname, ec->num_trans);
3296 int j = ec->num_trans;
3299 arg.index = ec->num_trans;
3301 ret = transcode_search_path(sname, dname, rb_econv_init_by_convpath_i, &arg);
3302 if (ret == -1 || arg.ret == -1) {
3303 VALUE msg =
rb_sprintf(
"adding conversion failed: %s to %s", sname, dname);
3311 *sname_p = ec->elems[j].tc->transcoder->src_encoding;
3314 *dname_p = ec->elems[ec->num_trans-1].tc->transcoder->dst_encoding;
3325 ec->source_encoding_name = *sname_p;
3326 ec->destination_encoding_name = *dname_p;
3438 econv_init(
int argc,
VALUE *argv,
VALUE self)
3441 VALUE snamev, dnamev;
3442 const char *sname, *dname;
3453 ec = rb_econv_init_by_convpath(
self, convpath, &sname, &dname, &senc, &denc);
3458 econv_args(argc, argv, &snamev, &dnamev, &sname, &dname, &senc, &denc, &ecflags, &ecopts);
3469 if (!DECORATOR_P(sname, dname)) {
3471 senc = make_dummy_encoding(sname);
3473 denc = make_dummy_encoding(dname);
3478 ec->source_encoding = senc;
3479 ec->destination_encoding = denc;
3497 econv_inspect(
VALUE self)
3504 return rb_sprintf(
"#<%s: uninitialized>", cname);
3506 const char *sname = ec->source_encoding_name;
3507 const char *dname = ec->destination_encoding_name;
3510 econv_description(sname, dname, ec->flags, str);
3517 check_econv(
VALUE self)
3535 econv_source_encoding(
VALUE self)
3538 if (!ec->source_encoding)
3550 econv_destination_encoding(
VALUE self)
3553 if (!ec->destination_encoding)
3581 econv_convpath(
VALUE self)
3588 for (i = 0; i < ec->num_trans; i++) {
3591 if (DECORATOR_P(
tr->src_encoding,
tr->dst_encoding))
3594 v =
rb_assoc_new(make_encobj(
tr->src_encoding), make_encobj(
tr->dst_encoding));
3616 if (ec1->source_encoding_name != ec2->source_encoding_name &&
3617 strcmp(ec1->source_encoding_name, ec2->source_encoding_name))
3619 if (ec1->destination_encoding_name != ec2->destination_encoding_name &&
3620 strcmp(ec1->destination_encoding_name, ec2->destination_encoding_name))
3622 if (ec1->flags != ec2->flags)
return Qfalse;
3623 if (ec1->replacement_enc != ec2->replacement_enc &&
3624 strcmp(ec1->replacement_enc, ec2->replacement_enc))
3626 if (ec1->replacement_len != ec2->replacement_len)
return Qfalse;
3627 if (ec1->replacement_str != ec2->replacement_str &&
3628 memcmp(ec1->replacement_str, ec2->replacement_str, ec2->replacement_len))
3631 if (ec1->num_trans != ec2->num_trans)
return Qfalse;
3632 for (i = 0; i < ec1->num_trans; i++) {
3633 if (ec1->elems[i].tc->transcoder != ec2->elems[i].tc->transcoder)
3749 econv_primitive_convert(
int argc,
VALUE *argv,
VALUE self)
3751 VALUE input, output, output_byteoffset_v, output_bytesize_v, opt, flags_v;
3754 const unsigned char *ip, *is;
3755 unsigned char *op, *os;
3756 long output_byteoffset, output_bytesize;
3757 unsigned long output_byteend;
3760 argc =
rb_scan_args(argc, argv,
"23:", &input, &output, &output_byteoffset_v, &output_bytesize_v, &flags_v, &opt);
3762 if (
NIL_P(output_byteoffset_v))
3763 output_byteoffset = 0;
3765 output_byteoffset =
NUM2LONG(output_byteoffset_v);
3767 if (
NIL_P(output_bytesize_v))
3768 output_bytesize = 0;
3770 output_bytesize =
NUM2LONG(output_bytesize_v);
3772 if (!
NIL_P(flags_v)) {
3774 rb_error_arity(argc + 1, 2, 5);
3778 else if (!
NIL_P(opt)) {
3797 if (
NIL_P(output_bytesize_v)) {
3809 if (
NIL_P(output_byteoffset_v))
3812 if (output_byteoffset < 0)
3818 if (output_bytesize < 0)
3821 output_byteend = (
unsigned long)output_byteoffset +
3822 (
unsigned long)output_bytesize;
3824 if (output_byteend < (
unsigned long)output_byteoffset ||
3825 LONG_MAX < output_byteend)
3839 op = (
unsigned char *)
RSTRING_PTR(output) + output_byteoffset;
3840 os = op + output_bytesize;
3844 if (!
NIL_P(input)) {
3849 if (LONG_MAX / 2 < output_bytesize)
3851 output_bytesize *= 2;
3852 output_byteoffset_v =
Qnil;
3856 if (ec->destination_encoding) {
3860 return econv_result_to_symbol(res);
3898 econv_convert(
VALUE self,
VALUE source_string)
3916 ret = econv_primitive_convert(ac, av,
self);
3918 if (ret == sym_invalid_byte_sequence ||
3919 ret == sym_undefined_conversion ||
3920 ret == sym_incomplete_input) {
3921 VALUE exc = make_econv_exception(ec);
3925 if (ret == sym_finished) {
3929 if (ret != sym_source_buffer_empty) {
3930 rb_bug(
"unexpected result of econv_primitive_convert");
3948 econv_finish(
VALUE self)
3964 ret = econv_primitive_convert(ac, av,
self);
3966 if (ret == sym_invalid_byte_sequence ||
3967 ret == sym_undefined_conversion ||
3968 ret == sym_incomplete_input) {
3969 VALUE exc = make_econv_exception(ec);
3973 if (ret != sym_finished) {
3974 rb_bug(
"unexpected result of econv_primitive_convert");
4056 econv_primitive_errinfo(
VALUE self)
4064 rb_ary_store(ary, 0, econv_result_to_symbol(ec->last_error.result));
4067 if (ec->last_error.source_encoding)
4070 if (ec->last_error.destination_encoding)
4073 if (ec->last_error.error_bytes_start) {
4074 rb_ary_store(ary, 3,
rb_str_new((
const char *)ec->last_error.error_bytes_start, ec->last_error.error_bytes_len));
4075 rb_ary_store(ary, 4,
rb_str_new((
const char *)ec->last_error.error_bytes_start + ec->last_error.error_bytes_len, ec->last_error.readagain_len));
4114 econv_insert_output(
VALUE self,
VALUE string)
4116 const char *insert_enc;
4159 econv_putback(
int argc,
VALUE *argv,
VALUE self)
4172 if (putbackable < n)
4179 if (ec->source_encoding) {
4207 econv_last_error(
VALUE self)
4212 exc = make_econv_exception(ec);
4231 econv_get_replacement(
VALUE self)
4237 ret = make_replacement(ec);
4239 rb_raise(rb_eUndefinedConversionError,
"replacement character setup failed");
4243 return rb_enc_str_new((
const char *)ec->replacement_str, (
long)ec->replacement_len, enc);
4274 rb_raise(rb_eUndefinedConversionError,
"replacement character setup failed");
4283 return make_econv_exception(ec);
4291 exc = make_econv_exception(ec);
4304 ecerr_source_encoding_name(
VALUE self)
4306 return rb_attr_get(
self, id_source_encoding_name);
4330 ecerr_source_encoding(
VALUE self)
4342 ecerr_destination_encoding_name(
VALUE self)
4344 return rb_attr_get(
self, id_destination_encoding_name);
4354 ecerr_destination_encoding(
VALUE self)
4356 return rb_attr_get(
self, id_destination_encoding);
4375 ecerr_error_char(
VALUE self)
4396 ecerr_error_bytes(
VALUE self)
4408 ecerr_readagain_bytes(
VALUE self)
4438 ecerr_incomplete_input(
VALUE self)
4466 Init_transcode(
void)
4468 transcoder_table = st_init_strcasetable();
4471 id_destination_encoding_name =
rb_intern_const(
"destination_encoding_name");
4499 #ifdef ENABLE_ECONV_NEWLINE_OPTION
4511 InitVM_transcode(
void)
4527 rb_define_method(rb_cEncodingConverter,
"source_encoding", econv_source_encoding, 0);
4528 rb_define_method(rb_cEncodingConverter,
"destination_encoding", econv_destination_encoding, 0);
4529 rb_define_method(rb_cEncodingConverter,
"primitive_convert", econv_primitive_convert, -1);
4532 rb_define_method(rb_cEncodingConverter,
"primitive_errinfo", econv_primitive_errinfo, 0);
4533 rb_define_method(rb_cEncodingConverter,
"insert_output", econv_insert_output, 1);
4535 rb_define_method(rb_cEncodingConverter,
"last_error", econv_last_error, 0);
4536 rb_define_method(rb_cEncodingConverter,
"replacement", econv_get_replacement, 0);
4537 rb_define_method(rb_cEncodingConverter,
"replacement=", econv_set_replacement, 1);
4623 rb_define_method(rb_eUndefinedConversionError,
"source_encoding_name", ecerr_source_encoding_name, 0);
4624 rb_define_method(rb_eUndefinedConversionError,
"destination_encoding_name", ecerr_destination_encoding_name, 0);
4625 rb_define_method(rb_eUndefinedConversionError,
"source_encoding", ecerr_source_encoding, 0);
4626 rb_define_method(rb_eUndefinedConversionError,
"destination_encoding", ecerr_destination_encoding, 0);
4627 rb_define_method(rb_eUndefinedConversionError,
"error_char", ecerr_error_char, 0);
4629 rb_define_method(rb_eInvalidByteSequenceError,
"source_encoding_name", ecerr_source_encoding_name, 0);
4630 rb_define_method(rb_eInvalidByteSequenceError,
"destination_encoding_name", ecerr_destination_encoding_name, 0);
4631 rb_define_method(rb_eInvalidByteSequenceError,
"source_encoding", ecerr_source_encoding, 0);
4632 rb_define_method(rb_eInvalidByteSequenceError,
"destination_encoding", ecerr_destination_encoding, 0);
4633 rb_define_method(rb_eInvalidByteSequenceError,
"error_bytes", ecerr_error_bytes, 0);
4634 rb_define_method(rb_eInvalidByteSequenceError,
"readagain_bytes", ecerr_readagain_bytes, 0);
4635 rb_define_method(rb_eInvalidByteSequenceError,
"incomplete_input?", ecerr_incomplete_input, 0);
ruby_coderange_type
What rb_enc_str_coderange() returns.
#define rb_define_singleton_method(klass, mid, func, arity)
Defines klass.mid.
VALUE rb_define_class_under(VALUE outer, const char *name, VALUE super)
Defines a class under the namespace of outer.
int rb_scan_args(int argc, const VALUE *argv, const char *fmt,...)
Retrieves argument from argc and argv to given VALUE references according to the format string.
void rb_define_method(VALUE klass, const char *name, VALUE(*func)(ANYARGS), int argc)
Defines a method.
#define ECONV_XML_ATTR_QUOTE_DECORATOR
Old name of RUBY_ECONV_XML_ATTR_QUOTE_DECORATOR.
#define ECONV_AFTER_OUTPUT
Old name of RUBY_ECONV_AFTER_OUTPUT.
#define rb_str_new2
Old name of rb_str_new_cstr.
#define ENC_CODERANGE_7BIT
Old name of RUBY_ENC_CODERANGE_7BIT.
#define ENC_CODERANGE_VALID
Old name of RUBY_ENC_CODERANGE_VALID.
#define ECONV_UNIVERSAL_NEWLINE_DECORATOR
Old name of RUBY_ECONV_UNIVERSAL_NEWLINE_DECORATOR.
#define REALLOC_N
Old name of RB_REALLOC_N.
#define ALLOC
Old name of RB_ALLOC.
#define xfree
Old name of ruby_xfree.
#define Qundef
Old name of RUBY_Qundef.
#define INT2FIX
Old name of RB_INT2FIX.
#define OBJ_FROZEN
Old name of RB_OBJ_FROZEN.
#define ECONV_XML_ATTR_CONTENT_DECORATOR
Old name of RUBY_ECONV_XML_ATTR_CONTENT_DECORATOR.
#define ECONV_INVALID_MASK
Old name of RUBY_ECONV_INVALID_MASK.
#define ECONV_CRLF_NEWLINE_DECORATOR
Old name of RUBY_ECONV_CRLF_NEWLINE_DECORATOR.
#define xrealloc
Old name of ruby_xrealloc.
#define ID2SYM
Old name of RB_ID2SYM.
#define OBJ_FREEZE
Old name of RB_OBJ_FREEZE.
#define ECONV_UNDEF_REPLACE
Old name of RUBY_ECONV_UNDEF_REPLACE.
#define ECONV_XML_TEXT_DECORATOR
Old name of RUBY_ECONV_XML_TEXT_DECORATOR.
#define rb_ary_new4
Old name of rb_ary_new_from_values.
#define ENC_CODERANGE_UNKNOWN
Old name of RUBY_ENC_CODERANGE_UNKNOWN.
#define ECONV_CR_NEWLINE_DECORATOR
Old name of RUBY_ECONV_CR_NEWLINE_DECORATOR.
#define xmalloc
Old name of ruby_xmalloc.
#define ECONV_INVALID_REPLACE
Old name of RUBY_ECONV_INVALID_REPLACE.
#define T_HASH
Old name of RUBY_T_HASH.
#define ALLOC_N
Old name of RB_ALLOC_N.
#define MBCLEN_CHARFOUND_LEN(ret)
Old name of ONIGENC_MBCLEN_CHARFOUND_LEN.
#define rb_exc_new3
Old name of rb_exc_new_str.
#define ECONV_UNDEF_MASK
Old name of RUBY_ECONV_UNDEF_MASK.
#define Qtrue
Old name of RUBY_Qtrue.
#define ECONV_PARTIAL_INPUT
Old name of RUBY_ECONV_PARTIAL_INPUT.
#define NUM2INT
Old name of RB_NUM2INT.
#define ECONV_ERROR_HANDLER_MASK
Old name of RUBY_ECONV_ERROR_HANDLER_MASK.
#define INT2NUM
Old name of RB_INT2NUM.
#define Qnil
Old name of RUBY_Qnil.
#define Qfalse
Old name of RUBY_Qfalse.
#define ENC_CODERANGE_BROKEN
Old name of RUBY_ENC_CODERANGE_BROKEN.
#define T_ARRAY
Old name of RUBY_T_ARRAY.
#define NIL_P
Old name of RB_NIL_P.
#define MBCLEN_CHARFOUND_P(ret)
Old name of ONIGENC_MBCLEN_CHARFOUND_P.
#define ECONV_UNDEF_HEX_CHARREF
Old name of RUBY_ECONV_UNDEF_HEX_CHARREF.
#define NUM2LONG
Old name of RB_NUM2LONG.
#define ECONV_NEWLINE_DECORATOR_MASK
Old name of RUBY_ECONV_NEWLINE_DECORATOR_MASK.
#define rb_ary_new2
Old name of rb_ary_new_capa.
#define ENC_CODERANGE_SET(obj, cr)
Old name of RB_ENC_CODERANGE_SET.
#define SYMBOL_P
Old name of RB_SYMBOL_P.
void * rb_check_typeddata(VALUE obj, const rb_data_type_t *data_type)
Identical to rb_typeddata_is_kind_of(), except it raises exceptions instead of returning false.
void rb_raise(VALUE exc, const char *fmt,...)
Exception entry point.
void rb_exc_raise(VALUE mesg)
Raises an exception in the current thread.
int rb_typeddata_is_kind_of(VALUE obj, const rb_data_type_t *data_type)
Checks if the given object is of given kind.
void rb_bug(const char *fmt,...)
Interpreter panic switch.
VALUE rb_eTypeError
TypeError exception.
VALUE rb_eRuntimeError
RuntimeError exception.
VALUE rb_exc_new_str(VALUE etype, VALUE str)
Identical to rb_exc_new_cstr(), except it takes a Ruby's string instead of C's.
VALUE rb_eArgError
ArgumentError exception.
VALUE rb_eEncodingError
EncodingError exception.
void rb_warning(const char *fmt,...)
Issues a warning.
VALUE rb_obj_class(VALUE obj)
Queries the class of an object.
VALUE rb_cEncoding
Encoding class.
VALUE rb_cString
String class.
VALUE rb_to_int(VALUE val)
Identical to rb_check_to_int(), except it raises in case of conversion mismatch.
int rb_enc_precise_mbclen(const char *p, const char *e, rb_encoding *enc)
Queries the number of bytes of the character at the passed pointer.
int rb_enc_get_index(VALUE obj)
Queries the index of the encoding of the passed object, if any.
int rb_to_encoding_index(VALUE obj)
Obtains a encoding index from a wider range of objects (than rb_enc_find_index()).
VALUE rb_enc_associate(VALUE obj, rb_encoding *enc)
Identical to rb_enc_associate(), except it takes an encoding itself instead of its index.
rb_encoding * rb_enc_find(const char *name)
Identical to rb_find_encoding(), except it takes a C's string instead of Ruby's.
rb_encoding * rb_to_encoding(VALUE obj)
Identical to rb_find_encoding(), except it raises an exception instead of returning NULL.
VALUE rb_enc_from_encoding(rb_encoding *enc)
Queries the Ruby-level counterpart instance of rb_cEncoding that corresponds to the passed encoding.
static bool rb_enc_asciicompat(rb_encoding *enc)
Queries if the passed encoding is in some sense compatible with ASCII.
int rb_define_dummy_encoding(const char *name)
Creates a new "dummy" encoding.
rb_encoding * rb_utf8_encoding(void)
Queries the encoding that represents UTF-8.
rb_encoding * rb_enc_from_index(int idx)
Identical to rb_find_encoding(), except it takes an encoding index instead of a Ruby object.
VALUE rb_enc_default_internal(void)
Identical to rb_default_internal_encoding(), except it returns the Ruby-level counterpart instance of...
VALUE rb_enc_associate_index(VALUE obj, int encindex)
Identical to rb_enc_set_index(), except it additionally does contents fix-up depending on the passed ...
rb_encoding * rb_enc_get(VALUE obj)
Identical to rb_enc_get_index(), except the return type.
static OnigCodePoint rb_enc_mbc_to_codepoint(const char *p, const char *e, rb_encoding *enc)
Identical to rb_enc_codepoint(), except it assumes the passed character is not broken.
static const char * rb_enc_name(rb_encoding *enc)
Queries the (canonical) name of the passed encoding.
int rb_enc_find_index(const char *name)
Queries the index of the encoding.
VALUE rb_str_conv_enc(VALUE str, rb_encoding *from, rb_encoding *to)
Encoding conversion main routine.
int rb_enc_str_coderange(VALUE str)
Scans the passed string to collect its code range.
VALUE rb_enc_str_new(const char *ptr, long len, rb_encoding *enc)
Identical to rb_enc_str_new(), except it additionally takes an encoding.
VALUE rb_obj_encoding(VALUE obj)
Identical to rb_enc_get_index(), except the return type.
long rb_str_coderange_scan_restartable(const char *str, const char *end, rb_encoding *enc, int *cr)
Scans the passed string until it finds something odd.
int rb_econv_prepare_options(VALUE opthash, VALUE *ecopts, int ecflags)
Identical to rb_econv_prepare_opts(), except it additionally takes the initial value of flags.
VALUE rb_econv_open_exc(const char *senc, const char *denc, int ecflags)
Creates a rb_eConverterNotFoundError exception object (but does not raise).
int rb_econv_prepare_opts(VALUE opthash, VALUE *ecopts)
Splits a keyword arguments hash (that for instance String#encode took) into a set of enum ruby_econv_...
rb_econv_result_t rb_econv_convert(rb_econv_t *ec, const unsigned char **source_buffer_ptr, const unsigned char *source_buffer_end, unsigned char **destination_buffer_ptr, unsigned char *destination_buffer_end, int flags)
Converts a string from an encoding to another.
rb_econv_result_t
return value of rb_econv_convert()
@ econv_incomplete_input
The conversion stopped in middle of reading a character, possibly due to a partial read of a socket e...
@ econv_finished
The conversion stopped after converting everything.
@ econv_undefined_conversion
The conversion stopped when it found a character in the input which cannot be representable in the ou...
@ econv_after_output
The conversion stopped after writing something to somewhere, before reading everything.
@ econv_source_buffer_empty
The conversion stopped because there is no input.
@ econv_destination_buffer_full
The conversion stopped because there is no destination.
@ econv_invalid_byte_sequence
The conversion stopped when it found an invalid sequence.
int rb_econv_putbackable(rb_econv_t *ec)
Queries if rb_econv_putback() makes sense, i.e.
int rb_econv_has_convpath_p(const char *from_encoding, const char *to_encoding)
Queries if there is more than one way to convert between the passed two encodings.
VALUE rb_econv_str_append(rb_econv_t *ec, VALUE src, VALUE dst, int flags)
Identical to rb_econv_str_convert(), except it appends the conversion result to the additionally pass...
VALUE rb_econv_substr_append(rb_econv_t *ec, VALUE src, long byteoff, long bytesize, VALUE dst, int flags)
Identical to rb_econv_str_append(), except it appends only a part of the passed string with conversio...
int rb_econv_insert_output(rb_econv_t *ec, const unsigned char *str, size_t len, const char *str_encoding)
Appends the passed string to the passed converter's output buffer.
VALUE rb_econv_str_convert(rb_econv_t *ec, VALUE src, int flags)
Identical to rb_econv_convert(), except it takes Ruby's string instead of C's pointer.
int rb_econv_decorate_at_last(rb_econv_t *ec, const char *decorator_name)
Identical to rb_econv_decorate_at_first(), except it adds to the opposite direction.
void rb_econv_binmode(rb_econv_t *ec)
This badly named function does not set the destination encoding to binary, but instead just nullifies...
int rb_econv_decorate_at_first(rb_econv_t *ec, const char *decorator_name)
"Decorate"s a converter.
VALUE rb_str_encode(VALUE str, VALUE to, int ecflags, VALUE ecopts)
Converts the contents of the passed string from its encoding to the passed one.
VALUE rb_econv_make_exception(rb_econv_t *ec)
This function makes sense right after rb_econv_convert() returns.
struct rb_econv_t rb_econv_t
An opaque struct that represents a lowest level of encoding conversion.
void rb_econv_check_error(rb_econv_t *ec)
This is a rb_econv_make_exception() + rb_exc_raise() combo.
const char * rb_econv_asciicompat_encoding(const char *encname)
Queries the passed encoding's corresponding ASCII compatible encoding.
VALUE rb_econv_substr_convert(rb_econv_t *ec, VALUE src, long byteoff, long bytesize, int flags)
Identical to rb_econv_str_convert(), except it converts only a part of the passed string.
rb_econv_t * rb_econv_open_opts(const char *source_encoding, const char *destination_encoding, int ecflags, VALUE ecopts)
Identical to rb_econv_open(), except it additionally takes a hash of optional strings.
void rb_econv_close(rb_econv_t *ec)
Destructs a converter.
VALUE rb_econv_append(rb_econv_t *ec, const char *bytesrc, long bytesize, VALUE dst, int flags)
Converts the passed C's pointer according to the passed converter, then append the conversion result ...
void rb_econv_putback(rb_econv_t *ec, unsigned char *p, int n)
Puts back the bytes.
int rb_econv_set_replacement(rb_econv_t *ec, const unsigned char *str, size_t len, const char *encname)
Assigns the replacement string.
rb_econv_t * rb_econv_open(const char *source_encoding, const char *destination_encoding, int ecflags)
Creates a new instance of struct rb_econv_t.
const char * rb_econv_encoding_to_insert_output(rb_econv_t *ec)
Queries an encoding name which best suits for rb_econv_insert_output()'s last parameter.
VALUE rb_funcallv_public(VALUE recv, ID mid, int argc, const VALUE *argv)
Identical to rb_funcallv(), except it only takes public methods into account.
VALUE rb_check_array_type(VALUE obj)
Try converting an object to its array representation using its to_ary method, if any.
VALUE rb_ary_new(void)
Allocates a new, empty array.
VALUE rb_ary_push(VALUE ary, VALUE elem)
Special case of rb_ary_cat() that it adds only one element.
VALUE rb_ary_entry(VALUE ary, long off)
Queries an element of an array.
VALUE rb_assoc_new(VALUE car, VALUE cdr)
Identical to rb_ary_new_from_values(), except it expects exactly two parameters.
void rb_ary_store(VALUE ary, long key, VALUE val)
Destructively stores the passed value to the passed array's passed index.
#define rb_check_frozen
Just another name of rb_check_frozen.
static int rb_check_arity(int argc, int min, int max)
Ensures that the passed integer is in the passed range.
VALUE rb_check_hash_type(VALUE obj)
Try converting an object to its hash representation using its to_hash method, if any.
VALUE rb_hash_freeze(VALUE obj)
Just another name of rb_obj_freeze.
VALUE rb_hash_aref(VALUE hash, VALUE key)
Queries the given key in the given hash table.
VALUE rb_hash_aset(VALUE hash, VALUE key, VALUE val)
Inserts or replaces ("upsert"s) the objects into the given hash table.
VALUE rb_hash_new(void)
Creates a new, empty hash object.
VALUE rb_proc_call(VALUE recv, VALUE args)
Evaluates the passed proc with the passed arguments.
VALUE rb_obj_is_method(VALUE recv)
Queries if the given object is a method.
VALUE rb_method_call(int argc, const VALUE *argv, VALUE recv)
Evaluates the passed method with the passed arguments.
VALUE rb_obj_is_proc(VALUE recv)
Queries if the given object is a proc.
VALUE rb_str_tmp_new(long len)
Allocates a "temporary" string.
void rb_str_shared_replace(VALUE dst, VALUE src)
Replaces the contents of the former with the latter.
size_t rb_str_capacity(VALUE str)
Queries the capacity of the given string.
VALUE rb_str_new_frozen(VALUE str)
Creates a frozen copy of the string, if necessary.
VALUE rb_str_cat2(VALUE, const char *)
Just another name of rb_str_cat_cstr.
VALUE rb_str_dup(VALUE str)
Duplicates a string.
void rb_str_modify(VALUE str)
Declares that the string is about to be modified.
void rb_str_set_len(VALUE str, long len)
Overwrites the length of the string.
VALUE rb_str_new(const char *ptr, long len)
Allocates an instance of rb_cString.
VALUE rb_str_new_cstr(const char *ptr)
Identical to rb_str_new(), except it assumes the passed pointer is a pointer to a C string.
VALUE rb_str_resize(VALUE str, long len)
Overwrites the length of the string.
void rb_str_modify_expand(VALUE str, long capa)
Identical to rb_str_modify(), except it additionally expands the capacity of the receiver.
VALUE rb_str_dump(VALUE str)
"Inverse" of rb_eval_string().
VALUE rb_str_buf_new(long capa)
Allocates a "string buffer".
VALUE rb_str_drop_bytes(VALUE str, long len)
Shrinks the given string for the given number of bytes.
VALUE rb_attr_get(VALUE obj, ID name)
Identical to rb_ivar_get()
VALUE rb_ivar_set(VALUE obj, ID name, VALUE val)
Identical to rb_iv_set(), except it accepts the name as an ID instead of a C string.
int rb_respond_to(VALUE obj, ID mid)
Queries if the object responds to the method.
void rb_define_alloc_func(VALUE klass, rb_alloc_func_t func)
Sets the allocator function of a class.
static ID rb_intern_const(const char *str)
This is a "tiny optimisation" over rb_intern().
VALUE rb_sym2str(VALUE id)
Identical to rb_id2str(), except it takes an instance of rb_cSymbol rather than an ID.
void rb_define_const(VALUE klass, const char *name, VALUE val)
Defines a Ruby level constant under a namespace.
VALUE rb_sprintf(const char *fmt,...)
Ruby's extended sprintf(3).
VALUE rb_str_catf(VALUE dst, const char *fmt,...)
Identical to rb_sprintf(), except it renders the output to the specified object rather than creating ...
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
#define ALLOCA_N(type, n)
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
#define MEMMOVE(p1, p2, type, n)
Handy macro to call memmove.
int st_foreach(st_table *q, int_type *w, st_data_t e)
Iteration over the given table.
#define RARRAY_LEN
Just another name of rb_array_len.
static int RARRAY_LENINT(VALUE ary)
Identical to rb_array_len(), except it differs for the return type.
#define RARRAY_AREF(a, i)
#define DATA_PTR(obj)
Convenient getter macro.
@ RSTRING_EMBED_LEN_MAX
Max possible number of characters that can be embedded.
#define StringValue(v)
Ensures that the parameter object is a String.
static char * RSTRING_END(VALUE str)
Queries the end of the contents pointer of the string.
static char * RSTRING_PTR(VALUE str)
Queries the contents pointer of the string.
static long RSTRING_LEN(VALUE str)
Queries the length of the string.
#define StringValueCStr(v)
Identical to StringValuePtr, except it additionally checks for the contents for viability as a C stri...
#define TypedData_Get_Struct(obj, type, data_type, sval)
Obtains a C struct from inside of a wrapper Ruby object.
#define TypedData_Wrap_Struct(klass, data_type, sval)
Converts sval, a pointer to your struct, into a Ruby object.
const char * rb_obj_classname(VALUE obj)
Queries the name of the class of the passed object.
#define InitVM(ext)
This macro is for internal use.
#define RTEST
This is an old name of RB_TEST.
This is the struct that holds necessary info for a struct.
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
uintptr_t VALUE
Type that represents a Ruby object.
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.