Project

General

Profile

Download (5.64 KB)

Bug #15839 » mixed-encoding-heredoc-fix-v2.patch

jeremyevans0 (Jeremy Evans), 05/15/2019 11:24 PM

View differences:

parse.y
# define yylval (*p->lval)
static int regx_options(struct parser_params*);
static int tokadd_string(struct parser_params*,int,int,int,long*,rb_encoding**);
static int tokadd_string(struct parser_params*,int,int,int,long*,rb_encoding**,rb_encoding**);
static void tokaddmbc(struct parser_params *p, int c, rb_encoding *enc);
static enum yytokentype parse_string(struct parser_params*,rb_strterm_literal_t*);
static enum yytokentype here_document(struct parser_params*,rb_strterm_heredoc_t*);
......
static int
tokadd_string(struct parser_params *p,
int func, int term, int paren, long *nest,
rb_encoding **encp)
rb_encoding **encp, rb_encoding **enc)
{
int c;
rb_encoding *enc = 0;
bool erred = false;
#define mixed_error(enc1, enc2) \
......
tokadd(p, '\\');
break;
}
if (!parser_tokadd_utf8(p, &enc, term,
if (!parser_tokadd_utf8(p, enc, term,
func & STR_FUNC_SYMBOL,
func & STR_FUNC_REGEXP)) {
return -1;
......
continue;
}
pushback(p, c);
if ((c = tokadd_escape(p, &enc)) < 0)
if ((c = tokadd_escape(p, enc)) < 0)
return -1;
if (enc && enc != *encp) {
mixed_escape(p->lex.ptok+2, enc, *encp);
if (*enc && *enc != *encp) {
mixed_escape(p->lex.ptok+2, *enc, *encp);
}
continue;
}
else if (func & STR_FUNC_EXPAND) {
pushback(p, c);
if (func & STR_FUNC_ESCAPE) tokadd(p, '\\');
c = read_escape(p, 0, &enc);
c = read_escape(p, 0, enc);
}
else if ((func & STR_FUNC_QWORDS) && ISSPACE(c)) {
/* ignore backslashed spaces in %w */
......
}
else if (!parser_isascii(p)) {
non_ascii:
if (!enc) {
enc = *encp;
if (!*enc) {
*enc = *encp;
}
else if (enc != *encp) {
mixed_error(enc, *encp);
else if (*enc != *encp) {
mixed_error(*enc, *encp);
continue;
}
if (tokadd_mbchar(p, c) == -1) return -1;
......
break;
}
if (c & 0x80) {
if (!enc) {
enc = *encp;
if (!*enc) {
*enc = *encp;
}
else if (enc != *encp) {
mixed_error(enc, *encp);
else if (*enc != *encp) {
mixed_error(*enc, *encp);
continue;
}
}
tokadd(p, c);
}
terminate:
if (enc) *encp = enc;
if (*enc) *encp = *enc;
return c;
}
......
int paren = (int)quote->u2.paren;
int c, space = 0;
rb_encoding *enc = p->enc;
rb_encoding *base_enc = 0;
VALUE lit;
if (func & STR_FUNC_TERM) {
......
}
pushback(p, c);
if (tokadd_string(p, func, term, paren, &quote->u0.nest,
&enc) == -1) {
&enc, &base_enc) == -1) {
if (p->eofp) {
#ifndef RIPPER
# define unterminated_literal(mesg) yyerror0(mesg)
......
long len;
VALUE str = 0;
rb_encoding *enc = p->enc;
rb_encoding *base_enc = 0;
int bol;
eos = RSTRING_PTR(here->lastline) + here->offset;
......
}
do {
pushback(p, c);
if ((c = tokadd_string(p, func, '\n', 0, NULL, &enc)) == -1) {
enc = p->enc;
if ((c = tokadd_string(p, func, '\n', 0, NULL, &enc, &base_enc)) == -1) {
if (p->eofp) goto error;
goto restore;
}
test/ruby/test_syntax.rb
assert_equal("\n0\n1", eval("<<~0 '1'\n \n0\#{}\n0"))
end
def test_heredoc_mixed_encoding
assert_syntax_error(<<-'HEREDOC', 'UTF-8 mixed within Windows-31J source')
#encoding: cp932
<<-TEXT
\xe9\x9d\u1234
TEXT
HEREDOC
assert_syntax_error(<<-'HEREDOC', 'UTF-8 mixed within Windows-31J source')
#encoding: cp932
<<-TEXT
\xe9\x9d
\u1234
TEXT
HEREDOC
assert_syntax_error(<<-'HEREDOC', 'UTF-8 mixed within Windows-31J source')
#encoding: cp932
<<-TEXT
\u1234\xe9\x9d
TEXT
HEREDOC
assert_syntax_error(<<-'HEREDOC', 'UTF-8 mixed within Windows-31J source')
#encoding: cp932
<<-TEXT
\u1234
\xe9\x9d
TEXT
HEREDOC
end
def test_lineno_operation_brace_block
expected = __LINE__ + 1
actual = caller_lineno\
(3-3/3)