Index: ext/ripper/lib/ripper/filter.rb
===================================================================
--- ext/ripper/lib/ripper/filter.rb	(revision 59171)
+++ ext/ripper/lib/ripper/filter.rb	(working copy)
@@ -25,6 +25,7 @@
       @__lexer = Lexer.new(src, filename, lineno)
       @__line = nil
       @__col = nil
+      @__state = nil
     end
 
     # The file name of the input.
@@ -46,13 +47,20 @@
       @__col
     end
 
+    # The scanner's state of the current token.
+    # This value is the bitwise OR of zero or more of the +Ripper::EXPR_*+ constants.
+    def state
+      @__state
+    end
+
     # Starts the parser.
     # +init+ is a data accumulator and is passed to the next event handler (as
     # of Enumerable#inject).
     def parse(init = nil)
       data = init
-      @__lexer.lex.each do |pos, event, tok|
+      @__lexer.lex.each do |pos, event, tok, state|
         @__line, @__col = *pos
+        @__state = state
         data = if respond_to?(event, true)
                then __send__(event, tok, data)
                else on_default(event, tok, data)
Index: ext/ripper/lib/ripper/lexer.rb
===================================================================
--- ext/ripper/lib/ripper/lexer.rb	(revision 59171)
+++ ext/ripper/lib/ripper/lexer.rb	(working copy)
@@ -23,22 +23,23 @@
   end
 
   # Tokenizes the Ruby program and returns an array of an array,
-  # which is formatted like <code>[[lineno, column], type, token]</code>.
+  # which is formatted like
+  # <code>[[lineno, column], type, token, state]</code>.
   #
   #   require 'ripper'
   #   require 'pp'
   #
   #   pp Ripper.lex("def m(a) nil end")
-  #     #=> [[[1,  0], :on_kw,     "def"],
-  #          [[1,  3], :on_sp,     " "  ],
-  #          [[1,  4], :on_ident,  "m"  ],
-  #          [[1,  5], :on_lparen, "("  ],
-  #          [[1,  6], :on_ident,  "a"  ],
-  #          [[1,  7], :on_rparen, ")"  ],
-  #          [[1,  8], :on_sp,     " "  ],
-  #          [[1,  9], :on_kw,     "nil"],
-  #          [[1, 12], :on_sp,     " "  ],
-  #          [[1, 13], :on_kw,     "end"]]
+  #   #=> [[[1,  0], :on_kw,     "def", Ripper::EXPR_FNAME                   ],
+  #        [[1,  3], :on_sp,     " ",   Ripper::EXPR_FNAME                   ],
+  #        [[1,  4], :on_ident,  "m",   Ripper::EXPR_ENDFN                   ],
+  #        [[1,  5], :on_lparen, "(",   Ripper::EXPR_LABEL | Ripper::EXPR_BEG],
+  #        [[1,  6], :on_ident,  "a",   Ripper::EXPR_ARG                     ],
+  #        [[1,  7], :on_rparen, ")",   Ripper::EXPR_ENDFN                   ],
+  #        [[1,  8], :on_sp,     " ",   Ripper::EXPR_BEG                     ],
+  #        [[1,  9], :on_kw,     "nil", Ripper::EXPR_END                     ],
+  #        [[1, 12], :on_sp,     " ",   Ripper::EXPR_END                     ],
+  #        [[1, 13], :on_kw,     "end", Ripper::EXPR_END                     ]]
   #
   def Ripper.lex(src, filename = '-', lineno = 1)
     Lexer.new(src, filename, lineno).lex
@@ -45,7 +46,7 @@
   end
 
   class Lexer < ::Ripper   #:nodoc: internal use only
-    Elem = Struct.new(:pos, :event, :tok)
+    Elem = Struct.new(:pos, :event, :tok, :state)
 
     def tokenize
       parse().sort_by(&:pos).map(&:tok)
@@ -72,7 +73,7 @@
         if Elem === e and e.event == :on_tstring_content
           tok = e.tok.dup if w > 0 and /\A\s/ =~ e.tok
           if (n = dedent_string(e.tok, w)) > 0
-            ignored_sp << [i, Elem.new(e.pos.dup, :on_ignored_sp, tok[0, n])]
+            ignored_sp << [i, Elem.new(e.pos.dup, :on_ignored_sp, tok[0, n], e.state)]
             e.pos[1] += n
           end
         end
@@ -88,16 +89,16 @@
       buf = []
       @buf << buf
       @buf = buf
-      @buf.push Elem.new([lineno(), column()], __callee__, tok)
+      @buf.push Elem.new([lineno(), column()], __callee__, tok, state())
     end
 
     def on_heredoc_end(tok)
-      @buf.push Elem.new([lineno(), column()], __callee__, tok)
+      @buf.push Elem.new([lineno(), column()], __callee__, tok, state())
       @buf = @stack.pop
     end
 
     def _push_token(tok)
-      @buf.push Elem.new([lineno(), column()], __callee__, tok)
+      @buf.push Elem.new([lineno(), column()], __callee__, tok, state())
     end
 
     (SCANNER_EVENTS.map {|event|:"on_#{event}"} - private_instance_methods(false)).each do |event|
Index: parse.y
===================================================================
--- parse.y	(revision 59171)
+++ parse.y	(working copy)
@@ -156,6 +156,7 @@
     const char *token;
     int linenum;
     int column;
+    enum lex_state_e state;
     int nonspc;
     struct token_info *next;
 } token_info;
@@ -4989,6 +4990,7 @@
     ptinfo->token = token;
     ptinfo->linenum = ruby_sourceline;
     ptinfo->column = token_info_get_column(parser, t);
+    ptinfo->state = lex_state;
     ptinfo->nonspc = token_info_has_nonspaces(parser, t);
     ptinfo->next = parser->token_info;
 
@@ -11389,6 +11391,25 @@
     return INT2NUM(ruby_sourceline);
 }
 
+/*
+ *  call-seq:
+ *    ripper#state   -> Integer
+ *
+ *  Return scanner state of current token.
+ */
+static VALUE
+ripper_state(VALUE self)
+{
+    struct parser_params *parser;
+
+    TypedData_Get_Struct(self, struct parser_params, &parser_data_type, parser);
+    if (!ripper_initialized_p(parser)) {
+        rb_raise(rb_eArgError, "method called for uninitialized object");
+    }
+    if (NIL_P(parser->parsing_thread)) return Qnil;
+    return INT2NUM(lex_state);
+}
+
 #ifdef RIPPER_DEBUG
 /* :nodoc: */
 static VALUE
@@ -11436,6 +11457,7 @@
     rb_define_method(Ripper, "column", ripper_column, 0);
     rb_define_method(Ripper, "filename", ripper_filename, 0);
     rb_define_method(Ripper, "lineno", ripper_lineno, 0);
+    rb_define_method(Ripper, "state", ripper_state, 0);
     rb_define_method(Ripper, "end_seen?", rb_parser_end_seen_p, 0);
     rb_define_method(Ripper, "encoding", rb_parser_encoding, 0);
     rb_define_method(Ripper, "yydebug", rb_parser_get_yydebug, 0);
@@ -11450,6 +11472,41 @@
     rb_define_singleton_method(Ripper, "dedent_string", parser_dedent_string, 2);
     rb_define_private_method(Ripper, "dedent_string", parser_dedent_string, 2);
 
+    /* ignore newline, +/- is a sign. */
+    rb_define_const(Ripper, "EXPR_BEG", INT2NUM(EXPR_BEG));
+    /* newline significant, +/- is an operator. */
+    rb_define_const(Ripper, "EXPR_END", INT2NUM(EXPR_END));
+    /* ditto, and unbound braces. */
+    rb_define_const(Ripper, "EXPR_ENDARG", INT2NUM(EXPR_ENDARG));
+    /* ditto, and unbound braces. */
+    rb_define_const(Ripper, "EXPR_ENDFN", INT2NUM(EXPR_ENDFN));
+    /* newline significant, +/- is an operator. */
+    rb_define_const(Ripper, "EXPR_ARG", INT2NUM(EXPR_ARG));
+    /* newline significant, +/- is an operator. */
+    rb_define_const(Ripper, "EXPR_CMDARG", INT2NUM(EXPR_CMDARG));
+    /* newline significant, +/- is an operator. */
+    rb_define_const(Ripper, "EXPR_MID", INT2NUM(EXPR_MID));
+    /* ignore newline, no reserved words. */
+    rb_define_const(Ripper, "EXPR_FNAME", INT2NUM(EXPR_FNAME));
+    /* right after `.' or `::', no reserved words. */
+    rb_define_const(Ripper, "EXPR_DOT", INT2NUM(EXPR_DOT));
+    /* immediate after `class', no here document. */
+    rb_define_const(Ripper, "EXPR_CLASS", INT2NUM(EXPR_CLASS));
+    /* flag bit, label is allowed. */
+    rb_define_const(Ripper, "EXPR_LABEL", INT2NUM(EXPR_LABEL));
+    /* flag bit, just after a label. */
+    rb_define_const(Ripper, "EXPR_LABELED", INT2NUM(EXPR_LABELED));
+    /* symbol literal as FNAME. */
+    rb_define_const(Ripper, "EXPR_FITEM", INT2NUM(EXPR_FITEM));
+    /* equals to +EXPR_BEG+ */
+    rb_define_const(Ripper, "EXPR_VALUE", INT2NUM(EXPR_VALUE));
+    /* equals to <tt>(EXPR_BEG | EXPR_MID | EXPR_CLASS)</tt> */
+    rb_define_const(Ripper, "EXPR_BEG_ANY", INT2NUM(EXPR_BEG_ANY));
+    /* equals to <tt>(EXPR_ARG | EXPR_CMDARG)</tt> */
+    rb_define_const(Ripper, "EXPR_ARG_ANY", INT2NUM(EXPR_ARG_ANY));
+    /* equals to <tt>(EXPR_END | EXPR_ENDARG | EXPR_ENDFN)</tt> */
+    rb_define_const(Ripper, "EXPR_END_ANY", INT2NUM(EXPR_END_ANY));
+
     ripper_init_eventids1_table(Ripper);
     ripper_init_eventids2_table(Ripper);
 
Index: test/ripper/test_filter.rb
===================================================================
--- test/ripper/test_filter.rb	(revision 59171)
+++ test/ripper/test_filter.rb	(working copy)
@@ -15,6 +15,7 @@
         data[:filename] = filename rescue nil
         data[:lineno] = lineno
         data[:column] = column
+        data[:state] = state
         data[:token] = token
       end
       data
@@ -75,6 +76,16 @@
     assert_equal(last_columns, filter.column)
   end
 
+  def test_filter_state
+    data = {}
+    src = File.read(filename)
+    filter = Filter.new(src)
+    assert_equal(nil, filter.state)
+    filter.parse(data)
+    assert_not_nil(data[:state])
+    assert_not_nil(filter.state)
+  end
+
   def test_filter_token
     data = {}
     filter = Filter.new("begin; puts 1; end")
Index: test/ripper/test_ripper.rb
===================================================================
--- test/ripper/test_ripper.rb	(revision 59171)
+++ test/ripper/test_ripper.rb	(working copy)
@@ -17,6 +17,10 @@
     assert_nil @ripper.column
   end
 
+  def test_state
+    assert_nil @ripper.state
+  end
+
   def test_encoding
     assert_equal Encoding::UTF_8, @ripper.encoding
     ripper = Ripper.new('# coding: iso-8859-15')
Index: test/ripper/test_scanner_events.rb
===================================================================
--- test/ripper/test_scanner_events.rb	(revision 59171)
+++ test/ripper/test_scanner_events.rb	(working copy)
@@ -48,70 +48,70 @@
   def test_lex
     assert_equal [],
                  Ripper.lex('')
-    assert_equal [[[1,0], :on_ident, "a"]],
+    assert_equal [[[1,0], :on_ident, "a", Ripper::EXPR_CMDARG]],
                  Ripper.lex('a')
-    assert_equal [[[1, 0], :on_kw, "nil"]],
+    assert_equal [[[1, 0], :on_kw, "nil", Ripper::EXPR_END]],
                  Ripper.lex("nil")
-    assert_equal [[[1, 0], :on_kw, "def"],
-                  [[1, 3], :on_sp, " "],
-                  [[1, 4], :on_ident, "m"],
-                  [[1, 5], :on_lparen, "("],
-                  [[1, 6], :on_ident, "a"],
-                  [[1, 7], :on_rparen, ")"],
-                  [[1, 8], :on_kw, "end"]],
+    assert_equal [[[1, 0], :on_kw, "def", Ripper::EXPR_FNAME],
+                  [[1, 3], :on_sp, " ", Ripper::EXPR_FNAME],
+                  [[1, 4], :on_ident, "m", Ripper::EXPR_ENDFN],
+                  [[1, 5], :on_lparen, "(", Ripper::EXPR_BEG | Ripper::EXPR_LABEL],
+                  [[1, 6], :on_ident, "a", Ripper::EXPR_ARG],
+                  [[1, 7], :on_rparen, ")", Ripper::EXPR_ENDFN],
+                  [[1, 8], :on_kw, "end", Ripper::EXPR_END]],
                  Ripper.lex("def m(a)end")
-    assert_equal [[[1, 0], :on_int, "1"],
-                  [[1, 1], :on_nl, "\n"],
-                  [[2, 0], :on_int, "2"],
-                  [[2, 1], :on_nl, "\n"],
-                  [[3, 0], :on_int, "3"]],
+    assert_equal [[[1, 0], :on_int, "1", Ripper::EXPR_END | Ripper::EXPR_ENDARG],
+                  [[1, 1], :on_nl, "\n", Ripper::EXPR_BEG],
+                  [[2, 0], :on_int, "2", Ripper::EXPR_END | Ripper::EXPR_ENDARG],
+                  [[2, 1], :on_nl, "\n", Ripper::EXPR_BEG],
+                  [[3, 0], :on_int, "3", Ripper::EXPR_END | Ripper::EXPR_ENDARG]],
                  Ripper.lex("1\n2\n3")
-    assert_equal [[[1, 0], :on_heredoc_beg, "<<""EOS"],
-                  [[1, 5], :on_nl, "\n"],
-                  [[2, 0], :on_tstring_content, "heredoc\n"],
-                  [[3, 0], :on_heredoc_end, "EOS"]],
+    assert_equal [[[1, 0], :on_heredoc_beg, "<<""EOS", Ripper::EXPR_BEG],
+                  [[1, 5], :on_nl, "\n", Ripper::EXPR_BEG],
+                  [[2, 0], :on_tstring_content, "heredoc\n", Ripper::EXPR_BEG],
+                  [[3, 0], :on_heredoc_end, "EOS", Ripper::EXPR_BEG]],
                  Ripper.lex("<<""EOS\nheredoc\nEOS")
-    assert_equal [[[1, 0], :on_heredoc_beg, "<<""EOS"],
-                  [[1, 5], :on_nl, "\n"],
-                  [[2, 0], :on_heredoc_end, "EOS"]],
+    assert_equal [[[1, 0], :on_heredoc_beg, "<<""EOS", Ripper::EXPR_BEG],
+                  [[1, 5], :on_nl, "\n", Ripper::EXPR_BEG],
+                  [[2, 0], :on_heredoc_end, "EOS", Ripper::EXPR_BEG]],
                  Ripper.lex("<<""EOS\nEOS"),
                  "bug#4543"
-    assert_equal [[[1, 0], :on_regexp_beg, "/"],
-                  [[1, 1], :on_tstring_content, "foo\nbar"],
-                  [[2, 3], :on_regexp_end, "/"]],
+    assert_equal [[[1, 0], :on_regexp_beg, "/", Ripper::EXPR_BEG],
+                  [[1, 1], :on_tstring_content, "foo\nbar", Ripper::EXPR_BEG],
+                  [[2, 3], :on_regexp_end, "/", Ripper::EXPR_BEG]],
                  Ripper.lex("/foo\nbar/")
-    assert_equal [[[1, 0], :on_regexp_beg, "/"],
-                  [[1, 1], :on_tstring_content, "foo\n\u3020"],
-                  [[2, 3], :on_regexp_end, "/"]],
+    assert_equal [[[1, 0], :on_regexp_beg, "/", Ripper::EXPR_BEG],
+                  [[1, 1], :on_tstring_content, "foo\n\u3020", Ripper::EXPR_BEG],
+                  [[2, 3], :on_regexp_end, "/", Ripper::EXPR_BEG]],
                  Ripper.lex("/foo\n\u3020/")
-    assert_equal [[[1, 0], :on_tstring_beg, "'"],
-                  [[1, 1], :on_tstring_content, "foo\n\xe3\x80\xa0"],
-                  [[2, 3], :on_tstring_end, "'"]],
+    assert_equal [[[1, 0], :on_tstring_beg, "'", Ripper::EXPR_BEG],
+                  [[1, 1], :on_tstring_content, "foo\n\xe3\x80\xa0", Ripper::EXPR_BEG],
+                  [[2, 3], :on_tstring_end, "'", Ripper::EXPR_END | Ripper::EXPR_ENDARG]],
                  Ripper.lex("'foo\n\xe3\x80\xa0'")
-    assert_equal [[[1, 0], :on_tstring_beg, "'"],
-                  [[1, 1], :on_tstring_content, "\u3042\n\u3044"],
-                  [[2, 3], :on_tstring_end, "'"]],
+    assert_equal [[[1, 0], :on_tstring_beg, "'", Ripper::EXPR_BEG],
+                  [[1, 1], :on_tstring_content, "\u3042\n\u3044", Ripper::EXPR_BEG],
+                  [[2, 3], :on_tstring_end, "'", Ripper::EXPR_END | Ripper::EXPR_ENDARG]],
                  Ripper.lex("'\u3042\n\u3044'")
-    assert_equal [[[1, 0], :on_rational, "1r"],
-                  [[1, 2], :on_nl, "\n"],
-                  [[2, 0], :on_imaginary, "2i"],
-                  [[2, 2], :on_nl, "\n"],
-                  [[3, 0], :on_imaginary, "3ri"],
-                  [[3, 3], :on_nl, "\n"],
-                  [[4, 0], :on_rational, "4.2r"],
-                  [[4, 4], :on_nl, "\n"],
-                  [[5, 0], :on_imaginary, "5.6ri"],
+    assert_equal [[[1, 0], :on_rational, "1r", Ripper::EXPR_END | Ripper::EXPR_ENDARG],
+                  [[1, 2], :on_nl, "\n", Ripper::EXPR_BEG],
+                  [[2, 0], :on_imaginary, "2i", Ripper::EXPR_END | Ripper::EXPR_ENDARG],
+                  [[2, 2], :on_nl, "\n", Ripper::EXPR_BEG],
+                  [[3, 0], :on_imaginary, "3ri", Ripper::EXPR_END | Ripper::EXPR_ENDARG],
+                  [[3, 3], :on_nl, "\n", Ripper::EXPR_BEG],
+                  [[4, 0], :on_rational, "4.2r", Ripper::EXPR_END | Ripper::EXPR_ENDARG],
+                  [[4, 4], :on_nl, "\n", Ripper::EXPR_BEG],
+                  [[5, 0], :on_imaginary, "5.6ri", Ripper::EXPR_END | Ripper::EXPR_ENDARG],
                  ],
                  Ripper.lex("1r\n2i\n3ri\n4.2r\n5.6ri")
-     assert_equal [[[1, 0], :on_heredoc_beg, "<<~EOS"],
-                   [[1, 6], :on_nl, "\n"],
-                   [[2, 0], :on_ignored_sp, "  "],
-                   [[2, 2], :on_tstring_content, "heredoc\n"],
-                   [[3, 0], :on_heredoc_end, "EOS"]
+     assert_equal [[[1, 0], :on_heredoc_beg, "<<~EOS", Ripper::EXPR_BEG],
+                   [[1, 6], :on_nl, "\n", Ripper::EXPR_BEG],
+                   [[2, 0], :on_ignored_sp, "  ", Ripper::EXPR_BEG],
+                   [[2, 2], :on_tstring_content, "heredoc\n", Ripper::EXPR_BEG],
+                   [[3, 0], :on_heredoc_end, "EOS", Ripper::EXPR_BEG]
                  ],
                  Ripper.lex("<<~EOS\n  heredoc\nEOS")
-    assert_equal [[[1, 0], :on_tstring_beg, "'"],
-                  [[1, 1], :on_tstring_content, "foo"]],
+    assert_equal [[[1, 0], :on_tstring_beg, "'", Ripper::EXPR_BEG],
+                  [[1, 1], :on_tstring_content, "foo", Ripper::EXPR_BEG]],
                  Ripper.lex("'foo")
   end
 
