Support multiple states when parsing rules (#2078)

lib/rouge/regex_lexer.rb

@@ -166,6 +166,19 @@ def rule(re, tok=nil, next_state=nil, &callback)
             puts "    yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug
             @output_stream.call(tok, stream[0])
           end
+        when Array
+          proc do |stream|
+            puts "    yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug
+            @output_stream.call(tok, stream[0])
+            for i_next_state in next_state do
+              next @stack.pop if i_next_state == :pop!
+              next @stack.push(@stack.last) if i_next_state == :push
+
+              state = @states[i_next_state] || self.class.get_state(i_next_state)
+              puts "    pushing :#{state.name}" if @debug
+              @stack.push(state)
+            end
+          end
         else
           raise "invalid next state: #{next_state.inspect}"
         end

spec/lexer_spec.rb

@@ -128,6 +128,33 @@ def incr
     assert { types == %w(digit lt gt) }
   end
 
+  it 'supports multiple states' do
+    lexer = Class.new(Rouge::RegexLexer) do
+      state :root do
+        rule %r/\w+/, 'text'
+        rule %r/(=)(\s*)/ do
+          groups 'operator', 'whitespace'
+          push :value
+        end
+      end
+
+      state :value do
+        rule %r/\d+/, 'digit', :pop!
+        rule %r/\[/, 'punctuation', [:pop!, :array]
+      end
+
+      state :array do
+        rule %r/\]/, 'punctuation', :pop!
+        rule %r//, 'token', :value
+      end
+    end
+
+    result = lexer.lex('numbers=[1]')
+
+    types = result.map { |(t,_)| t }
+    assert { types == %w(text operator punctuation digit punctuation) }
+  end
+
   it 'delegates' do
     class MasterLexer < Rouge::RegexLexer
       state :root do