Ruby lexer adapted from irb.

The internals are not documented because they are scary.

Namespace
Methods
N
T
Class Public methods
new(content, options)

Creates a new lexer for content. options is an RDoc::Options, only +tab_width is used.

Source: show | on GitHub 

# File lib/rdoc/ruby_lex.rb, line 81
def initialize(content, options)
  lex_init

  if /\t/ =~ content then
    tab_width = options.tab_width
    content = content.split(/\n/).map do |line|
      1 while line.gsub!(/\t+/) {
        ' ' * (tab_width*$&.length - $`.length % tab_width)
      }  && $~
      line
    end.join("\n")
  end

  content << "\n" unless content[-1, 1] == "\n"

  set_input StringIO.new content

  @base_char_no = 0
  @char_no = 0
  @exp_line_no = @line_no = 1
  @here_readed = []
  @readed = []
  @rests = []
  @seek = 0

  @here_header = false
  @indent = 0
  @indent_stack = []
  @lex_state = EXPR_BEG
  @space_seen = false

  @continue = false
  @line = ""

  @skip_space = false
  @readed_auto_clean_up = false
  @exception_on_syntax_error = true

  @prompt = nil
  @prev_seek = nil
  @ltype = nil
end
tokenize(ruby, options)

Returns an Array of ruby tokens. See ::new for a description of options.

Source: show | on GitHub 

# File lib/rdoc/ruby_lex.rb, line 64
def self.tokenize ruby, options
  tokens = []

  scanner = RDoc::RubyLex.new ruby, options
  scanner.exception_on_syntax_error = true

  while token = scanner.token do
    tokens << token
  end

  tokens
end