Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 17 additions & 12 deletions lib/prism/translation/ripper.rb
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ def self.parse(src, filename = "(ripper)", lineno = 1)
# [[1, 13], :on_kw, "end", END ]]
#
def self.lex(src, filename = "-", lineno = 1, raise_errors: false)
result = Prism.lex_compat(src, filepath: filename, line: lineno, version: "current")
result = Prism.lex_compat(coerce_source(src), filepath: filename, line: lineno, version: "current")

if result.failure? && raise_errors
raise SyntaxError, result.errors.first.message
Expand All @@ -91,6 +91,21 @@ def self.tokenize(...)
lex(...).map { |token| token[2] }
end

# Mirros the various lex_types that ripper supports
def self.coerce_source(source) # :nodoc:
if source.is_a?(IO)
source.read
elsif source.respond_to?(:gets)
src = +""
while line = source.gets
src << line
end
src
else
source.to_str
end
end

# This contains a table of all of the parser events and their
# corresponding arity.
PARSER_EVENT_TABLE = {
Expand Down Expand Up @@ -480,17 +495,7 @@ def self.lex_state_name(state)

# Create a new Translation::Ripper object with the given source.
def initialize(source, filename = "(ripper)", lineno = 1)
if source.is_a?(IO)
@source = source.read
elsif source.respond_to?(:gets)
@source = +""
while line = source.gets
@source << line
end
else
@source = source.to_str
end

@source = Ripper.coerce_source(source)
@filename = filename
@lineno = lineno
@column = 0
Expand Down
8 changes: 8 additions & 0 deletions test/prism/ruby/ripper_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,14 @@ def string_like.to_str
end
end

def test_lex_coersion
string_like = Object.new
def string_like.to_str
"a"
end
assert_equal Ripper.lex(string_like), Translation::Ripper.lex(string_like)
end

# Check that the hardcoded values don't change without us noticing.
def test_internals
actual = Translation::Ripper.constants.select { |name| name.start_with?("EXPR_") }.sort
Expand Down