| Class | RubyLex |
| In: |
lib/irb/ruby-lex.rb
lib/rdoc/parsers/parse_rb.rb |
| Parent: | Object |
Lexical analyzer for Ruby source
| ENINDENT_CLAUSE | = | [ "case", "class", "def", "do", "for", "if", "module", "unless", "until", "while", "begin" |
| DEINDENT_CLAUSE | = | ["end" |
| PERCENT_LTYPE | = | { "q" => "\'", "Q" => "\"", "x" => "\`", "r" => "/", "w" => "]", "W" => "]", "s" => ":" |
| PERCENT_PAREN | = | { "{" => "}", "[" => "]", "<" => ">", "(" => ")" |
| Ltype2Token | = | { "\'" => TkSTRING, "\"" => TkSTRING, "\`" => TkXSTRING, "/" => TkREGEXP, "]" => TkDSTRING, ":" => TkSYMBOL |
| DLtype2Token | = | { "\"" => TkDSTRING, "\`" => TkDXSTRING, "/" => TkDREGEXP, } |
| ENINDENT_CLAUSE | = | [ "case", "class", "def", "do", "for", "if", "module", "unless", "until", "while", "begin" |
| DEINDENT_CLAUSE | = | ["end" |
| PERCENT_LTYPE | = | { "q" => "\'", "Q" => "\"", "x" => "\`", "r" => "/", "w" => "]" |
| PERCENT_PAREN | = | { "{" => "}", "[" => "]", "<" => ">", "(" => ")" |
| Ltype2Token | = | { "\'" => TkSTRING, "\"" => TkSTRING, "\`" => TkXSTRING, "/" => TkREGEXP, "]" => TkDSTRING |
| DLtype2Token | = | { "\"" => TkDSTRING, "\`" => TkDXSTRING, "/" => TkDREGEXP, } |
| char_no | [R] | |
| continue | [R] | |
| debug_level | [RW] | |
| exception_on_syntax_error | [RW] | |
| exception_on_syntax_error | [RW] | |
| indent | [R] | |
| indent | [R] | |
| lex_state | [R] | |
| line_no | [R] | |
| read_auto_clean_up | [RW] | |
| readed_auto_clean_up | [RW] | |
| seek | [R] | |
| skip_space | [RW] | |
| skip_space | [RW] |
# File lib/irb/ruby-lex.rb, line 40
40: def initialize
41: lex_init
42: set_input(STDIN)
43:
44: @seek = 0
45: @exp_line_no = @line_no = 1
46: @base_char_no = 0
47: @char_no = 0
48: @rests = []
49: @readed = []
50: @here_readed = []
51:
52: @indent = 0
53: @indent_stack = []
54: @lex_state = EXPR_BEG
55: @space_seen = false
56: @here_header = false
57:
58: @continue = false
59: @line = ""
60:
61: @skip_space = false
62: @readed_auto_clean_up = false
63: @exception_on_syntax_error = true
64:
65: @prompt = nil
66: end
# File lib/rdoc/parsers/parse_rb.rb, line 447
447: def initialize(content)
448: lex_init
449:
450: @reader = BufferedReader.new(content)
451:
452: @exp_line_no = @line_no = 1
453: @base_char_no = 0
454: @indent = 0
455:
456: @ltype = nil
457: @quoted = nil
458: @lex_state = EXPR_BEG
459: @space_seen = false
460:
461: @continue = false
462: @line = ""
463:
464: @skip_space = false
465: @read_auto_clean_up = false
466: @exception_on_syntax_error = true
467: end
# File lib/irb/ruby-lex.rb, line 227
227: def each_top_level_statement
228: initialize_input
229: catch(:TERM_INPUT) do
230: loop do
231: begin
232: @continue = false
233: prompt
234: unless l = lex
235: throw :TERM_INPUT if @line == ''
236: else
237: #p l
238: @line.concat l
239: if @ltype or @continue or @indent > 0
240: next
241: end
242: end
243: if @line != "\n"
244: yield @line, @exp_line_no
245: end
246: break unless l
247: @line = ''
248: @exp_line_no = @line_no
249:
250: @indent = 0
251: @indent_stack = []
252: prompt
253: rescue TerminateLineInput
254: initialize_input
255: prompt
256: get_readed
257: end
258: end
259: end
260: end
# File lib/irb/ruby-lex.rb, line 89
89: def get_readed
90: if idx = @readed.reverse.index("\n")
91: @base_char_no = idx
92: else
93: @base_char_no += @readed.size
94: end
95:
96: readed = @readed.join("")
97: @readed = []
98: readed
99: end
# File lib/irb/ruby-lex.rb, line 101
101: def getc
102: while @rests.empty?
103: # return nil unless buf_input
104: @rests.push nil unless buf_input
105: end
106: c = @rests.shift
107: if @here_header
108: @here_readed.push c
109: else
110: @readed.push c
111: end
112: @seek += 1
113: if c == "\n"
114: @line_no += 1
115: @char_no = 0
116: else
117: @char_no += 1
118: end
119: c
120: end
# File lib/rdoc/parsers/parse_rb.rb, line 492
492: def getc_of_rests
493: @reader.getc_already_read
494: end
# File lib/irb/ruby-lex.rb, line 136
136: def getc_of_rests
137: if @rests.empty?
138: nil
139: else
140: getc
141: end
142: end
# File lib/irb/ruby-lex.rb, line 122
122: def gets
123: l = ""
124: while c = getc
125: l.concat(c)
126: break if c == "\n"
127: end
128: return nil if l == "" and c.nil?
129: l
130: end
# File lib/rdoc/parsers/parse_rb.rb, line 496
496: def gets
497: c = getc or return
498: l = ""
499: begin
500: l.concat c unless c == "\r"
501: break if c == "\n"
502: end while c = getc
503: l
504: end
# File lib/rdoc/parsers/parse_rb.rb, line 1269
1269: def identify_comment
1270: @ltype = "#"
1271: comment = "#"
1272: while ch = getc
1273: if ch == "\\"
1274: ch = getc
1275: if ch == "\n"
1276: ch = " "
1277: else
1278: comment << "\\"
1279: end
1280: else
1281: if ch == "\n"
1282: @ltype = nil
1283: ungetc
1284: break
1285: end
1286: end
1287: comment << ch
1288: end
1289: return Token(TkCOMMENT).set_text(comment)
1290: end
# File lib/irb/ruby-lex.rb, line 1085
1085: def identify_comment
1086: @ltype = "#"
1087:
1088: while ch = getc
1089: # if ch == "\\" #"
1090: # read_escape
1091: # end
1092: if ch == "\n"
1093: @ltype = nil
1094: ungetc
1095: break
1096: end
1097: end
1098: return Token(TkCOMMENT)
1099: end
# File lib/rdoc/parsers/parse_rb.rb, line 964
964: def identify_gvar
965: @lex_state = EXPR_END
966: str = "$"
967:
968: tk = case ch = getc
969: when /[~_*$?!@\/\\;,=:<>".]/ #"
970: str << ch
971: Token(TkGVAR, str)
972:
973: when "-"
974: str << "-" << getc
975: Token(TkGVAR, str)
976:
977: when "&", "`", "'", "+"
978: str << ch
979: Token(TkBACK_REF, str)
980:
981: when /[1-9]/
982: str << ch
983: while (ch = getc) =~ /[0-9]/
984: str << ch
985: end
986: ungetc
987: Token(TkNTH_REF)
988: when /\w/
989: ungetc
990: ungetc
991: return identify_identifier
992: else
993: ungetc
994: Token("$")
995: end
996: tk.set_text(str)
997: end
# File lib/irb/ruby-lex.rb, line 746
746: def identify_gvar
747: @lex_state = EXPR_END
748:
749: case ch = getc
750: when /[~_*$?!@\/\\;,=:<>".]/ #"
751: Token(TkGVAR, "$" + ch)
752: when "-"
753: Token(TkGVAR, "$-" + getc)
754: when "&", "`", "'", "+"
755: Token(TkBACK_REF, "$"+ch)
756: when /[1-9]/
757: while getc =~ /[0-9]/; end
758: ungetc
759: Token(TkNTH_REF)
760: when /\w/
761: ungetc
762: ungetc
763: identify_identifier
764: else
765: ungetc
766: Token("$")
767: end
768: end
# File lib/rdoc/parsers/parse_rb.rb, line 1074
1074: def identify_here_document
1075: ch = getc
1076: if ch == "-"
1077: ch = getc
1078: indent = true
1079: end
1080: if /['"`]/ =~ ch # '
1081: lt = ch
1082: quoted = ""
1083: while (c = getc) && c != lt
1084: quoted.concat c
1085: end
1086: else
1087: lt = '"'
1088: quoted = ch.dup
1089: while (c = getc) && c =~ /\w/
1090: quoted.concat c
1091: end
1092: ungetc
1093: end
1094:
1095: ltback, @ltype = @ltype, lt
1096: reserve = ""
1097:
1098: while ch = getc
1099: reserve << ch
1100: if ch == "\\" #"
1101: ch = getc
1102: reserve << ch
1103: elsif ch == "\n"
1104: break
1105: end
1106: end
1107:
1108: str = ""
1109: while (l = gets)
1110: l.chomp!
1111: l.strip! if indent
1112: break if l == quoted
1113: str << l.chomp << "\n"
1114: end
1115:
1116: @reader.divert_read_from(reserve)
1117:
1118: @ltype = ltback
1119: @lex_state = EXPR_END
1120: Token(Ltype2Token[lt], str).set_text(str.dump)
1121: end
# File lib/irb/ruby-lex.rb, line 879
879: def identify_here_document
880: ch = getc
881: # if lt = PERCENT_LTYPE[ch]
882: if ch == "-"
883: ch = getc
884: indent = true
885: end
886: if /['"`]/ =~ ch
887: lt = ch
888: quoted = ""
889: while (c = getc) && c != lt
890: quoted.concat c
891: end
892: else
893: lt = '"'
894: quoted = ch.dup
895: while (c = getc) && c =~ /\w/
896: quoted.concat c
897: end
898: ungetc
899: end
900:
901: ltback, @ltype = @ltype, lt
902: reserve = []
903: while ch = getc
904: reserve.push ch
905: if ch == "\\"
906: reserve.push ch = getc
907: elsif ch == "\n"
908: break
909: end
910: end
911:
912: @here_header = false
913: while l = gets
914: l = l.sub(/(:?\r)?\n\z/, '')
915: if (indent ? l.strip : l) == quoted
916: break
917: end
918: end
919:
920: @here_header = true
921: @here_readed.concat reserve
922: while ch = reserve.pop
923: ungetc ch
924: end
925:
926: @ltype = ltback
927: @lex_state = EXPR_END
928: Token(Ltype2Token[lt])
929: end
# File lib/rdoc/parsers/parse_rb.rb, line 999
999: def identify_identifier
1000: token = ""
1001: token.concat getc if peek(0) =~ /[$@]/
1002: token.concat getc if peek(0) == "@"
1003:
1004: while (ch = getc) =~ /\w|_/
1005: print ":", ch, ":" if RubyLex.debug?
1006: token.concat ch
1007: end
1008: ungetc
1009:
1010: if ch == "!" or ch == "?"
1011: token.concat getc
1012: end
1013: # fix token
1014:
1015: # $stderr.puts "identifier - #{token}, state = #@lex_state"
1016:
1017: case token
1018: when /^\$/
1019: return Token(TkGVAR, token).set_text(token)
1020: when /^\@/
1021: @lex_state = EXPR_END
1022: return Token(TkIVAR, token).set_text(token)
1023: end
1024:
1025: if @lex_state != EXPR_DOT
1026: print token, "\n" if RubyLex.debug?
1027:
1028: token_c, *trans = TkReading2Token[token]
1029: if token_c
1030: # reserved word?
1031:
1032: if (@lex_state != EXPR_BEG &&
1033: @lex_state != EXPR_FNAME &&
1034: trans[1])
1035: # modifiers
1036: token_c = TkSymbol2Token[trans[1]]
1037: @lex_state = trans[0]
1038: else
1039: if @lex_state != EXPR_FNAME
1040: if ENINDENT_CLAUSE.include?(token)
1041: @indent += 1
1042: elsif DEINDENT_CLAUSE.include?(token)
1043: @indent -= 1
1044: end
1045: @lex_state = trans[0]
1046: else
1047: @lex_state = EXPR_END
1048: end
1049: end
1050: return Token(token_c, token).set_text(token)
1051: end
1052: end
1053:
1054: if @lex_state == EXPR_FNAME
1055: @lex_state = EXPR_END
1056: if peek(0) == '='
1057: token.concat getc
1058: end
1059: elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
1060: @lex_state = EXPR_ARG
1061: else
1062: @lex_state = EXPR_END
1063: end
1064:
1065: if token[0, 1] =~ /[A-Z]/
1066: return Token(TkCONSTANT, token).set_text(token)
1067: elsif token[token.size - 1, 1] =~ /[!?]/
1068: return Token(TkFID, token).set_text(token)
1069: else
1070: return Token(TkIDENTIFIER, token).set_text(token)
1071: end
1072: end
# File lib/irb/ruby-lex.rb, line 770
770: def identify_identifier
771: token = ""
772: if peek(0) =~ /[$@]/
773: token.concat(c = getc)
774: if c == "@" and peek(0) == "@"
775: token.concat getc
776: end
777: end
778:
779: while (ch = getc) =~ /\w|_/
780: print ":", ch, ":" if RubyLex.debug?
781: token.concat ch
782: end
783: ungetc
784:
785: if (ch == "!" || ch == "?") && token[0,1] =~ /\w/ && peek(0) != "="
786: token.concat getc
787: end
788:
789: # almost fix token
790:
791: case token
792: when /^\$/
793: return Token(TkGVAR, token)
794: when /^\@\@/
795: @lex_state = EXPR_END
796: # p Token(TkCVAR, token)
797: return Token(TkCVAR, token)
798: when /^\@/
799: @lex_state = EXPR_END
800: return Token(TkIVAR, token)
801: end
802:
803: if @lex_state != EXPR_DOT
804: print token, "\n" if RubyLex.debug?
805:
806: token_c, *trans = TkReading2Token[token]
807: if token_c
808: # reserved word?
809:
810: if (@lex_state != EXPR_BEG &&
811: @lex_state != EXPR_FNAME &&
812: trans[1])
813: # modifiers
814: token_c = TkSymbol2Token[trans[1]]
815: @lex_state = trans[0]
816: else
817: if @lex_state != EXPR_FNAME
818: if ENINDENT_CLAUSE.include?(token)
819: # check for ``class = val'' etc.
820: valid = true
821: case token
822: when "class"
823: valid = false unless peek_match?(/^\s*(<<|\w|::)/)
824: when "def"
825: valid = false if peek_match?(/^\s*(([+-\/*&\|^]|<<|>>|\|\||\&\&)=|\&\&|\|\|)/)
826: when "do"
827: valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&)/)
828: when *ENINDENT_CLAUSE
829: valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&|\|)/)
830: else
831: # no nothing
832: end
833: if valid
834: if token == "do"
835: if ![TkFOR, TkWHILE, TkUNTIL].include?(@indent_stack.last)
836: @indent += 1
837: @indent_stack.push token_c
838: end
839: else
840: @indent += 1
841: @indent_stack.push token_c
842: end
843: # p @indent_stack
844: end
845:
846: elsif DEINDENT_CLAUSE.include?(token)
847: @indent -= 1
848: @indent_stack.pop
849: end
850: @lex_state = trans[0]
851: else
852: @lex_state = EXPR_END
853: end
854: end
855: return Token(token_c, token)
856: end
857: end
858:
859: if @lex_state == EXPR_FNAME
860: @lex_state = EXPR_END
861: if peek(0) == '='
862: token.concat getc
863: end
864: elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
865: @lex_state = EXPR_ARG
866: else
867: @lex_state = EXPR_END
868: end
869:
870: if token[0, 1] =~ /[A-Z]/
871: return Token(TkCONSTANT, token)
872: elsif token[token.size - 1, 1] =~ /[!?]/
873: return Token(TkFID, token)
874: else
875: return Token(TkIDENTIFIER, token)
876: end
877: end
# File lib/irb/ruby-lex.rb, line 949
949: def identify_number
950: @lex_state = EXPR_END
951:
952: if peek(0) == "0" && peek(1) !~ /[.eE]/
953: getc
954: case peek(0)
955: when /[xX]/
956: ch = getc
957: match = /[0-9a-fA-F_]/
958: when /[bB]/
959: ch = getc
960: match = /[01_]/
961: when /[oO]/
962: ch = getc
963: match = /[0-7_]/
964: when /[dD]/
965: ch = getc
966: match = /[0-9_]/
967: when /[0-7]/
968: match = /[0-7_]/
969: when /[89]/
970: RubyLex.fail SyntaxError, "Illegal octal digit"
971: else
972: return Token(TkINTEGER)
973: end
974:
975: len0 = true
976: non_digit = false
977: while ch = getc
978: if match =~ ch
979: if ch == "_"
980: if non_digit
981: RubyLex.fail SyntaxError, "trailing `#{ch}' in number"
982: else
983: non_digit = ch
984: end
985: else
986: non_digit = false
987: len0 = false
988: end
989: else
990: ungetc
991: if len0
992: RubyLex.fail SyntaxError, "numeric literal without digits"
993: end
994: if non_digit
995: RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
996: end
997: break
998: end
999: end
1000: return Token(TkINTEGER)
1001: end
1002:
1003: type = TkINTEGER
1004: allow_point = true
1005: allow_e = true
1006: non_digit = false
1007: while ch = getc
1008: case ch
1009: when /[0-9]/
1010: non_digit = false
1011: when "_"
1012: non_digit = ch
1013: when allow_point && "."
1014: if non_digit
1015: RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
1016: end
1017: type = TkFLOAT
1018: if peek(0) !~ /[0-9]/
1019: type = TkINTEGER
1020: ungetc
1021: break
1022: end
1023: allow_point = false
1024: when allow_e && "e", allow_e && "E"
1025: if non_digit
1026: RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
1027: end
1028: type = TkFLOAT
1029: if peek(0) =~ /[+-]/
1030: getc
1031: end
1032: allow_e = false
1033: allow_point = false
1034: non_digit = ch
1035: else
1036: if non_digit
1037: RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
1038: end
1039: ungetc
1040: break
1041: end
1042: end
1043: Token(type)
1044: end
# File lib/rdoc/parsers/parse_rb.rb, line 1142
1142: def identify_number(start)
1143: str = start.dup
1144:
1145: if start == "+" or start == "-" or start == ""
1146: start = getc
1147: str << start
1148: end
1149:
1150: @lex_state = EXPR_END
1151:
1152: if start == "0"
1153: if peek(0) == "x"
1154: ch = getc
1155: str << ch
1156: match = /[0-9a-f_]/
1157: else
1158: match = /[0-7_]/
1159: end
1160: while ch = getc
1161: if ch !~ match
1162: ungetc
1163: break
1164: else
1165: str << ch
1166: end
1167: end
1168: return Token(TkINTEGER).set_text(str)
1169: end
1170:
1171: type = TkINTEGER
1172: allow_point = TRUE
1173: allow_e = TRUE
1174: while ch = getc
1175: case ch
1176: when /[0-9_]/
1177: str << ch
1178:
1179: when allow_point && "."
1180: type = TkFLOAT
1181: if peek(0) !~ /[0-9]/
1182: ungetc
1183: break
1184: end
1185: str << ch
1186: allow_point = false
1187:
1188: when allow_e && "e", allow_e && "E"
1189: str << ch
1190: type = TkFLOAT
1191: if peek(0) =~ /[+-]/
1192: str << getc
1193: end
1194: allow_e = false
1195: allow_point = false
1196: else
1197: ungetc
1198: break
1199: end
1200: end
1201: Token(type).set_text(str)
1202: end
# File lib/rdoc/parsers/parse_rb.rb, line 1123
1123: def identify_quotation(initial_char)
1124: ch = getc
1125: if lt = PERCENT_LTYPE[ch]
1126: initial_char += ch
1127: ch = getc
1128: elsif ch =~ /\W/
1129: lt = "\""
1130: else
1131: RubyLex.fail SyntaxError, "unknown type of %string ('#{ch}')"
1132: end
1133: # if ch !~ /\W/
1134: # ungetc
1135: # next
1136: # end
1137: #@ltype = lt
1138: @quoted = ch unless @quoted = PERCENT_PAREN[ch]
1139: identify_string(lt, @quoted, ch, initial_char)
1140: end
# File lib/irb/ruby-lex.rb, line 931
931: def identify_quotation
932: ch = getc
933: if lt = PERCENT_LTYPE[ch]
934: ch = getc
935: elsif ch =~ /\W/
936: lt = "\""
937: else
938: RubyLex.fail SyntaxError, "unknown type of %string"
939: end
940: # if ch !~ /\W/
941: # ungetc
942: # next
943: # end
944: #@ltype = lt
945: @quoted = ch unless @quoted = PERCENT_PAREN[ch]
946: identify_string(lt, @quoted)
947: end
# File lib/rdoc/parsers/parse_rb.rb, line 1204
1204: def identify_string(ltype, quoted = ltype, opener=nil, initial_char = nil)
1205: @ltype = ltype
1206: @quoted = quoted
1207: subtype = nil
1208:
1209: str = ""
1210: str << initial_char if initial_char
1211: str << (opener||quoted)
1212:
1213: nest = 0
1214: begin
1215: while ch = getc
1216: str << ch
1217: if @quoted == ch
1218: if nest == 0
1219: break
1220: else
1221: nest -= 1
1222: end
1223: elsif opener == ch
1224: nest += 1
1225: elsif @ltype != "'" && @ltype != "]" and ch == "#"
1226: ch = getc
1227: if ch == "{"
1228: subtype = true
1229: str << ch << skip_inner_expression
1230: else
1231: ungetc(ch)
1232: end
1233: elsif ch == '\\' #'
1234: str << read_escape
1235: end
1236: end
1237: if @ltype == "/"
1238: if peek(0) =~ /i|o|n|e|s/
1239: str << getc
1240: end
1241: end
1242: if subtype
1243: Token(DLtype2Token[ltype], str)
1244: else
1245: Token(Ltype2Token[ltype], str)
1246: end.set_text(str)
1247: ensure
1248: @ltype = nil
1249: @quoted = nil
1250: @lex_state = EXPR_END
1251: end
1252: end
# File lib/irb/ruby-lex.rb, line 1046
1046: def identify_string(ltype, quoted = ltype)
1047: @ltype = ltype
1048: @quoted = quoted
1049: subtype = nil
1050: begin
1051: nest = 0
1052: while ch = getc
1053: if @quoted == ch and nest == 0
1054: break
1055: elsif @ltype != "'" && @ltype != "]" && @ltype != ":" and ch == "#"
1056: subtype = true
1057: elsif ch == '\\' #'
1058: read_escape
1059: end
1060: if PERCENT_PAREN.values.include?(@quoted)
1061: if PERCENT_PAREN[ch] == @quoted
1062: nest += 1
1063: elsif ch == @quoted
1064: nest -= 1
1065: end
1066: end
1067: end
1068: if @ltype == "/"
1069: if peek(0) =~ /i|m|x|o|e|s|u|n/
1070: getc
1071: end
1072: end
1073: if subtype
1074: Token(DLtype2Token[ltype])
1075: else
1076: Token(Ltype2Token[ltype])
1077: end
1078: ensure
1079: @ltype = nil
1080: @quoted = nil
1081: @lex_state = EXPR_END
1082: end
1083: end
# File lib/irb/ruby-lex.rb, line 211
211: def initialize_input
212: @ltype = nil
213: @quoted = nil
214: @indent = 0
215: @indent_stack = []
216: @lex_state = EXPR_BEG
217: @space_seen = false
218: @here_header = false
219:
220: @continue = false
221: prompt
222:
223: @line = ""
224: @exp_line_no = @line_no
225: end
# File lib/irb/ruby-lex.rb, line 262
262: def lex
263: until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) &&
264: !@continue or
265: tk.nil?)
266: #p tk
267: #p @lex_state
268: #p self
269: end
270: line = get_readed
271: # print self.inspect
272: if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
273: nil
274: else
275: line
276: end
277: end
# File lib/rdoc/parsers/parse_rb.rb, line 519
519: def lex
520: until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) &&
521: !@continue or
522: tk.nil?)
523: end
524: line = get_read
525:
526: if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
527: nil
528: else
529: line
530: end
531: end
# File lib/irb/ruby-lex.rb, line 339
339: def lex_init()
340: @OP = IRB::SLex.new
341: @OP.def_rules("\0", "\004", "\032") do |op, io|
342: Token(TkEND_OF_SCRIPT)
343: end
344:
345: @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |op, io|
346: @space_seen = true
347: while getc =~ /[ \t\f\r\13]/; end
348: ungetc
349: Token(TkSPACE)
350: end
351:
352: @OP.def_rule("#") do |op, io|
353: identify_comment
354: end
355:
356: @OP.def_rule("=begin",
357: proc{|op, io| @prev_char_no == 0 && peek(0) =~ /\s/}) do
358: |op, io|
359: @ltype = "="
360: until getc == "\n"; end
361: until peek_equal?("=end") && peek(4) =~ /\s/
362: until getc == "\n"; end
363: end
364: gets
365: @ltype = nil
366: Token(TkRD_COMMENT)
367: end
368:
369: @OP.def_rule("\n") do |op, io|
370: print "\\n\n" if RubyLex.debug?
371: case @lex_state
372: when EXPR_BEG, EXPR_FNAME, EXPR_DOT
373: @continue = true
374: else
375: @continue = false
376: @lex_state = EXPR_BEG
377: until (@indent_stack.empty? ||
378: [TkLPAREN, TkLBRACK, TkLBRACE,
379: TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
380: @indent_stack.pop
381: end
382: end
383: @here_header = false
384: @here_readed = []
385: Token(TkNL)
386: end
387:
388: @OP.def_rules("*", "**",
389: "=", "==", "===",
390: "=~", "<=>",
391: "<", "<=",
392: ">", ">=", ">>") do
393: |op, io|
394: case @lex_state
395: when EXPR_FNAME, EXPR_DOT
396: @lex_state = EXPR_ARG
397: else
398: @lex_state = EXPR_BEG
399: end
400: Token(op)
401: end
402:
403: @OP.def_rules("!", "!=", "!~") do
404: |op, io|
405: @lex_state = EXPR_BEG
406: Token(op)
407: end
408:
409: @OP.def_rules("<<") do
410: |op, io|
411: tk = nil
412: if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
413: (@lex_state != EXPR_ARG || @space_seen)
414: c = peek(0)
415: if /\S/ =~ c && (/["'`]/ =~ c || /[\w_]/ =~ c || c == "-")
416: tk = identify_here_document
417: end
418: end
419: unless tk
420: tk = Token(op)
421: case @lex_state
422: when EXPR_FNAME, EXPR_DOT
423: @lex_state = EXPR_ARG
424: else
425: @lex_state = EXPR_BEG
426: end
427: end
428: tk
429: end
430:
431: @OP.def_rules("'", '"') do
432: |op, io|
433: identify_string(op)
434: end
435:
436: @OP.def_rules("`") do
437: |op, io|
438: if @lex_state == EXPR_FNAME
439: @lex_state = EXPR_END
440: Token(op)
441: else
442: identify_string(op)
443: end
444: end
445:
446: @OP.def_rules('?') do
447: |op, io|
448: if @lex_state == EXPR_END
449: @lex_state = EXPR_BEG
450: Token(TkQUESTION)
451: else
452: ch = getc
453: if @lex_state == EXPR_ARG && ch =~ /\s/
454: ungetc
455: @lex_state = EXPR_BEG;
456: Token(TkQUESTION)
457: else
458: if (ch == '\\')
459: read_escape
460: end
461: @lex_state = EXPR_END
462: Token(TkINTEGER)
463: end
464: end
465: end
466:
467: @OP.def_rules("&", "&&", "|", "||") do
468: |op, io|
469: @lex_state = EXPR_BEG
470: Token(op)
471: end
472:
473: @OP.def_rules("+=", "-=", "*=", "**=",
474: "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
475: |op, io|
476: @lex_state = EXPR_BEG
477: op =~ /^(.*)=$/
478: Token(TkOPASGN, $1)
479: end
480:
481: @OP.def_rule("+@", proc{|op, io| @lex_state == EXPR_FNAME}) do
482: |op, io|
483: @lex_state = EXPR_ARG
484: Token(op)
485: end
486:
487: @OP.def_rule("-@", proc{|op, io| @lex_state == EXPR_FNAME}) do
488: |op, io|
489: @lex_state = EXPR_ARG
490: Token(op)
491: end
492:
493: @OP.def_rules("+", "-") do
494: |op, io|
495: catch(:RET) do
496: if @lex_state == EXPR_ARG
497: if @space_seen and peek(0) =~ /[0-9]/
498: throw :RET, identify_number
499: else
500: @lex_state = EXPR_BEG
501: end
502: elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/
503: throw :RET, identify_number
504: else
505: @lex_state = EXPR_BEG
506: end
507: Token(op)
508: end
509: end
510:
511: @OP.def_rule(".") do
512: |op, io|
513: @lex_state = EXPR_BEG
514: if peek(0) =~ /[0-9]/
515: ungetc
516: identify_number
517: else
518: # for "obj.if" etc.
519: @lex_state = EXPR_DOT
520: Token(TkDOT)
521: end
522: end
523:
524: @OP.def_rules("..", "...") do
525: |op, io|
526: @lex_state = EXPR_BEG
527: Token(op)
528: end
529:
530: lex_int2
531: end
# File lib/rdoc/parsers/parse_rb.rb, line 589
589: def lex_init()
590: @OP = SLex.new
591: @OP.def_rules("\0", "\004", "\032") do |chars, io|
592: Token(TkEND_OF_SCRIPT).set_text(chars)
593: end
594:
595: @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |chars, io|
596: @space_seen = TRUE
597: while (ch = getc) =~ /[ \t\f\r\13]/
598: chars << ch
599: end
600: ungetc
601: Token(TkSPACE).set_text(chars)
602: end
603:
604: @OP.def_rule("#") do
605: |op, io|
606: identify_comment
607: end
608:
609: @OP.def_rule("=begin", proc{@prev_char_no == 0 && peek(0) =~ /\s/}) do
610: |op, io|
611: str = op
612: @ltype = "="
613:
614:
615: begin
616: line = ""
617: begin
618: ch = getc
619: line << ch
620: end until ch == "\n"
621: str << line
622: end until line =~ /^=end/
623:
624: ungetc
625:
626: @ltype = nil
627:
628: if str =~ /\A=begin\s+rdoc/i
629: str.sub!(/\A=begin.*\n/, '')
630: str.sub!(/^=end.*/m, '')
631: Token(TkCOMMENT).set_text(str)
632: else
633: Token(TkRD_COMMENT)#.set_text(str)
634: end
635: end
636:
637: @OP.def_rule("\n") do
638: print "\\n\n" if RubyLex.debug?
639: case @lex_state
640: when EXPR_BEG, EXPR_FNAME, EXPR_DOT
641: @continue = TRUE
642: else
643: @continue = FALSE
644: @lex_state = EXPR_BEG
645: end
646: Token(TkNL).set_text("\n")
647: end
648:
649: @OP.def_rules("*", "**",
650: "!", "!=", "!~",
651: "=", "==", "===",
652: "=~", "<=>",
653: "<", "<=",
654: ">", ">=", ">>") do
655: |op, io|
656: @lex_state = EXPR_BEG
657: Token(op).set_text(op)
658: end
659:
660: @OP.def_rules("<<") do
661: |op, io|
662: tk = nil
663: if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
664: (@lex_state != EXPR_ARG || @space_seen)
665: c = peek(0)
666: if /[-\w_\"\'\`]/ =~ c
667: tk = identify_here_document
668: end
669: end
670: if !tk
671: @lex_state = EXPR_BEG
672: tk = Token(op).set_text(op)
673: end
674: tk
675: end
676:
677: @OP.def_rules("'", '"') do
678: |op, io|
679: identify_string(op)
680: end
681:
682: @OP.def_rules("`") do
683: |op, io|
684: if @lex_state == EXPR_FNAME
685: Token(op).set_text(op)
686: else
687: identify_string(op)
688: end
689: end
690:
691: @OP.def_rules('?') do
692: |op, io|
693: if @lex_state == EXPR_END
694: @lex_state = EXPR_BEG
695: Token(TkQUESTION).set_text(op)
696: else
697: ch = getc
698: if @lex_state == EXPR_ARG && ch !~ /\s/
699: ungetc
700: @lex_state = EXPR_BEG;
701: Token(TkQUESTION).set_text(op)
702: else
703: str = op
704: str << ch
705: if (ch == '\\') #'
706: str << read_escape
707: end
708: @lex_state = EXPR_END
709: Token(TkINTEGER).set_text(str)
710: end
711: end
712: end
713:
714: @OP.def_rules("&", "&&", "|", "||") do
715: |op, io|
716: @lex_state = EXPR_BEG
717: Token(op).set_text(op)
718: end
719:
720: @OP.def_rules("+=", "-=", "*=", "**=",
721: "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
722: |op, io|
723: @lex_state = EXPR_BEG
724: op =~ /^(.*)=$/
725: Token(TkOPASGN, $1).set_text(op)
726: end
727:
728: @OP.def_rule("+@", proc{@lex_state == EXPR_FNAME}) do |op, io|
729: Token(TkUPLUS).set_text(op)
730: end
731:
732: @OP.def_rule("-@", proc{@lex_state == EXPR_FNAME}) do |op, io|
733: Token(TkUMINUS).set_text(op)
734: end
735:
736: @OP.def_rules("+", "-") do
737: |op, io|
738: catch(:RET) do
739: if @lex_state == EXPR_ARG
740: if @space_seen and peek(0) =~ /[0-9]/
741: throw :RET, identify_number(op)
742: else
743: @lex_state = EXPR_BEG
744: end
745: elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/
746: throw :RET, identify_number(op)
747: else
748: @lex_state = EXPR_BEG
749: end
750: Token(op).set_text(op)
751: end
752: end
753:
754: @OP.def_rule(".") do
755: @lex_state = EXPR_BEG
756: if peek(0) =~ /[0-9]/
757: ungetc
758: identify_number("")
759: else
760: # for obj.if
761: @lex_state = EXPR_DOT
762: Token(TkDOT).set_text(".")
763: end
764: end
765:
766: @OP.def_rules("..", "...") do
767: |op, io|
768: @lex_state = EXPR_BEG
769: Token(op).set_text(op)
770: end
771:
772: lex_int2
773: end
# File lib/irb/ruby-lex.rb, line 533
533: def lex_int2
534: @OP.def_rules("]", "}", ")") do
535: |op, io|
536: @lex_state = EXPR_END
537: @indent -= 1
538: @indent_stack.pop
539: Token(op)
540: end
541:
542: @OP.def_rule(":") do
543: |op, io|
544: if @lex_state == EXPR_END || peek(0) =~ /\s/
545: @lex_state = EXPR_BEG
546: Token(TkCOLON)
547: else
548: @lex_state = EXPR_FNAME;
549: Token(TkSYMBEG)
550: end
551: end
552:
553: @OP.def_rule("::") do
554: |op, io|
555: # p @lex_state.id2name, @space_seen
556: if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen
557: @lex_state = EXPR_BEG
558: Token(TkCOLON3)
559: else
560: @lex_state = EXPR_DOT
561: Token(TkCOLON2)
562: end
563: end
564:
565: @OP.def_rule("/") do
566: |op, io|
567: if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
568: identify_string(op)
569: elsif peek(0) == '='
570: getc
571: @lex_state = EXPR_BEG
572: Token(TkOPASGN, "/") #/)
573: elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
574: identify_string(op)
575: else
576: @lex_state = EXPR_BEG
577: Token("/") #/)
578: end
579: end
580:
581: @OP.def_rules("^") do
582: |op, io|
583: @lex_state = EXPR_BEG
584: Token("^")
585: end
586:
587: # @OP.def_rules("^=") do
588: # @lex_state = EXPR_BEG
589: # Token(OP_ASGN, :^)
590: # end
591:
592: @OP.def_rules(",") do
593: |op, io|
594: @lex_state = EXPR_BEG
595: Token(op)
596: end
597:
598: @OP.def_rules(";") do
599: |op, io|
600: @lex_state = EXPR_BEG
601: until (@indent_stack.empty? ||
602: [TkLPAREN, TkLBRACK, TkLBRACE,
603: TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
604: @indent_stack.pop
605: end
606: Token(op)
607: end
608:
609: @OP.def_rule("~") do
610: |op, io|
611: @lex_state = EXPR_BEG
612: Token("~")
613: end
614:
615: @OP.def_rule("~@", proc{|op, io| @lex_state == EXPR_FNAME}) do
616: |op, io|
617: @lex_state = EXPR_BEG
618: Token("~")
619: end
620:
621: @OP.def_rule("(") do
622: |op, io|
623: @indent += 1
624: if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
625: @lex_state = EXPR_BEG
626: tk_c = TkfLPAREN
627: else
628: @lex_state = EXPR_BEG
629: tk_c = TkLPAREN
630: end
631: @indent_stack.push tk_c
632: tk = Token(tk_c)
633: end
634:
635: @OP.def_rule("[]", proc{|op, io| @lex_state == EXPR_FNAME}) do
636: |op, io|
637: @lex_state = EXPR_ARG
638: Token("[]")
639: end
640:
641: @OP.def_rule("[]=", proc{|op, io| @lex_state == EXPR_FNAME}) do
642: |op, io|
643: @lex_state = EXPR_ARG
644: Token("[]=")
645: end
646:
647: @OP.def_rule("[") do
648: |op, io|
649: @indent += 1
650: if @lex_state == EXPR_FNAME
651: tk_c = TkfLBRACK
652: else
653: if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
654: tk_c = TkLBRACK
655: elsif @lex_state == EXPR_ARG && @space_seen
656: tk_c = TkLBRACK
657: else
658: tk_c = TkfLBRACK
659: end
660: @lex_state = EXPR_BEG
661: end
662: @indent_stack.push tk_c
663: Token(tk_c)
664: end
665:
666: @OP.def_rule("{") do
667: |op, io|
668: @indent += 1
669: if @lex_state != EXPR_END && @lex_state != EXPR_ARG
670: tk_c = TkLBRACE
671: else
672: tk_c = TkfLBRACE
673: end
674: @lex_state = EXPR_BEG
675: @indent_stack.push tk_c
676: Token(tk_c)
677: end
678:
679: @OP.def_rule('\\') do
680: |op, io|
681: if getc == "\n"
682: @space_seen = true
683: @continue = true
684: Token(TkSPACE)
685: else
686: ungetc
687: Token("\\")
688: end
689: end
690:
691: @OP.def_rule('%') do
692: |op, io|
693: if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
694: identify_quotation
695: elsif peek(0) == '='
696: getc
697: Token(TkOPASGN, :%)
698: elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
699: identify_quotation
700: else
701: @lex_state = EXPR_BEG
702: Token("%") #))
703: end
704: end
705:
706: @OP.def_rule('$') do
707: |op, io|
708: identify_gvar
709: end
710:
711: @OP.def_rule('@') do
712: |op, io|
713: if peek(0) =~ /[\w_@]/
714: ungetc
715: identify_identifier
716: else
717: Token("@")
718: end
719: end
720:
721: # @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do
722: # |op, io|
723: # @indent += 1
724: # @lex_state = EXPR_FNAME
725: # # @lex_state = EXPR_END
726: # # until @rests[0] == "\n" or @rests[0] == ";"
727: # # rests.shift
728: # # end
729: # end
730:
731: @OP.def_rule("") do
732: |op, io|
733: printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
734: if peek(0) =~ /[0-9]/
735: t = identify_number
736: elsif peek(0) =~ /[\w_]/
737: t = identify_identifier
738: end
739: printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
740: t
741: end
742:
743: p @OP if RubyLex.debug?
744: end
# File lib/rdoc/parsers/parse_rb.rb, line 775
775: def lex_int2
776: @OP.def_rules("]", "}", ")") do
777: |op, io|
778: @lex_state = EXPR_END
779: @indent -= 1
780: Token(op).set_text(op)
781: end
782:
783: @OP.def_rule(":") do
784: if @lex_state == EXPR_END || peek(0) =~ /\s/
785: @lex_state = EXPR_BEG
786: tk = Token(TkCOLON)
787: else
788: @lex_state = EXPR_FNAME;
789: tk = Token(TkSYMBEG)
790: end
791: tk.set_text(":")
792: end
793:
794: @OP.def_rule("::") do
795: # p @lex_state.id2name, @space_seen
796: if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen
797: @lex_state = EXPR_BEG
798: tk = Token(TkCOLON3)
799: else
800: @lex_state = EXPR_DOT
801: tk = Token(TkCOLON2)
802: end
803: tk.set_text("::")
804: end
805:
806: @OP.def_rule("/") do
807: |op, io|
808: if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
809: identify_string(op)
810: elsif peek(0) == '='
811: getc
812: @lex_state = EXPR_BEG
813: Token(TkOPASGN, :/).set_text("/=") #")
814: elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
815: identify_string(op)
816: else
817: @lex_state = EXPR_BEG
818: Token("/").set_text(op)
819: end
820: end
821:
822: @OP.def_rules("^") do
823: @lex_state = EXPR_BEG
824: Token("^").set_text("^")
825: end
826:
827: # @OP.def_rules("^=") do
828: # @lex_state = EXPR_BEG
829: # Token(TkOPASGN, :^)
830: # end
831:
832: @OP.def_rules(",", ";") do
833: |op, io|
834: @lex_state = EXPR_BEG
835: Token(op).set_text(op)
836: end
837:
838: @OP.def_rule("~") do
839: @lex_state = EXPR_BEG
840: Token("~").set_text("~")
841: end
842:
843: @OP.def_rule("~@", proc{@lex_state = EXPR_FNAME}) do
844: @lex_state = EXPR_BEG
845: Token("~").set_text("~@")
846: end
847:
848: @OP.def_rule("(") do
849: @indent += 1
850: if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
851: @lex_state = EXPR_BEG
852: tk = Token(TkfLPAREN)
853: else
854: @lex_state = EXPR_BEG
855: tk = Token(TkLPAREN)
856: end
857: tk.set_text("(")
858: end
859:
860: @OP.def_rule("[]", proc{@lex_state == EXPR_FNAME}) do
861: Token("[]").set_text("[]")
862: end
863:
864: @OP.def_rule("[]=", proc{@lex_state == EXPR_FNAME}) do
865: Token("[]=").set_text("[]=")
866: end
867:
868: @OP.def_rule("[") do
869: @indent += 1
870: if @lex_state == EXPR_FNAME
871: t = Token(TkfLBRACK)
872: else
873: if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
874: t = Token(TkLBRACK)
875: elsif @lex_state == EXPR_ARG && @space_seen
876: t = Token(TkLBRACK)
877: else
878: t = Token(TkfLBRACK)
879: end
880: @lex_state = EXPR_BEG
881: end
882: t.set_text("[")
883: end
884:
885: @OP.def_rule("{") do
886: @indent += 1
887: if @lex_state != EXPR_END && @lex_state != EXPR_ARG
888: t = Token(TkLBRACE)
889: else
890: t = Token(TkfLBRACE)
891: end
892: @lex_state = EXPR_BEG
893: t.set_text("{")
894: end
895:
896: @OP.def_rule('\\') do #'
897: if getc == "\n"
898: @space_seen = true
899: @continue = true
900: Token(TkSPACE).set_text("\\\n")
901: else
902: ungetc
903: Token("\\").set_text("\\") #"
904: end
905: end
906:
907: @OP.def_rule('%') do
908: |op, io|
909: if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
910: identify_quotation('%')
911: elsif peek(0) == '='
912: getc
913: Token(TkOPASGN, "%").set_text("%=")
914: elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
915: identify_quotation('%')
916: else
917: @lex_state = EXPR_BEG
918: Token("%").set_text("%")
919: end
920: end
921:
922: @OP.def_rule('$') do #'
923: identify_gvar
924: end
925:
926: @OP.def_rule('@') do
927: if peek(0) =~ /[@\w_]/
928: ungetc
929: identify_identifier
930: else
931: Token("@").set_text("@")
932: end
933: end
934:
935: # @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do
936: # |op, io|
937: # @indent += 1
938: # @lex_state = EXPR_FNAME
939: # # @lex_state = EXPR_END
940: # # until @rests[0] == "\n" or @rests[0] == ";"
941: # # rests.shift
942: # # end
943: # end
944:
945: @OP.def_rule("__END__", proc{@prev_char_no == 0 && peek(0) =~ /[\r\n]/}) do
946: throw :eof
947: end
948:
949: @OP.def_rule("") do
950: |op, io|
951: printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
952: if peek(0) =~ /[0-9]/
953: t = identify_number("")
954: elsif peek(0) =~ /[\w_]/
955: t = identify_identifier
956: end
957: printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
958: t
959: end
960:
961: p @OP if RubyLex.debug?
962: end
io functions
# File lib/rdoc/parsers/parse_rb.rb, line 476
476: def line_no
477: @reader.line_num
478: end
# File lib/irb/ruby-lex.rb, line 180
180: def peek(i = 0)
181: while @rests.size <= i
182: return nil unless buf_input
183: end
184: @rests[i]
185: end
# File lib/irb/ruby-lex.rb, line 165
165: def peek_equal?(str)
166: chrs = str.split(//)
167: until @rests.size >= chrs.size
168: return false unless buf_input
169: end
170: @rests[0, chrs.size] == chrs
171: end
# File lib/rdoc/parsers/parse_rb.rb, line 511
511: def peek_equal?(str)
512: @reader.peek_equal(str)
513: end
# File lib/irb/ruby-lex.rb, line 173
173: def peek_match?(regexp)
174: while @rests.empty?
175: return false unless buf_input
176: end
177: regexp =~ @rests.join("")
178: end
# File lib/irb/ruby-lex.rb, line 205
205: def prompt
206: if @prompt
207: @prompt.call(@ltype, @indent, @continue, @line_no)
208: end
209: end
# File lib/irb/ruby-lex.rb, line 1101
1101: def read_escape
1102: case ch = getc
1103: when "\n", "\r", "\f"
1104: when "\\", "n", "t", "r", "f", "v", "a", "e", "b", "s" #"
1105: when /[0-7]/
1106: ungetc ch
1107: 3.times do
1108: case ch = getc
1109: when /[0-7]/
1110: when nil
1111: break
1112: else
1113: ungetc
1114: break
1115: end
1116: end
1117:
1118: when "x"
1119: 2.times do
1120: case ch = getc
1121: when /[0-9a-fA-F]/
1122: when nil
1123: break
1124: else
1125: ungetc
1126: break
1127: end
1128: end
1129:
1130: when "M"
1131: if (ch = getc) != '-'
1132: ungetc
1133: else
1134: if (ch = getc) == "\\" #"
1135: read_escape
1136: end
1137: end
1138:
1139: when "C", "c" #, "^"
1140: if ch == "C" and (ch = getc) != "-"
1141: ungetc
1142: elsif (ch = getc) == "\\" #"
1143: read_escape
1144: end
1145: else
1146: # other characters
1147: end
1148: end
# File lib/rdoc/parsers/parse_rb.rb, line 1292
1292: def read_escape
1293: res = ""
1294: case ch = getc
1295: when /[0-7]/
1296: ungetc ch
1297: 3.times do
1298: case ch = getc
1299: when /[0-7]/
1300: when nil
1301: break
1302: else
1303: ungetc
1304: break
1305: end
1306: res << ch
1307: end
1308:
1309: when "x"
1310: res << ch
1311: 2.times do
1312: case ch = getc
1313: when /[0-9a-fA-F]/
1314: when nil
1315: break
1316: else
1317: ungetc
1318: break
1319: end
1320: res << ch
1321: end
1322:
1323: when "M"
1324: res << ch
1325: if (ch = getc) != '-'
1326: ungetc
1327: else
1328: res << ch
1329: if (ch = getc) == "\\" #"
1330: res << ch
1331: res << read_escape
1332: else
1333: res << ch
1334: end
1335: end
1336:
1337: when "C", "c" #, "^"
1338: res << ch
1339: if ch == "C" and (ch = getc) != "-"
1340: ungetc
1341: else
1342: res << ch
1343: if (ch = getc) == "\\" #"
1344: res << ch
1345: res << read_escape
1346: else
1347: res << ch
1348: end
1349: end
1350: else
1351: res << ch
1352: end
1353: res
1354: end
io functions
# File lib/irb/ruby-lex.rb, line 78
78: def set_input(io, p = nil, &block)
79: @io = io
80: if p.respond_to?(:call)
81: @input = p
82: elsif block_given?
83: @input = block
84: else
85: @input = Proc.new{@io.gets}
86: end
87: end
# File lib/irb/ruby-lex.rb, line 196
196: def set_prompt(p = nil, &block)
197: p = block if block_given?
198: if p.respond_to?(:call)
199: @prompt = p
200: else
201: @prompt = Proc.new{print p}
202: end
203: end
# File lib/rdoc/parsers/parse_rb.rb, line 1254
1254: def skip_inner_expression
1255: res = ""
1256: nest = 0
1257: while (ch = getc)
1258: res << ch
1259: if ch == '}'
1260: break if nest.zero?
1261: nest -= 1
1262: elsif ch == '{'
1263: nest += 1
1264: end
1265: end
1266: res
1267: end
# File lib/irb/ruby-lex.rb, line 279
279: def token
280: # require "tracer"
281: # Tracer.on
282: @prev_seek = @seek
283: @prev_line_no = @line_no
284: @prev_char_no = @char_no
285: begin
286: begin
287: tk = @OP.match(self)
288: @space_seen = tk.kind_of?(TkSPACE)
289: rescue SyntaxError
290: raise if @exception_on_syntax_error
291: tk = TkError.new(@seek, @line_no, @char_no)
292: end
293: end while @skip_space and tk.kind_of?(TkSPACE)
294: if @readed_auto_clean_up
295: get_readed
296: end
297: # Tracer.off
298: tk
299: end
# File lib/rdoc/parsers/parse_rb.rb, line 533
533: def token
534: set_token_position(line_no, char_no)
535: begin
536: begin
537: tk = @OP.match(self)
538: @space_seen = tk.kind_of?(TkSPACE)
539: rescue SyntaxError
540: abort if @exception_on_syntax_error
541: tk = TkError.new(line_no, char_no)
542: end
543: end while @skip_space and tk.kind_of?(TkSPACE)
544: if @read_auto_clean_up
545: get_read
546: end
547: # throw :eof unless tk
548: p tk if $DEBUG
549: tk
550: end
# File lib/irb/ruby-lex.rb, line 144
144: def ungetc(c = nil)
145: if @here_readed.empty?
146: c2 = @readed.pop
147: else
148: c2 = @here_readed.pop
149: end
150: c = c2 unless c
151: @rests.unshift c #c =
152: @seek -= 1
153: if c == "\n"
154: @line_no -= 1
155: if idx = @readed.reverse.index("\n")
156: @char_no = @readed.size - idx
157: else
158: @char_no = @base_char_no + @readed.size
159: end
160: else
161: @char_no -= 1
162: end
163: end
# File lib/rdoc/parsers/parse_rb.rb, line 507
507: def ungetc(c = nil)
508: @reader.ungetc(c)
509: end