Java org.antlr.v4.runtime.CommonTokenStream 代码实例

・10 分钟阅读

以下是展示如何使用org.antlr.v4.runtime.CommonTokenStream的最佳示例。 我们使用了代码质量辨别算法从开源项目中提取出了最佳的优秀示例。

实例 1


private List<Token> getTokens(String input) {
    ANTLRInputStream inputStream = new ANTLRInputStream(input);
    TurkishLexer lexer = new TurkishLexer(inputStream);
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    tokenStream.fill();
    return tokenStream.getTokens();
}
 

实例 2


private void loadTokens(final Document document, LexerInterpreterData interpreterData, LexerTraceAnalyzer analyzer) {
    try {
        TracingCharStream charStream = new TracingCharStream(analyzer, document.getText(0, document.getLength()));
        TracingLexer lexer = new TracingLexer(interpreterData, analyzer, charStream);
        ATN atn = new ATNDeserializer().deserialize(interpreterData.serializedAtn.toCharArray());
        TracingLexerATNSimulator atnSimulator = new TracingLexerATNSimulator(analyzer, lexer, atn);
        lexer.setInterpreter(atnSimulator);
        CommonTokenStream commonTokenStream = new CommonTokenStream(lexer);
        commonTokenStream.fill();
    } catch (BadLocationException ex) {
        Exceptions.printStackTrace(ex);
    }
}
 

实例 3


public static void printLex(InputStream in) throws Exception {
    Java7Lexer lex = new Java7Lexer(new ANTLRInputStream(in));
    CommonTokenStream tokens = new CommonTokenStream(lex);
    tokens.fill();
    for (Token token : tokens.getTokens()) {
        System.out.println(token.getType() + " " + token.getText());
    }
}
 

实例 4


public static void main(String[] args) throws Exception {
        String code = "identifier = 5;// supn/*derpnderp*/";
        ScratchCLikeLexer lexer = new ScratchCLikeLexer(new ANTLRInputStream(
                        new StringReader(code)));
        CommonTokenStream stream = new CommonTokenStream(lexer);
        stream.fill();
        Class<ScratchCLikeLexer> lexerClass = ScratchCLikeLexer.class;
        Map<Integer, String> tokenIds = new HashMap<>();
        for (String s : lexer.getRuleNames()) {
                if (s.equals("AChar") || s.equals("AEscapeSequence"))
                        continue;
                int num = lexerClass.getDeclaredField(s).getInt((Object) null);
                tokenIds.put(num, s);
        }
        tokenIds.put(Token.EOF, "<EOF>");
        for (Token t : stream.getTokens()) {
                System.out.println(t.getType() + " "+tokenIds.get(t.getType()));
        }
}
 

实例 5


private void highlight(FilterBypass fb) {
        try {
                String text;
                try {
                        text = doc.getText(doc.getStartPosition().getOffset(), doc
                                        .getEndPosition().getOffset());
                } catch (BadLocationException e) {
                        e.printStackTrace();
                        return;
                }
                lexer.reset();
                lexer.setInputStream(new ANTLRInputStream(
                                new StringReader(text)));
                CommonTokenStream tokens = new CommonTokenStream(lexer);
                tokens.fill();
                doc.setCharacterAttributes(0, text.length(), defaultStyle, true);
                List<Token> tokenList = tokens.getTokens();
                for (int i = 0; i < tokenList.size(); i++) {
                        Token t = tokenList.get(i);
                        int type = t.getType();
                        if (styles.containsKey(type)) {
                                if (type == IDENTIFIER && i + 1 < tokenList.size()) {
                                        if (tokenList.get(i + 1).getType() == OPENP) {
                                                type = 999;
                                        }
                                }
                                int offs = t.getStartIndex();
                                int len = t.getStopIndex() - offs + 1;
                                doc.setCharacterAttributes(offs, len, styles.get(type),
                                                true);
                        }
                }
        } catch (Exception e1) {
                e1.printStackTrace();
        }
}
 

实例 6


private static StyleSpans<Collection<String>> computeHighlighting(String text) {
        StyleSpansBuilder<Collection<String>> spansBuilder = new StyleSpansBuilder<>();
        int lastTokenEnd = 0;
        ANTLRInputStream input = new ANTLRInputStream(text);
        ChurchLexer lexer = new ChurchLexer(input);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        tokens.fill();  
        for (int i = 0; i < tokens.size(); i++) {
                Token t = tokens.get(i);
                if (t.getType() == Token.EOF) {
                        break;
                }                       
                String styleClass;
                if (t.getType() == ChurchLexer.COMMENT || t.getType() == ChurchLexer.LINE_COMMENT) {
                        styleClass = "churchCodeComment";
                }
                else if (t.getType() == ChurchLexer.NUM_10) {
                        styleClass = "churchCodeNumberLiteral";
                }
                else if (t.getType() == ChurchLexer.STRING) {
                        styleClass = "churchCodeStringLiteral";
                }
                else if (t.getText().equals("(") || t.getText().equals(")")) {
                        styleClass = "churchCodeParenthesis";
                }
                else if (ChurchTerminalSymbols.isTerminalSymbol(t.getText())) {
                        styleClass = "churchCodeKeyword";
                }
                else {
                        styleClass = "churchCodeOther";
                }
                int spacing = t.getStartIndex() - lastTokenEnd;
                if (spacing > 0) {                   
                        spansBuilder.add(Collections.emptyList(), spacing);
                }
                int stylesize = (t.getStopIndex() - t.getStartIndex()) +1;                      
                spansBuilder.add(Collections.singleton(styleClass), stylesize);
                lastTokenEnd = t.getStopIndex()+1;
        }
        return spansBuilder.create();
}
 

实例 7


private static StyleSpans<Collection<String>> computeHighlighting(String text) {
        StyleSpansBuilder<Collection<String>> spansBuilder = new StyleSpansBuilder<>();
        int lastTokenEnd = 0;
        ANTLRInputStream input = new ANTLRInputStream(text);
        HOGMLexer lexer = new HOGMLexer(input);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        tokens.fill();  
        for (int i = 0; i < tokens.size(); i++) {
                Token t = tokens.get(i);
                if (t.getType() == Token.EOF) {
                        break;
                }                       
                String styleClass;
                if (t.getType() == HOGMLexer.COMMENT || t.getType() == HOGMLexer.LINE_COMMENT) {
                        styleClass = "hogmCodeComment";
                }
                else if (HOGMTerminalSymbols.isTerminalSymbol(t.getText())) {
                        styleClass = "hogmCodeKeyword";
                }
                else {
                        styleClass = "hogmCodeOther";
                }
                int spacing = t.getStartIndex() - lastTokenEnd;
                if (spacing > 0) {                   
                        spansBuilder.add(Collections.emptyList(), spacing);
                }
                int stylesize = (t.getStopIndex() - t.getStartIndex()) +1;                      
                spansBuilder.add(Collections.singleton(styleClass), stylesize);
                lastTokenEnd = t.getStopIndex()+1;
        }
        return spansBuilder.create();
}
 
讨论
淘淘あ西西 profile image