pub fn lex(source: Rc<Source>) -> Result<Spanned<Tokens>, Syntax> {
        // get a span that spans the entire source file:
        let span = Span::new(&source, 0, source.contents.len());

        // build a base lexer for this file
        let mut lexer = Lexer {
            source,
            index: 0,
            tokens: vec![],
        };

        // prime the lexer
        lexer.strip();

        // consume all!
        while lexer.index < lexer.source.contents.len() {
            let token = lexer.next_token()?;
            lexer.tokens.push(token);
            lexer.strip();
        }

        // phew, nothing broke. Your tokens, sir!
        Ok(Spanned::new(lexer.tokens, span))
    }
Lexes a source file into a stream of tokens.
passerine/src/compiler/lex.rs