class Lexer

Declaration

class Lexer : public PreprocessorLexer { /* full declaration omitted */ };

Description

Lexer - This provides a simple interface that turns a text buffer into a stream of tokens. This provides no support for file reading or buffering, or buffering/seeking of tokens, only forward lexing is supported. It relies on the specified Preprocessor object to handle preprocessor directives, etc.

Declared at: clang/include/clang/Lex/Lexer.h:78

Inherits from: PreprocessorLexer

Member Variables

private const char* BufferStart
private const char* BufferEnd
private clang::SourceLocation FileLoc
private const clang::LangOptions& LangOpts
private bool LineComment
private bool Is_PragmaLexer
private unsigned char ExtendedTokenMode
When this is set to 2 it returns comments and whitespace. When set to 1 it returns comments, when it is set to 0 it returns normal tokens only.
private const char* BufferPtr
private bool IsAtStartOfLine
private bool IsAtPhysicalStartOfLine
private bool HasLeadingSpace
private bool HasLeadingEmptyMacro
private bool IsFirstTimeLexingFile
True if this is the first time we're lexing the input file.
private const char* NewLinePtr
private clang::ConflictMarkerKind CurrentConflictMarkerState
private ArrayRef<dependency_directives_scan::Directive> DepDirectives
Non-empty if this \p Lexer is \p isDependencyDirectivesLexer().
private unsigned int NextDepDirectiveTokenIndex = 0
If this \p Lexer is \p isDependencyDirectivesLexer(), it represents the next token to use from the current dependency directive.

Inherited from PreprocessorLexer:

protected PP = nullptr
protected FID
protected InitialNumSLocEntries = 0
protected ParsingPreprocessorDirective = false
protected ParsingFilename = false
protected LexingRawMode = false
protected MIOpt
protected ConditionalStack

Method Overview

  • public static clang::SourceLocation AdvanceToTokenCharacter(clang::SourceLocation TokStart, unsigned int Characters, const clang::SourceManager & SM, const clang::LangOptions & LangOpts)
  • private bool CheckUnicodeWhitespace(clang::Token & Result, uint32_t C, const char * CurPtr)
  • public static clang::PreambleBounds ComputePreamble(llvm::StringRef Buffer, const clang::LangOptions & LangOpts, unsigned int MaxLines = 0)
  • private const char * ConsumeChar(const char * Ptr, unsigned int Size, clang::Token & Tok)
  • public static clang::Lexer * Create_PragmaLexer(clang::SourceLocation SpellingLoc, clang::SourceLocation ExpansionLocStart, clang::SourceLocation ExpansionLocEnd, unsigned int TokLen, clang::Preprocessor & PP)
  • public clang::DiagnosticBuilder Diag(const char * Loc, unsigned int DiagID) const
  • private void FormTokenWithChars(clang::Token & Result, const char * TokEnd, tok::TokenKind Kind)
  • public static clang::SourceLocation GetBeginningOfToken(clang::SourceLocation Loc, const clang::SourceManager & SM, const clang::LangOptions & LangOpts)
  • private bool HandleEndOfConflictMarker(const char * CurPtr)
  • private void IndirectLex(clang::Token & Result)
  • private void InitLexer(const char * BufStart, const char * BufPtr, const char * BufEnd)
  • private bool IsStartOfConflictMarker(const char * CurPtr)
  • private bool Lex(clang::Token & Result)
  • private bool LexAngledStringLiteral(clang::Token & Result, const char * CurPtr)
  • private bool LexCharConstant(clang::Token & Result, const char * CurPtr, tok::TokenKind Kind)
  • private bool LexDependencyDirectiveToken(clang::Token & Result)
  • private bool LexDependencyDirectiveTokenWhileSkipping(clang::Token & Result)
  • private bool LexEndOfFile(clang::Token & Result, const char * CurPtr)
  • public bool LexFromRawLexer(clang::Token & Result)
  • private bool LexIdentifierContinue(clang::Token & Result, const char * CurPtr)
  • private bool LexNumericConstant(clang::Token & Result, const char * CurPtr)
  • private bool LexRawStringLiteral(clang::Token & Result, const char * CurPtr, tok::TokenKind Kind)
  • private bool LexStringLiteral(clang::Token & Result, const char * CurPtr, tok::TokenKind Kind)
  • private bool LexTokenInternal(clang::Token & Result, bool TokAtPhysicalStartOfLine)
  • private const char * LexUDSuffix(clang::Token & Result, const char * CurPtr, bool IsStringLiteral)
  • private bool LexUnicodeIdentifierStart(clang::Token & Result, uint32_t C, const char * CurPtr)
  • public Lexer(clang::SourceLocation FileLoc, const clang::LangOptions & LangOpts, const char * BufStart, const char * BufPtr, const char * BufEnd, bool IsFirstIncludeOfFile = true)
  • public Lexer(clang::FileID FID, const llvm::MemoryBufferRef & FromFile, const clang::SourceManager & SM, const clang::LangOptions & LangOpts, bool IsFirstIncludeOfFile = true)
  • public Lexer(clang::FileID FID, const llvm::MemoryBufferRef & InputFile, clang::Preprocessor & PP, bool IsFirstIncludeOfFile = true)
  • public Lexer(const clang::Lexer &)
  • public static unsigned int MeasureTokenLength(clang::SourceLocation Loc, const clang::SourceManager & SM, const clang::LangOptions & LangOpts)
  • private void PropagateLineStartLeadingSpaceInfo(clang::Token & Result)
  • public void ReadToEndOfLine(SmallVectorImpl<char> * Result = nullptr)
  • private bool SaveLineComment(clang::Token & Result, const char * CurPtr)
  • private void SetByteOffset(unsigned int Offset, bool StartOfLine)
  • public void SetCommentRetentionState(bool Mode)
  • public void SetKeepWhitespaceMode(bool Val)
  • private bool SkipBlockComment(clang::Token & Result, const char * CurPtr, bool & TokAtPhysicalStartOfLine)
  • private static const char * SkipEscapedNewLines(const char * P)
  • private bool SkipLineComment(clang::Token & Result, const char * CurPtr, bool & TokAtPhysicalStartOfLine)
  • private bool SkipWhitespace(clang::Token & Result, const char * CurPtr, bool & TokAtPhysicalStartOfLine)
  • public static void Stringify(SmallVectorImpl<char> & Str)
  • public static std::string Stringify(llvm::StringRef Str, bool Charify = false)
  • private void anchor()
  • private void codeCompleteIncludedFile(const char * PathStart, const char * CompletionPoint, bool IsAngled)
  • private const char * convertDependencyDirectiveToken(const dependency_directives_scan::Token & DDTok, clang::Token & Result)
  • private void cutOffLexing()
  • public static clang::SourceLocation findLocationAfterToken(clang::SourceLocation loc, tok::TokenKind TKind, const clang::SourceManager & SM, const clang::LangOptions & LangOpts, bool SkipTrailingWhitespaceAndNewLine)
  • public static Optional<clang::Token> findNextToken(clang::SourceLocation Loc, const clang::SourceManager & SM, const clang::LangOptions & LangOpts)
  • private inline char getAndAdvanceChar(const char *& Ptr, clang::Token & Tok)
  • public static clang::CharSourceRange getAsCharRange(clang::CharSourceRange Range, const clang::SourceManager & SM, const clang::LangOptions & LangOpts)
  • public static clang::CharSourceRange getAsCharRange(clang::SourceRange Range, const clang::SourceManager & SM, const clang::LangOptions & LangOpts)
  • public llvm::StringRef getBuffer() const
  • public const char * getBufferLocation() const
  • private inline char getCharAndSize(const char * Ptr, unsigned int & Size)
  • public static inline char getCharAndSizeNoWarn(const char * Ptr, unsigned int & Size, const clang::LangOptions & LangOpts)
  • private char getCharAndSizeSlow(const char * Ptr, unsigned int & Size, clang::Token * Tok = nullptr)
  • private static char getCharAndSizeSlowNoWarn(const char * Ptr, unsigned int & Size, const clang::LangOptions & LangOpts)
  • public unsigned int getCurrentBufferOffset()
  • private static unsigned int getEscapedNewLineSize(const char * P)
  • public clang::SourceLocation getFileLoc() const
  • public static llvm::StringRef getImmediateMacroName(clang::SourceLocation Loc, const clang::SourceManager & SM, const clang::LangOptions & LangOpts)
  • public static llvm::StringRef getImmediateMacroNameForDiagnostics(clang::SourceLocation Loc, const clang::SourceManager & SM, const clang::LangOptions & LangOpts)
  • public static llvm::StringRef getIndentationForLine(clang::SourceLocation Loc, const clang::SourceManager & SM)
  • public static clang::SourceLocation getLocForEndOfToken(clang::SourceLocation Loc, unsigned int Offset, const clang::SourceManager & SM, const clang::LangOptions & LangOpts)
  • public static bool getRawToken(clang::SourceLocation Loc, clang::Token & Result, const clang::SourceManager & SM, const clang::LangOptions & LangOpts, bool IgnoreWhiteSpace = false)
  • public clang::SourceLocation getSourceLocation(const char * Loc, unsigned int TokLen = 1) const
  • public clang::SourceLocation getSourceLocation()
  • public static llvm::StringRef getSourceText(clang::CharSourceRange Range, const clang::SourceManager & SM, const clang::LangOptions & LangOpts, bool * Invalid = nullptr)
  • public static llvm::StringRef getSpelling(clang::SourceLocation loc, SmallVectorImpl<char> & buffer, const clang::SourceManager & SM, const clang::LangOptions & options, bool * invalid = nullptr)
  • public static std::string getSpelling(const clang::Token & Tok, const clang::SourceManager & SourceMgr, const clang::LangOptions & LangOpts, bool * Invalid = nullptr)
  • public static unsigned int getSpelling(const clang::Token & Tok, const char *& Buffer, const clang::SourceManager & SourceMgr, const clang::LangOptions & LangOpts, bool * Invalid = nullptr)
  • public static unsigned int getTokenPrefixLength(clang::SourceLocation TokStart, unsigned int CharNo, const clang::SourceManager & SM, const clang::LangOptions & LangOpts)
  • public bool inKeepCommentMode() const
  • public static bool isAsciiIdentifierContinueChar(char c, const clang::LangOptions & LangOpts)
  • public static bool isAtEndOfMacroExpansion(clang::SourceLocation loc, const clang::SourceManager & SM, const clang::LangOptions & LangOpts, clang::SourceLocation * MacroEnd = nullptr)
  • public static bool isAtStartOfMacroExpansion(clang::SourceLocation loc, const clang::SourceManager & SM, const clang::LangOptions & LangOpts, clang::SourceLocation * MacroBegin = nullptr)
  • private bool isCodeCompletionPoint(const char * CurPtr) const
  • private bool isDependencyDirectivesLexer() const
  • public bool isFirstTimeLexingFile() const
  • private bool isHexaLiteral(const char * Start, const clang::LangOptions & LangOpts)
  • public bool isKeepWhitespaceMode() const
  • public static bool isNewLineEscaped(const char * BufferStart, const char * Str)
  • private unsigned int isNextPPTokenLParen()
  • private static bool isObviouslySimpleCharacter(char C)
  • public bool isPragmaLexer() const
  • private bool lexEditorPlaceholder(clang::Token & Result, const char * CurPtr)
  • public static clang::CharSourceRange makeFileCharRange(clang::CharSourceRange Range, const clang::SourceManager & SM, const clang::LangOptions & LangOpts)
  • public void resetExtendedTokenMode()
  • public void seek(unsigned int Offset, bool IsAtStartOfLine)
  • private bool tryConsumeIdentifierUCN(const char *& CurPtr, unsigned int Size, clang::Token & Result)
  • private bool tryConsumeIdentifierUTF8Char(const char *& CurPtr)
  • private llvm::Optional<uint32_t> tryReadNamedUCN(const char *& StartPtr, clang::Token * Result)
  • private llvm::Optional<uint32_t> tryReadNumericUCN(const char *& StartPtr, const char * SlashLoc, clang::Token * Result)
  • private uint32_t tryReadUCN(const char *& StartPtr, const char * SlashLoc, clang::Token * Result)

Inherited from PreprocessorLexer:

Methods

static clang::SourceLocation
AdvanceToTokenCharacter(
    clang::SourceLocation TokStart,
    unsigned int Characters,
    const clang::SourceManager& SM,
    const clang::LangOptions& LangOpts)

Description

AdvanceToTokenCharacter - If the current SourceLocation specifies a location at the start of a token, return a new location that specifies a character within the token. This handles trigraphs and escaped newlines.

Declared at: clang/include/clang/Lex/Lexer.h:399

Parameters

clang::SourceLocation TokStart
unsigned int Characters
const clang::SourceManager& SM
const clang::LangOptions& LangOpts

bool CheckUnicodeWhitespace(clang::Token& Result,
                            uint32_t C,
                            const char* CurPtr)

Declared at: clang/include/clang/Lex/Lexer.h:610

Parameters

clang::Token& Result
uint32_t C
const char* CurPtr

static clang::PreambleBounds ComputePreamble(
    llvm::StringRef Buffer,
    const clang::LangOptions& LangOpts,
    unsigned int MaxLines = 0)

Description

Compute the preamble of the given file. The preamble of a file contains the initial comments, include directives, and other preprocessor directives that occur before the code in this particular file actually begins. The preamble of the main source file is a potential prefix header.

Declared at: clang/include/clang/Lex/Lexer.h:548

Parameters

llvm::StringRef Buffer
The memory buffer containing the file's contents.
const clang::LangOptions& LangOpts
unsigned int MaxLines = 0
If non-zero, restrict the length of the preamble to fewer than this number of lines.

Returns

The offset into the file where the preamble ends and the rest of the file begins along with a boolean value indicating whether the preamble ends at the beginning of a new line.

const char* ConsumeChar(const char* Ptr,
                        unsigned int Size,
                        clang::Token& Tok)

Description

ConsumeChar - When a character (identified by getCharAndSize) is consumed and added to a given token, check to see if there are diagnostics that need to be emitted or flags that need to be set on the token. If so, do it.

Declared at: clang/include/clang/Lex/Lexer.h:678

Parameters

const char* Ptr
unsigned int Size
clang::Token& Tok

static clang::Lexer* Create_PragmaLexer(
    clang::SourceLocation SpellingLoc,
    clang::SourceLocation ExpansionLocStart,
    clang::SourceLocation ExpansionLocEnd,
    unsigned int TokLen,
    clang::Preprocessor& PP)

Description

Create_PragmaLexer: Lexer constructor - Create a new lexer object for _Pragma expansion. This has a variety of magic semantics that this method sets up. It returns a new'd Lexer that must be delete'd when done.

Declared at: clang/include/clang/Lex/Lexer.h:190

Parameters

clang::SourceLocation SpellingLoc
clang::SourceLocation ExpansionLocStart
clang::SourceLocation ExpansionLocEnd
unsigned int TokLen
clang::Preprocessor& PP

clang::DiagnosticBuilder Diag(
    const char* Loc,
    unsigned int DiagID) const

Description

Diag - Forwarding function for diagnostics. This translate a source position in the current buffer into a SourceLocation object for rendering.

Declared at: clang/include/clang/Lex/Lexer.h:295

Parameters

const char* Loc
unsigned int DiagID

void FormTokenWithChars(clang::Token& Result,
                        const char* TokEnd,
                        tok::TokenKind Kind)

Description

FormTokenWithChars - When we lex a token, we have identified a span starting at BufferPtr, going to TokEnd that forms the token. This method takes that range and assigns it to the token as its location and size. In addition, since tokens cannot overlap, this also updates BufferPtr to be TokEnd.

Declared at: clang/include/clang/Lex/Lexer.h:619

Parameters

clang::Token& Result
const char* TokEnd
tok::TokenKind Kind

static clang::SourceLocation GetBeginningOfToken(
    clang::SourceLocation Loc,
    const clang::SourceManager& SM,
    const clang::LangOptions& LangOpts)

Description

Given a location any where in a source buffer, find the location that corresponds to the beginning of the token in which the original source location lands.

Declared at: clang/include/clang/Lex/Lexer.h:385

Parameters

clang::SourceLocation Loc
const clang::SourceManager& SM
const clang::LangOptions& LangOpts

bool HandleEndOfConflictMarker(const char* CurPtr)

Declared at: clang/include/clang/Lex/Lexer.h:760

Parameters

const char* CurPtr

void IndirectLex(clang::Token& Result)

Description

IndirectLex - An indirect call to 'Lex' that can be invoked via the PreprocessorLexer interface.

Declared at: clang/include/clang/Lex/Lexer.h:230

Parameters

clang::Token& Result

void InitLexer(const char* BufStart,
               const char* BufPtr,
               const char* BufEnd)

Declared at: clang/include/clang/Lex/Lexer.h:160

Parameters

const char* BufStart
const char* BufPtr
const char* BufEnd

bool IsStartOfConflictMarker(const char* CurPtr)

Declared at: clang/include/clang/Lex/Lexer.h:759

Parameters

const char* CurPtr

bool Lex(clang::Token& Result)

Description

Lex - Return the next token in the file. If this is the end of file, it return the tok::eof token. This implicitly involves the preprocessor.

Declared at: clang/include/clang/Lex/Lexer.h:204

Parameters

clang::Token& Result

bool LexAngledStringLiteral(clang::Token& Result,
                            const char* CurPtr)

Declared at: clang/include/clang/Lex/Lexer.h:747

Parameters

clang::Token& Result
const char* CurPtr

bool LexCharConstant(clang::Token& Result,
                     const char* CurPtr,
                     tok::TokenKind Kind)

Declared at: clang/include/clang/Lex/Lexer.h:748

Parameters

clang::Token& Result
const char* CurPtr
tok::TokenKind Kind

bool LexDependencyDirectiveToken(
    clang::Token& Result)

Description

Called when the preprocessor is in 'dependency scanning lexing mode'.

Declared at: clang/include/clang/Lex/Lexer.h:207

Parameters

clang::Token& Result

bool LexDependencyDirectiveTokenWhileSkipping(
    clang::Token& Result)

Description

Called when the preprocessor is in 'dependency scanning lexing mode' and is skipping a conditional block.

Declared at: clang/include/clang/Lex/Lexer.h:211

Parameters

clang::Token& Result

bool LexEndOfFile(clang::Token& Result,
                  const char* CurPtr)

Declared at: clang/include/clang/Lex/Lexer.h:750

Parameters

clang::Token& Result
const char* CurPtr

bool LexFromRawLexer(clang::Token& Result)

Description

LexFromRawLexer - Lex a token from a designated raw lexer (one with no associated preprocessor object. Return true if the 'next character to read' pointer points at the end of the lexer buffer, false otherwise.

Declared at: clang/include/clang/Lex/Lexer.h:236

Parameters

clang::Token& Result

bool LexIdentifierContinue(clang::Token& Result,
                           const char* CurPtr)

Declared at: clang/include/clang/Lex/Lexer.h:740

Parameters

clang::Token& Result
const char* CurPtr

bool LexNumericConstant(clang::Token& Result,
                        const char* CurPtr)

Declared at: clang/include/clang/Lex/Lexer.h:742

Parameters

clang::Token& Result
const char* CurPtr

bool LexRawStringLiteral(clang::Token& Result,
                         const char* CurPtr,
                         tok::TokenKind Kind)

Declared at: clang/include/clang/Lex/Lexer.h:745

Parameters

clang::Token& Result
const char* CurPtr
tok::TokenKind Kind

bool LexStringLiteral(clang::Token& Result,
                      const char* CurPtr,
                      tok::TokenKind Kind)

Declared at: clang/include/clang/Lex/Lexer.h:743

Parameters

clang::Token& Result
const char* CurPtr
tok::TokenKind Kind

bool LexTokenInternal(
    clang::Token& Result,
    bool TokAtPhysicalStartOfLine)

Description

LexTokenInternal - Internal interface to lex a preprocessing token. Called by Lex.

Declared at: clang/include/clang/Lex/Lexer.h:608

Parameters

clang::Token& Result
bool TokAtPhysicalStartOfLine

const char* LexUDSuffix(clang::Token& Result,
                        const char* CurPtr,
                        bool IsStringLiteral)

Declared at: clang/include/clang/Lex/Lexer.h:733

Parameters

clang::Token& Result
const char* CurPtr
bool IsStringLiteral

bool LexUnicodeIdentifierStart(
    clang::Token& Result,
    uint32_t C,
    const char* CurPtr)

Declared at: clang/include/clang/Lex/Lexer.h:612

Parameters

clang::Token& Result
uint32_t C
const char* CurPtr

Lexer(clang::SourceLocation FileLoc,
      const clang::LangOptions& LangOpts,
      const char* BufStart,
      const char* BufPtr,
      const char* BufEnd,
      bool IsFirstIncludeOfFile = true)

Description

Lexer constructor - Create a new raw lexer object. This object is only suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text range will outlive it, so it doesn't take ownership of it.

Declared at: clang/include/clang/Lex/Lexer.h:173

Parameters

clang::SourceLocation FileLoc
const clang::LangOptions& LangOpts
const char* BufStart
const char* BufPtr
const char* BufEnd
bool IsFirstIncludeOfFile = true

Lexer(clang::FileID FID,
      const llvm::MemoryBufferRef& FromFile,
      const clang::SourceManager& SM,
      const clang::LangOptions& LangOpts,
      bool IsFirstIncludeOfFile = true)

Description

Lexer constructor - Create a new raw lexer object. This object is only suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text range will outlive it, so it doesn't take ownership of it.

Declared at: clang/include/clang/Lex/Lexer.h:180

Parameters

clang::FileID FID
const llvm::MemoryBufferRef& FromFile
const clang::SourceManager& SM
const clang::LangOptions& LangOpts
bool IsFirstIncludeOfFile = true

Lexer(clang::FileID FID,
      const llvm::MemoryBufferRef& InputFile,
      clang::Preprocessor& PP,
      bool IsFirstIncludeOfFile = true)

Description

Lexer constructor - Create a new lexer object for the specified buffer with the specified preprocessor managing the lexing process. This lexer assumes that the associated file buffer and Preprocessor objects will outlive it, so it doesn't take ownership of either of them.

Declared at: clang/include/clang/Lex/Lexer.h:167

Parameters

clang::FileID FID
const llvm::MemoryBufferRef& InputFile
clang::Preprocessor& PP
bool IsFirstIncludeOfFile = true

Lexer(const clang::Lexer&)

Declared at: clang/include/clang/Lex/Lexer.h:184

Parameters

const clang::Lexer& 

static unsigned int MeasureTokenLength(
    clang::SourceLocation Loc,
    const clang::SourceManager& SM,
    const clang::LangOptions& LangOpts)

Description

MeasureTokenLength - Relex the token at the specified location and return its length in bytes in the input file. If the token needs cleaning (e.g. includes a trigraph or an escaped newline) then this count includes bytes that are part of that.

Declared at: clang/include/clang/Lex/Lexer.h:371

Parameters

clang::SourceLocation Loc
const clang::SourceManager& SM
const clang::LangOptions& LangOpts

void PropagateLineStartLeadingSpaceInfo(
    clang::Token& Result)

Declared at: clang/include/clang/Lex/Lexer.h:731

Parameters

clang::Token& Result

void ReadToEndOfLine(
    SmallVectorImpl<char>* Result = nullptr)

Description

ReadToEndOfLine - Read the rest of the current preprocessor line as an uninterpreted string. This switches the lexer out of directive mode.

Declared at: clang/include/clang/Lex/Lexer.h:290

Parameters

SmallVectorImpl<char>* Result = nullptr

bool SaveLineComment(clang::Token& Result,
                     const char* CurPtr)

Declared at: clang/include/clang/Lex/Lexer.h:757

Parameters

clang::Token& Result
const char* CurPtr

void SetByteOffset(unsigned int Offset,
                   bool StartOfLine)

Declared at: clang/include/clang/Lex/Lexer.h:729

Parameters

unsigned int Offset
bool StartOfLine

void SetCommentRetentionState(bool Mode)

Description

SetCommentRetentionMode - Change the comment retention mode of the lexer to the specified mode. This is really only useful when lexing in raw mode, because otherwise the lexer needs to manage this.

Declared at: clang/include/clang/Lex/Lexer.h:269

Parameters

bool Mode

void SetKeepWhitespaceMode(bool Val)

Description

SetKeepWhitespaceMode - This method lets clients enable or disable whitespace retention mode.

Declared at: clang/include/clang/Lex/Lexer.h:254

Parameters

bool Val

bool SkipBlockComment(
    clang::Token& Result,
    const char* CurPtr,
    bool& TokAtPhysicalStartOfLine)

Declared at: clang/include/clang/Lex/Lexer.h:755

Parameters

clang::Token& Result
const char* CurPtr
bool& TokAtPhysicalStartOfLine

static const char* SkipEscapedNewLines(
    const char* P)

Description

SkipEscapedNewLines - If P points to an escaped newline (or a series of them), skip over them and return the first non-escaped-newline found, otherwise return P.

Declared at: clang/include/clang/Lex/Lexer.h:719

Parameters

const char* P

bool SkipLineComment(
    clang::Token& Result,
    const char* CurPtr,
    bool& TokAtPhysicalStartOfLine)

Declared at: clang/include/clang/Lex/Lexer.h:753

Parameters

clang::Token& Result
const char* CurPtr
bool& TokAtPhysicalStartOfLine

bool SkipWhitespace(
    clang::Token& Result,
    const char* CurPtr,
    bool& TokAtPhysicalStartOfLine)

Declared at: clang/include/clang/Lex/Lexer.h:751

Parameters

clang::Token& Result
const char* CurPtr
bool& TokAtPhysicalStartOfLine

static void Stringify(SmallVectorImpl<char>& Str)

Description

Stringify - Convert the specified string into a C string by i) escaping ' \ ' and " characters and ii) replacing newline character(s) with " \ n".

Declared at: clang/include/clang/Lex/Lexer.h:326

Parameters

SmallVectorImpl<char>& Str

static std::string Stringify(llvm::StringRef Str,
                             bool Charify = false)

Description

Stringify - Convert the specified string into a C string by i) escaping ' \ ' and " characters and ii) replacing newline character(s) with " \ n". If Charify is true, this escapes the ' character instead of ".

Declared at: clang/include/clang/Lex/Lexer.h:322

Parameters

llvm::StringRef Str
bool Charify = false

void anchor()

Declared at: clang/include/clang/Lex/Lexer.h:81

void codeCompleteIncludedFile(
    const char* PathStart,
    const char* CompletionPoint,
    bool IsAngled)

Declared at: clang/include/clang/Lex/Lexer.h:769

Parameters

const char* PathStart
const char* CompletionPoint
bool IsAngled

const char* convertDependencyDirectiveToken(
    const dependency_directives_scan::Token&
        DDTok,
    clang::Token& Result)

Description

Initializes \p Result with data from \p DDTok and advances \p BufferPtr to the position just after the token.

Declared at: clang/include/clang/Lex/Lexer.h:220

Parameters

const dependency_directives_scan::Token& DDTok
clang::Token& Result

Returns

the buffer pointer at the beginning of the token.

void cutOffLexing()

Declared at: clang/include/clang/Lex/Lexer.h:765

static clang::SourceLocation
findLocationAfterToken(
    clang::SourceLocation loc,
    tok::TokenKind TKind,
    const clang::SourceManager& SM,
    const clang::LangOptions& LangOpts,
    bool SkipTrailingWhitespaceAndNewLine)

Description

Checks that the given token is the first token that occurs after the given location (this excludes comments and whitespace). Returns the location immediately after the specified token. If the token is not found or the location is inside a macro, the returned source location will be invalid.

Declared at: clang/include/clang/Lex/Lexer.h:564

Parameters

clang::SourceLocation loc
tok::TokenKind TKind
const clang::SourceManager& SM
const clang::LangOptions& LangOpts
bool SkipTrailingWhitespaceAndNewLine

static Optional<clang::Token> findNextToken(
    clang::SourceLocation Loc,
    const clang::SourceManager& SM,
    const clang::LangOptions& LangOpts)

Description

Finds the token that comes right after the given location. Returns the next token, or none if the location is inside a macro.

Declared at: clang/include/clang/Lex/Lexer.h:555

Parameters

clang::SourceLocation Loc
const clang::SourceManager& SM
const clang::LangOptions& LangOpts

inline char getAndAdvanceChar(const char*& Ptr,
                              clang::Token& Tok)

Description

getAndAdvanceChar - Read a single 'character' from the specified buffer, advance over it, and return it. This is tricky in several cases. Here we just handle the trivial case and fall-back to the non-inlined getCharAndSizeSlow method to handle the hard case.

Declared at: clang/include/clang/Lex/Lexer.h:663

Parameters

const char*& Ptr
clang::Token& Tok

static clang::CharSourceRange getAsCharRange(
    clang::CharSourceRange Range,
    const clang::SourceManager& SM,
    const clang::LangOptions& LangOpts)

Declared at: clang/include/clang/Lex/Lexer.h:438

Parameters

clang::CharSourceRange Range
const clang::SourceManager& SM
const clang::LangOptions& LangOpts

static clang::CharSourceRange getAsCharRange(
    clang::SourceRange Range,
    const clang::SourceManager& SM,
    const clang::LangOptions& LangOpts)

Description

Given a token range, produce a corresponding CharSourceRange that is not a token range. This allows the source range to be used by components that don't have access to the lexer and thus can't find the end of the range for themselves.

Declared at: clang/include/clang/Lex/Lexer.h:430

Parameters

clang::SourceRange Range
const clang::SourceManager& SM
const clang::LangOptions& LangOpts

llvm::StringRef getBuffer() const

Description

Gets source code buffer.

Declared at: clang/include/clang/Lex/Lexer.h:284

const char* getBufferLocation() const

Description

Return the current location in the buffer.

Declared at: clang/include/clang/Lex/Lexer.h:308

inline char getCharAndSize(const char* Ptr,
                           unsigned int& Size)

Description

getCharAndSize - Peek a single 'character' from the specified buffer, get its size, and return it. This is tricky in several cases. Here we just handle the trivial case and fall-back to the non-inlined getCharAndSizeSlow method to handle the hard case.

Declared at: clang/include/clang/Lex/Lexer.h:694

Parameters

const char* Ptr
unsigned int& Size

static inline char getCharAndSizeNoWarn(
    const char* Ptr,
    unsigned int& Size,
    const clang::LangOptions& LangOpts)

Description

getCharAndSizeNoWarn - Like the getCharAndSize method, but does not ever emit a warning.

Declared at: clang/include/clang/Lex/Lexer.h:580

Parameters

const char* Ptr
unsigned int& Size
const clang::LangOptions& LangOpts

char getCharAndSizeSlow(
    const char* Ptr,
    unsigned int& Size,
    clang::Token* Tok = nullptr)

Description

getCharAndSizeSlow - Handle the slow/uncommon case of the getCharAndSize method.

Declared at: clang/include/clang/Lex/Lexer.h:708

Parameters

const char* Ptr
unsigned int& Size
clang::Token* Tok = nullptr

static char getCharAndSizeSlowNoWarn(
    const char* Ptr,
    unsigned int& Size,
    const clang::LangOptions& LangOpts)

Description

getCharAndSizeSlowNoWarn - Same as getCharAndSizeSlow, but never emits a diagnostic.

Declared at: clang/include/clang/Lex/Lexer.h:723

Parameters

const char* Ptr
unsigned int& Size
const clang::LangOptions& LangOpts

unsigned int getCurrentBufferOffset()

Description

Returns the current lexing offset.

Declared at: clang/include/clang/Lex/Lexer.h:311

static unsigned int getEscapedNewLineSize(
    const char* P)

Description

getEscapedNewLineSize - Return the size of the specified escaped newline, or 0 if it is not an escaped newline. P[-1] is known to be a "\" on entry to this function.

Declared at: clang/include/clang/Lex/Lexer.h:714

Parameters

const char* P

clang::SourceLocation getFileLoc() const

Description

getFileLoc - Return the File Location for the file we are lexing out of. The physical location encodes the location where the characters come from, the virtual location encodes where we should *claim* the characters came from. Currently this is only used by _Pragma handling.

Declared at: clang/include/clang/Lex/Lexer.h:199

static llvm::StringRef getImmediateMacroName(
    clang::SourceLocation Loc,
    const clang::SourceManager& SM,
    const clang::LangOptions& LangOpts)

Description

Retrieve the name of the immediate macro expansion. This routine starts from a source location, and finds the name of the macro responsible for its immediate expansion. It looks through any intervening macro argument expansions to compute this. It returns a StringRef which refers to the SourceManager-owned buffer of the source where that macro name is spelled. Thus, the result shouldn't out-live that SourceManager.

Declared at: clang/include/clang/Lex/Lexer.h:509

Parameters

clang::SourceLocation Loc
const clang::SourceManager& SM
const clang::LangOptions& LangOpts

static llvm::StringRef
getImmediateMacroNameForDiagnostics(
    clang::SourceLocation Loc,
    const clang::SourceManager& SM,
    const clang::LangOptions& LangOpts)

Description

Retrieve the name of the immediate macro expansion. This routine starts from a source location, and finds the name of the macro responsible for its immediate expansion. It looks through any intervening macro argument expansions to compute this. It returns a StringRef which refers to the SourceManager-owned buffer of the source where that macro name is spelled. Thus, the result shouldn't out-live that SourceManager. This differs from Lexer::getImmediateMacroName in that any macro argument location will result in the topmost function macro that accepted it. e.g. for location of 'foo' token, this function will return "MAC1" while Lexer::getImmediateMacroName will return "MAC2".

Declared at: clang/include/clang/Lex/Lexer.h:530

Parameters

clang::SourceLocation Loc
const clang::SourceManager& SM
const clang::LangOptions& LangOpts

static llvm::StringRef getIndentationForLine(
    clang::SourceLocation Loc,
    const clang::SourceManager& SM)

Description

Returns the leading whitespace for line that corresponds to the given location \p Loc.

Declared at: clang/include/clang/Lex/Lexer.h:595

Parameters

clang::SourceLocation Loc
const clang::SourceManager& SM

static clang::SourceLocation getLocForEndOfToken(
    clang::SourceLocation Loc,
    unsigned int Offset,
    const clang::SourceManager& SM,
    const clang::LangOptions& LangOpts)

Description

Computes the source location just past the end of the token at this source location. This routine can be used to produce a source location that points just past the end of the token referenced by \p Loc, and is generally used when a diagnostic needs to point just after a token where it expected something different that it received. If the returned source location would not be meaningful (e.g., if it points into a macro), this routine returns an invalid source location.

Declared at: clang/include/clang/Lex/Lexer.h:422

Parameters

clang::SourceLocation Loc
unsigned int Offset
an offset from the end of the token, where the source location should refer to. The default offset (0) produces a source location pointing just past the end of the token; an offset of 1 produces a source location pointing to the last character in the token, etc.
const clang::SourceManager& SM
const clang::LangOptions& LangOpts

static bool getRawToken(
    clang::SourceLocation Loc,
    clang::Token& Result,
    const clang::SourceManager& SM,
    const clang::LangOptions& LangOpts,
    bool IgnoreWhiteSpace = false)

Description

Relex the token at the specified location.

Declared at: clang/include/clang/Lex/Lexer.h:377

Parameters

clang::SourceLocation Loc
clang::Token& Result
const clang::SourceManager& SM
const clang::LangOptions& LangOpts
bool IgnoreWhiteSpace = false

Returns

true if there was a failure, false on success.

clang::SourceLocation getSourceLocation(
    const char* Loc,
    unsigned int TokLen = 1) const

Description

getSourceLocation - Return a source location identifier for the specified offset in the current file.

Declared at: clang/include/clang/Lex/Lexer.h:299

Parameters

const char* Loc
unsigned int TokLen = 1

clang::SourceLocation getSourceLocation()

Description

getSourceLocation - Return a source location for the next character in the current file.

Declared at: clang/include/clang/Lex/Lexer.h:303

static llvm::StringRef getSourceText(
    clang::CharSourceRange Range,
    const clang::SourceManager& SM,
    const clang::LangOptions& LangOpts,
    bool* Invalid = nullptr)

Description

Returns a string for the source that the range encompasses.

Declared at: clang/include/clang/Lex/Lexer.h:497

Parameters

clang::CharSourceRange Range
const clang::SourceManager& SM
const clang::LangOptions& LangOpts
bool* Invalid = nullptr

static llvm::StringRef getSpelling(
    clang::SourceLocation loc,
    SmallVectorImpl<char>& buffer,
    const clang::SourceManager& SM,
    const clang::LangOptions& options,
    bool* invalid = nullptr)

Description

getSpelling - This method is used to get the spelling of the token at the given source location. If, as is usually true, it is not necessary to copy any data, then the returned string may not point into the provided buffer. This method lexes at the expansion depth of the given location and does not jump to the expansion or spelling location.

Declared at: clang/include/clang/Lex/Lexer.h:361

Parameters

clang::SourceLocation loc
SmallVectorImpl<char>& buffer
const clang::SourceManager& SM
const clang::LangOptions& options
bool* invalid = nullptr

static std::string getSpelling(
    const clang::Token& Tok,
    const clang::SourceManager& SourceMgr,
    const clang::LangOptions& LangOpts,
    bool* Invalid = nullptr)

Description

getSpelling() - Return the 'spelling' of the Tok token. The spelling of a token is the characters used to represent the token in the source file after trigraph expansion and escaped-newline folding. In particular, this wants to get the true, uncanonicalized, spelling of things like digraphs UCNs, etc.

Declared at: clang/include/clang/Lex/Lexer.h:348

Parameters

const clang::Token& Tok
const clang::SourceManager& SourceMgr
const clang::LangOptions& LangOpts
bool* Invalid = nullptr

static unsigned int getSpelling(
    const clang::Token& Tok,
    const char*& Buffer,
    const clang::SourceManager& SourceMgr,
    const clang::LangOptions& LangOpts,
    bool* Invalid = nullptr)

Description

getSpelling - This method is used to get the spelling of a token into a preallocated buffer, instead of as an std::string. The caller is required to allocate enough space for the token, which is guaranteed to be at least Tok.getLength() bytes long. The length of the actual result is returned. Note that this method may do two possible things: it may either fill in the buffer specified with characters, or it may *change the input pointer* to point to a constant buffer with the data already in it (avoiding a copy). The caller is not allowed to modify the returned buffer pointer if an internal buffer is returned.

Declared at: clang/include/clang/Lex/Lexer.h:338

Parameters

const clang::Token& Tok
const char*& Buffer
const clang::SourceManager& SourceMgr
const clang::LangOptions& LangOpts
bool* Invalid = nullptr

static unsigned int getTokenPrefixLength(
    clang::SourceLocation TokStart,
    unsigned int CharNo,
    const clang::SourceManager& SM,
    const clang::LangOptions& LangOpts)

Description

Get the physical length (including trigraphs and escaped newlines) of the first \p Characters characters of the token starting at TokStart.

Declared at: clang/include/clang/Lex/Lexer.h:391

Parameters

clang::SourceLocation TokStart
unsigned int CharNo
const clang::SourceManager& SM
const clang::LangOptions& LangOpts

bool inKeepCommentMode() const

Description

inKeepCommentMode - Return true if the lexer should return comments as tokens.

Declared at: clang/include/clang/Lex/Lexer.h:262

static bool isAsciiIdentifierContinueChar(
    char c,
    const clang::LangOptions& LangOpts)

Description

Returns true if the given character could appear in an identifier.

Declared at: clang/include/clang/Lex/Lexer.h:571

Parameters

char c
const clang::LangOptions& LangOpts

static bool isAtEndOfMacroExpansion(
    clang::SourceLocation loc,
    const clang::SourceManager& SM,
    const clang::LangOptions& LangOpts,
    clang::SourceLocation* MacroEnd = nullptr)

Description

Returns true if the given MacroID location points at the last token of the macro expansion.

Declared at: clang/include/clang/Lex/Lexer.h:461

Parameters

clang::SourceLocation loc
const clang::SourceManager& SM
const clang::LangOptions& LangOpts
clang::SourceLocation* MacroEnd = nullptr
If non-null and function returns true, it is set to end location of the macro.

static bool isAtStartOfMacroExpansion(
    clang::SourceLocation loc,
    const clang::SourceManager& SM,
    const clang::LangOptions& LangOpts,
    clang::SourceLocation* MacroBegin = nullptr)

Description

Returns true if the given MacroID location points at the first token of the macro expansion.

Declared at: clang/include/clang/Lex/Lexer.h:451

Parameters

clang::SourceLocation loc
const clang::SourceManager& SM
const clang::LangOptions& LangOpts
clang::SourceLocation* MacroBegin = nullptr
If non-null and function returns true, it is set to begin location of the macro.

bool isCodeCompletionPoint(
    const char* CurPtr) const

Declared at: clang/include/clang/Lex/Lexer.h:764

Parameters

const char* CurPtr

bool isDependencyDirectivesLexer() const

Description

True when the preprocessor is in 'dependency scanning lexing mode' and created this \p Lexer for lexing a set of dependency directive tokens.

Declared at: clang/include/clang/Lex/Lexer.h:215

bool isFirstTimeLexingFile() const

Description

Check if this is the first time we're lexing the input file.

Declared at: clang/include/clang/Lex/Lexer.h:599

bool isHexaLiteral(
    const char* Start,
    const clang::LangOptions& LangOpts)

Declared at: clang/include/clang/Lex/Lexer.h:767

Parameters

const char* Start
const clang::LangOptions& LangOpts

bool isKeepWhitespaceMode() const

Description

isKeepWhitespaceMode - Return true if the lexer should return tokens for every character in the file, including whitespace and comments. This should only be used in raw mode, as the preprocessor is not prepared to deal with the excess tokens.

Declared at: clang/include/clang/Lex/Lexer.h:248

static bool isNewLineEscaped(
    const char* BufferStart,
    const char* Str)

Description

Checks whether new line pointed by Str is preceded by escape sequence.

Declared at: clang/include/clang/Lex/Lexer.h:576

Parameters

const char* BufferStart
const char* Str

unsigned int isNextPPTokenLParen()

Description

isNextPPTokenLParen - Return 1 if the next unexpanded token will return a tok::l_paren token, 0 if it is something else and 2 if there are no more tokens in the buffer controlled by this lexer.

Declared at: clang/include/clang/Lex/Lexer.h:631

static bool isObviouslySimpleCharacter(char C)

Description

isObviouslySimpleCharacter - Return true if the specified character is obviously the same in translation phase 1 and translation phase 3. This can return false for characters that end up being the same, but it will never return true for something that needs to be mapped.

Declared at: clang/include/clang/Lex/Lexer.h:655

Parameters

char C

bool isPragmaLexer() const

Description

isPragmaLexer - Returns true if this Lexer is being used to lex a pragma.

Declared at: clang/include/clang/Lex/Lexer.h:225

bool lexEditorPlaceholder(clang::Token& Result,
                          const char* CurPtr)

Declared at: clang/include/clang/Lex/Lexer.h:762

Parameters

clang::Token& Result
const char* CurPtr

static clang::CharSourceRange makeFileCharRange(
    clang::CharSourceRange Range,
    const clang::SourceManager& SM,
    const clang::LangOptions& LangOpts)

Description

Accepts a range and returns a character range with file locations. Returns a null range if a part of the range resides inside a macro expansion or the range does not reside on the same FileID. This function is trying to deal with macros and return a range based on file locations. The cases where it can successfully handle macros are: -begin or end range lies at the start or end of a macro expansion, in which case the location will be set to the expansion point, e.g: # define M 1 2 a M If you have a range [a, 2] (where 2 came from the macro), the function will return a range for "a M" if you have range [a, 1], the function will fail because the range overlaps with only a part of the macro -The macro is a function macro and the range can be mapped to the macro arguments, e.g: # define M 1 2 # define FM(x) x FM(a b M) if you have range [b, 2], the function will return the file range "b M" inside the macro arguments. if you have range [a, 2], the function will return the file range "FM(a b M)" since the range includes all of the macro expansion.

Declared at: clang/include/clang/Lex/Lexer.h:492

Parameters

clang::CharSourceRange Range
const clang::SourceManager& SM
const clang::LangOptions& LangOpts

void resetExtendedTokenMode()

Description

Sets the extended token mode back to its initial value, according to the language options and preprocessor. This controls whether the lexer produces comment and whitespace tokens. This requires the lexer to have an associated preprocessor. A standalone lexer has nothing to reset to.

Declared at: clang/include/clang/Lex/Lexer.h:281

void seek(unsigned int Offset,
          bool IsAtStartOfLine)

Description

Set the lexer's buffer pointer to \p Offset.

Declared at: clang/include/clang/Lex/Lexer.h:317

Parameters

unsigned int Offset
bool IsAtStartOfLine

bool tryConsumeIdentifierUCN(const char*& CurPtr,
                             unsigned int Size,
                             clang::Token& Result)

Description

Try to consume a UCN as part of an identifier at the current location.

Declared at: clang/include/clang/Lex/Lexer.h:802

Parameters

const char*& CurPtr
Initially points to the range of characters in the source buffer containing the ' \ '. Updated to point past the end of the UCN on success.
unsigned int Size
The number of characters occupied by the ' \ ' (including trigraphs and escaped newlines).
clang::Token& Result
The token being produced. Marked as containing a UCN on success.

Returns

\c true if a UCN was lexed and it produced an acceptable identifier character, \c false otherwise.

bool tryConsumeIdentifierUTF8Char(
    const char*& CurPtr)

Description

Try to consume an identifier character encoded in UTF-8.

Declared at: clang/include/clang/Lex/Lexer.h:810

Parameters

const char*& CurPtr
Points to the start of the (potential) UTF-8 code unit sequence. On success, updated to point past the end of it.

Returns

\c true if a UTF-8 sequence mapping to an acceptable identifier character was lexed, \c false otherwise.

llvm::Optional<uint32_t> tryReadNamedUCN(
    const char*& StartPtr,
    clang::Token* Result)

Declared at: clang/include/clang/Lex/Lexer.h:774

Parameters

const char*& StartPtr
clang::Token* Result

llvm::Optional<uint32_t> tryReadNumericUCN(
    const char*& StartPtr,
    const char* SlashLoc,
    clang::Token* Result)

Declared at: clang/include/clang/Lex/Lexer.h:773

Parameters

const char*& StartPtr
const char* SlashLoc
clang::Token* Result

uint32_t tryReadUCN(const char*& StartPtr,
                    const char* SlashLoc,
                    clang::Token* Result)

Description

Read a universal character name.

Declared at: clang/include/clang/Lex/Lexer.h:789

Parameters

const char*& StartPtr
The position in the source buffer after the initial ' \ '. If the UCN is syntactically well-formed (but not necessarily valid), this parameter will be updated to point to the character after the UCN.
const char* SlashLoc
The position in the source buffer of the ' \ '.
clang::Token* Result
The token being formed. Pass \c nullptr to suppress diagnostics and handle token formation in the caller.

Returns

The Unicode codepoint specified by the UCN, or 0 if the UCN is invalid.