1	//===- Preprocessor.h - C Language Family Preprocessor ----------*- C++ -*-===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	/// \file
10	/// Defines the clang::Preprocessor interface.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15	#define LLVM_CLANG_LEX_PREPROCESSOR_H
16
17	#include "clang/Basic/Diagnostic.h"
18	#include "clang/Basic/DiagnosticIDs.h"
19	#include "clang/Basic/IdentifierTable.h"
20	#include "clang/Basic/LLVM.h"
21	#include "clang/Basic/LangOptions.h"
22	#include "clang/Basic/Module.h"
23	#include "clang/Basic/SourceLocation.h"
24	#include "clang/Basic/SourceManager.h"
25	#include "clang/Basic/TokenKinds.h"
26	#include "clang/Lex/HeaderSearch.h"
27	#include "clang/Lex/Lexer.h"
28	#include "clang/Lex/MacroInfo.h"
29	#include "clang/Lex/ModuleLoader.h"
30	#include "clang/Lex/ModuleMap.h"
31	#include "clang/Lex/PPCallbacks.h"
32	#include "clang/Lex/PPEmbedParameters.h"
33	#include "clang/Lex/Token.h"
34	#include "clang/Lex/TokenLexer.h"
35	#include "clang/Support/Compiler.h"
36	#include "llvm/ADT/APSInt.h"
37	#include "llvm/ADT/ArrayRef.h"
38	#include "llvm/ADT/DenseMap.h"
39	#include "llvm/ADT/FoldingSet.h"
40	#include "llvm/ADT/FunctionExtras.h"
41	#include "llvm/ADT/PointerUnion.h"
42	#include "llvm/ADT/STLExtras.h"
43	#include "llvm/ADT/SmallPtrSet.h"
44	#include "llvm/ADT/SmallVector.h"
45	#include "llvm/ADT/StringRef.h"
46	#include "llvm/ADT/TinyPtrVector.h"
47	#include "llvm/ADT/iterator_range.h"
48	#include "llvm/Support/Allocator.h"
49	#include "llvm/Support/Casting.h"
50	#include "llvm/Support/Registry.h"
51	#include <cassert>
52	#include <cstddef>
53	#include <cstdint>
54	#include <map>
55	#include <memory>
56	#include <optional>
57	#include <string>
58	#include <utility>
59	#include <vector>
60
61	namespace llvm {
62
63	template<unsigned InternalLen> class SmallString;
64
65	} // namespace llvm
66
67	namespace clang {
68
69	class CodeCompletionHandler;
70	class CommentHandler;
71	class DirectoryEntry;
72	class EmptylineHandler;
73	class ExternalPreprocessorSource;
74	class FileEntry;
75	class FileManager;
76	class HeaderSearch;
77	class MacroArgs;
78	class PragmaHandler;
79	class PragmaNamespace;
80	class PreprocessingRecord;
81	class PreprocessorLexer;
82	class PreprocessorOptions;
83	class ScratchBuffer;
84	class TargetInfo;
85
86	namespace Builtin {
87	class Context;
88	}
89
90	/// Stores token information for comparing actual tokens with
91	/// predefined values. Only handles simple tokens and identifiers.
92	class TokenValue {
93	tok::TokenKind Kind;
94	IdentifierInfo *II;
95
96	public:
97	TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
98	assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
99	assert(Kind != tok::identifier &&
100	"Identifiers should be created by TokenValue(IdentifierInfo *)");
101	assert(!tok::isLiteral(Kind) && "Literals are not supported.");
102	assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
103	}
104
105	TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
106
107	bool operator==(const Token &Tok) const {
108	return Tok.getKind() == Kind &&
109	(!II || II == Tok.getIdentifierInfo());
110	}
111	};
112
113	/// Context in which macro name is used.
114	enum MacroUse {
115	// other than #define or #undef
116	MU_Other = 0,
117
118	// macro name specified in #define
119	MU_Define = 1,
120
121	// macro name specified in #undef
122	MU_Undef = 2
123	};
124
125	enum class EmbedResult {
126	Invalid = -1, // Parsing error occurred.
127	NotFound = 0, // Corresponds to __STDC_EMBED_NOT_FOUND__
128	Found = 1, // Corresponds to __STDC_EMBED_FOUND__
129	Empty = 2, // Corresponds to __STDC_EMBED_EMPTY__
130	};
131
132	struct CXXStandardLibraryVersionInfo {
133	enum Library { Unknown, LibStdCXX };
134	Library Lib;
135	std::uint64_t Version;
136	};
137
138	/// Engages in a tight little dance with the lexer to efficiently
139	/// preprocess tokens.
140	///
141	/// Lexers know only about tokens within a single source file, and don't
142	/// know anything about preprocessor-level issues like the \#include stack,
143	/// token expansion, etc.
144	class Preprocessor {
145	friend class VAOptDefinitionContext;
146	friend class VariadicMacroScopeGuard;
147
148	llvm::unique_function<void(const clang::Token &)> OnToken;
149	/// Functor for getting the dependency preprocessor directives of a file.
150	///
151	/// These are directives derived from a special form of lexing where the
152	/// source input is scanned for the preprocessor directives that might have an
153	/// effect on the dependencies for a compilation unit.
154	DependencyDirectivesGetter *GetDependencyDirectives = nullptr;
155	const PreprocessorOptions &PPOpts;
156	DiagnosticsEngine *Diags;
157	const LangOptions &LangOpts;
158	const TargetInfo *Target = nullptr;
159	const TargetInfo *AuxTarget = nullptr;
160	FileManager &FileMgr;
161	SourceManager &SourceMgr;
162	std::unique_ptr<ScratchBuffer> ScratchBuf;
163	HeaderSearch &HeaderInfo;
164	ModuleLoader &TheModuleLoader;
165
166	/// External source of macros.
167	ExternalPreprocessorSource *ExternalSource;
168
169	/// A BumpPtrAllocator object used to quickly allocate and release
170	/// objects internal to the Preprocessor.
171	llvm::BumpPtrAllocator BP;
172
173	/// Identifiers for builtin macros and other builtins.
174	IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__
175	IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__
176	IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__
177	IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__
178	IdentifierInfo *Ident__FILE_NAME__; // __FILE_NAME__
179	IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__
180	IdentifierInfo *Ident__COUNTER__; // __COUNTER__
181	IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma
182	IdentifierInfo *Ident__identifier; // __identifier
183	IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__
184	IdentifierInfo *Ident__VA_OPT__; // __VA_OPT__
185	IdentifierInfo *Ident__has_feature; // __has_feature
186	IdentifierInfo *Ident__has_extension; // __has_extension
187	IdentifierInfo *Ident__has_builtin; // __has_builtin
188	IdentifierInfo *Ident__has_constexpr_builtin; // __has_constexpr_builtin
189	IdentifierInfo *Ident__has_attribute; // __has_attribute
190	IdentifierInfo *Ident__has_embed; // __has_embed
191	IdentifierInfo *Ident__has_include; // __has_include
192	IdentifierInfo *Ident__has_include_next; // __has_include_next
193	IdentifierInfo *Ident__has_warning; // __has_warning
194	IdentifierInfo *Ident__is_identifier; // __is_identifier
195	IdentifierInfo *Ident__building_module; // __building_module
196	IdentifierInfo *Ident__MODULE__; // __MODULE__
197	IdentifierInfo *Ident__has_cpp_attribute; // __has_cpp_attribute
198	IdentifierInfo *Ident__has_c_attribute; // __has_c_attribute
199	IdentifierInfo *Ident__has_declspec; // __has_declspec_attribute
200	IdentifierInfo *Ident__is_target_arch; // __is_target_arch
201	IdentifierInfo *Ident__is_target_vendor; // __is_target_vendor
202	IdentifierInfo *Ident__is_target_os; // __is_target_os
203	IdentifierInfo *Ident__is_target_environment; // __is_target_environment
204	IdentifierInfo *Ident__is_target_variant_os;
205	IdentifierInfo *Ident__is_target_variant_environment;
206	IdentifierInfo *Ident__FLT_EVAL_METHOD__; // __FLT_EVAL_METHOD
207
208	// Weak, only valid (and set) while InMacroArgs is true.
209	Token* ArgMacro;
210
211	SourceLocation DATELoc, TIMELoc;
212
213	// FEM_UnsetOnCommandLine means that an explicit evaluation method was
214	// not specified on the command line. The target is queried to set the
215	// default evaluation method.
216	LangOptions::FPEvalMethodKind CurrentFPEvalMethod =
217	LangOptions::FPEvalMethodKind::FEM_UnsetOnCommandLine;
218
219	// The most recent pragma location where the floating point evaluation
220	// method was modified. This is used to determine whether the
221	// 'pragma clang fp eval_method' was used whithin the current scope.
222	SourceLocation LastFPEvalPragmaLocation;
223
224	LangOptions::FPEvalMethodKind TUFPEvalMethod =
225	LangOptions::FPEvalMethodKind::FEM_UnsetOnCommandLine;
226
227	// Next __COUNTER__ value, starts at 0.
228	unsigned CounterValue = 0;
229
230	enum {
231	/// Maximum depth of \#includes.
232	MaxAllowedIncludeStackDepth = 200
233	};
234
235	// State that is set before the preprocessor begins.
236	bool KeepComments : 1;
237	bool KeepMacroComments : 1;
238	bool SuppressIncludeNotFoundError : 1;
239
240	// State that changes while the preprocessor runs:
241	bool InMacroArgs : 1; // True if parsing fn macro invocation args.
242
243	/// Whether the preprocessor owns the header search object.
244	bool OwnsHeaderSearch : 1;
245
246	/// True if macro expansion is disabled.
247	bool DisableMacroExpansion : 1;
248
249	/// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
250	/// when parsing preprocessor directives.
251	bool MacroExpansionInDirectivesOverride : 1;
252
253	class ResetMacroExpansionHelper;
254
255	/// Whether we have already loaded macros from the external source.
256	mutable bool ReadMacrosFromExternalSource : 1;
257
258	/// True if pragmas are enabled.
259	bool PragmasEnabled : 1;
260
261	/// True if the current build action is a preprocessing action.
262	bool PreprocessedOutput : 1;
263
264	/// True if we are currently preprocessing a #if or #elif directive
265	bool ParsingIfOrElifDirective;
266
267	/// True if we are pre-expanding macro arguments.
268	bool InMacroArgPreExpansion;
269
270	/// Mapping/lookup information for all identifiers in
271	/// the program, including program keywords.
272	mutable IdentifierTable Identifiers;
273
274	/// This table contains all the selectors in the program.
275	///
276	/// Unlike IdentifierTable above, this table *isn't* populated by the
277	/// preprocessor. It is declared/expanded here because its role/lifetime is
278	/// conceptually similar to the IdentifierTable. In addition, the current
279	/// control flow (in clang::ParseAST()), make it convenient to put here.
280	///
281	/// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
282	/// the lifetime of the preprocessor.
283	SelectorTable Selectors;
284
285	/// Information about builtins.
286	std::unique_ptr<Builtin::Context> BuiltinInfo;
287
288	/// Tracks all of the pragmas that the client registered
289	/// with this preprocessor.
290	std::unique_ptr<PragmaNamespace> PragmaHandlers;
291
292	/// Pragma handlers of the original source is stored here during the
293	/// parsing of a model file.
294	std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
295
296	/// Tracks all of the comment handlers that the client registered
297	/// with this preprocessor.
298	std::vector<CommentHandler *> CommentHandlers;
299
300	/// Empty line handler.
301	EmptylineHandler *Emptyline = nullptr;
302
303	/// True to avoid tearing down the lexer etc on EOF
304	bool IncrementalProcessing = false;
305
306	public:
307	/// The kind of translation unit we are processing.
308	const TranslationUnitKind TUKind;
309
310	/// Returns a pointer into the given file's buffer that's guaranteed
311	/// to be between tokens. The returned pointer is always before \p Start.
312	/// The maximum distance betweenthe returned pointer and \p Start is
313	/// limited by a constant value, but also an implementation detail.
314	/// If no such check point exists, \c nullptr is returned.
315	const char *getCheckPoint(FileID FID, const char *Start) const;
316
317	private:
318	/// The code-completion handler.
319	CodeCompletionHandler *CodeComplete = nullptr;
320
321	/// The file that we're performing code-completion for, if any.
322	const FileEntry *CodeCompletionFile = nullptr;
323
324	/// The offset in file for the code-completion point.
325	unsigned CodeCompletionOffset = 0;
326
327	/// The location for the code-completion point. This gets instantiated
328	/// when the CodeCompletionFile gets \#include'ed for preprocessing.
329	SourceLocation CodeCompletionLoc;
330
331	/// The start location for the file of the code-completion point.
332	///
333	/// This gets instantiated when the CodeCompletionFile gets \#include'ed
334	/// for preprocessing.
335	SourceLocation CodeCompletionFileLoc;
336
337	/// The source location of the \c import contextual keyword we just
338	/// lexed, if any.
339	SourceLocation ModuleImportLoc;
340
341	/// The import path for named module that we're currently processing.
342	SmallVector<IdentifierLoc, 2> NamedModuleImportPath;
343
344	llvm::DenseMap<FileID, SmallVector<const char *>> CheckPoints;
345	unsigned CheckPointCounter = 0;
346
347	/// Whether the import is an `@import` or a standard c++ modules import.
348	bool IsAtImport = false;
349
350	/// Whether the last token we lexed was an '@'.
351	bool LastTokenWasAt = false;
352
353	/// A position within a C++20 import-seq.
354	class StdCXXImportSeq {
355	public:
356	enum State : int {
357	// Positive values represent a number of unclosed brackets.
358	AtTopLevel = 0,
359	AfterTopLevelTokenSeq = -1,
360	AfterExport = -2,
361	AfterImportSeq = -3,
362	};
363
364	StdCXXImportSeq(State S) : S(S) {}
365
366	/// Saw any kind of open bracket.
367	void handleOpenBracket() {
368	S = static_cast<State>(std::max<int>(a: S, b: 0) + 1);
369	}
370	/// Saw any kind of close bracket other than '}'.
371	void handleCloseBracket() {
372	S = static_cast<State>(std::max<int>(a: S, b: 1) - 1);
373	}
374	/// Saw a close brace.
375	void handleCloseBrace() {
376	handleCloseBracket();
377	if (S == AtTopLevel && !AfterHeaderName)
378	S = AfterTopLevelTokenSeq;
379	}
380	/// Saw a semicolon.
381	void handleSemi() {
382	if (atTopLevel()) {
383	S = AfterTopLevelTokenSeq;
384	AfterHeaderName = false;
385	}
386	}
387
388	/// Saw an 'export' identifier.
389	void handleExport() {
390	if (S == AfterTopLevelTokenSeq)
391	S = AfterExport;
392	else if (S <= 0)
393	S = AtTopLevel;
394	}
395	/// Saw an 'import' identifier.
396	void handleImport() {
397	if (S == AfterTopLevelTokenSeq || S == AfterExport)
398	S = AfterImportSeq;
399	else if (S <= 0)
400	S = AtTopLevel;
401	}
402
403	/// Saw a 'header-name' token; do not recognize any more 'import' tokens
404	/// until we reach a top-level semicolon.
405	void handleHeaderName() {
406	if (S == AfterImportSeq)
407	AfterHeaderName = true;
408	handleMisc();
409	}
410
411	/// Saw any other token.
412	void handleMisc() {
413	if (S <= 0)
414	S = AtTopLevel;
415	}
416
417	bool atTopLevel() { return S <= 0; }
418	bool afterImportSeq() { return S == AfterImportSeq; }
419	bool afterTopLevelSeq() { return S == AfterTopLevelTokenSeq; }
420
421	private:
422	State S;
423	/// Whether we're in the pp-import-suffix following the header-name in a
424	/// pp-import. If so, a close-brace is not sufficient to end the
425	/// top-level-token-seq of an import-seq.
426	bool AfterHeaderName = false;
427	};
428
429	/// Our current position within a C++20 import-seq.
430	StdCXXImportSeq StdCXXImportSeqState = StdCXXImportSeq::AfterTopLevelTokenSeq;
431
432	/// Track whether we are in a Global Module Fragment
433	class TrackGMF {
434	public:
435	enum GMFState : int {
436	GMFActive = 1,
437	MaybeGMF = 0,
438	BeforeGMFIntroducer = -1,
439	GMFAbsentOrEnded = -2,
440	};
441
442	TrackGMF(GMFState S) : S(S) {}
443
444	/// Saw a semicolon.
445	void handleSemi() {
446	// If it is immediately after the first instance of the module keyword,
447	// then that introduces the GMF.
448	if (S == MaybeGMF)
449	S = GMFActive;
450	}
451
452	/// Saw an 'export' identifier.
453	void handleExport() {
454	// The presence of an 'export' keyword always ends or excludes a GMF.
455	S = GMFAbsentOrEnded;
456	}
457
458	/// Saw an 'import' identifier.
459	void handleImport(bool AfterTopLevelTokenSeq) {
460	// If we see this before any 'module' kw, then we have no GMF.
461	if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
462	S = GMFAbsentOrEnded;
463	}
464
465	/// Saw a 'module' identifier.
466	void handleModule(bool AfterTopLevelTokenSeq) {
467	// This was the first module identifier and not preceded by any token
468	// that would exclude a GMF. It could begin a GMF, but only if directly
469	// followed by a semicolon.
470	if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
471	S = MaybeGMF;
472	else
473	S = GMFAbsentOrEnded;
474	}
475
476	/// Saw any other token.
477	void handleMisc() {
478	// We saw something other than ; after the 'module' kw, so not a GMF.
479	if (S == MaybeGMF)
480	S = GMFAbsentOrEnded;
481	}
482
483	bool inGMF() { return S == GMFActive; }
484
485	private:
486	/// Track the transitions into and out of a Global Module Fragment,
487	/// if one is present.
488	GMFState S;
489	};
490
491	TrackGMF TrackGMFState = TrackGMF::BeforeGMFIntroducer;
492
493	/// Track the status of the c++20 module decl.
494	///
495	/// module-declaration:
496	/// 'export'[opt] 'module' module-name module-partition[opt]
497	/// attribute-specifier-seq[opt] ';'
498	///
499	/// module-name:
500	/// module-name-qualifier[opt] identifier
501	///
502	/// module-partition:
503	/// ':' module-name-qualifier[opt] identifier
504	///
505	/// module-name-qualifier:
506	/// identifier '.'
507	/// module-name-qualifier identifier '.'
508	///
509	/// Transition state:
510	///
511	/// NotAModuleDecl --- export ---> FoundExport
512	/// NotAModuleDecl --- module ---> ImplementationCandidate
513	/// FoundExport --- module ---> InterfaceCandidate
514	/// ImplementationCandidate --- Identifier ---> ImplementationCandidate
515	/// ImplementationCandidate --- period ---> ImplementationCandidate
516	/// ImplementationCandidate --- colon ---> ImplementationCandidate
517	/// InterfaceCandidate --- Identifier ---> InterfaceCandidate
518	/// InterfaceCandidate --- period ---> InterfaceCandidate
519	/// InterfaceCandidate --- colon ---> InterfaceCandidate
520	/// ImplementationCandidate --- Semi ---> NamedModuleImplementation
521	/// NamedModuleInterface --- Semi ---> NamedModuleInterface
522	/// NamedModuleImplementation --- Anything ---> NamedModuleImplementation
523	/// NamedModuleInterface --- Anything ---> NamedModuleInterface
524	///
525	/// FIXME: We haven't handle attribute-specifier-seq here. It may not be bad
526	/// soon since we don't support any module attributes yet.
527	class ModuleDeclSeq {
528	enum ModuleDeclState : int {
529	NotAModuleDecl,
530	FoundExport,
531	InterfaceCandidate,
532	ImplementationCandidate,
533	NamedModuleInterface,
534	NamedModuleImplementation,
535	};
536
537	public:
538	ModuleDeclSeq() = default;
539
540	void handleExport() {
541	if (State == NotAModuleDecl)
542	State = FoundExport;
543	else if (!isNamedModule())
544	reset();
545	}
546
547	void handleModule() {
548	if (State == FoundExport)
549	State = InterfaceCandidate;
550	else if (State == NotAModuleDecl)
551	State = ImplementationCandidate;
552	else if (!isNamedModule())
553	reset();
554	}
555
556	void handleIdentifier(IdentifierInfo *Identifier) {
557	if (isModuleCandidate() && Identifier)
558	Name += Identifier->getName().str();
559	else if (!isNamedModule())
560	reset();
561	}
562
563	void handleColon() {
564	if (isModuleCandidate())
565	Name += ":";
566	else if (!isNamedModule())
567	reset();
568	}
569
570	void handlePeriod() {
571	if (isModuleCandidate())
572	Name += ".";
573	else if (!isNamedModule())
574	reset();
575	}
576
577	void handleSemi() {
578	if (!Name.empty() && isModuleCandidate()) {
579	if (State == InterfaceCandidate)
580	State = NamedModuleInterface;
581	else if (State == ImplementationCandidate)
582	State = NamedModuleImplementation;
583	else
584	llvm_unreachable("Unimaged ModuleDeclState.");
585	} else if (!isNamedModule())
586	reset();
587	}
588
589	void handleMisc() {
590	if (!isNamedModule())
591	reset();
592	}
593
594	bool isModuleCandidate() const {
595	return State == InterfaceCandidate || State == ImplementationCandidate;
596	}
597
598	bool isNamedModule() const {
599	return State == NamedModuleInterface ||
600	State == NamedModuleImplementation;
601	}
602
603	bool isNamedInterface() const { return State == NamedModuleInterface; }
604
605	bool isImplementationUnit() const {
606	return State == NamedModuleImplementation && !getName().contains(C: ':');
607	}
608
609	StringRef getName() const {
610	assert(isNamedModule() && "Can't get name from a non named module");
611	return Name;
612	}
613
614	StringRef getPrimaryName() const {
615	assert(isNamedModule() && "Can't get name from a non named module");
616	return getName().split(Separator: ':').first;
617	}
618
619	void reset() {
620	Name.clear();
621	State = NotAModuleDecl;
622	}
623
624	private:
625	ModuleDeclState State = NotAModuleDecl;
626	std::string Name;
627	};
628
629	ModuleDeclSeq ModuleDeclState;
630
631	/// Whether the module import expects an identifier next. Otherwise,
632	/// it expects a '.' or ';'.
633	bool ModuleImportExpectsIdentifier = false;
634
635	/// The identifier and source location of the currently-active
636	/// \#pragma clang arc_cf_code_audited begin.
637	IdentifierLoc PragmaARCCFCodeAuditedInfo;
638
639	/// The source location of the currently-active
640	/// \#pragma clang assume_nonnull begin.
641	SourceLocation PragmaAssumeNonNullLoc;
642
643	/// Set only for preambles which end with an active
644	/// \#pragma clang assume_nonnull begin.
645	///
646	/// When the preamble is loaded into the main file,
647	/// `PragmaAssumeNonNullLoc` will be set to this to
648	/// replay the unterminated assume_nonnull.
649	SourceLocation PreambleRecordedPragmaAssumeNonNullLoc;
650
651	/// True if we hit the code-completion point.
652	bool CodeCompletionReached = false;
653
654	/// The code completion token containing the information
655	/// on the stem that is to be code completed.
656	IdentifierInfo *CodeCompletionII = nullptr;
657
658	/// Range for the code completion token.
659	SourceRange CodeCompletionTokenRange;
660
661	/// The directory that the main file should be considered to occupy,
662	/// if it does not correspond to a real file (as happens when building a
663	/// module).
664	OptionalDirectoryEntryRef MainFileDir;
665
666	/// The number of bytes that we will initially skip when entering the
667	/// main file, along with a flag that indicates whether skipping this number
668	/// of bytes will place the lexer at the start of a line.
669	///
670	/// This is used when loading a precompiled preamble.
671	std::pair<int, bool> SkipMainFilePreamble;
672
673	/// Whether we hit an error due to reaching max allowed include depth. Allows
674	/// to avoid hitting the same error over and over again.
675	bool HasReachedMaxIncludeDepth = false;
676
677	/// The number of currently-active calls to Lex.
678	///
679	/// Lex is reentrant, and asking for an (end-of-phase-4) token can often
680	/// require asking for multiple additional tokens. This counter makes it
681	/// possible for Lex to detect whether it's producing a token for the end
682	/// of phase 4 of translation or for some other situation.
683	unsigned LexLevel = 0;
684
685	/// The number of (LexLevel 0) preprocessor tokens.
686	unsigned TokenCount = 0;
687
688	/// Preprocess every token regardless of LexLevel.
689	bool PreprocessToken = false;
690
691	/// The maximum number of (LexLevel 0) tokens before issuing a -Wmax-tokens
692	/// warning, or zero for unlimited.
693	unsigned MaxTokens = 0;
694	SourceLocation MaxTokensOverrideLoc;
695
696	public:
697	struct PreambleSkipInfo {
698	SourceLocation HashTokenLoc;
699	SourceLocation IfTokenLoc;
700	bool FoundNonSkipPortion;
701	bool FoundElse;
702	SourceLocation ElseLoc;
703
704	PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc,
705	bool FoundNonSkipPortion, bool FoundElse,
706	SourceLocation ElseLoc)
707	: HashTokenLoc(HashTokenLoc), IfTokenLoc(IfTokenLoc),
708	FoundNonSkipPortion(FoundNonSkipPortion), FoundElse(FoundElse),
709	ElseLoc(ElseLoc) {}
710	};
711
712	using IncludedFilesSet = llvm::DenseSet<const FileEntry *>;
713
714	private:
715	friend class ASTReader;
716	friend class MacroArgs;
717
718	class PreambleConditionalStackStore {
719	enum State {
720	Off = 0,
721	Recording = 1,
722	Replaying = 2,
723	};
724
725	public:
726	PreambleConditionalStackStore() = default;
727
728	void startRecording() { ConditionalStackState = Recording; }
729	void startReplaying() { ConditionalStackState = Replaying; }
730	bool isRecording() const { return ConditionalStackState == Recording; }
731	bool isReplaying() const { return ConditionalStackState == Replaying; }
732
733	ArrayRef<PPConditionalInfo> getStack() const {
734	return ConditionalStack;
735	}
736
737	void doneReplaying() {
738	ConditionalStack.clear();
739	ConditionalStackState = Off;
740	}
741
742	void setStack(ArrayRef<PPConditionalInfo> s) {
743	if (!isRecording() && !isReplaying())
744	return;
745	ConditionalStack.clear();
746	ConditionalStack.append(in_start: s.begin(), in_end: s.end());
747	}
748
749	bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
750
751	bool reachedEOFWhileSkipping() const { return SkipInfo.has_value(); }
752
753	void clearSkipInfo() { SkipInfo.reset(); }
754
755	std::optional<PreambleSkipInfo> SkipInfo;
756
757	private:
758	SmallVector<PPConditionalInfo, 4> ConditionalStack;
759	State ConditionalStackState = Off;
760	} PreambleConditionalStack;
761
762	/// The current top of the stack that we're lexing from if
763	/// not expanding a macro and we are lexing directly from source code.
764	///
765	/// Only one of CurLexer, or CurTokenLexer will be non-null.
766	std::unique_ptr<Lexer> CurLexer;
767
768	/// The current top of the stack that we're lexing from
769	/// if not expanding a macro.
770	///
771	/// This is an alias for CurLexer.
772	PreprocessorLexer *CurPPLexer = nullptr;
773
774	/// Used to find the current FileEntry, if CurLexer is non-null
775	/// and if applicable.
776	///
777	/// This allows us to implement \#include_next and find directory-specific
778	/// properties.
779	ConstSearchDirIterator CurDirLookup = nullptr;
780
781	/// The current macro we are expanding, if we are expanding a macro.
782	///
783	/// One of CurLexer and CurTokenLexer must be null.
784	std::unique_ptr<TokenLexer> CurTokenLexer;
785
786	/// The kind of lexer we're currently working with.
787	typedef bool (*LexerCallback)(Preprocessor &, Token &);
788	LexerCallback CurLexerCallback = &CLK_Lexer;
789
790	/// If the current lexer is for a submodule that is being built, this
791	/// is that submodule.
792	Module *CurLexerSubmodule = nullptr;
793
794	/// Keeps track of the stack of files currently
795	/// \#included, and macros currently being expanded from, not counting
796	/// CurLexer/CurTokenLexer.
797	struct IncludeStackInfo {
798	LexerCallback CurLexerCallback;
799	Module *TheSubmodule;
800	std::unique_ptr<Lexer> TheLexer;
801	PreprocessorLexer *ThePPLexer;
802	std::unique_ptr<TokenLexer> TheTokenLexer;
803	ConstSearchDirIterator TheDirLookup;
804
805	// The following constructors are completely useless copies of the default
806	// versions, only needed to pacify MSVC.
807	IncludeStackInfo(LexerCallback CurLexerCallback, Module *TheSubmodule,
808	std::unique_ptr<Lexer> &&TheLexer,
809	PreprocessorLexer *ThePPLexer,
810	std::unique_ptr<TokenLexer> &&TheTokenLexer,
811	ConstSearchDirIterator TheDirLookup)
812	: CurLexerCallback(std::move(CurLexerCallback)),
813	TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
814	ThePPLexer(std::move(ThePPLexer)),
815	TheTokenLexer(std::move(TheTokenLexer)),
816	TheDirLookup(std::move(TheDirLookup)) {}
817	};
818	std::vector<IncludeStackInfo> IncludeMacroStack;
819
820	/// Actions invoked when some preprocessor activity is
821	/// encountered (e.g. a file is \#included, etc).
822	std::unique_ptr<PPCallbacks> Callbacks;
823
824	struct MacroExpandsInfo {
825	Token Tok;
826	MacroDefinition MD;
827	SourceRange Range;
828
829	MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
830	: Tok(Tok), MD(MD), Range(Range) {}
831	};
832	SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
833
834	/// Information about a name that has been used to define a module macro.
835	struct ModuleMacroInfo {
836	/// The most recent macro directive for this identifier.
837	MacroDirective *MD;
838
839	/// The active module macros for this identifier.
840	llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
841
842	/// The generation number at which we last updated ActiveModuleMacros.
843	/// \see Preprocessor::VisibleModules.
844	unsigned ActiveModuleMacrosGeneration = 0;
845
846	/// Whether this macro name is ambiguous.
847	bool IsAmbiguous = false;
848
849	/// The module macros that are overridden by this macro.
850	llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
851
852	ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
853	};
854
855	/// The state of a macro for an identifier.
856	class MacroState {
857	mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
858
859	ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
860	const IdentifierInfo *II) const {
861	if (II->isOutOfDate())
862	PP.updateOutOfDateIdentifier(II: *II);
863	// FIXME: Find a spare bit on IdentifierInfo and store a
864	// HasModuleMacros flag.
865	if (!II->hasMacroDefinition() ||
866	(!PP.getLangOpts().Modules &&
867	!PP.getLangOpts().ModulesLocalVisibility) ||
868	!PP.CurSubmoduleState->VisibleModules.getGeneration())
869	return nullptr;
870
871	auto *Info = dyn_cast_if_present<ModuleMacroInfo *>(Val&: State);
872	if (!Info) {
873	Info = new (PP.getPreprocessorAllocator())
874	ModuleMacroInfo(cast<MacroDirective *>(Val&: State));
875	State = Info;
876	}
877
878	if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
879	Info->ActiveModuleMacrosGeneration)
880	PP.updateModuleMacroInfo(II, Info&: *Info);
881	return Info;
882	}
883
884	public:
885	MacroState() : MacroState(nullptr) {}
886	MacroState(MacroDirective *MD) : State(MD) {}
887
888	MacroState(MacroState &&O) noexcept : State(O.State) {
889	O.State = (MacroDirective *)nullptr;
890	}
891
892	MacroState &operator=(MacroState &&O) noexcept {
893	auto S = O.State;
894	O.State = (MacroDirective *)nullptr;
895	State = S;
896	return *this;
897	}
898
899	~MacroState() {
900	if (auto *Info = dyn_cast_if_present<ModuleMacroInfo *>(Val&: State))
901	Info->~ModuleMacroInfo();
902	}
903
904	MacroDirective *getLatest() const {
905	if (auto *Info = dyn_cast_if_present<ModuleMacroInfo *>(Val&: State))
906	return Info->MD;
907	return cast<MacroDirective *>(Val&: State);
908	}
909
910	void setLatest(MacroDirective *MD) {
911	if (auto *Info = dyn_cast_if_present<ModuleMacroInfo *>(Val&: State))
912	Info->MD = MD;
913	else
914	State = MD;
915	}
916
917	bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
918	auto *Info = getModuleInfo(PP, II);
919	return Info ? Info->IsAmbiguous : false;
920	}
921
922	ArrayRef<ModuleMacro *>
923	getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
924	if (auto *Info = getModuleInfo(PP, II))
925	return Info->ActiveModuleMacros;
926	return {};
927	}
928
929	MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
930	SourceManager &SourceMgr) const {
931	// FIXME: Incorporate module macros into the result of this.
932	if (auto *Latest = getLatest())
933	return Latest->findDirectiveAtLoc(L: Loc, SM: SourceMgr);
934	return {};
935	}
936
937	void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
938	if (auto *Info = getModuleInfo(PP, II)) {
939	Info->OverriddenMacros.insert(I: Info->OverriddenMacros.end(),
940	From: Info->ActiveModuleMacros.begin(),
941	To: Info->ActiveModuleMacros.end());
942	Info->ActiveModuleMacros.clear();
943	Info->IsAmbiguous = false;
944	}
945	}
946
947	ArrayRef<ModuleMacro*> getOverriddenMacros() const {
948	if (auto *Info = dyn_cast_if_present<ModuleMacroInfo *>(Val&: State))
949	return Info->OverriddenMacros;
950	return {};
951	}
952
953	void setOverriddenMacros(Preprocessor &PP,
954	ArrayRef<ModuleMacro *> Overrides) {
955	auto *Info = dyn_cast_if_present<ModuleMacroInfo *>(Val&: State);
956	if (!Info) {
957	if (Overrides.empty())
958	return;
959	Info = new (PP.getPreprocessorAllocator())
960	ModuleMacroInfo(cast<MacroDirective *>(Val&: State));
961	State = Info;
962	}
963	Info->OverriddenMacros.clear();
964	Info->OverriddenMacros.insert(I: Info->OverriddenMacros.end(),
965	From: Overrides.begin(), To: Overrides.end());
966	Info->ActiveModuleMacrosGeneration = 0;
967	}
968	};
969
970	/// For each IdentifierInfo that was associated with a macro, we
971	/// keep a mapping to the history of all macro definitions and #undefs in
972	/// the reverse order (the latest one is in the head of the list).
973	///
974	/// This mapping lives within the \p CurSubmoduleState.
975	using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
976
977	struct SubmoduleState;
978
979	/// Information about a submodule that we're currently building.
980	struct BuildingSubmoduleInfo {
981	/// The module that we are building.
982	Module *M;
983
984	/// The location at which the module was included.
985	SourceLocation ImportLoc;
986
987	/// Whether we entered this submodule via a pragma.
988	bool IsPragma;
989
990	/// The previous SubmoduleState.
991	SubmoduleState *OuterSubmoduleState;
992
993	/// The number of pending module macro names when we started building this.
994	unsigned OuterPendingModuleMacroNames;
995
996	BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
997	SubmoduleState *OuterSubmoduleState,
998	unsigned OuterPendingModuleMacroNames)
999	: M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
1000	OuterSubmoduleState(OuterSubmoduleState),
1001	OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
1002	};
1003	SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
1004
1005	/// Information about a submodule's preprocessor state.
1006	struct SubmoduleState {
1007	/// The macros for the submodule.
1008	MacroMap Macros;
1009
1010	/// The set of modules that are visible within the submodule.
1011	VisibleModuleSet VisibleModules;
1012
1013	// FIXME: CounterValue?
1014	// FIXME: PragmaPushMacroInfo?
1015	};
1016	std::map<Module *, SubmoduleState> Submodules;
1017
1018	/// The preprocessor state for preprocessing outside of any submodule.
1019	SubmoduleState NullSubmoduleState;
1020
1021	/// The current submodule state. Will be \p NullSubmoduleState if we're not
1022	/// in a submodule.
1023	SubmoduleState *CurSubmoduleState;
1024
1025	/// The files that have been included.
1026	IncludedFilesSet IncludedFiles;
1027
1028	/// The set of top-level modules that affected preprocessing, but were not
1029	/// imported.
1030	llvm::SmallSetVector<Module *, 2> AffectingClangModules;
1031
1032	/// The set of known macros exported from modules.
1033	llvm::FoldingSet<ModuleMacro> ModuleMacros;
1034
1035	/// The names of potential module macros that we've not yet processed.
1036	llvm::SmallVector<IdentifierInfo *, 32> PendingModuleMacroNames;
1037
1038	/// The list of module macros, for each identifier, that are not overridden by
1039	/// any other module macro.
1040	llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
1041	LeafModuleMacros;
1042
1043	/// Macros that we want to warn because they are not used at the end
1044	/// of the translation unit.
1045	///
1046	/// We store just their SourceLocations instead of
1047	/// something like MacroInfo*. The benefit of this is that when we are
1048	/// deserializing from PCH, we don't need to deserialize identifier & macros
1049	/// just so that we can report that they are unused, we just warn using
1050	/// the SourceLocations of this set (that will be filled by the ASTReader).
1051	using WarnUnusedMacroLocsTy = llvm::SmallDenseSet<SourceLocation, 32>;
1052	WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
1053
1054	/// This is a pair of an optional message and source location used for pragmas
1055	/// that annotate macros like pragma clang restrict_expansion and pragma clang
1056	/// deprecated. This pair stores the optional message and the location of the
1057	/// annotation pragma for use producing diagnostics and notes.
1058	using MsgLocationPair = std::pair<std::string, SourceLocation>;
1059
1060	struct MacroAnnotationInfo {
1061	SourceLocation Location;
1062	std::string Message;
1063	};
1064
1065	struct MacroAnnotations {
1066	std::optional<MacroAnnotationInfo> DeprecationInfo;
1067	std::optional<MacroAnnotationInfo> RestrictExpansionInfo;
1068	std::optional<SourceLocation> FinalAnnotationLoc;
1069	};
1070
1071	/// Warning information for macro annotations.
1072	llvm::DenseMap<const IdentifierInfo *, MacroAnnotations> AnnotationInfos;
1073
1074	/// A "freelist" of MacroArg objects that can be
1075	/// reused for quick allocation.
1076	MacroArgs *MacroArgCache = nullptr;
1077
1078	/// For each IdentifierInfo used in a \#pragma push_macro directive,
1079	/// we keep a MacroInfo stack used to restore the previous macro value.
1080	llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
1081	PragmaPushMacroInfo;
1082
1083	// Various statistics we track for performance analysis.
1084	unsigned NumDirectives = 0;
1085	unsigned NumDefined = 0;
1086	unsigned NumUndefined = 0;
1087	unsigned NumPragma = 0;
1088	unsigned NumIf = 0;
1089	unsigned NumElse = 0;
1090	unsigned NumEndif = 0;
1091	unsigned NumEnteredSourceFiles = 0;
1092	unsigned MaxIncludeStackDepth = 0;
1093	unsigned NumMacroExpanded = 0;
1094	unsigned NumFnMacroExpanded = 0;
1095	unsigned NumBuiltinMacroExpanded = 0;
1096	unsigned NumFastMacroExpanded = 0;
1097	unsigned NumTokenPaste = 0;
1098	unsigned NumFastTokenPaste = 0;
1099	unsigned NumSkipped = 0;
1100
1101	/// The predefined macros that preprocessor should use from the
1102	/// command line etc.
1103	std::string Predefines;
1104
1105	/// The file ID for the preprocessor predefines.
1106	FileID PredefinesFileID;
1107
1108	/// The file ID for the PCH through header.
1109	FileID PCHThroughHeaderFileID;
1110
1111	/// Whether tokens are being skipped until a #pragma hdrstop is seen.
1112	bool SkippingUntilPragmaHdrStop = false;
1113
1114	/// Whether tokens are being skipped until the through header is seen.
1115	bool SkippingUntilPCHThroughHeader = false;
1116
1117	/// \{
1118	/// Cache of macro expanders to reduce malloc traffic.
1119	enum { TokenLexerCacheSize = 8 };
1120	unsigned NumCachedTokenLexers;
1121	std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
1122	/// \}
1123
1124	/// Keeps macro expanded tokens for TokenLexers.
1125	//
1126	/// Works like a stack; a TokenLexer adds the macro expanded tokens that is
1127	/// going to lex in the cache and when it finishes the tokens are removed
1128	/// from the end of the cache.
1129	SmallVector<Token, 16> MacroExpandedTokens;
1130	std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
1131
1132	/// A record of the macro definitions and expansions that
1133	/// occurred during preprocessing.
1134	///
1135	/// This is an optional side structure that can be enabled with
1136	/// \c createPreprocessingRecord() prior to preprocessing.
1137	PreprocessingRecord *Record = nullptr;
1138
1139	/// Cached tokens state.
1140	using CachedTokensTy = SmallVector<Token, 1>;
1141
1142	/// Cached tokens are stored here when we do backtracking or
1143	/// lookahead. They are "lexed" by the CachingLex() method.
1144	CachedTokensTy CachedTokens;
1145
1146	/// The position of the cached token that CachingLex() should
1147	/// "lex" next.
1148	///
1149	/// If it points beyond the CachedTokens vector, it means that a normal
1150	/// Lex() should be invoked.
1151	CachedTokensTy::size_type CachedLexPos = 0;
1152
1153	/// Stack of backtrack positions, allowing nested backtracks.
1154	///
1155	/// The EnableBacktrackAtThisPos() method pushes a position to
1156	/// indicate where CachedLexPos should be set when the BackTrack() method is
1157	/// invoked (at which point the last position is popped).
1158	std::vector<CachedTokensTy::size_type> BacktrackPositions;
1159
1160	/// Stack of cached tokens/initial number of cached tokens pairs, allowing
1161	/// nested unannotated backtracks.
1162	std::vector<std::pair<CachedTokensTy, CachedTokensTy::size_type>>
1163	UnannotatedBacktrackTokens;
1164
1165	/// True if \p Preprocessor::SkipExcludedConditionalBlock() is running.
1166	/// This is used to guard against calling this function recursively.
1167	///
1168	/// See comments at the use-site for more context about why it is needed.
1169	bool SkippingExcludedConditionalBlock = false;
1170
1171	/// Keeps track of skipped range mappings that were recorded while skipping
1172	/// excluded conditional directives. It maps the source buffer pointer at
1173	/// the beginning of a skipped block, to the number of bytes that should be
1174	/// skipped.
1175	llvm::DenseMap<const char *, unsigned> RecordedSkippedRanges;
1176
1177	void updateOutOfDateIdentifier(const IdentifierInfo &II) const;
1178
1179	public:
1180	Preprocessor(const PreprocessorOptions &PPOpts, DiagnosticsEngine &diags,
1181	const LangOptions &LangOpts, SourceManager &SM,
1182	HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
1183	IdentifierInfoLookup *IILookup = nullptr,
1184	bool OwnsHeaderSearch = false,
1185	TranslationUnitKind TUKind = TU_Complete);
1186
1187	~Preprocessor();
1188
1189	/// Initialize the preprocessor using information about the target.
1190	///
1191	/// \param Target is owned by the caller and must remain valid for the
1192	/// lifetime of the preprocessor.
1193	/// \param AuxTarget is owned by the caller and must remain valid for
1194	/// the lifetime of the preprocessor.
1195	void Initialize(const TargetInfo &Target,
1196	const TargetInfo *AuxTarget = nullptr);
1197
1198	/// Initialize the preprocessor to parse a model file
1199	///
1200	/// To parse model files the preprocessor of the original source is reused to
1201	/// preserver the identifier table. However to avoid some duplicate
1202	/// information in the preprocessor some cleanup is needed before it is used
1203	/// to parse model files. This method does that cleanup.
1204	void InitializeForModelFile();
1205
1206	/// Cleanup after model file parsing
1207	void FinalizeForModelFile();
1208
1209	/// Retrieve the preprocessor options used to initialize this preprocessor.
1210	const PreprocessorOptions &getPreprocessorOpts() const { return PPOpts; }
1211
1212	DiagnosticsEngine &getDiagnostics() const { return *Diags; }
1213	void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
1214
1215	const LangOptions &getLangOpts() const { return LangOpts; }
1216	const TargetInfo &getTargetInfo() const { return *Target; }
1217	const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
1218	FileManager &getFileManager() const { return FileMgr; }
1219	SourceManager &getSourceManager() const { return SourceMgr; }
1220	HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
1221
1222	IdentifierTable &getIdentifierTable() { return Identifiers; }
1223	const IdentifierTable &getIdentifierTable() const { return Identifiers; }
1224	SelectorTable &getSelectorTable() { return Selectors; }
1225	Builtin::Context &getBuiltinInfo() { return *BuiltinInfo; }
1226	llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
1227
1228	void setExternalSource(ExternalPreprocessorSource *Source) {
1229	ExternalSource = Source;
1230	}
1231
1232	ExternalPreprocessorSource *getExternalSource() const {
1233	return ExternalSource;
1234	}
1235
1236	/// Retrieve the module loader associated with this preprocessor.
1237	ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
1238
1239	bool hadModuleLoaderFatalFailure() const {
1240	return TheModuleLoader.HadFatalFailure;
1241	}
1242
1243	/// Retrieve the number of Directives that have been processed by the
1244	/// Preprocessor.
1245	unsigned getNumDirectives() const {
1246	return NumDirectives;
1247	}
1248
1249	/// True if we are currently preprocessing a #if or #elif directive
1250	bool isParsingIfOrElifDirective() const {
1251	return ParsingIfOrElifDirective;
1252	}
1253
1254	/// Control whether the preprocessor retains comments in output.
1255	void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
1256	this->KeepComments = KeepComments | KeepMacroComments;
1257	this->KeepMacroComments = KeepMacroComments;
1258	}
1259
1260	bool getCommentRetentionState() const { return KeepComments; }
1261
1262	void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
1263	bool getPragmasEnabled() const { return PragmasEnabled; }
1264
1265	void SetSuppressIncludeNotFoundError(bool Suppress) {
1266	SuppressIncludeNotFoundError = Suppress;
1267	}
1268
1269	bool GetSuppressIncludeNotFoundError() {
1270	return SuppressIncludeNotFoundError;
1271	}
1272
1273	/// Sets whether the preprocessor is responsible for producing output or if
1274	/// it is producing tokens to be consumed by Parse and Sema.
1275	void setPreprocessedOutput(bool IsPreprocessedOutput) {
1276	PreprocessedOutput = IsPreprocessedOutput;
1277	}
1278
1279	/// Returns true if the preprocessor is responsible for generating output,
1280	/// false if it is producing tokens to be consumed by Parse and Sema.
1281	bool isPreprocessedOutput() const { return PreprocessedOutput; }
1282
1283	/// Return true if we are lexing directly from the specified lexer.
1284	bool isCurrentLexer(const PreprocessorLexer *L) const {
1285	return CurPPLexer == L;
1286	}
1287
1288	/// Return the current lexer being lexed from.
1289	///
1290	/// Note that this ignores any potentially active macro expansions and _Pragma
1291	/// expansions going on at the time.
1292	PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
1293
1294	/// Return the current file lexer being lexed from.
1295	///
1296	/// Note that this ignores any potentially active macro expansions and _Pragma
1297	/// expansions going on at the time.
1298	PreprocessorLexer *getCurrentFileLexer() const;
1299
1300	/// Return the submodule owning the file being lexed. This may not be
1301	/// the current module if we have changed modules since entering the file.
1302	Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; }
1303
1304	/// Returns the FileID for the preprocessor predefines.
1305	FileID getPredefinesFileID() const { return PredefinesFileID; }
1306
1307	/// \{
1308	/// Accessors for preprocessor callbacks.
1309	///
1310	/// Note that this class takes ownership of any PPCallbacks object given to
1311	/// it.
1312	PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
1313	void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
1314	if (Callbacks)
1315	C = std::make_unique<PPChainedCallbacks>(args: std::move(C),
1316	args: std::move(Callbacks));
1317	Callbacks = std::move(C);
1318	}
1319	/// \}
1320
1321	/// Get the number of tokens processed so far.
1322	unsigned getTokenCount() const { return TokenCount; }
1323
1324	/// Get the max number of tokens before issuing a -Wmax-tokens warning.
1325	unsigned getMaxTokens() const { return MaxTokens; }
1326
1327	void overrideMaxTokens(unsigned Value, SourceLocation Loc) {
1328	MaxTokens = Value;
1329	MaxTokensOverrideLoc = Loc;
1330	};
1331
1332	SourceLocation getMaxTokensOverrideLoc() const { return MaxTokensOverrideLoc; }
1333
1334	/// Register a function that would be called on each token in the final
1335	/// expanded token stream.
1336	/// This also reports annotation tokens produced by the parser.
1337	void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
1338	OnToken = std::move(F);
1339	}
1340
1341	void setDependencyDirectivesGetter(DependencyDirectivesGetter &Get) {
1342	GetDependencyDirectives = &Get;
1343	}
1344
1345	void setPreprocessToken(bool Preprocess) { PreprocessToken = Preprocess; }
1346
1347	bool isMacroDefined(StringRef Id) {
1348	return isMacroDefined(&Identifiers.get(Id));
1349	}
1350	bool isMacroDefined(const IdentifierInfo *II) {
1351	return II->hasMacroDefinition() &&
1352	(!getLangOpts().Modules || (bool)getMacroDefinition(II));
1353	}
1354
1355	/// Determine whether II is defined as a macro within the module M,
1356	/// if that is a module that we've already preprocessed. Does not check for
1357	/// macros imported into M.
1358	bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M) {
1359	if (!II->hasMacroDefinition())
1360	return false;
1361	auto I = Submodules.find(x: M);
1362	if (I == Submodules.end())
1363	return false;
1364	auto J = I->second.Macros.find(Val: II);
1365	if (J == I->second.Macros.end())
1366	return false;
1367	auto *MD = J->second.getLatest();
1368	return MD && MD->isDefined();
1369	}
1370
1371	MacroDefinition getMacroDefinition(const IdentifierInfo *II) {
1372	if (!II->hasMacroDefinition())
1373	return {};
1374
1375	MacroState &S = CurSubmoduleState->Macros[II];
1376	auto *MD = S.getLatest();
1377	while (isa_and_nonnull<VisibilityMacroDirective>(Val: MD))
1378	MD = MD->getPrevious();
1379	return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(Val: MD),
1380	S.getActiveModuleMacros(PP&: *this, II),
1381	S.isAmbiguous(PP&: *this, II));
1382	}
1383
1384	MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II,
1385	SourceLocation Loc) {
1386	if (!II->hadMacroDefinition())
1387	return {};
1388
1389	MacroState &S = CurSubmoduleState->Macros[II];
1390	MacroDirective::DefInfo DI;
1391	if (auto *MD = S.getLatest())
1392	DI = MD->findDirectiveAtLoc(L: Loc, SM: getSourceManager());
1393	// FIXME: Compute the set of active module macros at the specified location.
1394	return MacroDefinition(DI.getDirective(),
1395	S.getActiveModuleMacros(PP&: *this, II),
1396	S.isAmbiguous(PP&: *this, II));
1397	}
1398
1399	/// Given an identifier, return its latest non-imported MacroDirective
1400	/// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
1401	MacroDirective *getLocalMacroDirective(const IdentifierInfo *II) const {
1402	if (!II->hasMacroDefinition())
1403	return nullptr;
1404
1405	auto *MD = getLocalMacroDirectiveHistory(II);
1406	if (!MD || MD->getDefinition().isUndefined())
1407	return nullptr;
1408
1409	return MD;
1410	}
1411
1412	const MacroInfo *getMacroInfo(const IdentifierInfo *II) const {
1413	return const_cast<Preprocessor*>(this)->getMacroInfo(II);
1414	}
1415
1416	MacroInfo *getMacroInfo(const IdentifierInfo *II) {
1417	if (!II->hasMacroDefinition())
1418	return nullptr;
1419	if (auto MD = getMacroDefinition(II))
1420	return MD.getMacroInfo();
1421	return nullptr;
1422	}
1423
1424	/// Given an identifier, return the latest non-imported macro
1425	/// directive for that identifier.
1426	///
1427	/// One can iterate over all previous macro directives from the most recent
1428	/// one.
1429	MacroDirective *getLocalMacroDirectiveHistory(const IdentifierInfo *II) const;
1430
1431	/// Add a directive to the macro directive history for this identifier.
1432	void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD);
1433	DefMacroDirective *appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI,
1434	SourceLocation Loc) {
1435	DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
1436	appendMacroDirective(II, MD);
1437	return MD;
1438	}
1439	DefMacroDirective *appendDefMacroDirective(IdentifierInfo *II,
1440	MacroInfo *MI) {
1441	return appendDefMacroDirective(II, MI, Loc: MI->getDefinitionLoc());
1442	}
1443
1444	/// Set a MacroDirective that was loaded from a PCH file.
1445	void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED,
1446	MacroDirective *MD);
1447
1448	/// Register an exported macro for a module and identifier.
1449	ModuleMacro *addModuleMacro(Module *Mod, IdentifierInfo *II,
1450	MacroInfo *Macro,
1451	ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
1452	ModuleMacro *getModuleMacro(Module *Mod, const IdentifierInfo *II);
1453
1454	/// Get the list of leaf (non-overridden) module macros for a name.
1455	ArrayRef<ModuleMacro*> getLeafModuleMacros(const IdentifierInfo *II) const {
1456	if (II->isOutOfDate())
1457	updateOutOfDateIdentifier(II: *II);
1458	auto I = LeafModuleMacros.find(Val: II);
1459	if (I != LeafModuleMacros.end())
1460	return I->second;
1461	return {};
1462	}
1463
1464	/// Get the list of submodules that we're currently building.
1465	ArrayRef<BuildingSubmoduleInfo> getBuildingSubmodules() const {
1466	return BuildingSubmoduleStack;
1467	}
1468
1469	/// \{
1470	/// Iterators for the macro history table. Currently defined macros have
1471	/// IdentifierInfo::hasMacroDefinition() set and an empty
1472	/// MacroInfo::getUndefLoc() at the head of the list.
1473	using macro_iterator = MacroMap::const_iterator;
1474
1475	macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
1476	macro_iterator macro_end(bool IncludeExternalMacros = true) const;
1477
1478	llvm::iterator_range<macro_iterator>
1479	macros(bool IncludeExternalMacros = true) const {
1480	macro_iterator begin = macro_begin(IncludeExternalMacros);
1481	macro_iterator end = macro_end(IncludeExternalMacros);
1482	return llvm::make_range(x: begin, y: end);
1483	}
1484
1485	/// \}
1486
1487	/// Mark the given clang module as affecting the current clang module or translation unit.
1488	void markClangModuleAsAffecting(Module *M) {
1489	assert(M->isModuleMapModule());
1490	if (!BuildingSubmoduleStack.empty()) {
1491	if (M != BuildingSubmoduleStack.back().M)
1492	BuildingSubmoduleStack.back().M->AffectingClangModules.insert(X: M);
1493	} else {
1494	AffectingClangModules.insert(X: M);
1495	}
1496	}
1497
1498	/// Get the set of top-level clang modules that affected preprocessing, but were not
1499	/// imported.
1500	const llvm::SmallSetVector<Module *, 2> &getAffectingClangModules() const {
1501	return AffectingClangModules;
1502	}
1503
1504	/// Mark the file as included.
1505	/// Returns true if this is the first time the file was included.
1506	bool markIncluded(FileEntryRef File) {
1507	HeaderInfo.getFileInfo(FE: File).IsLocallyIncluded = true;
1508	return IncludedFiles.insert(V: File).second;
1509	}
1510
1511	/// Return true if this header has already been included.
1512	bool alreadyIncluded(FileEntryRef File) const {
1513	HeaderInfo.getFileInfo(FE: File);
1514	return IncludedFiles.count(V: File);
1515	}
1516
1517	/// Get the set of included files.
1518	IncludedFilesSet &getIncludedFiles() { return IncludedFiles; }
1519	const IncludedFilesSet &getIncludedFiles() const { return IncludedFiles; }
1520
1521	/// Return the name of the macro defined before \p Loc that has
1522	/// spelling \p Tokens. If there are multiple macros with same spelling,
1523	/// return the last one defined.
1524	StringRef getLastMacroWithSpelling(SourceLocation Loc,
1525	ArrayRef<TokenValue> Tokens) const;
1526
1527	/// Get the predefines for this processor.
1528	/// Used by some third-party tools to inspect and add predefines (see
1529	/// https://github.com/llvm/llvm-project/issues/57483).
1530	const std::string &getPredefines() const { return Predefines; }
1531
1532	/// Set the predefines for this Preprocessor.
1533	///
1534	/// These predefines are automatically injected when parsing the main file.
1535	void setPredefines(std::string P) { Predefines = std::move(P); }
1536
1537	/// Return information about the specified preprocessor
1538	/// identifier token.
1539	IdentifierInfo *getIdentifierInfo(StringRef Name) const {
1540	return &Identifiers.get(Name);
1541	}
1542
1543	/// Add the specified pragma handler to this preprocessor.
1544	///
1545	/// If \p Namespace is non-null, then it is a token required to exist on the
1546	/// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
1547	void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1548	void AddPragmaHandler(PragmaHandler *Handler) {
1549	AddPragmaHandler(Namespace: StringRef(), Handler);
1550	}
1551
1552	/// Remove the specific pragma handler from this preprocessor.
1553	///
1554	/// If \p Namespace is non-null, then it should be the namespace that
1555	/// \p Handler was added to. It is an error to remove a handler that
1556	/// has not been registered.
1557	void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1558	void RemovePragmaHandler(PragmaHandler *Handler) {
1559	RemovePragmaHandler(Namespace: StringRef(), Handler);
1560	}
1561
1562	/// Install empty handlers for all pragmas (making them ignored).
1563	void IgnorePragmas();
1564
1565	/// Set empty line handler.
1566	void setEmptylineHandler(EmptylineHandler *Handler) { Emptyline = Handler; }
1567
1568	EmptylineHandler *getEmptylineHandler() const { return Emptyline; }
1569
1570	/// Add the specified comment handler to the preprocessor.
1571	void addCommentHandler(CommentHandler *Handler);
1572
1573	/// Remove the specified comment handler.
1574	///
1575	/// It is an error to remove a handler that has not been registered.
1576	void removeCommentHandler(CommentHandler *Handler);
1577
1578	/// Set the code completion handler to the given object.
1579	void setCodeCompletionHandler(CodeCompletionHandler &Handler) {
1580	CodeComplete = &Handler;
1581	}
1582
1583	/// Retrieve the current code-completion handler.
1584	CodeCompletionHandler *getCodeCompletionHandler() const {
1585	return CodeComplete;
1586	}
1587
1588	/// Clear out the code completion handler.
1589	void clearCodeCompletionHandler() {
1590	CodeComplete = nullptr;
1591	}
1592
1593	/// Hook used by the lexer to invoke the "included file" code
1594	/// completion point.
1595	void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled);
1596
1597	/// Hook used by the lexer to invoke the "natural language" code
1598	/// completion point.
1599	void CodeCompleteNaturalLanguage();
1600
1601	/// Set the code completion token for filtering purposes.
1602	void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter) {
1603	CodeCompletionII = Filter;
1604	}
1605
1606	/// Set the code completion token range for detecting replacement range later
1607	/// on.
1608	void setCodeCompletionTokenRange(const SourceLocation Start,
1609	const SourceLocation End) {
1610	CodeCompletionTokenRange = {Start, End};
1611	}
1612	SourceRange getCodeCompletionTokenRange() const {
1613	return CodeCompletionTokenRange;
1614	}
1615
1616	/// Get the code completion token for filtering purposes.
1617	StringRef getCodeCompletionFilter() {
1618	if (CodeCompletionII)
1619	return CodeCompletionII->getName();
1620	return {};
1621	}
1622
1623	/// Retrieve the preprocessing record, or NULL if there is no
1624	/// preprocessing record.
1625	PreprocessingRecord *getPreprocessingRecord() const { return Record; }
1626
1627	/// Create a new preprocessing record, which will keep track of
1628	/// all macro expansions, macro definitions, etc.
1629	void createPreprocessingRecord();
1630
1631	/// Returns true if the FileEntry is the PCH through header.
1632	bool isPCHThroughHeader(const FileEntry *FE);
1633
1634	/// True if creating a PCH with a through header.
1635	bool creatingPCHWithThroughHeader();
1636
1637	/// True if using a PCH with a through header.
1638	bool usingPCHWithThroughHeader();
1639
1640	/// True if creating a PCH with a #pragma hdrstop.
1641	bool creatingPCHWithPragmaHdrStop();
1642
1643	/// True if using a PCH with a #pragma hdrstop.
1644	bool usingPCHWithPragmaHdrStop();
1645
1646	/// Skip tokens until after the #include of the through header or
1647	/// until after a #pragma hdrstop.
1648	void SkipTokensWhileUsingPCH();
1649
1650	/// Process directives while skipping until the through header or
1651	/// #pragma hdrstop is found.
1652	void HandleSkippedDirectiveWhileUsingPCH(Token &Result,
1653	SourceLocation HashLoc);
1654
1655	/// Enter the specified FileID as the main source file,
1656	/// which implicitly adds the builtin defines etc.
1657	void EnterMainSourceFile();
1658
1659	/// Inform the preprocessor callbacks that processing is complete.
1660	void EndSourceFile();
1661
1662	/// Add a source file to the top of the include stack and
1663	/// start lexing tokens from it instead of the current buffer.
1664	///
1665	/// Emits a diagnostic, doesn't enter the file, and returns true on error.
1666	bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir,
1667	SourceLocation Loc, bool IsFirstIncludeOfFile = true);
1668
1669	/// Add a Macro to the top of the include stack and start lexing
1670	/// tokens from it instead of the current buffer.
1671	///
1672	/// \param Args specifies the tokens input to a function-like macro.
1673	/// \param ILEnd specifies the location of the ')' for a function-like macro
1674	/// or the identifier for an object-like macro.
1675	void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro,
1676	MacroArgs *Args);
1677
1678	private:
1679	/// Add a "macro" context to the top of the include stack,
1680	/// which will cause the lexer to start returning the specified tokens.
1681	///
1682	/// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1683	/// will not be subject to further macro expansion. Otherwise, these tokens
1684	/// will be re-macro-expanded when/if expansion is enabled.
1685	///
1686	/// If \p OwnsTokens is false, this method assumes that the specified stream
1687	/// of tokens has a permanent owner somewhere, so they do not need to be
1688	/// copied. If it is true, it assumes the array of tokens is allocated with
1689	/// \c new[] and the Preprocessor will delete[] it.
1690	///
1691	/// If \p IsReinject the resulting tokens will have Token::IsReinjected flag
1692	/// set, see the flag documentation for details.
1693	void EnterTokenStream(const Token *Toks, unsigned NumToks,
1694	bool DisableMacroExpansion, bool OwnsTokens,
1695	bool IsReinject);
1696
1697	public:
1698	void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
1699	bool DisableMacroExpansion, bool IsReinject) {
1700	EnterTokenStream(Toks: Toks.release(), NumToks, DisableMacroExpansion, OwnsTokens: true,
1701	IsReinject);
1702	}
1703
1704	void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion,
1705	bool IsReinject) {
1706	EnterTokenStream(Toks: Toks.data(), NumToks: Toks.size(), DisableMacroExpansion, OwnsTokens: false,
1707	IsReinject);
1708	}
1709
1710	/// Pop the current lexer/macro exp off the top of the lexer stack.
1711	///
1712	/// This should only be used in situations where the current state of the
1713	/// top-of-stack lexer is known.
1714	void RemoveTopOfLexerStack();
1715
1716	/// From the point that this method is called, and until
1717	/// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1718	/// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1719	/// make the Preprocessor re-lex the same tokens.
1720	///
1721	/// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1722	/// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1723	/// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1724	///
1725	/// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
1726	/// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1727	/// tokens will continue indefinitely.
1728	///
1729	/// \param Unannotated Whether token annotations are reverted upon calling
1730	/// Backtrack().
1731	void EnableBacktrackAtThisPos(bool Unannotated = false);
1732
1733	private:
1734	std::pair<CachedTokensTy::size_type, bool> LastBacktrackPos();
1735
1736	CachedTokensTy PopUnannotatedBacktrackTokens();
1737
1738	public:
1739	/// Disable the last EnableBacktrackAtThisPos call.
1740	void CommitBacktrackedTokens();
1741
1742	/// Make Preprocessor re-lex the tokens that were lexed since
1743	/// EnableBacktrackAtThisPos() was previously called.
1744	void Backtrack();
1745
1746	/// True if EnableBacktrackAtThisPos() was called and
1747	/// caching of tokens is on.
1748	bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1749
1750	/// True if EnableBacktrackAtThisPos() was called and
1751	/// caching of unannotated tokens is on.
1752	bool isUnannotatedBacktrackEnabled() const {
1753	return !UnannotatedBacktrackTokens.empty();
1754	}
1755
1756	/// Lex the next token for this preprocessor.
1757	void Lex(Token &Result);
1758
1759	/// Lex all tokens for this preprocessor until (and excluding) end of file.
1760	void LexTokensUntilEOF(std::vector<Token> *Tokens = nullptr);
1761
1762	/// Lex a token, forming a header-name token if possible.
1763	bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
1764
1765	/// Lex the parameters for an #embed directive, returns nullopt on error.
1766	std::optional<LexEmbedParametersResult> LexEmbedParameters(Token &Current,
1767	bool ForHasEmbed);
1768
1769	bool LexAfterModuleImport(Token &Result);
1770	void CollectPpImportSuffix(SmallVectorImpl<Token> &Toks);
1771
1772	void makeModuleVisible(Module *M, SourceLocation Loc,
1773	bool IncludeExports = true);
1774
1775	SourceLocation getModuleImportLoc(Module *M) const {
1776	return CurSubmoduleState->VisibleModules.getImportLoc(M);
1777	}
1778
1779	/// Lex a string literal, which may be the concatenation of multiple
1780	/// string literals and may even come from macro expansion.
1781	/// \returns true on success, false if a error diagnostic has been generated.
1782	bool LexStringLiteral(Token &Result, std::string &String,
1783	const char *DiagnosticTag, bool AllowMacroExpansion) {
1784	if (AllowMacroExpansion)
1785	Lex(Result);
1786	else
1787	LexUnexpandedToken(Result);
1788	return FinishLexStringLiteral(Result, String, DiagnosticTag,
1789	AllowMacroExpansion);
1790	}
1791
1792	/// Complete the lexing of a string literal where the first token has
1793	/// already been lexed (see LexStringLiteral).
1794	bool FinishLexStringLiteral(Token &Result, std::string &String,
1795	const char *DiagnosticTag,
1796	bool AllowMacroExpansion);
1797
1798	/// Lex a token. If it's a comment, keep lexing until we get
1799	/// something not a comment.
1800	///
1801	/// This is useful in -E -C mode where comments would foul up preprocessor
1802	/// directive handling.
1803	void LexNonComment(Token &Result) {
1804	do
1805	Lex(Result);
1806	while (Result.getKind() == tok::comment);
1807	}
1808
1809	/// Just like Lex, but disables macro expansion of identifier tokens.
1810	void LexUnexpandedToken(Token &Result) {
1811	// Disable macro expansion.
1812	bool OldVal = DisableMacroExpansion;
1813	DisableMacroExpansion = true;
1814	// Lex the token.
1815	Lex(Result);
1816
1817	// Reenable it.
1818	DisableMacroExpansion = OldVal;
1819	}
1820
1821	/// Like LexNonComment, but this disables macro expansion of
1822	/// identifier tokens.
1823	void LexUnexpandedNonComment(Token &Result) {
1824	do
1825	LexUnexpandedToken(Result);
1826	while (Result.getKind() == tok::comment);
1827	}
1828
1829	/// Parses a simple integer literal to get its numeric value. Floating
1830	/// point literals and user defined literals are rejected. Used primarily to
1831	/// handle pragmas that accept integer arguments.
1832	bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value);
1833
1834	/// Disables macro expansion everywhere except for preprocessor directives.
1835	void SetMacroExpansionOnlyInDirectives() {
1836	DisableMacroExpansion = true;
1837	MacroExpansionInDirectivesOverride = true;
1838	}
1839
1840	/// Peeks ahead N tokens and returns that token without consuming any
1841	/// tokens.
1842	///
1843	/// LookAhead(0) returns the next token that would be returned by Lex(),
1844	/// LookAhead(1) returns the token after it, etc. This returns normal
1845	/// tokens after phase 5. As such, it is equivalent to using
1846	/// 'Lex', not 'LexUnexpandedToken'.
1847	const Token &LookAhead(unsigned N) {
1848	assert(LexLevel == 0 && "cannot use lookahead while lexing");
1849	if (CachedLexPos + N < CachedTokens.size())
1850	return CachedTokens[CachedLexPos+N];
1851	else
1852	return PeekAhead(N: N+1);
1853	}
1854
1855	/// When backtracking is enabled and tokens are cached,
1856	/// this allows to revert a specific number of tokens.
1857	///
1858	/// Note that the number of tokens being reverted should be up to the last
1859	/// backtrack position, not more.
1860	void RevertCachedTokens(unsigned N) {
1861	assert(isBacktrackEnabled() &&
1862	"Should only be called when tokens are cached for backtracking");
1863	assert(signed(CachedLexPos) - signed(N) >=
1864	signed(LastBacktrackPos().first) &&
1865	"Should revert tokens up to the last backtrack position, not more");
1866	assert(signed(CachedLexPos) - signed(N) >= 0 &&
1867	"Corrupted backtrack positions ?");
1868	CachedLexPos -= N;
1869	}
1870
1871	/// Enters a token in the token stream to be lexed next.
1872	///
1873	/// If BackTrack() is called afterwards, the token will remain at the
1874	/// insertion point.
1875	/// If \p IsReinject is true, resulting token will have Token::IsReinjected
1876	/// flag set. See the flag documentation for details.
1877	void EnterToken(const Token &Tok, bool IsReinject) {
1878	if (LexLevel) {
1879	// It's not correct in general to enter caching lex mode while in the
1880	// middle of a nested lexing action.
1881	auto TokCopy = std::make_unique<Token[]>(num: 1);
1882	TokCopy[0] = Tok;
1883	EnterTokenStream(Toks: std::move(TokCopy), NumToks: 1, DisableMacroExpansion: true, IsReinject);
1884	} else {
1885	EnterCachingLexMode();
1886	assert(IsReinject && "new tokens in the middle of cached stream");
1887	CachedTokens.insert(I: CachedTokens.begin()+CachedLexPos, Elt: Tok);
1888	}
1889	}
1890
1891	/// We notify the Preprocessor that if it is caching tokens (because
1892	/// backtrack is enabled) it should replace the most recent cached tokens
1893	/// with the given annotation token. This function has no effect if
1894	/// backtracking is not enabled.
1895	///
1896	/// Note that the use of this function is just for optimization, so that the
1897	/// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1898	/// invoked.
1899	void AnnotateCachedTokens(const Token &Tok) {
1900	assert(Tok.isAnnotation() && "Expected annotation token");
1901	if (CachedLexPos != 0 && isBacktrackEnabled())
1902	AnnotatePreviousCachedTokens(Tok);
1903	}
1904
1905	/// Get the location of the last cached token, suitable for setting the end
1906	/// location of an annotation token.
1907	SourceLocation getLastCachedTokenLocation() const {
1908	assert(CachedLexPos != 0);
1909	return CachedTokens[CachedLexPos-1].getLastLoc();
1910	}
1911
1912	/// Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
1913	/// CachedTokens.
1914	bool IsPreviousCachedToken(const Token &Tok) const;
1915
1916	/// Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
1917	/// in \p NewToks.
1918	///
1919	/// Useful when a token needs to be split in smaller ones and CachedTokens
1920	/// most recent token must to be updated to reflect that.
1921	void ReplacePreviousCachedToken(ArrayRef<Token> NewToks);
1922
1923	/// Replace the last token with an annotation token.
1924	///
1925	/// Like AnnotateCachedTokens(), this routine replaces an
1926	/// already-parsed (and resolved) token with an annotation
1927	/// token. However, this routine only replaces the last token with
1928	/// the annotation token; it does not affect any other cached
1929	/// tokens. This function has no effect if backtracking is not
1930	/// enabled.
1931	void ReplaceLastTokenWithAnnotation(const Token &Tok) {
1932	assert(Tok.isAnnotation() && "Expected annotation token");
1933	if (CachedLexPos != 0 && isBacktrackEnabled())
1934	CachedTokens[CachedLexPos-1] = Tok;
1935	}
1936
1937	/// Enter an annotation token into the token stream.
1938	void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind,
1939	void *AnnotationVal);
1940
1941	/// Determine whether it's possible for a future call to Lex to produce an
1942	/// annotation token created by a previous call to EnterAnnotationToken.
1943	bool mightHavePendingAnnotationTokens() {
1944	return CurLexerCallback != CLK_Lexer;
1945	}
1946
1947	/// Update the current token to represent the provided
1948	/// identifier, in order to cache an action performed by typo correction.
1949	void TypoCorrectToken(const Token &Tok) {
1950	assert(Tok.getIdentifierInfo() && "Expected identifier token");
1951	if (CachedLexPos != 0 && isBacktrackEnabled())
1952	CachedTokens[CachedLexPos-1] = Tok;
1953	}
1954
1955	/// Recompute the current lexer kind based on the CurLexer/
1956	/// CurTokenLexer pointers.
1957	void recomputeCurLexerKind();
1958
1959	/// Returns true if incremental processing is enabled
1960	bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
1961
1962	/// Enables the incremental processing
1963	void enableIncrementalProcessing(bool value = true) {
1964	IncrementalProcessing = value;
1965	}
1966
1967	/// Specify the point at which code-completion will be performed.
1968	///
1969	/// \param File the file in which code completion should occur. If
1970	/// this file is included multiple times, code-completion will
1971	/// perform completion the first time it is included. If NULL, this
1972	/// function clears out the code-completion point.
1973	///
1974	/// \param Line the line at which code completion should occur
1975	/// (1-based).
1976	///
1977	/// \param Column the column at which code completion should occur
1978	/// (1-based).
1979	///
1980	/// \returns true if an error occurred, false otherwise.
1981	bool SetCodeCompletionPoint(FileEntryRef File, unsigned Line,
1982	unsigned Column);
1983
1984	/// Determine if we are performing code completion.
1985	bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
1986
1987	/// Returns the location of the code-completion point.
1988	///
1989	/// Returns an invalid location if code-completion is not enabled or the file
1990	/// containing the code-completion point has not been lexed yet.
1991	SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
1992
1993	/// Returns the start location of the file of code-completion point.
1994	///
1995	/// Returns an invalid location if code-completion is not enabled or the file
1996	/// containing the code-completion point has not been lexed yet.
1997	SourceLocation getCodeCompletionFileLoc() const {
1998	return CodeCompletionFileLoc;
1999	}
2000
2001	/// Returns true if code-completion is enabled and we have hit the
2002	/// code-completion point.
2003	bool isCodeCompletionReached() const { return CodeCompletionReached; }
2004
2005	/// Note that we hit the code-completion point.
2006	void setCodeCompletionReached() {
2007	assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
2008	CodeCompletionReached = true;
2009	// Silence any diagnostics that occur after we hit the code-completion.
2010	getDiagnostics().setSuppressAllDiagnostics(true);
2011	}
2012
2013	/// The location of the currently-active \#pragma clang
2014	/// arc_cf_code_audited begin.
2015	///
2016	/// Returns an invalid location if there is no such pragma active.
2017	IdentifierLoc getPragmaARCCFCodeAuditedInfo() const {
2018	return PragmaARCCFCodeAuditedInfo;
2019	}
2020
2021	/// Set the location of the currently-active \#pragma clang
2022	/// arc_cf_code_audited begin. An invalid location ends the pragma.
2023	void setPragmaARCCFCodeAuditedInfo(IdentifierInfo *Ident,
2024	SourceLocation Loc) {
2025	PragmaARCCFCodeAuditedInfo = IdentifierLoc(Loc, Ident);
2026	}
2027
2028	/// The location of the currently-active \#pragma clang
2029	/// assume_nonnull begin.
2030	///
2031	/// Returns an invalid location if there is no such pragma active.
2032	SourceLocation getPragmaAssumeNonNullLoc() const {
2033	return PragmaAssumeNonNullLoc;
2034	}
2035
2036	/// Set the location of the currently-active \#pragma clang
2037	/// assume_nonnull begin. An invalid location ends the pragma.
2038	void setPragmaAssumeNonNullLoc(SourceLocation Loc) {
2039	PragmaAssumeNonNullLoc = Loc;
2040	}
2041
2042	/// Get the location of the recorded unterminated \#pragma clang
2043	/// assume_nonnull begin in the preamble, if one exists.
2044	///
2045	/// Returns an invalid location if the premable did not end with
2046	/// such a pragma active or if there is no recorded preamble.
2047	SourceLocation getPreambleRecordedPragmaAssumeNonNullLoc() const {
2048	return PreambleRecordedPragmaAssumeNonNullLoc;
2049	}
2050
2051	/// Record the location of the unterminated \#pragma clang
2052	/// assume_nonnull begin in the preamble.
2053	void setPreambleRecordedPragmaAssumeNonNullLoc(SourceLocation Loc) {
2054	PreambleRecordedPragmaAssumeNonNullLoc = Loc;
2055	}
2056
2057	/// Set the directory in which the main file should be considered
2058	/// to have been found, if it is not a real file.
2059	void setMainFileDir(DirectoryEntryRef Dir) { MainFileDir = Dir; }
2060
2061	/// Instruct the preprocessor to skip part of the main source file.
2062	///
2063	/// \param Bytes The number of bytes in the preamble to skip.
2064	///
2065	/// \param StartOfLine Whether skipping these bytes puts the lexer at the
2066	/// start of a line.
2067	void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
2068	SkipMainFilePreamble.first = Bytes;
2069	SkipMainFilePreamble.second = StartOfLine;
2070	}
2071
2072	/// Forwarding function for diagnostics. This emits a diagnostic at
2073	/// the specified Token's location, translating the token's start
2074	/// position in the current buffer into a SourcePosition object for rendering.
2075	DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
2076	return Diags->Report(Loc, DiagID);
2077	}
2078
2079	DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
2080	return Diags->Report(Loc: Tok.getLocation(), DiagID);
2081	}
2082
2083	/// Return the 'spelling' of the token at the given
2084	/// location; does not go up to the spelling location or down to the
2085	/// expansion location.
2086	///
2087	/// \param buffer A buffer which will be used only if the token requires
2088	/// "cleaning", e.g. if it contains trigraphs or escaped newlines
2089	/// \param invalid If non-null, will be set \c true if an error occurs.
2090	StringRef getSpelling(SourceLocation loc,
2091	SmallVectorImpl<char> &buffer,
2092	bool *invalid = nullptr) const {
2093	return Lexer::getSpelling(loc, buffer, SM: SourceMgr, options: LangOpts, invalid);
2094	}
2095
2096	/// Return the 'spelling' of the Tok token.
2097	///
2098	/// The spelling of a token is the characters used to represent the token in
2099	/// the source file after trigraph expansion and escaped-newline folding. In
2100	/// particular, this wants to get the true, uncanonicalized, spelling of
2101	/// things like digraphs, UCNs, etc.
2102	///
2103	/// \param Invalid If non-null, will be set \c true if an error occurs.
2104	std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
2105	return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
2106	}
2107
2108	/// Get the spelling of a token into a preallocated buffer, instead
2109	/// of as an std::string.
2110	///
2111	/// The caller is required to allocate enough space for the token, which is
2112	/// guaranteed to be at least Tok.getLength() bytes long. The length of the
2113	/// actual result is returned.
2114	///
2115	/// Note that this method may do two possible things: it may either fill in
2116	/// the buffer specified with characters, or it may *change the input pointer*
2117	/// to point to a constant buffer with the data already in it (avoiding a
2118	/// copy). The caller is not allowed to modify the returned buffer pointer
2119	/// if an internal buffer is returned.
2120	unsigned getSpelling(const Token &Tok, const char *&Buffer,
2121	bool *Invalid = nullptr) const {
2122	return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
2123	}
2124
2125	/// Get the spelling of a token into a SmallVector.
2126	///
2127	/// Note that the returned StringRef may not point to the
2128	/// supplied buffer if a copy can be avoided.
2129	StringRef getSpelling(const Token &Tok,
2130	SmallVectorImpl<char> &Buffer,
2131	bool *Invalid = nullptr) const;
2132
2133	/// Relex the token at the specified location.
2134	/// \returns true if there was a failure, false on success.
2135	bool getRawToken(SourceLocation Loc, Token &Result,
2136	bool IgnoreWhiteSpace = false) {
2137	return Lexer::getRawToken(Loc, Result, SM: SourceMgr, LangOpts, IgnoreWhiteSpace);
2138	}
2139
2140	/// Given a Token \p Tok that is a numeric constant with length 1,
2141	/// return the value of constant as an unsigned 8-bit integer.
2142	uint8_t
2143	getSpellingOfSingleCharacterNumericConstant(const Token &Tok,
2144	bool *Invalid = nullptr) const {
2145	assert((Tok.is(tok::numeric_constant) || Tok.is(tok::binary_data)) &&
2146	Tok.getLength() == 1 && "Called on unsupported token");
2147	assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
2148
2149	// If the token is carrying a literal data pointer, just use it.
2150	if (const char *D = Tok.getLiteralData())
2151	return (Tok.getKind() == tok::binary_data) ? *D : *D - '0';
2152
2153	assert(Tok.is(tok::numeric_constant) && "binary data with no data");
2154	// Otherwise, fall back on getCharacterData, which is slower, but always
2155	// works.
2156	return *SourceMgr.getCharacterData(SL: Tok.getLocation(), Invalid) - '0';
2157	}
2158
2159	/// Retrieve the name of the immediate macro expansion.
2160	///
2161	/// This routine starts from a source location, and finds the name of the
2162	/// macro responsible for its immediate expansion. It looks through any
2163	/// intervening macro argument expansions to compute this. It returns a
2164	/// StringRef that refers to the SourceManager-owned buffer of the source
2165	/// where that macro name is spelled. Thus, the result shouldn't out-live
2166	/// the SourceManager.
2167	StringRef getImmediateMacroName(SourceLocation Loc) {
2168	return Lexer::getImmediateMacroName(Loc, SM: SourceMgr, LangOpts: getLangOpts());
2169	}
2170
2171	/// Plop the specified string into a scratch buffer and set the
2172	/// specified token's location and length to it.
2173	///
2174	/// If specified, the source location provides a location of the expansion
2175	/// point of the token.
2176	void CreateString(StringRef Str, Token &Tok,
2177	SourceLocation ExpansionLocStart = SourceLocation(),
2178	SourceLocation ExpansionLocEnd = SourceLocation());
2179
2180	/// Split the first Length characters out of the token starting at TokLoc
2181	/// and return a location pointing to the split token. Re-lexing from the
2182	/// split token will return the split token rather than the original.
2183	SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length);
2184
2185	/// Computes the source location just past the end of the
2186	/// token at this source location.
2187	///
2188	/// This routine can be used to produce a source location that
2189	/// points just past the end of the token referenced by \p Loc, and
2190	/// is generally used when a diagnostic needs to point just after a
2191	/// token where it expected something different that it received. If
2192	/// the returned source location would not be meaningful (e.g., if
2193	/// it points into a macro), this routine returns an invalid
2194	/// source location.
2195	///
2196	/// \param Offset an offset from the end of the token, where the source
2197	/// location should refer to. The default offset (0) produces a source
2198	/// location pointing just past the end of the token; an offset of 1 produces
2199	/// a source location pointing to the last character in the token, etc.
2200	SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset = 0) {
2201	return Lexer::getLocForEndOfToken(Loc, Offset, SM: SourceMgr, LangOpts);
2202	}
2203
2204	/// Returns true if the given MacroID location points at the first
2205	/// token of the macro expansion.
2206	///
2207	/// \param MacroBegin If non-null and function returns true, it is set to
2208	/// begin location of the macro.
2209	bool isAtStartOfMacroExpansion(SourceLocation loc,
2210	SourceLocation *MacroBegin = nullptr) const {
2211	return Lexer::isAtStartOfMacroExpansion(loc, SM: SourceMgr, LangOpts,
2212	MacroBegin);
2213	}
2214
2215	/// Returns true if the given MacroID location points at the last
2216	/// token of the macro expansion.
2217	///
2218	/// \param MacroEnd If non-null and function returns true, it is set to
2219	/// end location of the macro.
2220	bool isAtEndOfMacroExpansion(SourceLocation loc,
2221	SourceLocation *MacroEnd = nullptr) const {
2222	return Lexer::isAtEndOfMacroExpansion(loc, SM: SourceMgr, LangOpts, MacroEnd);
2223	}
2224
2225	/// Print the token to stderr, used for debugging.
2226	void DumpToken(const Token &Tok, bool DumpFlags = false) const;
2227	void DumpLocation(SourceLocation Loc) const;
2228	void DumpMacro(const MacroInfo &MI) const;
2229	void dumpMacroInfo(const IdentifierInfo *II);
2230
2231	/// Given a location that specifies the start of a
2232	/// token, return a new location that specifies a character within the token.
2233	SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart,
2234	unsigned Char) const {
2235	return Lexer::AdvanceToTokenCharacter(TokStart, Characters: Char, SM: SourceMgr, LangOpts);
2236	}
2237
2238	/// Increment the counters for the number of token paste operations
2239	/// performed.
2240	///
2241	/// If fast was specified, this is a 'fast paste' case we handled.
2242	void IncrementPasteCounter(bool isFast) {
2243	if (isFast)
2244	++NumFastTokenPaste;
2245	else
2246	++NumTokenPaste;
2247	}
2248
2249	void PrintStats();
2250
2251	size_t getTotalMemory() const;
2252
2253	/// When the macro expander pastes together a comment (/##/) in Microsoft
2254	/// mode, this method handles updating the current state, returning the
2255	/// token on the next source line.
2256	void HandleMicrosoftCommentPaste(Token &Tok);
2257
2258	//===--------------------------------------------------------------------===//
2259	// Preprocessor callback methods. These are invoked by a lexer as various
2260	// directives and events are found.
2261
2262	/// Given a tok::raw_identifier token, look up the
2263	/// identifier information for the token and install it into the token,
2264	/// updating the token kind accordingly.
2265	IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const;
2266
2267	private:
2268	llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
2269
2270	public:
2271	/// Specifies the reason for poisoning an identifier.
2272	///
2273	/// If that identifier is accessed while poisoned, then this reason will be
2274	/// used instead of the default "poisoned" diagnostic.
2275	void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
2276
2277	/// Display reason for poisoned identifier.
2278	void HandlePoisonedIdentifier(Token & Identifier);
2279
2280	void MaybeHandlePoisonedIdentifier(Token & Identifier) {
2281	if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
2282	if(II->isPoisoned()) {
2283	HandlePoisonedIdentifier(Identifier);
2284	}
2285	}
2286	}
2287
2288	/// Determine whether the next preprocessor token to be
2289	/// lexed is a '('. If so, consume the token and return true, if not, this
2290	/// method should have no observable side-effect on the lexed tokens.
2291	bool isNextPPTokenLParen();
2292
2293	private:
2294	/// Identifiers used for SEH handling in Borland. These are only
2295	/// allowed in particular circumstances
2296	// __except block
2297	IdentifierInfo *Ident__exception_code,
2298	*Ident___exception_code,
2299	*Ident_GetExceptionCode;
2300	// __except filter expression
2301	IdentifierInfo *Ident__exception_info,
2302	*Ident___exception_info,
2303	*Ident_GetExceptionInfo;
2304	// __finally
2305	IdentifierInfo *Ident__abnormal_termination,
2306	*Ident___abnormal_termination,
2307	*Ident_AbnormalTermination;
2308
2309	const char *getCurLexerEndPos();
2310	void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
2311
2312	public:
2313	void PoisonSEHIdentifiers(bool Poison = true); // Borland
2314
2315	/// Callback invoked when the lexer reads an identifier and has
2316	/// filled in the tokens IdentifierInfo member.
2317	///
2318	/// This callback potentially macro expands it or turns it into a named
2319	/// token (like 'for').
2320	///
2321	/// \returns true if we actually computed a token, false if we need to
2322	/// lex again.
2323	bool HandleIdentifier(Token &Identifier);
2324
2325	/// Callback invoked when the lexer hits the end of the current file.
2326	///
2327	/// This either returns the EOF token and returns true, or
2328	/// pops a level off the include stack and returns false, at which point the
2329	/// client should call lex again.
2330	bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
2331
2332	/// Callback invoked when the current TokenLexer hits the end of its
2333	/// token stream.
2334	bool HandleEndOfTokenLexer(Token &Result);
2335
2336	/// Callback invoked when the lexer sees a # token at the start of a
2337	/// line.
2338	///
2339	/// This consumes the directive, modifies the lexer/preprocessor state, and
2340	/// advances the lexer(s) so that the next token read is the correct one.
2341	void HandleDirective(Token &Result);
2342
2343	/// Ensure that the next token is a tok::eod token.
2344	///
2345	/// If not, emit a diagnostic and consume up until the eod.
2346	/// If \p EnableMacros is true, then we consider macros that expand to zero
2347	/// tokens as being ok.
2348	///
2349	/// \return The location of the end of the directive (the terminating
2350	/// newline).
2351	SourceLocation CheckEndOfDirective(const char *DirType,
2352	bool EnableMacros = false);
2353
2354	/// Read and discard all tokens remaining on the current line until
2355	/// the tok::eod token is found. Returns the range of the skipped tokens.
2356	SourceRange DiscardUntilEndOfDirective() {
2357	Token Tmp;
2358	return DiscardUntilEndOfDirective(Tok&: Tmp);
2359	}
2360
2361	/// Same as above except retains the token that was found.
2362	SourceRange DiscardUntilEndOfDirective(Token &Tok);
2363
2364	/// Returns true if the preprocessor has seen a use of
2365	/// __DATE__ or __TIME__ in the file so far.
2366	bool SawDateOrTime() const {
2367	return DATELoc != SourceLocation() || TIMELoc != SourceLocation();
2368	}
2369	unsigned getCounterValue() const { return CounterValue; }
2370	void setCounterValue(unsigned V) { CounterValue = V; }
2371
2372	LangOptions::FPEvalMethodKind getCurrentFPEvalMethod() const {
2373	assert(CurrentFPEvalMethod != LangOptions::FEM_UnsetOnCommandLine &&
2374	"FPEvalMethod should be set either from command line or from the "
2375	"target info");
2376	return CurrentFPEvalMethod;
2377	}
2378
2379	LangOptions::FPEvalMethodKind getTUFPEvalMethod() const {
2380	return TUFPEvalMethod;
2381	}
2382
2383	SourceLocation getLastFPEvalPragmaLocation() const {
2384	return LastFPEvalPragmaLocation;
2385	}
2386
2387	void setCurrentFPEvalMethod(SourceLocation PragmaLoc,
2388	LangOptions::FPEvalMethodKind Val) {
2389	assert(Val != LangOptions::FEM_UnsetOnCommandLine &&
2390	"FPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2391	// This is the location of the '#pragma float_control" where the
2392	// execution state is modifed.
2393	LastFPEvalPragmaLocation = PragmaLoc;
2394	CurrentFPEvalMethod = Val;
2395	TUFPEvalMethod = Val;
2396	}
2397
2398	void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val) {
2399	assert(Val != LangOptions::FEM_UnsetOnCommandLine &&
2400	"TUPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2401	TUFPEvalMethod = Val;
2402	}
2403
2404	/// Retrieves the module that we're currently building, if any.
2405	Module *getCurrentModule();
2406
2407	/// Retrieves the module whose implementation we're current compiling, if any.
2408	Module *getCurrentModuleImplementation();
2409
2410	/// If we are preprocessing a named module.
2411	bool isInNamedModule() const { return ModuleDeclState.isNamedModule(); }
2412
2413	/// If we are proprocessing a named interface unit.
2414	/// Note that a module implementation partition is not considered as an
2415	/// named interface unit here although it is importable
2416	/// to ease the parsing.
2417	bool isInNamedInterfaceUnit() const {
2418	return ModuleDeclState.isNamedInterface();
2419	}
2420
2421	/// Get the named module name we're preprocessing.
2422	/// Requires we're preprocessing a named module.
2423	StringRef getNamedModuleName() const { return ModuleDeclState.getName(); }
2424
2425	/// If we are implementing an implementation module unit.
2426	/// Note that the module implementation partition is not considered as an
2427	/// implementation unit.
2428	bool isInImplementationUnit() const {
2429	return ModuleDeclState.isImplementationUnit();
2430	}
2431
2432	/// If we're importing a standard C++20 Named Modules.
2433	bool isInImportingCXXNamedModules() const {
2434	// NamedModuleImportPath will be non-empty only if we're importing
2435	// Standard C++ named modules.
2436	return !NamedModuleImportPath.empty() && getLangOpts().CPlusPlusModules &&
2437	!IsAtImport;
2438	}
2439
2440	/// Allocate a new MacroInfo object with the provided SourceLocation.
2441	MacroInfo *AllocateMacroInfo(SourceLocation L);
2442
2443	/// Turn the specified lexer token into a fully checked and spelled
2444	/// filename, e.g. as an operand of \#include.
2445	///
2446	/// The caller is expected to provide a buffer that is large enough to hold
2447	/// the spelling of the filename, but is also expected to handle the case
2448	/// when this method decides to use a different buffer.
2449	///
2450	/// \returns true if the input filename was in <>'s or false if it was
2451	/// in ""'s.
2452	bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Buffer);
2453
2454	/// Given a "foo" or \<foo> reference, look up the indicated file.
2455	///
2456	/// Returns std::nullopt on failure. \p isAngled indicates whether the file
2457	/// reference is for system \#include's or not (i.e. using <> instead of "").
2458	OptionalFileEntryRef
2459	LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled,
2460	ConstSearchDirIterator FromDir, const FileEntry *FromFile,
2461	ConstSearchDirIterator *CurDir, SmallVectorImpl<char> *SearchPath,
2462	SmallVectorImpl<char> *RelativePath,
2463	ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped,
2464	bool *IsFrameworkFound, bool SkipCache = false,
2465	bool OpenFile = true, bool CacheFailures = true);
2466
2467	/// Given a "Filename" or \<Filename> reference, look up the indicated embed
2468	/// resource. \p isAngled indicates whether the file reference is for
2469	/// system \#include's or not (i.e. using <> instead of ""). If \p OpenFile
2470	/// is true, the file looked up is opened for reading, otherwise it only
2471	/// validates that the file exists. Quoted filenames are looked up relative
2472	/// to \p LookupFromFile if it is nonnull.
2473	///
2474	/// Returns std::nullopt on failure.
2475	OptionalFileEntryRef
2476	LookupEmbedFile(StringRef Filename, bool isAngled, bool OpenFile,
2477	const FileEntry *LookupFromFile = nullptr);
2478
2479	/// Return true if we're in the top-level file, not in a \#include.
2480	bool isInPrimaryFile() const;
2481
2482	/// Lex an on-off-switch (C99 6.10.6p2) and verify that it is
2483	/// followed by EOD. Return true if the token is not a valid on-off-switch.
2484	bool LexOnOffSwitch(tok::OnOffSwitch &Result);
2485
2486	bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
2487	bool *ShadowFlag = nullptr);
2488
2489	void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
2490	Module *LeaveSubmodule(bool ForPragma);
2491
2492	private:
2493	friend void TokenLexer::ExpandFunctionArguments();
2494
2495	void PushIncludeMacroStack() {
2496	assert(CurLexerCallback != CLK_CachingLexer &&
2497	"cannot push a caching lexer");
2498	IncludeMacroStack.emplace_back(args&: CurLexerCallback, args&: CurLexerSubmodule,
2499	args: std::move(CurLexer), args&: CurPPLexer,
2500	args: std::move(CurTokenLexer), args&: CurDirLookup);
2501	CurPPLexer = nullptr;
2502	}
2503
2504	void PopIncludeMacroStack() {
2505	CurLexer = std::move(IncludeMacroStack.back().TheLexer);
2506	CurPPLexer = IncludeMacroStack.back().ThePPLexer;
2507	CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
2508	CurDirLookup = IncludeMacroStack.back().TheDirLookup;
2509	CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
2510	CurLexerCallback = IncludeMacroStack.back().CurLexerCallback;
2511	IncludeMacroStack.pop_back();
2512	}
2513
2514	void PropagateLineStartLeadingSpaceInfo(Token &Result);
2515
2516	/// Determine whether we need to create module macros for #defines in the
2517	/// current context.
2518	bool needModuleMacros() const;
2519
2520	/// Update the set of active module macros and ambiguity flag for a module
2521	/// macro name.
2522	void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
2523
2524	DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
2525	SourceLocation Loc);
2526	UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
2527	VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
2528	bool isPublic);
2529
2530	/// Lex and validate a macro name, which occurs after a
2531	/// \#define or \#undef.
2532	///
2533	/// \param MacroNameTok Token that represents the name defined or undefined.
2534	/// \param IsDefineUndef Kind if preprocessor directive.
2535	/// \param ShadowFlag Points to flag that is set if macro name shadows
2536	/// a keyword.
2537	///
2538	/// This emits a diagnostic, sets the token kind to eod,
2539	/// and discards the rest of the macro line if the macro name is invalid.
2540	void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
2541	bool *ShadowFlag = nullptr);
2542
2543	/// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
2544	/// entire line) of the macro's tokens and adds them to MacroInfo, and while
2545	/// doing so performs certain validity checks including (but not limited to):
2546	/// - # (stringization) is followed by a macro parameter
2547	/// \param MacroNameTok - Token that represents the macro name
2548	/// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
2549	///
2550	/// Either returns a pointer to a MacroInfo object OR emits a diagnostic and
2551	/// returns a nullptr if an invalid sequence of tokens is encountered.
2552	MacroInfo *ReadOptionalMacroParameterListAndBody(
2553	const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
2554
2555	/// The ( starting an argument list of a macro definition has just been read.
2556	/// Lex the rest of the parameters and the closing ), updating \p MI with
2557	/// what we learn and saving in \p LastTok the last token read.
2558	/// Return true if an error occurs parsing the arg list.
2559	bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
2560
2561	/// Provide a suggestion for a typoed directive. If there is no typo, then
2562	/// just skip suggesting.
2563	///
2564	/// \param Tok - Token that represents the directive
2565	/// \param Directive - String reference for the directive name
2566	void SuggestTypoedDirective(const Token &Tok, StringRef Directive) const;
2567
2568	/// We just read a \#if or related directive and decided that the
2569	/// subsequent tokens are in the \#if'd out portion of the
2570	/// file. Lex the rest of the file, until we see an \#endif. If \p
2571	/// FoundNonSkipPortion is true, then we have already emitted code for part of
2572	/// this \#if directive, so \#else/\#elif blocks should never be entered. If
2573	/// \p FoundElse is false, then \#else directives are ok, if not, then we have
2574	/// already seen one so a \#else directive is a duplicate. When this returns,
2575	/// the caller can lex the first valid token.
2576	void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
2577	SourceLocation IfTokenLoc,
2578	bool FoundNonSkipPortion, bool FoundElse,
2579	SourceLocation ElseLoc = SourceLocation());
2580
2581	/// Information about the result for evaluating an expression for a
2582	/// preprocessor directive.
2583	struct DirectiveEvalResult {
2584	/// The integral value of the expression.
2585	std::optional<llvm::APSInt> Value;
2586
2587	/// Whether the expression was evaluated as true or not.
2588	bool Conditional;
2589
2590	/// True if the expression contained identifiers that were undefined.
2591	bool IncludedUndefinedIds;
2592
2593	/// The source range for the expression.
2594	SourceRange ExprRange;
2595	};
2596
2597	/// Evaluate an integer constant expression that may occur after a
2598	/// \#if or \#elif directive and return a \p DirectiveEvalResult object.
2599	///
2600	/// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
2601	DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro,
2602	bool CheckForEoD = true);
2603
2604	/// Evaluate an integer constant expression that may occur after a
2605	/// \#if or \#elif directive and return a \p DirectiveEvalResult object.
2606	///
2607	/// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
2608	/// \p EvaluatedDefined will contain the result of whether "defined" appeared
2609	/// in the evaluated expression or not.
2610	DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro,
2611	Token &Tok,
2612	bool &EvaluatedDefined,
2613	bool CheckForEoD = true);
2614
2615	/// Process a '__has_embed("path" [, ...])' expression.
2616	///
2617	/// Returns predefined `__STDC_EMBED_*` macro values if
2618	/// successful.
2619	EmbedResult EvaluateHasEmbed(Token &Tok, IdentifierInfo *II);
2620
2621	/// Process a '__has_include("path")' expression.
2622	///
2623	/// Returns true if successful.
2624	bool EvaluateHasInclude(Token &Tok, IdentifierInfo *II);
2625
2626	/// Process '__has_include_next("path")' expression.
2627	///
2628	/// Returns true if successful.
2629	bool EvaluateHasIncludeNext(Token &Tok, IdentifierInfo *II);
2630
2631	/// Get the directory and file from which to start \#include_next lookup.
2632	std::pair<ConstSearchDirIterator, const FileEntry *>
2633	getIncludeNextStart(const Token &IncludeNextTok) const;
2634
2635	/// Install the standard preprocessor pragmas:
2636	/// \#pragma GCC poison/system_header/dependency and \#pragma once.
2637	void RegisterBuiltinPragmas();
2638
2639	/// RegisterBuiltinMacro - Register the specified identifier in the identifier
2640	/// table and mark it as a builtin macro to be expanded.
2641	IdentifierInfo *RegisterBuiltinMacro(const char *Name) {
2642	// Get the identifier.
2643	IdentifierInfo *Id = getIdentifierInfo(Name);
2644
2645	// Mark it as being a macro that is builtin.
2646	MacroInfo *MI = AllocateMacroInfo(L: SourceLocation());
2647	MI->setIsBuiltinMacro();
2648	appendDefMacroDirective(II: Id, MI);
2649	return Id;
2650	}
2651
2652	/// Register builtin macros such as __LINE__ with the identifier table.
2653	void RegisterBuiltinMacros();
2654
2655	/// If an identifier token is read that is to be expanded as a macro, handle
2656	/// it and return the next token as 'Tok'. If we lexed a token, return true;
2657	/// otherwise the caller should lex again.
2658	bool HandleMacroExpandedIdentifier(Token &Identifier, const MacroDefinition &MD);
2659
2660	/// Cache macro expanded tokens for TokenLexers.
2661	//
2662	/// Works like a stack; a TokenLexer adds the macro expanded tokens that is
2663	/// going to lex in the cache and when it finishes the tokens are removed
2664	/// from the end of the cache.
2665	Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2666	ArrayRef<Token> tokens);
2667
2668	void removeCachedMacroExpandedTokensOfLastLexer();
2669
2670	/// After reading "MACRO(", this method is invoked to read all of the formal
2671	/// arguments specified for the macro invocation. Returns null on error.
2672	MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2673	SourceLocation &MacroEnd);
2674
2675	/// If an identifier token is read that is to be expanded
2676	/// as a builtin macro, handle it and return the next token as 'Tok'.
2677	void ExpandBuiltinMacro(Token &Tok);
2678
2679	/// Read a \c _Pragma directive, slice it up, process it, then
2680	/// return the first token after the directive.
2681	/// This assumes that the \c _Pragma token has just been read into \p Tok.
2682	void Handle_Pragma(Token &Tok);
2683
2684	/// Like Handle_Pragma except the pragma text is not enclosed within
2685	/// a string literal.
2686	void HandleMicrosoft__pragma(Token &Tok);
2687
2688	/// Add a lexer to the top of the include stack and
2689	/// start lexing tokens from it instead of the current buffer.
2690	void EnterSourceFileWithLexer(Lexer *TheLexer, ConstSearchDirIterator Dir);
2691
2692	/// Set the FileID for the preprocessor predefines.
2693	void setPredefinesFileID(FileID FID) {
2694	assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
2695	PredefinesFileID = FID;
2696	}
2697
2698	/// Set the FileID for the PCH through header.
2699	void setPCHThroughHeaderFileID(FileID FID);
2700
2701	/// Returns true if we are lexing from a file and not a
2702	/// pragma or a macro.
2703	static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
2704	return L ? !L->isPragmaLexer() : P != nullptr;
2705	}
2706
2707	static bool IsFileLexer(const IncludeStackInfo& I) {
2708	return IsFileLexer(L: I.TheLexer.get(), P: I.ThePPLexer);
2709	}
2710
2711	bool IsFileLexer() const {
2712	return IsFileLexer(L: CurLexer.get(), P: CurPPLexer);
2713	}
2714
2715	//===--------------------------------------------------------------------===//
2716	// Standard Library Identification
2717	std::optional<CXXStandardLibraryVersionInfo> CXXStandardLibraryVersion;
2718
2719	public:
2720	std::optional<std::uint64_t> getStdLibCxxVersion();
2721	bool NeedsStdLibCxxWorkaroundBefore(std::uint64_t FixedVersion);
2722
2723	private:
2724	//===--------------------------------------------------------------------===//
2725	// Caching stuff.
2726	void CachingLex(Token &Result);
2727
2728	bool InCachingLexMode() const {
2729	// If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
2730	// that we are past EOF, not that we are in CachingLex mode.
2731	return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
2732	}
2733
2734	void EnterCachingLexMode();
2735	void EnterCachingLexModeUnchecked();
2736
2737	void ExitCachingLexMode() {
2738	if (InCachingLexMode())
2739	RemoveTopOfLexerStack();
2740	}
2741
2742	const Token &PeekAhead(unsigned N);
2743	void AnnotatePreviousCachedTokens(const Token &Tok);
2744
2745	//===--------------------------------------------------------------------===//
2746	/// Handle*Directive - implement the various preprocessor directives. These
2747	/// should side-effect the current preprocessor object so that the next call
2748	/// to Lex() will return the appropriate token next.
2749	void HandleLineDirective();
2750	void HandleDigitDirective(Token &Tok);
2751	void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
2752	void HandleIdentSCCSDirective(Token &Tok);
2753	void HandleMacroPublicDirective(Token &Tok);
2754	void HandleMacroPrivateDirective();
2755
2756	/// An additional notification that can be produced by a header inclusion or
2757	/// import to tell the parser what happened.
2758	struct ImportAction {
2759	enum ActionKind {
2760	None,
2761	ModuleBegin,
2762	ModuleImport,
2763	HeaderUnitImport,
2764	SkippedModuleImport,
2765	Failure,
2766	} Kind;
2767	Module *ModuleForHeader = nullptr;
2768
2769	ImportAction(ActionKind AK, Module *Mod = nullptr)
2770	: Kind(AK), ModuleForHeader(Mod) {
2771	assert((AK == None || Mod || AK == Failure) &&
2772	"no module for module action");
2773	}
2774	};
2775
2776	OptionalFileEntryRef LookupHeaderIncludeOrImport(
2777	ConstSearchDirIterator *CurDir, StringRef &Filename,
2778	SourceLocation FilenameLoc, CharSourceRange FilenameRange,
2779	const Token &FilenameTok, bool &IsFrameworkFound, bool IsImportDecl,
2780	bool &IsMapped, ConstSearchDirIterator LookupFrom,
2781	const FileEntry *LookupFromFile, StringRef &LookupFilename,
2782	SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath,
2783	ModuleMap::KnownHeader &SuggestedModule, bool isAngled);
2784	// Binary data inclusion
2785	void HandleEmbedDirective(SourceLocation HashLoc, Token &Tok,
2786	const FileEntry *LookupFromFile = nullptr);
2787	void HandleEmbedDirectiveImpl(SourceLocation HashLoc,
2788	const LexEmbedParametersResult &Params,
2789	StringRef BinaryContents, StringRef FileName);
2790
2791	// File inclusion.
2792	void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok,
2793	ConstSearchDirIterator LookupFrom = nullptr,
2794	const FileEntry *LookupFromFile = nullptr);
2795	ImportAction
2796	HandleHeaderIncludeOrImport(SourceLocation HashLoc, Token &IncludeTok,
2797	Token &FilenameTok, SourceLocation EndLoc,
2798	ConstSearchDirIterator LookupFrom = nullptr,
2799	const FileEntry *LookupFromFile = nullptr);
2800	void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2801	void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2802	void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2803	void HandleMicrosoftImportDirective(Token &Tok);
2804
2805	public:
2806	/// Check that the given module is available, producing a diagnostic if not.
2807	/// \return \c true if the check failed (because the module is not available).
2808	/// \c false if the module appears to be usable.
2809	static bool checkModuleIsAvailable(const LangOptions &LangOpts,
2810	const TargetInfo &TargetInfo,
2811	const Module &M, DiagnosticsEngine &Diags);
2812
2813	// Module inclusion testing.
2814	/// Find the module that owns the source or header file that
2815	/// \p Loc points to. If the location is in a file that was included
2816	/// into a module, or is outside any module, returns nullptr.
2817	Module *getModuleForLocation(SourceLocation Loc, bool AllowTextual);
2818
2819	/// We want to produce a diagnostic at location IncLoc concerning an
2820	/// unreachable effect at location MLoc (eg, where a desired entity was
2821	/// declared or defined). Determine whether the right way to make MLoc
2822	/// reachable is by #include, and if so, what header should be included.
2823	///
2824	/// This is not necessarily fast, and might load unexpected module maps, so
2825	/// should only be called by code that intends to produce an error.
2826	///
2827	/// \param IncLoc The location at which the missing effect was detected.
2828	/// \param MLoc A location within an unimported module at which the desired
2829	/// effect occurred.
2830	/// \return A file that can be #included to provide the desired effect. Null
2831	/// if no such file could be determined or if a #include is not
2832	/// appropriate (eg, if a module should be imported instead).
2833	OptionalFileEntryRef getHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
2834	SourceLocation MLoc);
2835
2836	bool isRecordingPreamble() const {
2837	return PreambleConditionalStack.isRecording();
2838	}
2839
2840	bool hasRecordedPreamble() const {
2841	return PreambleConditionalStack.hasRecordedPreamble();
2842	}
2843
2844	ArrayRef<PPConditionalInfo> getPreambleConditionalStack() const {
2845	return PreambleConditionalStack.getStack();
2846	}
2847
2848	void setRecordedPreambleConditionalStack(ArrayRef<PPConditionalInfo> s) {
2849	PreambleConditionalStack.setStack(s);
2850	}
2851
2852	void setReplayablePreambleConditionalStack(
2853	ArrayRef<PPConditionalInfo> s, std::optional<PreambleSkipInfo> SkipInfo) {
2854	PreambleConditionalStack.startReplaying();
2855	PreambleConditionalStack.setStack(s);
2856	PreambleConditionalStack.SkipInfo = SkipInfo;
2857	}
2858
2859	std::optional<PreambleSkipInfo> getPreambleSkipInfo() const {
2860	return PreambleConditionalStack.SkipInfo;
2861	}
2862
2863	private:
2864	/// After processing predefined file, initialize the conditional stack from
2865	/// the preamble.
2866	void replayPreambleConditionalStack();
2867
2868	// Macro handling.
2869	void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterHeaderGuard);
2870	void HandleUndefDirective();
2871
2872	// Conditional Inclusion.
2873	void HandleIfdefDirective(Token &Result, const Token &HashToken,
2874	bool isIfndef, bool ReadAnyTokensBeforeDirective);
2875	void HandleIfDirective(Token &IfToken, const Token &HashToken,
2876	bool ReadAnyTokensBeforeDirective);
2877	void HandleEndifDirective(Token &EndifToken);
2878	void HandleElseDirective(Token &Result, const Token &HashToken);
2879	void HandleElifFamilyDirective(Token &ElifToken, const Token &HashToken,
2880	tok::PPKeywordKind Kind);
2881
2882	// Pragmas.
2883	void HandlePragmaDirective(PragmaIntroducer Introducer);
2884
2885	public:
2886	void HandlePragmaOnce(Token &OnceTok);
2887	void HandlePragmaMark(Token &MarkTok);
2888	void HandlePragmaPoison();
2889	void HandlePragmaSystemHeader(Token &SysHeaderTok);
2890	void HandlePragmaDependency(Token &DependencyTok);
2891	void HandlePragmaPushMacro(Token &Tok);
2892	void HandlePragmaPopMacro(Token &Tok);
2893	void HandlePragmaIncludeAlias(Token &Tok);
2894	void HandlePragmaModuleBuild(Token &Tok);
2895	void HandlePragmaHdrstop(Token &Tok);
2896	IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok);
2897
2898	// Return true and store the first token only if any CommentHandler
2899	// has inserted some tokens and getCommentRetentionState() is false.
2900	bool HandleComment(Token &result, SourceRange Comment);
2901
2902	/// A macro is used, update information about macros that need unused
2903	/// warnings.
2904	void markMacroAsUsed(MacroInfo *MI);
2905
2906	void addMacroDeprecationMsg(const IdentifierInfo *II, std::string Msg,
2907	SourceLocation AnnotationLoc) {
2908	AnnotationInfos[II].DeprecationInfo =
2909	MacroAnnotationInfo{.Location: AnnotationLoc, .Message: std::move(Msg)};
2910	}
2911
2912	void addRestrictExpansionMsg(const IdentifierInfo *II, std::string Msg,
2913	SourceLocation AnnotationLoc) {
2914	AnnotationInfos[II].RestrictExpansionInfo =
2915	MacroAnnotationInfo{.Location: AnnotationLoc, .Message: std::move(Msg)};
2916	}
2917
2918	void addFinalLoc(const IdentifierInfo *II, SourceLocation AnnotationLoc) {
2919	AnnotationInfos[II].FinalAnnotationLoc = AnnotationLoc;
2920	}
2921
2922	const MacroAnnotations &getMacroAnnotations(const IdentifierInfo *II) const {
2923	return AnnotationInfos.find(Val: II)->second;
2924	}
2925
2926	void emitMacroExpansionWarnings(const Token &Identifier,
2927	bool IsIfnDef = false) const {
2928	IdentifierInfo *Info = Identifier.getIdentifierInfo();
2929	if (Info->isDeprecatedMacro())
2930	emitMacroDeprecationWarning(Identifier);
2931
2932	if (Info->isRestrictExpansion() &&
2933	!SourceMgr.isInMainFile(Loc: Identifier.getLocation()))
2934	emitRestrictExpansionWarning(Identifier);
2935
2936	if (!IsIfnDef) {
2937	if (Info->getName() == "INFINITY" && getLangOpts().NoHonorInfs)
2938	emitRestrictInfNaNWarning(Identifier, DiagSelection: 0);
2939	if (Info->getName() == "NAN" && getLangOpts().NoHonorNaNs)
2940	emitRestrictInfNaNWarning(Identifier, DiagSelection: 1);
2941	}
2942	}
2943
2944	static void processPathForFileMacro(SmallVectorImpl<char> &Path,
2945	const LangOptions &LangOpts,
2946	const TargetInfo &TI);
2947
2948	static void processPathToFileName(SmallVectorImpl<char> &FileName,
2949	const PresumedLoc &PLoc,
2950	const LangOptions &LangOpts,
2951	const TargetInfo &TI);
2952
2953	private:
2954	void emitMacroDeprecationWarning(const Token &Identifier) const;
2955	void emitRestrictExpansionWarning(const Token &Identifier) const;
2956	void emitFinalMacroWarning(const Token &Identifier, bool IsUndef) const;
2957	void emitRestrictInfNaNWarning(const Token &Identifier,
2958	unsigned DiagSelection) const;
2959
2960	/// This boolean state keeps track if the current scanned token (by this PP)
2961	/// is in an "-Wunsafe-buffer-usage" opt-out region. Assuming PP scans a
2962	/// translation unit in a linear order.
2963	bool InSafeBufferOptOutRegion = false;
2964
2965	/// Hold the start location of the current "-Wunsafe-buffer-usage" opt-out
2966	/// region if PP is currently in such a region. Hold undefined value
2967	/// otherwise.
2968	SourceLocation CurrentSafeBufferOptOutStart; // It is used to report the start location of an never-closed region.
2969
2970	using SafeBufferOptOutRegionsTy =
2971	SmallVector<std::pair<SourceLocation, SourceLocation>, 16>;
2972	// An ordered sequence of "-Wunsafe-buffer-usage" opt-out regions in this
2973	// translation unit. Each region is represented by a pair of start and
2974	// end locations.
2975	SafeBufferOptOutRegionsTy SafeBufferOptOutMap;
2976
2977	// The "-Wunsafe-buffer-usage" opt-out regions in loaded ASTs. We use the
2978	// following structure to manage them by their ASTs.
2979	struct {
2980	// A map from unique IDs to region maps of loaded ASTs. The ID identifies a
2981	// loaded AST. See `SourceManager::getUniqueLoadedASTID`.
2982	llvm::DenseMap<FileID, SafeBufferOptOutRegionsTy> LoadedRegions;
2983
2984	// Returns a reference to the safe buffer opt-out regions of the loaded
2985	// AST where `Loc` belongs to. (Construct if absent)
2986	SafeBufferOptOutRegionsTy &
2987	findAndConsLoadedOptOutMap(SourceLocation Loc, SourceManager &SrcMgr) {
2988	return LoadedRegions[SrcMgr.getUniqueLoadedASTFileID(Loc)];
2989	}
2990
2991	// Returns a reference to the safe buffer opt-out regions of the loaded
2992	// AST where `Loc` belongs to. (This const function returns nullptr if
2993	// absent.)
2994	const SafeBufferOptOutRegionsTy *
2995	lookupLoadedOptOutMap(SourceLocation Loc,
2996	const SourceManager &SrcMgr) const {
2997	FileID FID = SrcMgr.getUniqueLoadedASTFileID(Loc);
2998	auto Iter = LoadedRegions.find(Val: FID);
2999
3000	if (Iter == LoadedRegions.end())
3001	return nullptr;
3002	return &Iter->getSecond();
3003	}
3004	} LoadedSafeBufferOptOutMap;
3005
3006	public:
3007	/// \return true iff the given `Loc` is in a "-Wunsafe-buffer-usage" opt-out
3008	/// region. This `Loc` must be a source location that has been pre-processed.
3009	bool isSafeBufferOptOut(const SourceManager&SourceMgr, const SourceLocation &Loc) const;
3010
3011	/// Alter the state of whether this PP currently is in a
3012	/// "-Wunsafe-buffer-usage" opt-out region.
3013	///
3014	/// \param isEnter true if this PP is entering a region; otherwise, this PP
3015	/// is exiting a region
3016	/// \param Loc the location of the entry or exit of a
3017	/// region
3018	/// \return true iff it is INVALID to enter or exit a region, i.e.,
3019	/// attempt to enter a region before exiting a previous region, or exiting a
3020	/// region that PP is not currently in.
3021	bool enterOrExitSafeBufferOptOutRegion(bool isEnter,
3022	const SourceLocation &Loc);
3023
3024	/// \return true iff this PP is currently in a "-Wunsafe-buffer-usage"
3025	/// opt-out region
3026	bool isPPInSafeBufferOptOutRegion();
3027
3028	/// \param StartLoc output argument. It will be set to the start location of
3029	/// the current "-Wunsafe-buffer-usage" opt-out region iff this function
3030	/// returns true.
3031	/// \return true iff this PP is currently in a "-Wunsafe-buffer-usage"
3032	/// opt-out region
3033	bool isPPInSafeBufferOptOutRegion(SourceLocation &StartLoc);
3034
3035	/// \return a sequence of SourceLocations representing ordered opt-out regions
3036	/// specified by
3037	/// `\#pragma clang unsafe_buffer_usage begin/end`s of this translation unit.
3038	SmallVector<SourceLocation, 64> serializeSafeBufferOptOutMap() const;
3039
3040	/// \param SrcLocSeqs a sequence of SourceLocations deserialized from a
3041	/// record of code `PP_UNSAFE_BUFFER_USAGE`.
3042	/// \return true iff the `Preprocessor` has been updated; false `Preprocessor`
3043	/// is same as itself before the call.
3044	bool setDeserializedSafeBufferOptOutMap(
3045	const SmallVectorImpl<SourceLocation> &SrcLocSeqs);
3046
3047	private:
3048	/// Helper functions to forward lexing to the actual lexer. They all share the
3049	/// same signature.
3050	static bool CLK_Lexer(Preprocessor &P, Token &Result) {
3051	return P.CurLexer->Lex(Result);
3052	}
3053	static bool CLK_TokenLexer(Preprocessor &P, Token &Result) {
3054	return P.CurTokenLexer->Lex(Tok&: Result);
3055	}
3056	static bool CLK_CachingLexer(Preprocessor &P, Token &Result) {
3057	P.CachingLex(Result);
3058	return true;
3059	}
3060	static bool CLK_DependencyDirectivesLexer(Preprocessor &P, Token &Result) {
3061	return P.CurLexer->LexDependencyDirectiveToken(Result);
3062	}
3063	static bool CLK_LexAfterModuleImport(Preprocessor &P, Token &Result) {
3064	return P.LexAfterModuleImport(Result);
3065	}
3066	};
3067
3068	/// Abstract base class that describes a handler that will receive
3069	/// source ranges for each of the comments encountered in the source file.
3070	class CommentHandler {
3071	public:
3072	virtual ~CommentHandler();
3073
3074	// The handler shall return true if it has pushed any tokens
3075	// to be read using e.g. EnterToken or EnterTokenStream.
3076	virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
3077	};
3078
3079	/// Abstract base class that describes a handler that will receive
3080	/// source ranges for empty lines encountered in the source file.
3081	class EmptylineHandler {
3082	public:
3083	virtual ~EmptylineHandler();
3084
3085	// The handler handles empty lines.
3086	virtual void HandleEmptyline(SourceRange Range) = 0;
3087	};
3088
3089	/// Helper class to shuttle information about #embed directives from the
3090	/// preprocessor to the parser through an annotation token.
3091	struct EmbedAnnotationData {
3092	StringRef BinaryData;
3093	StringRef FileName;
3094	};
3095
3096	/// Registry of pragma handlers added by plugins
3097	using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
3098
3099	} // namespace clang
3100
3101	namespace llvm {
3102	extern template class CLANG_TEMPLATE_ABI Registry<clang::PragmaHandler>;
3103	} // namespace llvm
3104
3105	#endif // LLVM_CLANG_LEX_PREPROCESSOR_H
3106