Skip to content

Commit a756f86

Browse files
tamasvajktamasvajk
authored
Merge pull request #16732 from tamasvajk/refactor/extraction-states
C#: Refactor extractor state classes and simplify extraction code
2 parents 4ee8065 + 0df6a1c commit a756f86

25 files changed

+163
-230
lines changed

‎csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs

Lines changed: 25 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -42,17 +42,17 @@ private static void AnalyseStandalone(
4242
(compilation, options) => analyser.Initialize(output.FullName, extractionInput.CompilationInfos, compilation, options),
4343
() =>
4444
{
45-
foreach (var type in analyser.MissingNamespaces)
45+
foreach (var type in analyser.ExtractionContext!.MissingNamespaces)
4646
{
4747
progressMonitor.MissingNamespace(type);
4848
}
4949

50-
foreach (var type in analyser.MissingTypes)
50+
foreach (var type in analyser.ExtractionContext!.MissingTypes)
5151
{
5252
progressMonitor.MissingType(type);
5353
}
5454

55-
progressMonitor.MissingSummary(analyser.MissingTypes.Count(), analyser.MissingNamespaces.Count());
55+
progressMonitor.MissingSummary(analyser.ExtractionContext!.MissingTypes.Count(), analyser.ExtractionContext!.MissingNamespaces.Count());
5656
});
5757
}
5858
finally
@@ -69,29 +69,6 @@ private static void AnalyseStandalone(
6969
}
7070
}
7171

72-
private static void ExtractStandalone(
73-
ExtractionInput extractionInput,
74-
IProgressMonitor pm,
75-
ILogger logger,
76-
CommonOptions options)
77-
{
78-
var stopwatch = new Stopwatch();
79-
stopwatch.Start();
80-
81-
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
82-
var pathTransformer = new PathTransformer(canonicalPathCache);
83-
84-
using var analyser = new StandaloneAnalyser(pm, logger, false, pathTransformer);
85-
try
86-
{
87-
AnalyseStandalone(analyser, extractionInput, options, pm, stopwatch);
88-
}
89-
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
90-
{
91-
analyser.Logger.Log(Severity.Error, " Unhandled exception: {0}", ex);
92-
}
93-
}
94-
9572
private class ExtractionProgress : IProgressMonitor
9673
{
9774
public ExtractionProgress(ILogger output)
@@ -141,8 +118,8 @@ public record ExtractionInput(IEnumerable<string> Sources, IEnumerable<string> R
141118

142119
public static ExitCode Run(Options options)
143120
{
144-
var stopwatch = new Stopwatch();
145-
stopwatch.Start();
121+
var overallStopwatch = new Stopwatch();
122+
overallStopwatch.Start();
146123

147124
using var logger = new ConsoleLogger(options.Verbosity, logThreadId: true);
148125
logger.Log(Severity.Info, "Extracting C# with build-mode set to 'none'");
@@ -158,12 +135,26 @@ public static ExitCode Run(Options options)
158135

159136
logger.Log(Severity.Info, "");
160137
logger.Log(Severity.Info, "Extracting...");
161-
ExtractStandalone(
162-
new ExtractionInput(dependencyManager.AllSourceFiles, dependencyManager.ReferenceFiles, dependencyManager.CompilationInfos),
163-
new ExtractionProgress(logger),
164-
fileLogger,
165-
options);
166-
logger.Log(Severity.Info, $"Extraction completed in {stopwatch.Elapsed}");
138+
139+
var analyzerStopwatch = new Stopwatch();
140+
analyzerStopwatch.Start();
141+
142+
var canonicalPathCache = CanonicalPathCache.Create(fileLogger, 1000);
143+
var pathTransformer = new PathTransformer(canonicalPathCache);
144+
145+
var progressMonitor = new ExtractionProgress(logger);
146+
using var analyser = new StandaloneAnalyser(progressMonitor, fileLogger, pathTransformer, canonicalPathCache, false);
147+
try
148+
{
149+
var extractionInput = new ExtractionInput(dependencyManager.AllSourceFiles, dependencyManager.ReferenceFiles, dependencyManager.CompilationInfos);
150+
AnalyseStandalone(analyser, extractionInput, options, progressMonitor, analyzerStopwatch);
151+
}
152+
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
153+
{
154+
fileLogger.Log(Severity.Error, " Unhandled exception: {0}", ex);
155+
}
156+
157+
logger.Log(Severity.Info, $"Extraction completed in {overallStopwatch.Elapsed}");
167158

168159
return ExitCode.Ok;
169160
}

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/Assembly.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,23 +17,23 @@ private Assembly(Context cx, Microsoft.CodeAnalysis.Location? init)
1717
isOutputAssembly = init is null;
1818
if (isOutputAssembly)
1919
{
20-
assemblyPath = cx.Extractor.OutputPath;
20+
assemblyPath = cx.ExtractionContext.OutputPath;
2121
assembly = cx.Compilation.Assembly;
2222
}
2323
else
2424
{
2525
assembly = init!.MetadataModule!.ContainingAssembly;
2626
var identity = assembly.Identity;
2727
var idString = identity.Name + " " + identity.Version;
28-
assemblyPath = cx.Extractor.GetAssemblyFile(idString);
28+
assemblyPath = cx.ExtractionContext.GetAssemblyFile(idString);
2929
}
3030
}
3131

3232
public override void Populate(TextWriter trapFile)
3333
{
3434
if (assemblyPath is not null)
3535
{
36-
var isBuildlessOutputAssembly = isOutputAssembly && Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone);
36+
var isBuildlessOutputAssembly = isOutputAssembly && Context.ExtractionContext.Mode.HasFlag(ExtractorMode.Standalone);
3737
var identifier = isBuildlessOutputAssembly
3838
? ""
3939
: assembly.ToString() ?? "";
@@ -74,7 +74,7 @@ public static Assembly CreateOutputAssembly(Context cx)
7474

7575
public override void WriteId(EscapingTextWriter trapFile)
7676
{
77-
if (isOutputAssembly && Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
77+
if (isOutputAssembly && Context.ExtractionContext.Mode.HasFlag(ExtractorMode.Standalone))
7878
{
7979
trapFile.Write("buildlessOutputAssembly");
8080
}

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ internal class Compilation : CachedEntity<object>
1818
#nullable disable warnings
1919
private Compilation(Context cx) : base(cx, null)
2020
{
21-
cwd = cx.Extractor.Cwd;
22-
args = cx.Extractor.Args;
21+
cwd = cx.ExtractionContext.Cwd;
22+
args = cx.ExtractionContext.Args;
2323
hashCode = cwd.GetHashCode();
2424
for (var i = 0; i < args.Length; i++)
2525
{

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/CompilerDiagnostic.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ protected override void Populate(TextWriter trapFile)
2828
{
2929
if (messageCount == limit + 1)
3030
{
31-
Context.Extractor.Logger.LogWarning($"Stopped logging {key} compiler diagnostics for the current compilation after reaching {limit}");
31+
Context.ExtractionContext.Logger.LogWarning($"Stopped logging {key} compiler diagnostics for the current compilation after reaching {limit}");
3232
}
3333

3434
return;

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Invocation.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ public IMethodSymbol? TargetSymbol
133133
.Where(method => method.Parameters.Length >= Syntax.ArgumentList.Arguments.Count)
134134
.Where(method => method.Parameters.Count(p => !p.HasExplicitDefaultValue) <= Syntax.ArgumentList.Arguments.Count);
135135

136-
return Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone) ?
136+
return Context.ExtractionContext.Mode.HasFlag(ExtractorMode.Standalone) ?
137137
candidates.FirstOrDefault() :
138138
candidates.SingleOrDefault();
139139
}

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/File.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ public override void Populate(TextWriter trapFile)
6161
}
6262
}
6363

64-
trapFile.file_extraction_mode(this, Context.Extractor.Mode);
64+
trapFile.file_extraction_mode(this, Context.ExtractionContext.Mode);
6565
}
6666

6767
private bool IsPossiblyTextFile()

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/NonGeneratedSourceLocation.cs

Lines changed: 1 addition & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ public override void Populate(TextWriter trapFile)
2727
var mapped = Symbol.GetMappedLineSpan();
2828
if (mapped.HasMappedPath && mapped.IsValid)
2929
{
30-
var path = TryAdjustRelativeMappedFilePath(mapped.Path, Position.Path, Context.Extractor.Logger);
30+
var path = Context.TryAdjustRelativeMappedFilePath(mapped.Path, Position.Path);
3131
var mappedLoc = Create(Context, Location.Create(path, default, mapped.Span));
3232

3333
trapFile.locations_mapped(this, mappedLoc);
@@ -64,25 +64,5 @@ private class SourceLocationFactory : CachedEntityFactory<Location, NonGenerated
6464

6565
public override NonGeneratedSourceLocation Create(Context cx, Location init) => new NonGeneratedSourceLocation(cx, init);
6666
}
67-
68-
public static string TryAdjustRelativeMappedFilePath(string mappedToPath, string mappedFromPath, ILogger logger)
69-
{
70-
if (!Path.IsPathRooted(mappedToPath))
71-
{
72-
try
73-
{
74-
var fullPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(mappedFromPath)!, mappedToPath));
75-
logger.LogDebug($"Found relative path in line mapping: '{mappedToPath}', interpreting it as '{fullPath}'");
76-
77-
mappedToPath = fullPath;
78-
}
79-
catch (Exception e)
80-
{
81-
logger.LogDebug($"Failed to compute absolute path for relative path in line mapping: '{mappedToPath}': {e}");
82-
}
83-
}
84-
85-
return mappedToPath;
86-
}
8767
}
8868
}

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ Symbol.ContainingType is INamedTypeSymbol nt &&
6363
{
6464
if (method.MethodKind == MethodKind.ReducedExtension)
6565
{
66-
cx.Extractor.Logger.Log(Semmle.Util.Logging.Severity.Warning, "Reduced extension method symbols should not be directly extracted.");
66+
cx.ExtractionContext.Logger.Log(Semmle.Util.Logging.Severity.Warning, "Reduced extension method symbols should not be directly extracted.");
6767
}
6868

6969
return OrdinaryMethodFactory.Instance.CreateEntityFromSymbol(cx, method);

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/LineOrSpanDirective.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ protected override void PopulatePreprocessor(TextWriter trapFile)
2828
var path = Symbol.File.ValueText;
2929
if (!string.IsNullOrWhiteSpace(path))
3030
{
31-
path = NonGeneratedSourceLocation.TryAdjustRelativeMappedFilePath(path, Symbol.SyntaxTree.FilePath, Context.Extractor.Logger);
31+
path = Context.TryAdjustRelativeMappedFilePath(path, Symbol.SyntaxTree.FilePath);
3232
var file = File.Create(Context, path);
3333
trapFile.directive_line_file(this, file);
3434
}

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PragmaChecksumDirective.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ private PragmaChecksumDirective(Context cx, PragmaChecksumDirectiveTriviaSyntax
1212

1313
protected override void PopulatePreprocessor(TextWriter trapFile)
1414
{
15-
var path = NonGeneratedSourceLocation.TryAdjustRelativeMappedFilePath(Symbol.File.ValueText, Symbol.SyntaxTree.FilePath, Context.Extractor.Logger);
15+
var path = Context.TryAdjustRelativeMappedFilePath(Symbol.File.ValueText, Symbol.SyntaxTree.FilePath);
1616
var file = File.Create(Context, path);
1717
trapFile.pragma_checksums(this, file, Symbol.Guid.ToString(), Symbol.Bytes.ToString());
1818
}

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ public override void Populate(TextWriter trapFile)
3636
if (Symbol.TypeKind == TypeKind.Error)
3737
{
3838
UnknownType.Create(Context); // make sure this exists so we can use it in `TypeRef::getReferencedType`
39-
Context.Extractor.MissingType(Symbol.ToString()!, Context.FromSource);
39+
Context.ExtractionContext.MissingType(Symbol.ToString()!, Context.FromSource);
4040
return;
4141
}
4242

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ protected override void Populate(TextWriter trapFile)
3636
}
3737
else
3838
{
39-
Context.Extractor.MissingNamespace(name.ToFullString(), Context.FromSource);
39+
Context.ExtractionContext.MissingNamespace(name.ToFullString(), Context.FromSource);
4040
Context.ModelError(node, "Namespace not found");
4141
return;
4242
}

‎csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

Lines changed: 40 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
using Semmle.Util;
1111
using Semmle.Util.Logging;
1212
using Semmle.Extraction.CSharp.Populators;
13+
using System.Reflection;
1314

1415
namespace Semmle.Extraction.CSharp
1516
{
@@ -18,7 +19,7 @@ namespace Semmle.Extraction.CSharp
1819
/// </summary>
1920
public class Analyser : IDisposable
2021
{
21-
protected Extraction.Extractor? extractor;
22+
public ExtractionContext? ExtractionContext { get; protected set; }
2223
protected CSharpCompilation? compilation;
2324
protected CommonOptions? options;
2425
private protected Entities.Compilation? compilationEntity;
@@ -38,14 +39,23 @@ public class Analyser : IDisposable
3839

3940
public PathTransformer PathTransformer { get; }
4041

41-
protected Analyser(IProgressMonitor pm, ILogger logger, bool addAssemblyTrapPrefix, PathTransformer pathTransformer)
42+
public IPathCache PathCache { get; }
43+
44+
protected Analyser(
45+
IProgressMonitor pm,
46+
ILogger logger,
47+
PathTransformer pathTransformer,
48+
IPathCache pathCache,
49+
bool addAssemblyTrapPrefix)
4250
{
4351
Logger = logger;
52+
PathTransformer = pathTransformer;
53+
PathCache = pathCache;
4454
this.addAssemblyTrapPrefix = addAssemblyTrapPrefix;
55+
this.progressMonitor = pm;
56+
4557
Logger.Log(Severity.Info, "EXTRACTION STARTING at {0}", DateTime.Now);
4658
stopWatch.Start();
47-
progressMonitor = pm;
48-
PathTransformer = pathTransformer;
4959
}
5060

5161
/// <summary>
@@ -98,12 +108,12 @@ protected void SetReferencePaths()
98108
var def = reader.GetAssemblyDefinition();
99109
assemblyIdentity = reader.GetString(def.Name) + " " + def.Version;
100110
}
101-
extractor.SetAssemblyFile(assemblyIdentity, refPath);
111+
ExtractionContext.SetAssemblyFile(assemblyIdentity, refPath);
102112

103113
}
104114
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
105115
{
106-
extractor.Message(new Message("Exception reading reference file", reference.FilePath, null, ex.StackTrace));
116+
ExtractionContext.Message(new Message("Exception reading reference file", reference.FilePath, null, ex.StackTrace));
107117
}
108118
}
109119
}
@@ -148,7 +158,7 @@ private void DoAnalyseReferenceAssembly(PortableExecutableReference r)
148158

149159
if (compilation.GetAssemblyOrModuleSymbol(r) is IAssemblySymbol assembly)
150160
{
151-
var cx = new Context(extractor, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
161+
var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
152162

153163
foreach (var module in assembly.Modules)
154164
{
@@ -191,7 +201,7 @@ private void DoExtractTree(SyntaxTree tree)
191201

192202
if (!upToDate)
193203
{
194-
var cx = new Context(extractor, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
204+
var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
195205
// Ensure that the file itself is populated in case the source file is totally empty
196206
var root = tree.GetRoot();
197207
Entities.File.Create(cx, root.SyntaxTree.FilePath);
@@ -213,15 +223,15 @@ private void DoExtractTree(SyntaxTree tree)
213223
}
214224
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
215225
{
216-
extractor.Message(new Message($"Unhandled exception processing syntax tree. {ex.Message}", tree.FilePath, null, ex.StackTrace));
226+
ExtractionContext.Message(new Message($"Unhandled exception processing syntax tree. {ex.Message}", tree.FilePath, null, ex.StackTrace));
217227
}
218228
}
219229

220230
private void DoAnalyseCompilation()
221231
{
222232
try
223233
{
224-
var assemblyPath = extractor.OutputPath;
234+
var assemblyPath = ExtractionContext.OutputPath;
225235
var stopwatch = new Stopwatch();
226236
stopwatch.Start();
227237
var currentTaskId = IncrementTaskCount();
@@ -231,11 +241,11 @@ private void DoAnalyseCompilation()
231241
var assembly = compilation.Assembly;
232242
var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
233243
compilationTrapFile = trapWriter; // Dispose later
234-
var cx = new Context(extractor, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
244+
var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
235245

236246
compilationEntity = Entities.Compilation.Create(cx);
237247

238-
extractor.CompilationInfos.ForEach(ci => trapWriter.Writer.compilation_info(compilationEntity, ci.key, ci.value));
248+
ExtractionContext.CompilationInfos.ForEach(ci => trapWriter.Writer.compilation_info(compilationEntity, ci.key, ci.value));
239249

240250
ReportProgressTaskDone(currentTaskId, assemblyPath, trapWriter.TrapFile, stopwatch.Elapsed, AnalysisAction.Extracted);
241251
}
@@ -318,7 +328,7 @@ public virtual void Dispose()
318328
/// <summary>
319329
/// Number of errors encountered during extraction.
320330
/// </summary>
321-
private int ExtractorErrors => extractor?.Errors ?? 0;
331+
private int ExtractorErrors => ExtractionContext?.Errors ?? 0;
322332

323333
/// <summary>
324334
/// Number of errors encountered by the compiler.
@@ -333,11 +343,26 @@ public virtual void Dispose()
333343
/// <summary>
334344
/// Logs information about the extractor.
335345
/// </summary>
336-
public void LogExtractorInfo(string extractorVersion)
346+
public void LogExtractorInfo()
337347
{
338348
Logger.Log(Severity.Info, " Extractor: {0}", Environment.GetCommandLineArgs().First());
339-
Logger.Log(Severity.Info, " Extractor version: {0}", extractorVersion);
349+
Logger.Log(Severity.Info, " Extractor version: {0}", Version);
340350
Logger.Log(Severity.Info, " Current working directory: {0}", Directory.GetCurrentDirectory());
341351
}
352+
353+
private static string Version
354+
{
355+
get
356+
{
357+
// the attribute for the git information are always attached to the entry assembly by our build system
358+
var assembly = Assembly.GetEntryAssembly();
359+
var versionString = assembly?.GetCustomAttribute<AssemblyInformationalVersionAttribute>();
360+
if (versionString == null)
361+
{
362+
return "unknown (not built from internal bazel workspace)";
363+
}
364+
return versionString.InformationalVersion;
365+
}
366+
}
342367
}
343368
}

0 commit comments

Comments
 (0)