Skip to content

Commit 239776b

Browse files
tamasvajktamasvajk
authored
Merge pull request #16124 from tamasvajk/buildless/nuget-feed-precheck
C#: Validate all nuget feeds to respond in reasonable time
2 parents 5f8eb7b + 80995ec commit 239776b

File tree

18 files changed

+726
-362
lines changed

18 files changed

+726
-362
lines changed

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs

Lines changed: 514 additions & 0 deletions
Large diffs are not rendered by default.

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

Lines changed: 23 additions & 353 deletions
Large diffs are not rendered by default.

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
1616
public partial class DotNet : IDotNet
1717
{
1818
private readonly IDotNetCliInvoker dotnetCliInvoker;
19+
private readonly ILogger logger;
1920
private readonly TemporaryDirectory? tempWorkingDirectory;
2021

2122
private DotNet(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, TemporaryDirectory? tempWorkingDirectory = null)
2223
{
2324
this.tempWorkingDirectory = tempWorkingDirectory;
2425
this.dotnetCliInvoker = dotnetCliInvoker;
26+
this.logger = logger;
2527
Info();
2628
}
2729

@@ -89,17 +91,18 @@ public bool AddPackage(string folder, string package)
8991
return dotnetCliInvoker.RunCommand(args);
9092
}
9193

92-
public IList<string> GetListedRuntimes() => GetListed("--list-runtimes");
94+
public IList<string> GetListedRuntimes() => GetResultList("--list-runtimes");
9395

94-
public IList<string> GetListedSdks() => GetListed("--list-sdks");
96+
public IList<string> GetListedSdks() => GetResultList("--list-sdks");
9597

96-
private IList<string> GetListed(string args)
98+
private IList<string> GetResultList(string args)
9799
{
98-
if (dotnetCliInvoker.RunCommand(args, out var artifacts))
100+
if (dotnetCliInvoker.RunCommand(args, out var results))
99101
{
100-
return artifacts;
102+
return results;
101103
}
102-
return new List<string>();
104+
logger.LogWarning($"Running 'dotnet {args}' failed.");
105+
return [];
103106
}
104107

105108
public bool Exec(string execArgs)
@@ -108,6 +111,8 @@ public bool Exec(string execArgs)
108111
return dotnetCliInvoker.RunCommand(args);
109112
}
110113

114+
public IList<string> GetNugetFeeds(string nugetConfig) => GetResultList($"nuget list source --format Short --configfile \"{nugetConfig}\"");
115+
111116
// The version number should be kept in sync with the version .NET version used for building the application.
112117
public const string LatestDotNetSdkVersion = "8.0.101";
113118

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,30 @@ internal class EnvironmentVariableNames
1616
/// Controls whether to use framework dependencies from subfolders.
1717
/// </summary>
1818
public const string DotnetFrameworkReferencesUseSubfolders = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES_USE_SUBFOLDERS";
19+
20+
/// <summary>
21+
/// Controls whether to check the responsiveness of NuGet feeds.
22+
/// </summary>
23+
public const string CheckNugetFeedResponsiveness = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK";
24+
25+
/// <summary>
26+
/// Specifies the NuGet feeds to exclude from the responsiveness check. The value is a space-separated list of feed URLs.
27+
/// </summary>
28+
public const string ExcludedNugetFeedsFromResponsivenessCheck = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_EXCLUDED";
29+
30+
/// <summary>
31+
/// Specifies the timeout (as an integer) in milliseconds for the initial check of NuGet feeds responsiveness. The value is then doubled for each subsequent check.
32+
/// </summary>
33+
public const string NugetFeedResponsivenessInitialTimeout = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT";
34+
35+
/// <summary>
36+
/// Specifies how many requests to make to the NuGet feed to check its responsiveness.
37+
/// </summary>
38+
public const string NugetFeedResponsivenessRequestCount = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT";
39+
40+
/// <summary>
41+
/// Specifies the location of the diagnostic directory.
42+
/// </summary>
43+
public const string DiagnosticDir = "CODEQL_EXTRACTOR_CSHARP_DIAGNOSTIC_DIR";
1944
}
2045
}

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ public interface IDotNet
1313
IList<string> GetListedRuntimes();
1414
IList<string> GetListedSdks();
1515
bool Exec(string execArgs);
16+
IList<string> GetNugetFeeds(string nugetConfig);
1617
}
1718

1819
public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);

‎csharp/extractor/Semmle.Extraction.Tests/Runtime.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ public DotNetStub(IList<string> runtimes, IList<string> sdks)
2626
public IList<string> GetListedSdks() => sdks;
2727

2828
public bool Exec(string execArgs) => true;
29+
30+
public IList<string> GetNugetFeeds(string nugetConfig) => [];
2931
}
3032

3133
public class RuntimeTests

‎csharp/extractor/Semmle.Util/EnvironmentVariables.cs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,5 +27,12 @@ public static int GetDefaultNumberOfThreads()
2727
}
2828
return threads;
2929
}
30+
31+
public static bool GetBoolean(string name)
32+
{
33+
var env = Environment.GetEnvironmentVariable(name);
34+
var _ = bool.TryParse(env, out var value);
35+
return value;
36+
}
3037
}
3138
}

‎csharp/extractor/Semmle.Util/FileUtils.cs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,8 +102,7 @@ public static string ComputeFileHash(string filePath)
102102
private static async Task DownloadFileAsync(string address, string filename)
103103
{
104104
using var httpClient = new HttpClient();
105-
using var request = new HttpRequestMessage(HttpMethod.Get, address);
106-
using var contentStream = await (await httpClient.SendAsync(request)).Content.ReadAsStreamAsync();
105+
using var contentStream = await httpClient.GetStreamAsync(address);
107106
using var stream = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None, 4096, true);
108107
await contentStream.CopyToAsync(stream);
109108
}
@@ -112,7 +111,7 @@ private static async Task DownloadFileAsync(string address, string filename)
112111
/// Downloads the file at <paramref name="address"/> to <paramref name="fileName"/>.
113112
/// </summary>
114113
public static void DownloadFile(string address, string fileName) =>
115-
DownloadFileAsync(address, fileName).Wait();
114+
DownloadFileAsync(address, fileName).GetAwaiter().GetResult();
116115

117116
public static string NestPaths(ILogger logger, string? outerpath, string innerpath)
118117
{

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/Assemblies.expected

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
| [...]/newtonsoft.json/13.0.3/lib/net6.0/Newtonsoft.Json.dll |

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/Assemblies.ql

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import csharp
2+
3+
private string getPath(Assembly a) {
4+
not a.getCompilation().getOutputAssembly() = a and
5+
exists(string s | s = a.getFile().getAbsolutePath() |
6+
result = "[...]/" + s.substring(s.indexOf("newtonsoft.json"), s.length())
7+
)
8+
}
9+
10+
from Assembly a
11+
select getPath(a)

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.expected

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
| All Nuget feeds reachable | 0.0 |
2+
| Fallback nuget restore | 1.0 |
3+
| Project files on filesystem | 1.0 |
4+
| Resolved assembly conflicts | 7.0 |
5+
| Restored .NET framework variants | 0.0 |
6+
| Solution files on filesystem | 1.0 |
7+
| Source files generated | 0.0 |
8+
| Source files on filesystem | 1.0 |
9+
| Successfully ran fallback nuget restore | 1.0 |
10+
| Unresolved references | 0.0 |
11+
| UseWPF set | 0.0 |
12+
| UseWindowsForms set | 0.0 |
13+
| WebView extraction enabled | 1.0 |

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.ql

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import csharp
2+
import semmle.code.csharp.commons.Diagnostics
3+
4+
query predicate compilationInfo(string key, float value) {
5+
key != "Resolved references" and
6+
not key.matches("Compiler diagnostic count for%") and
7+
exists(Compilation c, string infoKey, string infoValue | infoValue = c.getInfo(infoKey) |
8+
key = infoKey and
9+
value = infoValue.toFloat()
10+
or
11+
not exists(infoValue.toFloat()) and
12+
key = infoKey + ": " + infoValue and
13+
value = 1
14+
)
15+
}

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/diagnostics.expected

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
{
2+
"markdownMessage": "C# analysis with build-mode 'none' completed.",
3+
"severity": "unknown",
4+
"source": {
5+
"extractorName": "csharp",
6+
"id": "csharp/autobuilder/buildless/complete",
7+
"name": "C# analysis with build-mode 'none' completed"
8+
},
9+
"visibility": {
10+
"cliSummaryTable": true,
11+
"statusPage": false,
12+
"telemetry": true
13+
}
14+
}
15+
{
16+
"markdownMessage": "C# with build-mode set to 'none'. This means that all C# source in the working directory will be scanned, with build tools, such as Nuget and Dotnet CLIs, only contributing information about external dependencies.",
17+
"severity": "note",
18+
"source": {
19+
"extractorName": "csharp",
20+
"id": "csharp/autobuilder/buildless/mode-active",
21+
"name": "C# with build-mode set to 'none'"
22+
},
23+
"visibility": {
24+
"cliSummaryTable": true,
25+
"statusPage": true,
26+
"telemetry": true
27+
}
28+
}
29+
{
30+
"markdownMessage": "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
31+
"severity": "warning",
32+
"source": {
33+
"extractorName": "csharp",
34+
"id": "csharp/autobuilder/buildless/unreachable-feed",
35+
"name": "Found unreachable Nuget feed in C# analysis with build-mode 'none'"
36+
},
37+
"visibility": {
38+
"cliSummaryTable": true,
39+
"statusPage": true,
40+
"telemetry": true
41+
}
42+
}

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/Program.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
class Program
2+
{
3+
static void Main(string[] args)
4+
{
5+
}
6+
}

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/nuget.config

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<configuration>
3+
<packageSources>
4+
<clear />
5+
<add key="x" value="https://localhost:53/packages/" />
6+
<add key="y" value="https://localhost:80/packages/" />
7+
</packageSources>
8+
</configuration>

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/proj.csproj

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFrameworks>net8.0</TargetFrameworks>
6+
</PropertyGroup>
7+
8+
<Target Name="DeleteBinObjFolders" BeforeTargets="Clean">
9+
<RemoveDir Directories=".\bin" />
10+
<RemoveDir Directories=".\obj" />
11+
</Target>
12+
13+
<ItemGroup>
14+
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
15+
</ItemGroup>
16+
</Project>

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/standalone.sln

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
2+
Microsoft Visual Studio Solution File, Format Version 12.00
3+
# Visual Studio Version 17
4+
VisualStudioVersion = 17.5.002.0
5+
MinimumVisualStudioVersion = 10.0.40219.1
6+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "proj", "proj\proj.csproj", "{6ED00460-7666-4AE9-A405-4B6C8B02279A}"
7+
EndProject
8+
Global
9+
GlobalSection(SolutionConfigurationPlatforms) = preSolution
10+
Debug|Any CPU = Debug|Any CPU
11+
Release|Any CPU = Release|Any CPU
12+
EndGlobalSection
13+
GlobalSection(SolutionProperties) = preSolution
14+
HideSolutionNode = FALSE
15+
EndGlobalSection
16+
GlobalSection(ExtensibilityGlobals) = postSolution
17+
SolutionGuid = {4ED55A1C-066C-43DF-B32E-7EAA035985EE}
18+
EndGlobalSection
19+
EndGlobal

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/test.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
from create_database_utils import *
2+
from diagnostics_test_utils import *
3+
import os
4+
5+
os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK"] = "true" # Enable NuGet feed check
6+
os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT"] = "1" # 1ms, the GET request should fail with such short timeout
7+
os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT"] = "1" # Limit the count of checks to 1
8+
os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_EXCLUDED"] = "https://abc.de:8000/packages/" # Exclude this feed from check
9+
run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
10+
check_diagnostics()

0 commit comments

Comments
 (0)