C#: Push the feed fetching logic into DownloadPackages, use allfeeds instead of a specific nuget.config file, and supply a list of nuget sources when using the TryRestore.

Author: michaelnebel

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs

@@ -78,6 +78,12 @@ internal sealed partial class FeedManager : IDisposable
         /// </summary>
         public ImmutableHashSet<string> ReachableFeeds => lazyReachableFeeds.Value;
 
+        private readonly Lazy<ImmutableHashSet<string>> lazyReachableFallbackFeeds;
+        /// <summary>
+        /// Gets the list of reachable NuGet feeds that are configured as fallback feeds.
+        /// </summary>
+        public ImmutableHashSet<string> ReachableFallbackFeeds => lazyReachableFallbackFeeds.Value;
+
         public FeedManager(ILogger logger, IDotNet dotnet, DependabotProxy? dependabotProxy, FileProvider fileProvider)
         {
             this.logger = logger;
@@ -101,6 +107,11 @@ public FeedManager(ILogger logger, IDotNet dotnet, DependabotProxy? dependabotPr
                 CheckSpecifiedFeeds(InheritedFeeds, out var reachableInheritedFeeds);
                 return ReachableExplicitFeeds.Union(reachableInheritedFeeds).ToImmutableHashSet();
             });
+            lazyReachableFallbackFeeds = new Lazy<ImmutableHashSet<string>>(() =>
+            {
+                var reachableFallbackFeeds = GetReachableFallbackNugetFeeds();
+                return reachableFallbackFeeds.ToImmutableHashSet();
+            });
         }
 
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -4,7 +4,6 @@
 using System.Collections.Immutable;
 using System.IO;
 using System.Linq;
-using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
@@ -53,7 +52,9 @@ public NugetPackageRestorer(
 
         public string? TryRestore(string package)
         {
-            if (TryRestorePackageManually(package))
+            var feeds = feedManager.CheckNugetFeedResponsiveness ? feedManager.ReachableFeeds : feedManager.AllFeeds;
+            var nugetSources = feedManager.FeedsToDotnetRestoreArgument(feeds);
+            if (TryRestorePackageManually(package, nugetSources))
             {
                 var packageDir = DependencyManager.GetPackageDirectory(package, missingPackageDirectory.DirInfo);
                 if (packageDir is not null)
@@ -133,7 +134,7 @@ public HashSet<AssemblyLookupLocation> Restore()
                 {
                     // If we experience a timeout, we use this fallback.
                     // todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too.
-                    var unresponsiveMissingPackageLocation = DownloadMissingPackagesAndUseFallback([]);
+                    var unresponsiveMissingPackageLocation = DownloadMissingPackages([]);
                     return unresponsiveMissingPackageLocation is null
                         ? []
                         : [unresponsiveMissingPackageLocation];
@@ -195,9 +196,7 @@ public HashSet<AssemblyLookupLocation> Restore()
 
             var usedPackageNames = GetAllUsedPackageDirNames(dependencies);
 
-            var missingPackageLocation = feedManager.CheckNugetFeedResponsiveness
-                ? DownloadMissingPackagesAndUseFallback(usedPackageNames)
-                : DownloadMissingPackages(usedPackageNames);
+            var missingPackageLocation = DownloadMissingPackages(usedPackageNames);
 
             if (missingPackageLocation is not null)
             {
@@ -303,22 +302,28 @@ private void RestoreProjects(IEnumerable<string> projects, out ConcurrentBag<Dep
             compilationInfoContainer.CompilationInfos.Add(("Failed project restore with missing package error", nugetMissingPackageFailures.ToString()));
         }
 
-        private AssemblyLookupLocation? DownloadMissingPackagesAndUseFallback(IEnumerable<string> usedPackageNames)
+        private AssemblyLookupLocation? DownloadMissingPackages(IEnumerable<string> usedPackageNames)
         {
-            var reachableFallbackFeeds = feedManager.GetReachableFallbackNugetFeeds();
-            compilationInfoContainer.CompilationInfos.Add(("Reachable fallback NuGet feed count", reachableFallbackFeeds.Count.ToString()));
+            ImmutableHashSet<string> feeds;
+            if (feedManager.CheckNugetFeedResponsiveness)
+            {
+                // Attempt to get the fallback configuration.
+                var reachableFallbackFeeds = feedManager.ReachableFallbackFeeds;
+                compilationInfoContainer.CompilationInfos.Add(("Reachable fallback NuGet feed count", reachableFallbackFeeds.Count.ToString()));
 
-            if (reachableFallbackFeeds.Count > 0)
+                if (reachableFallbackFeeds.Count == 0)
+                {
+                    logger.LogWarning("Skipping download of missing packages from specific feeds as no fallback NuGet feeds are reachable.");
+                    return null;
+                }
+                feeds = reachableFallbackFeeds;
+            }
+            else
             {
-                return DownloadMissingPackages(usedPackageNames, fallbackNugetFeeds: reachableFallbackFeeds);
+                feeds = feedManager.AllFeeds;
             }
 
-            logger.LogWarning("Skipping download of missing packages from specific feeds as no fallback NuGet feeds are reachable.");
-            return null;
-        }
-
-        private AssemblyLookupLocation? DownloadMissingPackages(IEnumerable<string> usedPackageNames, IEnumerable<string>? fallbackNugetFeeds = null)
-        {
+            var nugetSources = feedManager.FeedsToDotnetRestoreArgument(feeds);
             var alreadyDownloadedPackages = usedPackageNames.Select(p => p.ToLowerInvariant());
             var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
 
@@ -351,26 +356,14 @@ private void RestoreProjects(IEnumerable<string> projects, out ConcurrentBag<Dep
 
             logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
 
-            IEnumerable<string> feeds = [];
-            if (fallbackNugetFeeds is not null)
-            {
-                feeds = fallbackNugetFeeds;
-            }
-            else if (GetNugetConfig() is string config)
-            {
-                feeds = feedManager.FeedsToUseFromConfig(config);
-            }
-
-            var nugetSources = feedManager.FeedsToDotnetRestoreArgument(feeds);
-
             compilationInfoContainer.CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString()));
 
             var successCount = 0;
             var sync = new Lock();
 
             Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = DependencyManager.Threads }, package =>
             {
-                var success = TryRestorePackageManually(package.Name, nugetSources, package.PackageReferenceSource, tryWithoutNugetConfig: fallbackNugetFeeds is null);
+                var success = TryRestorePackageManually(package.Name, nugetSources, package.PackageReferenceSource);
                 if (!success)
                 {
                     return;
@@ -387,32 +380,6 @@ private void RestoreProjects(IEnumerable<string> projects, out ConcurrentBag<Dep
             return missingPackageDirectory.DirInfo.FullName;
         }
 
-        private string? GetNugetConfig()
-        {
-            var nugetConfigs = fileProvider.NugetConfigs;
-            string? nugetConfig;
-            if (nugetConfigs.Count > 1)
-            {
-                logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
-                nugetConfig = fileProvider.RootNugetConfig;
-                if (nugetConfig == null)
-                {
-                    logger.LogInfo("Could not find a top-level nuget.config file.");
-                }
-            }
-            else
-            {
-                nugetConfig = nugetConfigs.FirstOrDefault();
-            }
-
-            if (nugetConfig != null)
-            {
-                logger.LogInfo($"Using nuget.config file {nugetConfig}.");
-            }
-
-            return nugetConfig;
-        }
-
         private IEnumerable<string> GetAllUsedPackageDirNames(DependencyContainer dependencies)
         {
             var allPackageDirectories = GetAllPackageDirectories();
@@ -476,8 +443,7 @@ private static IEnumerable<string> GetRestoredPackageDirectoryNames(DirectoryInf
                 .Select(d => Path.GetFileName(d).ToLowerInvariant());
         }
 
-        private bool TryRestorePackageManually(string package, string? nugetSources = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj,
-            bool tryWithoutNugetConfig = true, bool tryPrereleaseVersion = true)
+        private bool TryRestorePackageManually(string package, string? nugetSources, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj, bool tryPrereleaseVersion = true)
         {
             logger.LogInfo($"Restoring package {package}...");
             using var tempDir = new TemporaryDirectory(
@@ -505,7 +471,7 @@ private bool TryRestorePackageManually(string package, string? nugetSources = nu
                 return true;
             }
 
-            if (tryWithoutNugetConfig && res.HasNugetPackageSourceError && nugetSources is not null && !feedManager.CheckNugetFeedResponsiveness)
+            if (!feedManager.CheckNugetFeedResponsiveness && res.HasNugetPackageSourceError && nugetSources is not null)
             {
                 logger.LogDebug($"Trying to restore '{package}' without nuget.config.");
                 // Restore could not be completed because the listed source is unavailable. Try without the nuget.config: