Merge branch 'main' into shati-patel/query-history-get-id

Author: shati-patel

.github/workflows/main.yml

@@ -139,7 +139,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        version: ['v2.6.3', 'v2.7.6', 'v2.8.5', 'v2.9.4', 'v2.10.5', 'v2.11.1', 'nightly']
+        version: ['v2.7.6', 'v2.8.5', 'v2.9.4', 'v2.10.5', 'v2.11.1', 'nightly']
     env:
       CLI_VERSION: ${{ matrix.version }}
       NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }}

extensions/ql-vscode/CHANGELOG.md

@@ -1,5 +1,7 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## [UNRELEASED]
+
 ## 1.7.2 - 14 October 2022
 
 - Fix a bug where results created in older versions were thought to be unsuccessful. [#1605](https://github.com/github/vscode-codeql/pull/1605)

extensions/ql-vscode/package-lock.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.7.2",
+  "version": "1.7.3",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -1230,6 +1230,7 @@
     "minimist": "~1.2.6",
     "nanoid": "^3.2.0",
     "node-fetch": "~2.6.7",
+    "p-queue": "^6.0.0",
     "path-browserify": "^1.0.1",
     "react": "^17.0.2",
     "react-dom": "^17.0.2",

extensions/ql-vscode/package.json

@@ -951,7 +951,7 @@ async function activateWithInstalledDistribution(
       variantAnalysisSummary: VariantAnalysisApiResponse,
       token: CancellationToken
     ) => {
-      await variantAnalysisManager.autoDownloadVariantAnalysisResult(scannedRepo, variantAnalysisSummary, token);
+      await variantAnalysisManager.enqueueDownload(scannedRepo, variantAnalysisSummary, token);
     })
   );
 

extensions/ql-vscode/src/extension.ts

@@ -22,6 +22,7 @@ import { VariantAnalysisResultsManager } from './variant-analysis-results-manage
 import { CodeQLCliServer } from '../cli';
 import { getControllerRepo } from './run-remote-query';
 import { processUpdatedVariantAnalysis } from './variant-analysis-processor';
+import PQueue from 'p-queue';
 
 export class VariantAnalysisManager extends DisposableObject implements VariantAnalysisViewManager<VariantAnalysisView> {
   private readonly _onVariantAnalysisAdded = this.push(new EventEmitter<VariantAnalysis>());
@@ -31,6 +32,8 @@ export class VariantAnalysisManager extends DisposableObject implements VariantA
   private readonly variantAnalysisResultsManager: VariantAnalysisResultsManager;
   private readonly variantAnalyses = new Map<number, VariantAnalysis>();
   private readonly views = new Map<number, VariantAnalysisView>();
+  private static readonly maxConcurrentDownloads = 3;
+  private readonly queue = new PQueue({ concurrency: VariantAnalysisManager.maxConcurrentDownloads });
 
   constructor(
     private readonly ctx: ExtensionContext,
@@ -39,7 +42,7 @@ export class VariantAnalysisManager extends DisposableObject implements VariantA
     logger: Logger,
   ) {
     super();
-    this.variantAnalysisMonitor = this.push(new VariantAnalysisMonitor(ctx, logger));
+    this.variantAnalysisMonitor = this.push(new VariantAnalysisMonitor(ctx));
     this.variantAnalysisMonitor.onVariantAnalysisChange(this.onVariantAnalysisUpdated.bind(this));
 
     this.variantAnalysisResultsManager = this.push(new VariantAnalysisResultsManager(cliServer, storagePath, logger));
@@ -161,6 +164,18 @@ export class VariantAnalysisManager extends DisposableObject implements VariantA
     await this.onRepoStateUpdated(variantAnalysisSummary.id, repoState);
   }
 
+  public async enqueueDownload(
+    scannedRepo: ApiVariantAnalysisScannedRepository,
+    variantAnalysisSummary: VariantAnalysisApiResponse,
+    token: CancellationToken
+  ): Promise<void> {
+    await this.queue.add(() => this.autoDownloadVariantAnalysisResult(scannedRepo, variantAnalysisSummary, token));
+  }
+
+  public downloadsQueueSize(): number {
+    return this.queue.pending;
+  }
+
   public async promptOpenVariantAnalysis() {
     const credentials = await Credentials.initialize(this.ctx);
     if (!credentials) { throw Error('Error authenticating with GitHub'); }

extensions/ql-vscode/src/remote-queries/variant-analysis-manager.ts

@@ -1,11 +1,11 @@
 import { ExtensionContext, CancellationToken, commands, EventEmitter } from 'vscode';
 import { Credentials } from '../authentication';
-import { Logger } from '../logging';
 import * as ghApiClient from './gh-api/gh-api-client';
 
 import { VariantAnalysis, VariantAnalysisStatus } from './shared/variant-analysis';
 import {
-  VariantAnalysis as VariantAnalysisApiResponse
+  VariantAnalysis as VariantAnalysisApiResponse,
+  VariantAnalysisScannedRepository
 } from './gh-api/variant-analysis';
 import { VariantAnalysisMonitorResult } from './shared/variant-analysis-monitor-result';
 import { processFailureReason, processUpdatedVariantAnalysis } from './variant-analysis-processor';
@@ -22,7 +22,6 @@ export class VariantAnalysisMonitor extends DisposableObject {
 
   constructor(
     private readonly extensionContext: ExtensionContext,
-    private readonly logger: Logger
   ) {
     super();
   }
@@ -73,16 +72,8 @@ export class VariantAnalysisMonitor extends DisposableObject {
 
       this._onVariantAnalysisChange.fire(variantAnalysis);
 
-      void this.logger.log('****** Retrieved variant analysis' + JSON.stringify(variantAnalysisSummary));
-
-      if (variantAnalysisSummary.scanned_repositories) {
-        variantAnalysisSummary.scanned_repositories.forEach(scannedRepo => {
-          if (!scannedReposDownloaded.includes(scannedRepo.repository.id) && scannedRepo.analysis_status === 'succeeded') {
-            void commands.executeCommand('codeQL.autoDownloadVariantAnalysisResult', scannedRepo, variantAnalysisSummary);
-            scannedReposDownloaded.push(scannedRepo.repository.id);
-          }
-        });
-      }
+      const downloadedRepos = this.downloadVariantAnalysisResults(variantAnalysisSummary, scannedReposDownloaded);
+      scannedReposDownloaded.push(...downloadedRepos);
 
       if (variantAnalysisSummary.status === 'completed') {
         break;
@@ -94,6 +85,46 @@ export class VariantAnalysisMonitor extends DisposableObject {
     return { status: 'CompletedSuccessfully', scannedReposDownloaded: scannedReposDownloaded };
   }
 
+  private scheduleForDownload(
+    scannedRepo: VariantAnalysisScannedRepository,
+    variantAnalysisSummary: VariantAnalysisApiResponse
+  ) {
+    void commands.executeCommand('codeQL.autoDownloadVariantAnalysisResult', scannedRepo, variantAnalysisSummary);
+  }
+
+  private shouldDownload(
+    scannedRepo: VariantAnalysisScannedRepository,
+    alreadyDownloaded: number[]
+  ): boolean {
+    return !alreadyDownloaded.includes(scannedRepo.repository.id) && scannedRepo.analysis_status === 'succeeded';
+  }
+
+  private getReposToDownload(
+    variantAnalysisSummary: VariantAnalysisApiResponse,
+    alreadyDownloaded: number[]
+  ): VariantAnalysisScannedRepository[] {
+    if (variantAnalysisSummary.scanned_repositories) {
+      return variantAnalysisSummary.scanned_repositories.filter(scannedRepo => this.shouldDownload(scannedRepo, alreadyDownloaded));
+    } else {
+      return [];
+    }
+  }
+
+  private downloadVariantAnalysisResults(
+    variantAnalysisSummary: VariantAnalysisApiResponse,
+    scannedReposDownloaded: number[]
+  ): number[] {
+    const repoResultsToDownload = this.getReposToDownload(variantAnalysisSummary, scannedReposDownloaded);
+    const downloadedRepos: number[] = [];
+
+    repoResultsToDownload.forEach(scannedRepo => {
+      downloadedRepos.push(scannedRepo.repository.id);
+      this.scheduleForDownload(scannedRepo, variantAnalysisSummary);
+    });
+
+    return downloadedRepos;
+  }
+
   private async sleep(ms: number) {
     return new Promise(resolve => setTimeout(resolve, ms));
   }

extensions/ql-vscode/src/remote-queries/variant-analysis-monitor.ts

@@ -149,6 +149,17 @@ describe('Variant Analysis Manager', async function() {
 
         expect(getVariantAnalysisRepoResultStub.calledOnce).to.be.true;
       });
+
+      it('should pop download tasks off the queue', async () => {
+        const getResultsSpy = sandbox.spy(variantAnalysisManager, 'autoDownloadVariantAnalysisResult');
+
+        await variantAnalysisManager.enqueueDownload(scannedRepos[0], variantAnalysis, cancellationTokenSource.token);
+        await variantAnalysisManager.enqueueDownload(scannedRepos[1], variantAnalysis, cancellationTokenSource.token);
+        await variantAnalysisManager.enqueueDownload(scannedRepos[2], variantAnalysis, cancellationTokenSource.token);
+
+        expect(variantAnalysisManager.downloadsQueueSize()).to.equal(0);
+        expect(getResultsSpy).to.have.been.calledThrice;
+      });
     });
   });
 });