Copy widget scripts to extension folder (#11082)

* Copy widget scripts to extension folder * Fix test
Don Jayamanne (DonJayamanne) · Apr 14, 2020 · Apr 10, 2020 · Apr 10, 2020 · Apr 10, 2020 · Apr 10, 2020
commit f75b565b14c93f75033f42f39c50486df6d70bad
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -3007,6 +3007,7 @@
         "request": "^2.87.0",
         "request-progress": "^3.0.0",
         "rxjs": "^5.5.9",
+        "sanitize-filename": "^1.6.3",
         "semver": "^5.5.0",
         "stack-trace": "0.0.10",
         "string-argv": "^0.3.1",
@@ -3207,7 +3208,6 @@
         "requirejs": "^2.3.6",
         "rewiremock": "^3.13.0",
         "rimraf": "^3.0.2",
-        "sanitize-filename": "^1.6.3",
         "sass-loader": "^7.1.0",
         "serialize-javascript": "^2.1.2",
         "shortid": "^2.2.8",

diff --git a/src/client/common/application/webPanels/webPanel.ts b/src/client/common/application/webPanels/webPanel.ts
@@ -28,11 +28,12 @@ export class WebPanel implements IWebPanel {
         private disposableRegistry: IDisposableRegistry,
         private port: number | undefined,
         private token: string | undefined,
-        private options: IWebPanelOptions
+        private options: IWebPanelOptions,
+        additionalRootPaths: Uri[] = []
     ) {
         const webViewOptions: WebviewOptions = {
             enableScripts: true,
-            localResourceRoots: [Uri.file(this.options.rootPath), Uri.file(this.options.cwd)],
+            localResourceRoots: [Uri.file(this.options.rootPath), Uri.file(this.options.cwd), ...additionalRootPaths],
             portMapping: port ? [{ webviewPort: RemappedPort, extensionHostPort: port }] : undefined
         };
         if (options.webViewPanel) {

diff --git a/src/client/common/application/webPanels/webPanelProvider.ts b/src/client/common/application/webPanels/webPanelProvider.ts
@@ -2,11 +2,12 @@
 // Licensed under the MIT License.
 'use strict';
 import { inject, injectable } from 'inversify';
+import * as path from 'path';
 import * as portfinder from 'portfinder';
 import * as uuid from 'uuid/v4';
-
+import { Uri } from 'vscode';
 import { IFileSystem } from '../../platform/types';
-import { IDisposableRegistry } from '../../types';
+import { IDisposableRegistry, IExtensionContext } from '../../types';
 import { IWebPanel, IWebPanelOptions, IWebPanelProvider } from '../types';
 import { WebPanel } from './webPanel';
 
@@ -16,17 +17,27 @@ export class WebPanelProvider implements IWebPanelProvider {
     private token: string | undefined;
 
     constructor(
-        @inject(IDisposableRegistry) private disposableRegistry: IDisposableRegistry,
-        @inject(IFileSystem) private fs: IFileSystem
+        @inject(IDisposableRegistry) private readonly disposableRegistry: IDisposableRegistry,
+        @inject(IFileSystem) private readonly fs: IFileSystem,
+        @inject(IExtensionContext) private readonly context: IExtensionContext
     ) {}
 
     // tslint:disable-next-line:no-any
     public async create(options: IWebPanelOptions): Promise<IWebPanel> {
         const serverData = options.startHttpServer
             ? await this.ensureServerIsRunning()
             : { port: undefined, token: undefined };
-
-        return new WebPanel(this.fs, this.disposableRegistry, serverData.port, serverData.token, options);
+        // Allow loading resources from the `<extension folder>/tmp` folder when in webiviews.
+        // Used by widgets to place files that are not otherwise accessible.
+        const additionalRootPaths = [Uri.file(path.join(this.context.extensionPath, 'tmp'))];
+        return new WebPanel(
+            this.fs,
+            this.disposableRegistry,
+            serverData.port,
+            serverData.token,
+            options,
+            additionalRootPaths
+        );
     }
 
     private async ensureServerIsRunning(): Promise<{ port: number; token: string }> {

diff --git a/src/client/datascience/ipywidgets/ipyWidgetScriptSource.ts b/src/client/datascience/ipywidgets/ipyWidgetScriptSource.ts
@@ -4,14 +4,22 @@
 'use strict';
 import type * as jupyterlabService from '@jupyterlab/services';
 import type * as serialize from '@jupyterlab/services/lib/kernel/serialize';
+import { sha256 } from 'hash.js';
 import { inject, injectable } from 'inversify';
 import { IDisposable } from 'monaco-editor';
+import * as path from 'path';
 import { Event, EventEmitter, Uri } from 'vscode';
 import type { Data as WebSocketData } from 'ws';
 import { IApplicationShell, IWorkspaceService } from '../../common/application/types';
-import { traceError } from '../../common/logger';
+import { traceError, traceInfo } from '../../common/logger';
 import { IFileSystem } from '../../common/platform/types';
-import { IConfigurationService, IDisposableRegistry, IHttpClient, IPersistentStateFactory } from '../../common/types';
+import {
+    IConfigurationService,
+    IDisposableRegistry,
+    IExtensionContext,
+    IHttpClient,
+    IPersistentStateFactory
+} from '../../common/types';
 import { createDeferred, Deferred } from '../../common/utils/async';
 import { IInterpreterService, PythonInterpreter } from '../../interpreter/contracts';
 import { sendTelemetryEvent } from '../../telemetry';
@@ -30,6 +38,8 @@ import {
 } from '../types';
 import { IPyWidgetScriptSourceProvider } from './ipyWidgetScriptSourceProvider';
 import { WidgetScriptSource } from './types';
+// tslint:disable: no-var-requires no-require-imports
+const sanitize = require('sanitize-filename');
 
 @injectable()
 export class IPyWidgetScriptSource implements IInteractiveWindowListener, ILocalResourceUriConverter {
@@ -41,6 +51,14 @@ export class IPyWidgetScriptSource implements IInteractiveWindowListener, ILocal
     public get postInternalMessage(): Event<{ message: string; payload: any }> {
         return this.postInternalMessageEmitter.event;
     }
+    private get deserialize(): typeof serialize.deserialize {
+        if (!this.jupyterSerialize) {
+            // tslint:disable-next-line: no-require-imports
+            this.jupyterSerialize = require('@jupyterlab/services/lib/kernel/serialize') as typeof serialize;
+        }
+        return this.jupyterSerialize.deserialize;
+    }
+    private readonly resourcesMappedToExtensionFolder = new Map<string, Promise<Uri>>();
     private notebookIdentity?: Uri;
     private postEmitter = new EventEmitter<{
         message: string;
@@ -64,14 +82,9 @@ export class IPyWidgetScriptSource implements IInteractiveWindowListener, ILocal
      */
     private pendingModuleRequests = new Map<string, string>();
     private jupyterSerialize?: typeof serialize;
-    private get deserialize(): typeof serialize.deserialize {
-        if (!this.jupyterSerialize) {
-            // tslint:disable-next-line: no-require-imports
-            this.jupyterSerialize = require('@jupyterlab/services/lib/kernel/serialize') as typeof serialize;
-        }
-        return this.jupyterSerialize.deserialize;
-    }
     private readonly uriConversionPromises = new Map<string, Deferred<Uri>>();
+    private readonly targetWidgetScriptsFolder: string;
+    private readonly createTargetWidgetScriptsFolder: Promise<string>;
     constructor(
         @inject(IDisposableRegistry) disposables: IDisposableRegistry,
         @inject(INotebookProvider) private readonly notebookProvider: INotebookProvider,
@@ -81,8 +94,18 @@ export class IPyWidgetScriptSource implements IInteractiveWindowListener, ILocal
         @inject(IHttpClient) private readonly httpClient: IHttpClient,
         @inject(IApplicationShell) private readonly appShell: IApplicationShell,
         @inject(IWorkspaceService) private readonly workspaceService: IWorkspaceService,
-        @inject(IPersistentStateFactory) private readonly stateFactory: IPersistentStateFactory
+        @inject(IPersistentStateFactory) private readonly stateFactory: IPersistentStateFactory,
+        @inject(IExtensionContext) extensionContext: IExtensionContext
     ) {
+        this.targetWidgetScriptsFolder = path.join(extensionContext.extensionPath, 'tmp', 'nbextensions');
+        this.createTargetWidgetScriptsFolder = this.fs
+            .directoryExists(this.targetWidgetScriptsFolder)
+            .then(async (exists) => {
+                if (!exists) {
+                    await this.fs.createDirectory(this.targetWidgetScriptsFolder);
+                }
+                return this.targetWidgetScriptsFolder;
+            });
         disposables.push(this);
         this.notebookProvider.onNotebookCreated(
             (e) => {
@@ -94,7 +117,39 @@ export class IPyWidgetScriptSource implements IInteractiveWindowListener, ILocal
             this.disposables
         );
     }
-    public asWebviewUri(localResource: Uri): Promise<Uri> {
+    /**
+     * This method is called to convert a Uri to a format such that it can be used in a webview.
+     * WebViews only allow files that are part of extension and the same directory where notebook lives.
+     * To ensure widgets can find the js files, we copy the script file to a into the extensionr folder  `tmp/nbextensions`.
+     * (storing files in `tmp/nbextensions` is relatively safe as this folder gets deleted when ever a user updates to a new version of VSC).
+     * Hence we need to copy for every version of the extension.
+     * Copying into global workspace folder would also work, but over time this folder size could grow (in an unmanaged way).
+     */
+    public async asWebviewUri(localResource: Uri): Promise<Uri> {
+        if (this.notebookIdentity && !this.resourcesMappedToExtensionFolder.has(localResource.fsPath)) {
+            const deferred = createDeferred<Uri>();
+            this.resourcesMappedToExtensionFolder.set(localResource.fsPath, deferred.promise);
+            try {
+                // Create a file name such that it will be unique and consistent across VSC reloads.
+                // Only if original file has been modified should we create a new copy of the sam file.
+                const fileHash: string = await this.fs.getFileHash(localResource.fsPath);
+                const uniqueFileName = sanitize(sha256().update(`${localResource.fsPath}${fileHash}`).digest('hex'));
+                const targetFolder = await this.createTargetWidgetScriptsFolder;
+                const mappedResource = Uri.file(
+                    path.join(targetFolder, `${uniqueFileName}${path.basename(localResource.fsPath)}`)
+                );
+                if (!(await this.fs.fileExists(mappedResource.fsPath))) {
+                    await this.fs.copyFile(localResource.fsPath, mappedResource.fsPath);
+                }
+                traceInfo(`Widget Script file ${localResource.fsPath} mapped to ${mappedResource.fsPath}`);
+                deferred.resolve(mappedResource);
+            } catch (ex) {
+                traceError(`Failed to map widget Script file ${localResource.fsPath}`);
+                deferred.reject(ex);
+            }
+        }
+        localResource = await this.resourcesMappedToExtensionFolder.get(localResource.fsPath)!;
+
         const key = localResource.toString();
         if (!this.uriConversionPromises.has(key)) {
             this.uriConversionPromises.set(key, createDeferred<Uri>());

diff --git a/src/client/datascience/ipywidgets/localWidgetScriptSourceProvider.ts b/src/client/datascience/ipywidgets/localWidgetScriptSourceProvider.ts
@@ -68,8 +68,7 @@ export class LocalWidgetScriptSourceProvider implements IWidgetScriptSourceProvi
             const parts = file.split(path.sep);
             const moduleName = parts[0];
 
-            // Drop the `.js`.
-            const fileUri = Uri.file(path.join(nbextensionsPath, moduleName, 'index'));
+            const fileUri = Uri.file(path.join(nbextensionsPath, file));
             const scriptUri = (await this.localResourceUriConverter.asWebviewUri(fileUri)).toString();
             // tslint:disable-next-line: no-unnecessary-local-variable
             const widgetScriptSource: WidgetScriptSource = { moduleName, scriptUri, source: 'local' };

diff --git a/src/client/datascience/ipywidgets/remoteWidgetScriptSourceProvider.ts b/src/client/datascience/ipywidgets/remoteWidgetScriptSourceProvider.ts
@@ -19,8 +19,8 @@ export class RemoteWidgetScriptSourceProvider implements IWidgetScriptSourceProv
         // Noop.
     }
     public async getWidgetScriptSource(moduleName: string, moduleVersion: string): Promise<WidgetScriptSource> {
-        const scriptUri = `${this.connection.baseUrl}nbextensions/${moduleName}/index`;
-        const exists = await this.getUrlForWidget(`${scriptUri}.js`);
+        const scriptUri = `${this.connection.baseUrl}nbextensions/${moduleName}/index.js`;
+        const exists = await this.getUrlForWidget(scriptUri);
         if (exists) {
             return { moduleName, scriptUri, source: 'cdn' };
         }

diff --git a/src/datascience-ui/ipywidgets/requirejsRegistry.ts b/src/datascience-ui/ipywidgets/requirejsRegistry.ts
@@ -55,8 +55,12 @@ function registerScriptsInRequireJs(scripts: NonPartial<WidgetScriptSource>[]) {
     };
     scripts.forEach((script) => {
         scriptsAlreadyRegisteredInRequireJs.set(script.moduleName, script.scriptUri);
+        // Drop the `.js` from the scriptUri.
+        const scriptUri = script.scriptUri.toLowerCase().endsWith('.js')
+            ? script.scriptUri.substring(0, script.scriptUri.length - 3)
+            : script.scriptUri;
         // Register the script source into requirejs so it gets loaded via requirejs.
-        config.paths[script.moduleName] = script.scriptUri;
+        config.paths[script.moduleName] = scriptUri;
     });
 
     requirejs.config(config);

diff --git a/src/ipywidgets/src/manager.ts b/src/ipywidgets/src/manager.ts
@@ -17,12 +17,7 @@ export const WIDGET_MIMETYPE = 'application/vnd.jupyter.widget-view+json';
 // Source borrowed from https://github.com/jupyter-widgets/ipywidgets/blob/master/examples/web3/src/manager.ts
 
 // These widgets can always be loaded from requirejs (as it is bundled).
-const widgetsRegisteredInRequireJs = [
-    '@jupyter-widgets/controls',
-    '@jupyter-widgets/base',
-    '@jupyter-widgets/output',
-    'azureml_widgets'
-];
+const widgetsRegisteredInRequireJs = ['@jupyter-widgets/controls', '@jupyter-widgets/base', '@jupyter-widgets/output'];
 
 export class WidgetManager extends jupyterlab.WidgetManager {
     public kernel: Kernel.IKernelConnection;

diff --git a/src/test/datascience/ipywidgets/localWidgetScriptSourceProvider.unit.test.ts b/src/test/datascience/ipywidgets/localWidgetScriptSourceProvider.unit.test.ts
@@ -121,7 +121,7 @@ suite('Data Science - ipywidget - Local Widget Script Source', () => {
         assert.deepEqual(value, {
             moduleName: 'widget2',
             source: 'local',
-            scriptUri: asVSCodeUri(Uri.file(path.join(searchDirectory, 'widget2', 'index')))
+            scriptUri: asVSCodeUri(Uri.file(path.join(searchDirectory, 'widget2', 'index.js')))
         });
         const value1 = await scriptSourceProvider.getWidgetScriptSource('widget2', '1');
         assert.deepEqual(value1, value);
@@ -149,7 +149,7 @@ suite('Data Science - ipywidget - Local Widget Script Source', () => {
         assert.deepEqual(value, {
             moduleName: 'widget1',
             source: 'local',
-            scriptUri: asVSCodeUri(Uri.file(path.join(searchDirectory, 'widget1', 'index')))
+            scriptUri: asVSCodeUri(Uri.file(path.join(searchDirectory, 'widget1', 'index.js')))
         });
 
         // Ensure we look for the right things in the right place.