Rich Chiodo (rchiodo) · May 14, 2020 · May 12, 2020 · May 12, 2020 · May 13, 2020 · May 13, 2020
diff --git a/build/ci/conda_base.yml b/build/ci/conda_base.yml
@@ -2,3 +2,4 @@ pandas
 jupyter
 numpy
 matplotlib
+pip
diff --git a/build/ci/conda_env_1.yml b/build/ci/conda_env_1.yml
@@ -5,3 +5,4 @@ dependencies:
   - jupyter
   - numpy
   - matplotlib
+  - pip
diff --git a/build/ci/conda_env_2.yml b/build/ci/conda_env_2.yml
@@ -5,3 +5,4 @@ dependencies:
   - jupyter
   - numpy
   - matplotlib
+  - pip
diff --git a/build/ci/templates/test_phases.yml b/build/ci/templates/test_phases.yml
@@ -161,7 +161,7 @@ steps:
   # Run the pip installs in the 3 environments (windows)
   - script: |
       call activate base
-      conda install --quiet -y --file build/ci/conda_base.yml
+      call conda install --quiet -y --file build/ci/conda_base.yml
       python -m pip install --upgrade -r build/conda-functional-requirements.txt
       call activate conda_env_1
       python -m pip install --upgrade -r build/conda-functional-requirements.txt

diff --git a/build/conda-functional-requirements.txt b/build/conda-functional-requirements.txt
@@ -8,7 +8,6 @@ black ; python_version>='3.6'
 yapf
 pylint
 pycodestyle
-prospector
 pydocstyle
 nose
 pytest==4.6.9 # Last version of pytest with Python 2.7 support

diff --git a/src/client/common/platform/fileSystem.ts b/src/client/common/platform/fileSystem.ts
@@ -577,8 +577,8 @@ export class FileSystem implements IFileSystem {
     public async search(globPattern: string, cwd?: string): Promise<string[]> {
         return this.utils.search(globPattern, cwd);
     }
-    public async createTemporaryFile(suffix: string): Promise<TemporaryFile> {
-        return this.utils.tmp.createFile(suffix);
+    public async createTemporaryFile(suffix: string, mode?: number): Promise<TemporaryFile> {
+        return this.utils.tmp.createFile(suffix, mode);
     }
     public async isDirReadonly(dirname: string): Promise<boolean> {
         return this.utils.isDirReadonly(dirname);

diff --git a/src/client/common/platform/fs-temp.ts b/src/client/common/platform/fs-temp.ts
@@ -30,9 +30,10 @@ export class TemporaryFileSystem implements ITempFileSystem {
     }
 
     // Create a new temp file with the given filename suffix.
-    public createFile(suffix: string): Promise<TemporaryFile> {
+    public createFile(suffix: string, mode?: number): Promise<TemporaryFile> {
         const opts = {
-            postfix: suffix
+            postfix: suffix,
+            mode
         };
         return new Promise<TemporaryFile>((resolve, reject) => {
             this.raw.file(opts, (err, filename, _fd, cleanUp) => {

diff --git a/src/client/common/platform/types.ts b/src/client/common/platform/types.ts
@@ -48,7 +48,7 @@ export type TemporaryFile = { filePath: string } & vscode.Disposable;
 export type TemporaryDirectory = { path: string } & vscode.Disposable;
 
 export interface ITempFileSystem {
-    createFile(suffix: string): Promise<TemporaryFile>;
+    createFile(suffix: string, mode?: number): Promise<TemporaryFile>;
 }
 
 //===========================
@@ -224,6 +224,6 @@ export interface IFileSystem {
     getFiles(rootDir: string): Promise<string[]>;
     getFileHash(filePath: string): Promise<string>;
     search(globPattern: string, cwd?: string): Promise<string[]>;
-    createTemporaryFile(extension: string): Promise<TemporaryFile>;
+    createTemporaryFile(extension: string, mode?: number): Promise<TemporaryFile>;
     isDirReadonly(dirname: string): Promise<boolean>;
 }
diff --git a/src/client/datascience/baseJupyterSession.ts b/src/client/datascience/baseJupyterSession.ts
@@ -358,7 +358,9 @@ export abstract class BaseJupyterSession implements IJupyterSession {
                 try {
                     suppressShutdownErrors(session.kernel);
                     // Shutdown may fail if the process has been killed
-                    await waitForPromise(session.shutdown(), 1000);
+                    if (!session.isDisposed) {
+                        await waitForPromise(session.shutdown(), 1000);
+                    }
                 } catch {
                     noop();
                 }

diff --git a/src/client/datascience/interactive-common/interactiveBase.ts b/src/client/datascience/interactive-common/interactiveBase.ts
@@ -1080,7 +1080,11 @@ export abstract class InteractiveBase extends WebViewHost<IInteractiveWindowMapp
             ]);
             try {
                 notebook = identity
-                    ? await this.notebookProvider.getOrCreateNotebook({ identity: identity.resource, metadata })
+                    ? await this.notebookProvider.getOrCreateNotebook({
+                          identity: identity.resource,
+                          resource,
+                          metadata
+                      })
                     : undefined;
                 if (notebook) {
                     const executionActivation = { ...identity, owningResource: resource };

diff --git a/src/client/datascience/interactive-common/notebookProvider.ts b/src/client/datascience/interactive-common/notebookProvider.ts
@@ -95,8 +95,8 @@ export class NotebookProvider implements INotebookProvider {
         }
 
         // Finally create if needed
-        let resource: Resource = options.identity;
-        if (options.identity.scheme === Identifiers.HistoryPurpose) {
+        let resource: Resource = options.resource;
+        if (options.identity.scheme === Identifiers.HistoryPurpose && !resource) {
             // If we have any workspaces, then use the first available workspace.
             // This is required, else using `undefined` as a resource when we have worksapce folders is a different meaning.
             // This means interactive window doesn't properly support mult-root workspaces as we pick first workspace.

diff --git a/src/client/datascience/interactive-window/interactiveWindow.ts b/src/client/datascience/interactive-window/interactiveWindow.ts
@@ -151,9 +151,6 @@ export class InteractiveWindow extends InteractiveBase implements IInteractiveWi
 
         // Send a telemetry event to indicate window is opening
         sendTelemetryEvent(Telemetry.OpenedInteractiveWindow);
-
-        // Start the server as soon as we open
-        this.ensureConnectionAndNotebook().ignoreErrors();
     }
 
     public dispose() {
@@ -170,6 +167,9 @@ export class InteractiveWindow extends InteractiveBase implements IInteractiveWi
     }
 
     public async show(): Promise<void> {
+        // Start the server as soon as we open
+        this.ensureConnectionAndNotebook().ignoreErrors();
+
         // When showing we have to load the web panel. Make sure
         // we use the last file sent through add code.
         await this.loadWebPanel(this.lastFile ? path.dirname(this.lastFile) : process.cwd());

diff --git a/src/client/datascience/jupyter/debuggerVariables.ts b/src/client/datascience/jupyter/debuggerVariables.ts
@@ -248,7 +248,7 @@ export class DebuggerVariables implements IConditionalJupyterVariables, DebugAda
             // tslint:disable-next-line: no-any
             (variable as any).frameId
         );
-        if (results) {
+        if (results && results.result) {
             // Results should be the updated variable.
             return {
                 ...variable,

diff --git a/src/client/datascience/jupyter/jupyterNotebookProvider.ts b/src/client/datascience/jupyter/jupyterNotebookProvider.ts
@@ -39,7 +39,7 @@ export class JupyterNotebookProvider implements IJupyterNotebookProvider {
         });
 
         if (server) {
-            return server.createNotebook(options.identity, options.identity, options.metadata, options.token);
+            return server.createNotebook(options.resource, options.identity, options.metadata, options.token);
         }
         // We want createNotebook to always return a notebook promise, so if we don't have a server
         // here throw our generic server disposed message that we use in server creatio n

diff --git a/src/client/datascience/jupyterDebugService.ts b/src/client/datascience/jupyterDebugService.ts
@@ -337,8 +337,11 @@ export class JupyterDebugService implements IJupyterDebugService, IDisposable {
 
     private sendMessage(command: string, args?: any): Promise<any> {
         const response = createDeferred<any>();
+        const disposable = this.sessionTerminatedEvent.event(() => response.resolve());
         this.protocolParser.once(`response_${command}`, (resp: any) => {
             this.sendToTrackers(resp);
+            traceInfo(`Received response from debugger: ${JSON.stringify(args)}`);
+            disposable.dispose();
             response.resolve(resp.body);
         });
         this.socket!.on('error', (err) => response.reject(err)); // NOSONAR

diff --git a/src/client/datascience/kernel-launcher/kernelProcess.ts b/src/client/datascience/kernel-launcher/kernelProcess.ts
@@ -145,7 +145,7 @@ export class KernelProcess implements IKernelProcess {
     }
 
     private async createAndUpdateConnectionFile() {
-        this.connectionFile = await this.file.createTemporaryFile('.json');
+        this.connectionFile = await this.file.createTemporaryFile('.json', 0o777);
         await this.file.writeFile(this.connectionFile.filePath, JSON.stringify(this._connection), {
             encoding: 'utf-8',
             flag: 'w'

diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts
@@ -1033,6 +1033,7 @@ export type GetServerOptions = {
  * Options for getting a notebook
  */
 export type GetNotebookOptions = {
+    resource?: Uri;
     identity: Uri;
     getOnly?: boolean;
     disableUI?: boolean;

diff --git a/src/test/common/platform/utils.ts b/src/test/common/platform/utils.ts
@@ -196,9 +196,6 @@ export class FSFixture extends CleanupFixture {
         try {
             await fsextra.remove(filename);
         } catch (err) {
-            if (!(await fsextra.pathExists(filename))) {
-                return;
-            }
             console.log(`failure while deleting ${filename}: ${err}`);
         }
     }

diff --git a/src/test/datascience/dataScienceIocContainer.ts b/src/test/datascience/dataScienceIocContainer.ts
@@ -1235,18 +1235,24 @@ export class DataScienceIocContainer extends UnitTestIocContainer {
         if (!this.mockJupyter) {
             const interpreterService = this.serviceManager.get<IInterpreterService>(IInterpreterService);
             const activeInterpreter = await interpreterService.getActiveInterpreter();
-            if (!activeInterpreter || !(await this.hasJupyter(activeInterpreter))) {
-                const list = await this.getJupyterInterpreters();
-                this.forceSettingsChanged(undefined, list[0].path);
-
-                // Log this all the time. Useful in determining why a test may not pass.
-                // tslint:disable-next-line: no-console
-                console.log(`Setting interpreter to ${list[0].displayName || list[0].path}`);
-
-                // Also set this as the interpreter to use for jupyter
-                await this.serviceManager
-                    .get<JupyterInterpreterService>(JupyterInterpreterService)
-                    .setAsSelectedInterpreter(list[0]);
+            if (!activeInterpreter || !(await this.hasFunctionalDependencies(activeInterpreter))) {
+                const list = await this.getFunctionalTestInterpreters();
+                if (list.length) {
+                    this.forceSettingsChanged(undefined, list[0].path);
+
+                    // Log this all the time. Useful in determining why a test may not pass.
+                    // tslint:disable-next-line: no-console
+                    console.log(`Setting interpreter to ${list[0].displayName || list[0].path} -> ${list[0].path}`);
+
+                    // Also set this as the interpreter to use for jupyter
+                    await this.serviceManager
+                        .get<JupyterInterpreterService>(JupyterInterpreterService)
+                        .setAsSelectedInterpreter(list[0]);
+                } else {
+                    throw new Error(
+                        'No jupyter capable interpreter found. Make sure you install all of the functional requirements before running a test'
+                    );
+                }
             }
         }
     }
@@ -1315,17 +1321,17 @@ export class DataScienceIocContainer extends UnitTestIocContainer {
     }
 
     public async getJupyterCapableInterpreter(): Promise<PythonInterpreter | undefined> {
-        const list = await this.getJupyterInterpreters();
+        const list = await this.getFunctionalTestInterpreters();
         return list ? list[0] : undefined;
     }
 
-    public async getJupyterInterpreters(): Promise<PythonInterpreter[]> {
+    public async getFunctionalTestInterpreters(): Promise<PythonInterpreter[]> {
         // This should be cacheable as we don't install new interpreters during tests
         if (DataScienceIocContainer.jupyterInterpreters.length > 0) {
             return DataScienceIocContainer.jupyterInterpreters;
         }
         const list = await this.get<IInterpreterService>(IInterpreterService).getInterpreters(undefined);
-        const promises = list.map((f) => this.hasJupyter(f).then((b) => (b ? f : undefined)));
+        const promises = list.map((f) => this.hasFunctionalDependencies(f).then((b) => (b ? f : undefined)));
         const resolved = await Promise.all(promises);
         DataScienceIocContainer.jupyterInterpreters = resolved.filter((r) => r) as PythonInterpreter[];
         return DataScienceIocContainer.jupyterInterpreters;
@@ -1587,12 +1593,22 @@ export class DataScienceIocContainer extends UnitTestIocContainer {
         return '';
     }
 
-    private async hasJupyter(interpreter: PythonInterpreter): Promise<boolean | undefined> {
+    private async hasFunctionalDependencies(interpreter: PythonInterpreter): Promise<boolean | undefined> {
         try {
             const dependencyChecker = this.serviceManager.get<JupyterInterpreterDependencyService>(
                 JupyterInterpreterDependencyService
             );
-            return dependencyChecker.areDependenciesInstalled(interpreter);
+            if (await dependencyChecker.areDependenciesInstalled(interpreter)) {
+                // Functional tests require livelossplot too. Make sure this interpreter has that value as well
+                const pythonProcess = await this.serviceContainer
+                    .get<IPythonExecutionFactory>(IPythonExecutionFactory)
+                    .createActivatedEnvironment({
+                        resource: undefined,
+                        interpreter,
+                        allowEnvironmentFetchExceptions: true
+                    });
+                return pythonProcess.isModuleInstalled('livelossplot'); // Should we check all dependencies?
+            }
         } catch (ex) {
             return false;
         }

diff --git a/src/test/datascience/interactiveWindow.functional.test.tsx b/src/test/datascience/interactiveWindow.functional.test.tsx
@@ -611,7 +611,7 @@ Type:      builtin_function_or_method`,
             if (!ioc.mockJupyter) {
                 const interactiveWindowProvider = ioc.get<IInteractiveWindowProvider>(IInteractiveWindowProvider);
                 const interpreterService = ioc.get<IInterpreterService>(IInterpreterService);
-                const interpreters = await ioc.getJupyterInterpreters();
+                const interpreters = await ioc.getFunctionalTestInterpreters();
                 if (interpreters.length < 2) {
                     // tslint:disable-next-line: no-console
                     console.log(

diff --git a/src/test/datascience/notebook.functional.test.ts b/src/test/datascience/notebook.functional.test.ts
@@ -365,7 +365,8 @@ suite('DataScience notebook tests', () => {
             runTest('Remote Self Certs', async (_this: Mocha.Context) => {
                 const pythonService = await createPythonService(2);
 
-                if (pythonService) {
+                // Skip test for older python and on windows. Getting E_PROTO on windows.
+                if (pythonService && os.platform() !== 'win32') {
                     // We will only connect if we allow for self signed cert connections
                     ioc.getSettings().datascience.allowUnauthorizedRemoteConnection = true;
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,3 +2,4 @@ pandas @@
     jupyter
     numpy
     matplotlib
+    pip