Skip to content

Commit a69be7a

Browse files
IanMatthewHuffIanMatthewHuff
authored
Dev/ianhu/fetch variables impl (microsoft#4446)
Add tests and implementation for fetching variables
1 parent 4434652 commit a69be7a

13 files changed

Lines changed: 331 additions & 51 deletions

File tree

‎package.nls.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@
139139
"DataScience.notebookVersionFormat": "Jupyter Notebook Version: {0}",
140140
"DataScience.jupyterKernelNotSupportedOnActive": "Jupyter kernel cannot be started from '{0}'. Using closest match {1} instead.",
141141
"DataScience.jupyterKernelSpecNotFound": "Cannot create a Jupyter kernel spec and none are available for use",
142+
"DataScience.jupyterGetVariablesBadResults": "Failed to fetch variables from the Jupyter server.",
142143
"DataScience.liveShareConnectFailure": "Cannot connect to host Jupyter session. URI not found.",
143144
"DataScience.liveShareCannotSpawnNotebooks": "Spawning Jupyter notebooks is not supported over a live share connection",
144145
"DataScience.liveShareCannotImportNotebooks": "Importing notebooks is not currently supported over a live share connection",

‎pythonFiles/datascience/getJupyterVariableList.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Query Jupyter server for defined variables list
2+
# Tested on 2.7 and 3.6
3+
from sys import getsizeof
4+
import json
5+
6+
# who_ls is a Jupyter line magic to fetch currently defined vars
7+
_VSCode_JupyterVars = %who_ls
8+
9+
json.dumps([{'name': var,
10+
'type': type(eval(var)).__name__,
11+
'size': getsizeof(var),
12+
'expensive': True
13+
} for var in _VSCode_JupyterVars])

‎src/client/common/utils/localize.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,7 @@ export namespace DataScience {
132132
export const liveShareSyncFailure = localize('DataScience.liveShareSyncFailure', 'Synchronization failure during live share startup.');
133133
export const liveShareServiceFailure = localize('DataScience.liveShareServiceFailure', 'Failure starting \'{0}\' service during live share connection.');
134134
export const documentMismatch = localize('DataScience.documentMismatch', 'Cannot run cells, duplicate documents for {0} found.');
135+
export const jupyterGetVariablesBadResults = localize('DataScience.jupyterGetVariablesBadResults', 'Failed to fetch variables from the Jupyter server.');
135136
}
136137

137138
export namespace DebugConfigurationPrompts {

‎src/client/datascience/history.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -424,7 +424,7 @@ export class History implements IHistory {
424424
}
425425

426426
// Attempt to evaluate this cell in the jupyter notebook
427-
const observable = this.jupyterServer.executeObservable(code, file, line, id);
427+
const observable = this.jupyterServer.executeObservable(code, file, line, id, false);
428428

429429
// Sign up for cell changes
430430
observable.subscribe(

‎src/client/datascience/jupyter/jupyterServer.ts‎

Lines changed: 33 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -158,15 +158,15 @@ export class JupyterServerBase implements INotebookServer {
158158
return this.session ? this.session.waitForIdle() : Promise.resolve();
159159
}
160160

161-
public execute(code: string, file: string, line: number, id: string, cancelToken?: CancellationToken): Promise<ICell[]> {
161+
public execute(code: string, file: string, line: number, id: string, cancelToken?: CancellationToken, silent?: boolean): Promise<ICell[]> {
162162
// Do initial setup if necessary
163163
this.initialNotebookSetup();
164164

165165
// Create a deferred that we'll fire when we're done
166166
const deferred = createDeferred<ICell[]>();
167167

168168
// Attempt to evaluate this cell in the jupyter notebook
169-
const observable = this.executeObservable(code, file, line, id);
169+
const observable = this.executeObservable(code, file, line, id, silent);
170170
let output: ICell[];
171171

172172
observable.subscribe(
@@ -196,40 +196,10 @@ export class JupyterServerBase implements INotebookServer {
196196
}
197197
}
198198

199-
public executeObservable(code: string, file: string, line: number, id: string): Observable<ICell[]> {
199+
public executeObservable(code: string, file: string, line: number, id: string, silent: boolean = false): Observable<ICell[]> {
200200
return this.executeObservableImpl(code, file, line, id, false);
201201
}
202202

203-
public executeSilently(code: string, cancelToken?: CancellationToken): Promise<ICell[]> {
204-
// Do initial setup if necessary
205-
this.initialNotebookSetup();
206-
207-
// Create a deferred that we'll fire when we're done
208-
const deferred = createDeferred<ICell[]>();
209-
210-
// Attempt to evaluate this cell in the jupyter notebook
211-
const observable = this.executeObservableImpl(code, Identifiers.EmptyFileName, 0, uuid(), true);
212-
let output: ICell[];
213-
214-
observable.subscribe(
215-
(cells: ICell[]) => {
216-
output = cells;
217-
},
218-
(error) => {
219-
deferred.reject(error);
220-
},
221-
() => {
222-
deferred.resolve(output);
223-
});
224-
225-
if (cancelToken) {
226-
this.disposableRegistry.push(cancelToken.onCancellationRequested(() => deferred.reject(new CancellationError())));
227-
}
228-
229-
// Wait for the execution to finish
230-
return deferred.promise;
231-
}
232-
233203
public async getSysInfo() : Promise<ICell> {
234204
// tslint:disable-next-line:no-multiline-string
235205
const versionCells = await this.executeSilently(`import sys\r\nsys.version`);
@@ -384,6 +354,36 @@ export class JupyterServerBase implements INotebookServer {
384354
};
385355
}
386356

357+
private executeSilently(code: string, cancelToken?: CancellationToken): Promise<ICell[]> {
358+
// Do initial setup if necessary
359+
this.initialNotebookSetup();
360+
361+
// Create a deferred that we'll fire when we're done
362+
const deferred = createDeferred<ICell[]>();
363+
364+
// Attempt to evaluate this cell in the jupyter notebook
365+
const observable = this.executeObservableImpl(code, Identifiers.EmptyFileName, 0, uuid(), true);
366+
let output: ICell[];
367+
368+
observable.subscribe(
369+
(cells: ICell[]) => {
370+
output = cells;
371+
},
372+
(error) => {
373+
deferred.reject(error);
374+
},
375+
() => {
376+
deferred.resolve(output);
377+
});
378+
379+
if (cancelToken) {
380+
this.disposableRegistry.push(cancelToken.onCancellationRequested(() => deferred.reject(new CancellationError())));
381+
}
382+
383+
// Wait for the execution to finish
384+
return deferred.promise;
385+
}
386+
387387
private extractStreamOutput(cell: ICell): string {
388388
let result = '';
389389
if (cell.state === CellState.error || cell.state === CellState.finished) {

‎src/client/datascience/jupyter/jupyterServerFactory.ts‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -85,21 +85,21 @@ export class JupyterServerFactory implements INotebookServer {
8585
return server.waitForIdle();
8686
}
8787

88-
public async execute(code: string, file: string, line: number, id: string, cancelToken?: CancellationToken): Promise<ICell[]> {
88+
public async execute(code: string, file: string, line: number, id: string, cancelToken?: CancellationToken, silent?: boolean): Promise<ICell[]> {
8989
const server = await this.serverFactory.get();
90-
return server.execute(code, file, line, id, cancelToken);
90+
return server.execute(code, file, line, id, cancelToken, silent);
9191
}
9292

9393
public async setInitialDirectory(directory: string): Promise<void> {
9494
const server = await this.serverFactory.get();
9595
return server.setInitialDirectory(directory);
9696
}
9797

98-
public executeObservable(code: string, file: string, line: number, id: string): Observable<ICell[]> {
98+
public executeObservable(code: string, file: string, line: number, id: string, silent: boolean = false): Observable<ICell[]> {
9999
// Create a wrapper observable around the actual server (because we have to wait for a promise)
100100
return new Observable<ICell[]>(subscriber => {
101101
this.serverFactory.get().then(s => {
102-
s.executeObservable(code, file, line, id)
102+
s.executeObservable(code, file, line, id, silent)
103103
.forEach(n => subscriber.next(n), Promise)
104104
.then(f => subscriber.complete())
105105
.catch(e => subscriber.error(e));
@@ -111,11 +111,6 @@ export class JupyterServerFactory implements INotebookServer {
111111
});
112112
}
113113

114-
public async executeSilently(code: string, cancelToken?: CancellationToken): Promise<void> {
115-
const server = await this.serverFactory.get();
116-
return server.dispose();
117-
}
118-
119114
public async restartKernel(): Promise<void> {
120115
const server = await this.serverFactory.get();
121116
return server.restartKernel();

‎src/client/datascience/jupyter/jupyterServerManager.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,12 @@ export class JupyterServerManager implements INotebookServerManager, IAsyncDispo
8181
}
8282
}
8383

84+
// Don't check the launch paramters, just return back the active
85+
// used for components that never create or control the active server like the variables view
86+
public getActiveServer(): INotebookServer | undefined {
87+
return this.activeServer;
88+
}
89+
8490
public dispose(): Promise<void> {
8591
if (this.activeServer) {
8692
return this.activeServer.dispose();

‎src/client/datascience/jupyter/jupyterVariables.ts‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
'use strict';
5+
6+
import { nbformat } from '@jupyterlab/coreutils';
7+
import { inject, injectable } from 'inversify';
8+
import * as path from 'path';
9+
import * as uuid from 'uuid/v4';
10+
import { IFileSystem } from '../../common/platform/types';
11+
import * as localize from '../../common/utils/localize';
12+
import { EXTENSION_ROOT_DIR } from '../../constants';
13+
import { Identifiers } from '../constants';
14+
import { ICell, IJupyterVariable, IJupyterVariables, INotebookServerManager } from '../types';
15+
16+
@injectable()
17+
export class JupyterVariables implements IJupyterVariables {
18+
private fetchVariablesScript?: string;
19+
20+
constructor(
21+
@inject(IFileSystem) private fileSystem: IFileSystem,
22+
@inject(INotebookServerManager) private jupyterServerManager: INotebookServerManager
23+
) {
24+
}
25+
26+
// IJupyterVariables implementation
27+
public async getVariables(): Promise<IJupyterVariable[]> {
28+
// First make sure our python file is loaded up
29+
if (!this.fetchVariablesScript) {
30+
await this.loadVariablesFile();
31+
}
32+
33+
const activeServer = this.jupyterServerManager.getActiveServer();
34+
if (!activeServer) {
35+
// No active server will just return an empty list
36+
return [];
37+
}
38+
39+
// Get our results and convert them to IJupyterVariable objects
40+
const results = await activeServer.execute(this.fetchVariablesScript!, Identifiers.EmptyFileName, 0, uuid(), undefined, true);
41+
return this.deserializeVariableData(results);
42+
}
43+
44+
// Private methods
45+
private async loadVariablesFile(): Promise<void> {
46+
if (this.fetchVariablesScript) {
47+
return;
48+
}
49+
50+
const file = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getJupyterVariableList.py');
51+
this.fetchVariablesScript = await this.fileSystem.readFile(file);
52+
}
53+
54+
private deserializeVariableData(cells: ICell[]): IJupyterVariable[] {
55+
// Verify that we have the correct cell type and outputs
56+
if (cells.length > 0 && cells[0].data) {
57+
const codeCell = cells[0].data as nbformat.ICodeCell;
58+
if (codeCell.outputs.length > 0) {
59+
const codeCellOutput = codeCell.outputs[0] as nbformat.IOutput;
60+
if (codeCellOutput.data && codeCellOutput.data.hasOwnProperty('text/plain')) {
61+
// tslint:disable-next-line:no-any
62+
let resultString = ((codeCellOutput.data as any)['text/plain']);
63+
64+
// Trim the excess ' character on the string
65+
resultString = resultString.slice(1, resultString.length - 1);
66+
67+
return JSON.parse(resultString) as IJupyterVariable[];
68+
}
69+
}
70+
}
71+
72+
throw new Error(localize.DataScience.jupyterGetVariablesBadResults());
73+
}
74+
}

‎src/client/datascience/jupyter/liveshare/guestJupyterServer.ts‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -109,11 +109,6 @@ export class GuestJupyterServer
109109
});
110110
}
111111

112-
public async executeSilently(code: string, cancelToken?: CancellationToken): Promise<void> {
113-
// We don't need the result from this. It should have already happened on the host side
114-
return Promise.resolve();
115-
}
116-
117112
public async restartKernel(): Promise<void> {
118113
await this.waitForResponse(ServerResponseType.Restart);
119114
}

‎src/client/datascience/serviceRegistry.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import { JupyterImporter } from './jupyter/jupyterImporter';
1717
import { JupyterServerFactory } from './jupyter/jupyterServerFactory';
1818
import { JupyterServerManager } from './jupyter/jupyterServerManager';
1919
import { JupyterSessionManager } from './jupyter/jupyterSessionManager';
20+
import { JupyterVariables } from './jupyter/jupyterVariables';
2021
import { StatusProvider } from './statusProvider';
2122
import {
2223
ICodeCssGenerator,
@@ -30,6 +31,7 @@ import {
3031
IJupyterCommandFactory,
3132
IJupyterExecution,
3233
IJupyterSessionManager,
34+
IJupyterVariables,
3335
INotebookExporter,
3436
INotebookImporter,
3537
INotebookServer,
@@ -52,6 +54,7 @@ export function registerTypes(serviceManager: IServiceManager) {
5254
serviceManager.addSingleton<ICodeCssGenerator>(ICodeCssGenerator, CodeCssGenerator);
5355
serviceManager.addSingleton<IStatusProvider>(IStatusProvider, StatusProvider);
5456
serviceManager.addSingleton<IJupyterSessionManager>(IJupyterSessionManager, JupyterSessionManager);
57+
serviceManager.addSingleton<IJupyterVariables>(IJupyterVariables, JupyterVariables);
5558
serviceManager.add<ICodeWatcher>(ICodeWatcher, CodeWatcher);
5659
serviceManager.add<IJupyterCommandFactory>(IJupyterCommandFactory, JupyterCommandFactory);
5760
}

0 commit comments

Comments
 (0)