Skip to content

Commit a0c9555

Browse files
IanMatthewHuffIanMatthewHuff
authored
Fetch variable values (microsoft#4659)
Add code to fetch variable values
1 parent d4ba5c6 commit a0c9555

8 files changed

Lines changed: 117 additions & 43 deletions

File tree

‎news/1 Enhancements/4291.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Added support for fetching variable values from the jupyter server

‎package.nls.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@
139139
"DataScience.notebookVersionFormat": "Jupyter Notebook Version: {0}",
140140
"DataScience.jupyterKernelNotSupportedOnActive": "Jupyter kernel cannot be started from '{0}'. Using closest match {1} instead.",
141141
"DataScience.jupyterKernelSpecNotFound": "Cannot create a Jupyter kernel spec and none are available for use",
142-
"DataScience.jupyterGetVariablesBadResults": "Failed to fetch variables from the Jupyter server.",
142+
"DataScience.jupyterGetVariablesBadResults": "Failed to fetch variable info from the Jupyter server.",
143143
"DataScience.liveShareConnectFailure": "Cannot connect to host Jupyter session. URI not found.",
144144
"DataScience.liveShareCannotSpawnNotebooks": "Spawning Jupyter notebooks is not supported over a live share connection",
145145
"DataScience.liveShareCannotImportNotebooks": "Importing notebooks is not currently supported over a live share connection",

‎pythonFiles/datascience/getJupyterVariableList.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@
66
# who_ls is a Jupyter line magic to fetch currently defined vars
77
_VSCode_JupyterVars = %who_ls
88

9-
json.dumps([{'name': var,
9+
print(json.dumps([{'name': var,
1010
'type': type(eval(var)).__name__,
1111
'size': getsizeof(var),
1212
'expensive': True
13-
} for var in _VSCode_JupyterVars])
13+
} for var in _VSCode_JupyterVars]))

‎pythonFiles/datascience/getJupyterVariableValue.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Query Jupyter server for the value of a variable
2+
import json
3+
_VSCODE_max_len = 200
4+
# In IJupyterVariables.getValue this '_VSCode_JupyterTestValue' will be replaced with the json stringified value of the target variable
5+
# Indexes off of _VSCODE_targetVariable need to index types that are part of IJupyterVariable
6+
_VSCODE_targetVariable = json.loads('_VSCode_JupyterTestValue')
7+
8+
_VSCODE_evalResult = eval(_VSCODE_targetVariable['name'])
9+
10+
# Find shape and count if available
11+
if _VSCODE_targetVariable['type'] in ['ndarray','DataFrame','Series']:
12+
_VSCODE_targetVariable['shape'] = str(_VSCODE_evalResult.shape)
13+
14+
if _VSCODE_targetVariable['type'] in ['tuple', 'str', 'dict', 'list', 'set', 'ndarray','DataFrame','Series']:
15+
_VSCODE_targetVariable['count'] = len(_VSCODE_evalResult)
16+
17+
# Get the string of the eval result, truncate it as it could be far too long
18+
_VSCODE_targetValue = str(_VSCODE_evalResult)
19+
if len(_VSCODE_targetValue) > _VSCODE_max_len:
20+
_VSCODE_targetVariable['truncated'] = True
21+
_VSCODE_targetVariable['value'] = _VSCODE_targetValue[:_VSCODE_max_len]
22+
else:
23+
_VSCODE_targetVariable['value'] = _VSCODE_targetValue
24+
25+
print(json.dumps(_VSCODE_targetVariable))

‎src/client/common/utils/localize.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ export namespace DataScience {
132132
export const liveShareSyncFailure = localize('DataScience.liveShareSyncFailure', 'Synchronization failure during live share startup.');
133133
export const liveShareServiceFailure = localize('DataScience.liveShareServiceFailure', 'Failure starting \'{0}\' service during live share connection.');
134134
export const documentMismatch = localize('DataScience.documentMismatch', 'Cannot run cells, duplicate documents for {0} found.');
135-
export const jupyterGetVariablesBadResults = localize('DataScience.jupyterGetVariablesBadResults', 'Failed to fetch variables from the Jupyter server.');
135+
export const jupyterGetVariablesBadResults = localize('DataScience.jupyterGetVariablesBadResults', 'Failed to fetch variable info from the Jupyter server.');
136136
}
137137

138138
export namespace DebugConfigurationPrompts {

‎src/client/datascience/jupyter/jupyterVariables.ts‎

Lines changed: 37 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ import { ICell, IHistoryProvider, IJupyterExecution, IJupyterVariable, IJupyterV
1616
@injectable()
1717
export class JupyterVariables implements IJupyterVariables {
1818
private fetchVariablesScript?: string;
19+
private fetchVariableValueScript?: string;
20+
private filesLoaded: boolean = false;
1921

2022
constructor(
2123
@inject(IFileSystem) private fileSystem: IFileSystem,
@@ -26,9 +28,8 @@ export class JupyterVariables implements IJupyterVariables {
2628

2729
// IJupyterVariables implementation
2830
public async getVariables(): Promise<IJupyterVariable[]> {
29-
// First make sure our python file is loaded up
30-
if (!this.fetchVariablesScript) {
31-
await this.loadVariablesFile();
31+
if (!this.filesLoaded) {
32+
await this.loadVariableFiles();
3233
}
3334

3435
const activeServer = await this.jupyterExecution.getServer(await this.historyProvider.getNotebookOptions());
@@ -39,33 +40,51 @@ export class JupyterVariables implements IJupyterVariables {
3940

4041
// Get our results and convert them to IJupyterVariable objects
4142
const results = await activeServer.execute(this.fetchVariablesScript!, Identifiers.EmptyFileName, 0, uuid(), undefined, true);
42-
return this.deserializeVariableData(results);
43+
return this.deserializeJupyterResult<IJupyterVariable[]>(results);
4344
}
4445

45-
// Private methods
46-
private async loadVariablesFile(): Promise<void> {
47-
if (this.fetchVariablesScript) {
48-
return;
46+
public async getValue(targetVariable: IJupyterVariable): Promise<IJupyterVariable> {
47+
if (!this.filesLoaded) {
48+
await this.loadVariableFiles();
4949
}
5050

51-
const file = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getJupyterVariableList.py');
51+
const activeServer = await this.jupyterExecution.getServer(await this.historyProvider.getNotebookOptions());
52+
if (!activeServer) {
53+
// No active server just return the unchanged target variable
54+
return targetVariable;
55+
}
56+
57+
// Prep our targetVariable to send over
58+
const variableString = JSON.stringify(targetVariable);
59+
60+
// Use just the name of the target variable to fetch the value
61+
const newScriptText = this.fetchVariableValueScript!.replace(/_VSCode_JupyterTestValue/g, variableString);
62+
const results = await activeServer.execute(newScriptText, Identifiers.EmptyFileName, 0, uuid(), undefined, true);
63+
return this.deserializeJupyterResult<IJupyterVariable>(results);
64+
}
65+
66+
// Private methods
67+
// Load our python files for fetching variables
68+
private async loadVariableFiles(): Promise<void> {
69+
let file = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getJupyterVariableList.py');
5270
this.fetchVariablesScript = await this.fileSystem.readFile(file);
71+
72+
file = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getJupyterVariableValue.py');
73+
this.fetchVariableValueScript = await this.fileSystem.readFile(file);
74+
75+
this.filesLoaded = true;
5376
}
5477

55-
private deserializeVariableData(cells: ICell[]): IJupyterVariable[] {
78+
// Pull our text result out of the Jupyter cell
79+
private deserializeJupyterResult<T>(cells: ICell[]): T {
5680
// Verify that we have the correct cell type and outputs
5781
if (cells.length > 0 && cells[0].data) {
5882
const codeCell = cells[0].data as nbformat.ICodeCell;
5983
if (codeCell.outputs.length > 0) {
6084
const codeCellOutput = codeCell.outputs[0] as nbformat.IOutput;
61-
if (codeCellOutput.data && codeCellOutput.data.hasOwnProperty('text/plain')) {
62-
// tslint:disable-next-line:no-any
63-
let resultString = ((codeCellOutput.data as any)['text/plain']);
64-
65-
// Trim the excess ' character on the string
66-
resultString = resultString.slice(1, resultString.length - 1);
67-
68-
return JSON.parse(resultString) as IJupyterVariable[];
85+
if (codeCellOutput && codeCellOutput.output_type === 'stream' && codeCellOutput.hasOwnProperty('text')) {
86+
const resultString = codeCellOutput['text'] as string;
87+
return JSON.parse(resultString) as T;
6988
}
7089
}
7190
}

‎src/client/datascience/types.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -246,16 +246,21 @@ export interface IDataScienceExtraSettings extends IDataScienceSettings {
246246
}
247247

248248
// Get variables from the currently running active Jupyter server
249+
// Note: This definition is used implicitly by getJupyterVariableValue.py file
250+
// Changes here may need to be reflected there as well
249251
export interface IJupyterVariable {
250252
name: string;
251-
shortValue: string | undefined;
252-
fullValue: string | undefined;
253+
value: string | undefined;
253254
type: string;
254255
size: number;
256+
shape: string;
257+
count: number;
258+
truncated: boolean;
255259
expensive: boolean;
256260
}
257261

258262
export const IJupyterVariables = Symbol('IJupyterVariables');
259263
export interface IJupyterVariables {
260264
getVariables(): Promise<IJupyterVariable[]>;
265+
getValue(targetVariable: IJupyterVariable): Promise<IJupyterVariable>;
261266
}

‎src/test/datascience/jupyterVariables.unit.test.ts‎

Lines changed: 43 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import * as typemoq from 'typemoq';
99
import { IFileSystem } from '../../client/common/platform/types';
1010
import { Identifiers } from '../../client/datascience/constants';
1111
import { JupyterVariables } from '../../client/datascience/jupyter/jupyterVariables';
12-
import { CellState, ICell, IHistoryProvider, IJupyterExecution, INotebookServer } from '../../client/datascience/types';
12+
import { CellState, ICell, IHistoryProvider, IJupyterExecution, IJupyterVariable, INotebookServer } from '../../client/datascience/types';
1313

1414
// tslint:disable:no-any max-func-body-length
1515
suite('JupyterVariables', () => {
@@ -19,20 +19,20 @@ suite('JupyterVariables', () => {
1919
let jupyterVariables: JupyterVariables;
2020
let fileSystem: typemoq.IMock<IFileSystem>;
2121

22-
function generateVariableOutput(outputData: nbformat.IMimeBundle): nbformat.IOutput {
22+
function generateVariableOutput(outputData: string, outputType: string): nbformat.IOutput {
2323
return {
24-
output_type: 'execute_result',
25-
data: outputData
24+
output_type: outputType,
25+
text: outputData
2626
};
2727
}
2828

29-
function generateCell(outputData: nbformat.IMimeBundle, hasOutput: boolean): ICell {
29+
function generateCell(outputData: string, outputType: string, hasOutput: boolean): ICell {
3030
return {
3131
data: {
3232
cell_type: 'code',
3333
execution_count: 0,
3434
metadata: {},
35-
outputs: hasOutput ? [generateVariableOutput(outputData)] : [],
35+
outputs: hasOutput ? [generateVariableOutput(outputData, outputType)] : [],
3636
source: ''
3737
},
3838
id: '0',
@@ -42,8 +42,8 @@ suite('JupyterVariables', () => {
4242
};
4343
}
4444

45-
function generateCells(outputData: nbformat.IMimeBundle, hasOutput: boolean = true): ICell[] {
46-
return [generateCell(outputData, hasOutput)];
45+
function generateCells(outputData: string, outputType: string, hasOutput: boolean = true): ICell[] {
46+
return [generateCell(outputData, outputType, hasOutput)];
4747
}
4848

4949
function createTypeMoq<T>(tag: string): typemoq.IMock<T> {
@@ -75,7 +75,8 @@ suite('JupyterVariables', () => {
7575

7676
fakeServer.setup(fs => fs.execute(typemoq.It.isAny(), typemoq.It.isAny(), typemoq.It.isAny(), typemoq.It.isAny(), undefined, typemoq.It.isAny()))
7777
.returns(() => Promise.resolve(generateCells(
78-
{ 'text/plain' : '"[{"name": "big_dataframe", "type": "DataFrame", "size": 62, "expensive": true}, {"name": "big_dict", "type": "dict", "size": 57, "expensive": true}, {"name": "big_list", "type": "list", "size": 57, "expensive": true}, {"name": "big_nparray", "type": "ndarray", "size": 60, "expensive": true}, {"name": "big_string", "type": "str", "size": 59, "expensive": true}, {"name": "getsizeof", "type": "builtin_function_or_method", "size": 58, "expensive": true}, {"name": "json", "type": "module", "size": 53, "expensive": true}, {"name": "notebook", "type": "module", "size": 57, "expensive": true}, {"name": "np", "type": "module", "size": 51, "expensive": true}, {"name": "pd", "type": "module", "size": 51, "expensive": true}, {"name": "plt", "type": "module", "size": 52, "expensive": true}, {"name": "style", "type": "module", "size": 54, "expensive": true}, {"name": "sys", "type": "module", "size": 52, "expensive": true}, {"name": "testing", "type": "str", "size": 56, "expensive": true}, {"name": "textFile", "type": "TextIOWrapper", "size": 57, "expensive": true}, {"name": "value", "type": "int", "size": 66, "expensive": true}]"'}
78+
'[{"name": "big_dataframe", "type": "DataFrame", "size": 62, "expensive": true}, {"name": "big_dict", "type": "dict", "size": 57, "expensive": true}, {"name": "big_float", "type": "float", "size": 58, "expensive": true}, {"name": "big_int", "type": "int", "size": 56, "expensive": true}, {"name": "big_list", "type": "list", "size": 57, "expensive": true}, {"name": "big_nparray", "type": "ndarray", "size": 60, "expensive": true}, {"name": "big_series", "type": "Series", "size": 59, "expensive": true}, {"name": "big_string", "type": "str", "size": 59, "expensive": true}, {"name": "big_tuple", "type": "tuple", "size": 58, "expensive": true}]',
79+
'stream'
7980
)))
8081
.verifiable(typemoq.Times.never());
8182

@@ -86,7 +87,6 @@ suite('JupyterVariables', () => {
8687
});
8788

8889
// No cells, no output, no text/plain
89-
9090
test('getVariables no cells', async() => {
9191
execution.setup(sm => sm.getServer(typemoq.It.isAny())).returns(() => {
9292
return Promise.resolve(fakeServer.object);
@@ -113,7 +113,7 @@ suite('JupyterVariables', () => {
113113
});
114114

115115
fakeServer.setup(fs => fs.execute(typemoq.It.isValue('test'), typemoq.It.isValue(Identifiers.EmptyFileName), typemoq.It.isValue(0), typemoq.It.isAnyString(), undefined, typemoq.It.isValue(true)))
116-
.returns(() => Promise.resolve(generateCells({}, false)))
116+
.returns(() => Promise.resolve(generateCells('', 'stream', false)))
117117
.verifiable(typemoq.Times.once());
118118

119119
let exceptionThrown = false;
@@ -127,14 +127,15 @@ suite('JupyterVariables', () => {
127127
fakeServer.verifyAll();
128128
});
129129

130-
test('getVariables bad mime', async() => {
130+
test('getVariables bad output type', async() => {
131131
execution.setup(sm => sm.getServer(typemoq.It.isAny())).returns(() => {
132132
return Promise.resolve(fakeServer.object);
133133
});
134134

135135
fakeServer.setup(fs => fs.execute(typemoq.It.isValue('test'), typemoq.It.isValue(Identifiers.EmptyFileName), typemoq.It.isValue(0), typemoq.It.isAnyString(), undefined, typemoq.It.isValue(true)))
136136
.returns(() => Promise.resolve(generateCells(
137-
{ 'text/html' : '' }
137+
'bogus string',
138+
'bogus output type'
138139
)))
139140
.verifiable(typemoq.Times.once());
140141

@@ -156,23 +157,46 @@ suite('JupyterVariables', () => {
156157

157158
fakeServer.setup(fs => fs.execute(typemoq.It.isValue('test'), typemoq.It.isValue(Identifiers.EmptyFileName), typemoq.It.isValue(0), typemoq.It.isAnyString(), undefined, typemoq.It.isValue(true)))
158159
.returns(() => Promise.resolve(generateCells(
159-
{ 'text/plain' : '"[{"name": "big_dataframe", "type": "DataFrame", "size": 62, "expensive": true}, {"name": "big_dict", "type": "dict", "size": 57, "expensive": true}, {"name": "big_list", "type": "list", "size": 57, "expensive": true}, {"name": "big_nparray", "type": "ndarray", "size": 60, "expensive": true}, {"name": "big_string", "type": "str", "size": 59, "expensive": true}, {"name": "getsizeof", "type": "builtin_function_or_method", "size": 58, "expensive": true}, {"name": "json", "type": "module", "size": 53, "expensive": true}, {"name": "notebook", "type": "module", "size": 57, "expensive": true}, {"name": "np", "type": "module", "size": 51, "expensive": true}, {"name": "pd", "type": "module", "size": 51, "expensive": true}, {"name": "plt", "type": "module", "size": 52, "expensive": true}, {"name": "style", "type": "module", "size": 54, "expensive": true}, {"name": "sys", "type": "module", "size": 52, "expensive": true}, {"name": "testing", "type": "str", "size": 56, "expensive": true}, {"name": "textFile", "type": "TextIOWrapper", "size": 57, "expensive": true}, {"name": "value", "type": "int", "size": 66, "expensive": true}]"'}
160+
'[{"name": "big_dataframe", "type": "DataFrame", "size": 62, "expensive": true}, {"name": "big_dict", "type": "dict", "size": 57, "expensive": true}, {"name": "big_int", "type": "int", "size": 56, "expensive": true}, {"name": "big_list", "type": "list", "size": 57, "expensive": true}, {"name": "big_nparray", "type": "ndarray", "size": 60, "expensive": true}, {"name": "big_string", "type": "str", "size": 59, "expensive": true}]',
161+
'stream'
160162
)))
161163
.verifiable(typemoq.Times.once());
162164

163165
const results = await jupyterVariables.getVariables();
164166

165167
// Check the results that we get back
166-
assert.equal(results.length, 16);
168+
assert.equal(results.length, 6);
167169

168170
// Check our items (just the first few real items, no need to check all 19)
169171
assert.deepEqual(results[0], {name: 'big_dataframe', size: 62, type: 'DataFrame', expensive: true});
170172
assert.deepEqual(results[1], {name: 'big_dict', size: 57, type: 'dict', expensive: true});
171-
assert.deepEqual(results[2], {name: 'big_list', size: 57, type: 'list', expensive: true});
172-
assert.deepEqual(results[3], {name: 'big_nparray', size: 60, type: 'ndarray', expensive: true});
173-
assert.deepEqual(results[4], {name: 'big_string', size: 59, type: 'str', expensive: true});
174-
assert.deepEqual(results[5], {name: 'getsizeof', size: 58, type: 'builtin_function_or_method', expensive: true});
173+
assert.deepEqual(results[2], {name: 'big_int', size: 56, type: 'int', expensive: true});
174+
assert.deepEqual(results[3], {name: 'big_list', size: 57, type: 'list', expensive: true});
175+
assert.deepEqual(results[4], {name: 'big_nparray', size: 60, type: 'ndarray', expensive: true});
176+
assert.deepEqual(results[5], {name: 'big_string', size: 59, type: 'str', expensive: true});
177+
178+
fakeServer.verifyAll();
179+
});
180+
181+
// getValue failure paths are shared with getVariables, so no need to test them here
182+
test('getValue fake data', async() => {
183+
execution.setup(sm => sm.getServer(typemoq.It.isAny())).returns(() => {
184+
return Promise.resolve(fakeServer.object);
185+
});
186+
187+
fakeServer.setup(fs => fs.execute(typemoq.It.isValue('test'), typemoq.It.isValue(Identifiers.EmptyFileName), typemoq.It.isValue(0), typemoq.It.isAnyString(), undefined, typemoq.It.isValue(true)))
188+
.returns(() => Promise.resolve(generateCells(
189+
'{"name": "big_complex", "type": "complex", "size": 60, "expensive": true, "value": "(1+1j)"}',
190+
'stream'
191+
)))
192+
.verifiable(typemoq.Times.once());
193+
194+
const testVariable: IJupyterVariable = { name: 'big_complex', type: 'complex', size: 60, expensive: true, truncated: false, count: 0, shape: '', value: '' };
195+
196+
const resultVariable = await jupyterVariables.getValue(testVariable);
175197

198+
// Verify the result value should be filled out from fake server result
199+
assert.deepEqual(resultVariable, {name: 'big_complex', size: 60, type: 'complex', expensive: true, value: '(1+1j)'});
176200
fakeServer.verifyAll();
177201
});
178202
});

0 commit comments

Comments
 (0)