RustPython/crates/vm/src/vm/interpreter.rs at ea68d616eeff8cd8ed717f166980f463ae5ca512 · youknowone/RustPython

Repository navigation

History

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

427

428

429

430

431

432

433

434

435

436

437

438

439

440

441

442

443

444

445

446

447

448

449

450

451

452

453

454

455

456

457

458

459

460

461

462

463

464

465

466

467

468

469

470

471

472

473

474

475

476

477

478

479

480

481

482

483

484

485

486

487

488

489

490

491

492

493

494

495

496

497

498

499

500

501

502

503

504

505

506

507

508

509

510

511

512

513

514

515

516

517

518

519

520

521

522

523

524

525

526

527

528

529

530

531

532

533

534

535

536

537

538

539

540

541

542

543

544

545

546

547

548

549

550

551

552

553

554

555

556

557

558

559

560

561

562

563

564

565

566

567

568

569

570

571

572

573

574

575

576

577

578

579

580

581

582

583

584

585

586

587

588

589

590

591

592

593

594

595

596

597

598

599

600

601

602

603

604

605

606

607

608

609

610

611

612

613

614

615

616

617

618

619

620

621

622

623

624

625

626

627

628

629

630

631

632

633

634

635

636

637

638

639

640

641

642

643

644

645

646

647

648

649

650

651

652

653

654

655

656

657

658

659

660

661

662

663

664

665

666

667

668

669

670

671

672

673

674

675

676

677

678

679

680

681

682

683

684

685

686

687

688

689

690

691

692

693

694

695

696

697

698

699

700

701

702

703

704

705

706

707

708

709

710

711

712

713

714

715

716

717

718

719

720

721

722

723

724

725

726

727

728

729

730

731

732

733

734

735

736

737

738

739

740

741

742

743

744

745

746

747

748

749

750

751

752

753

754

755

756

757

758

759

760

761

762

763

764

765

766

767

768

769

770

771

772

773

774

775

776

777

778

779

780

781

782

783

784

785

786

787

788

789

790

791

792

793

794

795

796

797

798

799

800

801

802

803

804

805

806

807

808

809

810

811

812

813

814

815

816

817

818

819

820

821

822

823

824

825

826

827

828

829

830

831

832

833

834

835

836

837

838

839

840

841

842

843

844

845

846

847

848

849

850

851

852

853

854

855

856

857

858

859

860

861

862

863

864

865

866

867

868

869

870

871

872

873

874

875

876

877

878

879

880

881

882

883

884

885

886

887

888

889

890

891

892

893

894

895

896

897

898

899

900

901

902

903

904

905

906

907

908

909

910

911

912

913

914

915

916

917

918

919

920

921

922

923

924

925

926

927

928

929

930

931

932

933

934

935

936

937

938

939

940

941

942

943

944

945

946

947

948

949

950

951

952

953

954

955

956

957

958

959

960

961

962

963

964

965

966

967

968

969

970

971

972

973

974

975

976

977

978

979

980

981

982

983

984

985

986

987

988

989

990

991

992

993

994

995

996

997

998

999

1000

#[cfg(feature = "threading")]

use super::StopTheWorldState;

use super::{

Context, PyConfig, PyGlobalState, VirtualMachine,

runtime::{self, InterpreterWhence},

setting::Settings,

thread,

};

use crate::{

PyResult, builtins, common::rc::PyRc, frozen::FrozenModule, getpath, py_freeze, stdlib::atexit,

vm::PyBaseExceptionRef,

};

use alloc::collections::BTreeMap;

use core::sync::atomic::Ordering;

type InitFunc = Box<dyn FnOnce(&mut VirtualMachine)>;

/// Exit code used when stdout/stderr flush fails during interpreter shutdown.

/// Matches CPython's behavior (see cpython/Python/pylifecycle.c).

const EXITCODE_FLUSH_FAILURE: u32 = 120;

/// Configuration builder for constructing an Interpreter.

///

/// This is the preferred way to configure and create an interpreter with custom modules.

/// Modules must be registered before the interpreter is built,

/// similar to CPython's `PyImport_AppendInittab` which must be called before `Py_Initialize`.

///

/// # Example

/// ```

/// use rustpython_vm::Interpreter;

///

/// let builder = Interpreter::builder(Default::default());

/// // In practice, add stdlib: builder.add_native_modules(&stdlib_module_defs(&builder.ctx))

/// let interp = builder.build();

/// ```

pub struct InterpreterBuilder {

settings: Settings,

pub ctx: PyRc<Context>,

module_defs: Vec<&'static builtins::PyModuleDef>,

frozen_modules: Vec<(&'static str, FrozenModule)>,

init_hooks: Vec<InitFunc>,

}

/// Options for constructing a main or sub-interpreter VM.

struct InitializeVmOpts<'a> {

settings: Settings,

ctx: PyRc<Context>,

module_defs: Vec<&'static builtins::PyModuleDef>,

frozen_modules: Vec<(&'static str, FrozenModule)>,

init_hooks: Vec<InitFunc>,

is_main: bool,

whence: InterpreterWhence,

/// When `Some`, reuse parent module_defs/frozen/config seeds for a subinterpreter.

parent_state: Option<&'a PyGlobalState>,

interp_config: runtime::InterpreterConfig,

}

/// Shared constructor for main and sub-interpreters.

fn initialize_vm<F>(opts: InitializeVmOpts<'_>, init: F) -> (VirtualMachine, PyRc<PyGlobalState>)

where

F: FnOnce(&mut VirtualMachine),

{

let InitializeVmOpts {

settings,

ctx,

module_defs,

frozen_modules,

init_hooks,

is_main,

whence,

parent_state,

interp_config,

} = opts;

use crate::codecs::CodecsRegistry;

use crate::common::hash::HashSecret;

use crate::common::lock::PyMutex;

use crate::warn::WarningsState;

use core::sync::atomic::{AtomicBool, AtomicI64, AtomicU64};

use crossbeam_utils::atomic::AtomicCell;

// Before any lock this interpreter's threads can contend on exists.

#[cfg(feature = "threading")]

thread::install_blocking_wait_hook();

let (config, all_module_defs, frozen, hash_secret, int_max_str_digits) =

if let Some(parent) = parent_state {

// Subinterpreter: clone config and module tables from parent, fresh runtime state.

let int_max_str_digits = AtomicCell::new(parent.int_max_str_digits.load());

(

parent.config.clone(),

parent.module_defs.clone(),

parent.frozen.clone(),

parent.hash_secret,

int_max_str_digits,

)

} else {

let paths = getpath::init_path_config(&settings);

let config = PyConfig::new(settings, paths);

// Build module_defs map from builtin modules + additional modules

let mut all_module_defs: BTreeMap<&'static str, &'static builtins::PyModuleDef> =

crate::stdlib::builtin_module_defs(&ctx)

.into_iter()

.chain(module_defs)

.map(|def| (def.name.as_str(), def))

.collect();

// Register sysconfigdata under platform-specific name as well

if let Some(&sysconfigdata_def) = all_module_defs.get("_sysconfigdata") {

use std::sync::OnceLock;

static SYSCONFIGDATA_NAME: OnceLock<&'static str> = OnceLock::new();

let leaked_name = *SYSCONFIGDATA_NAME.get_or_init(|| {

let name = crate::stdlib::sys::sysconfigdata_name();

Box::leak(name.into_boxed_str())

});

all_module_defs.insert(leaked_name, sysconfigdata_def);

}

let seed = match config.settings.hash_seed {

Some(seed) => seed,

None => super::process_hash_secret_seed(),

};

let hash_secret = HashSecret::new(seed);

let int_max_str_digits = AtomicCell::new(match config.settings.int_max_str_digits {

-1 => 4300,

other => other,

} as usize);

let mut frozen: std::collections::HashMap<

&'static str,

FrozenModule,

rapidhash::quality::RandomState,

> = core_frozen_inits().collect();

frozen.extend(frozen_modules);

(

config,

all_module_defs,

frozen,

hash_secret,

int_max_str_digits,

)

};

// Per-interpreter ephemeral state (must not be shared across interpreters).

let codec_registry = CodecsRegistry::new(&ctx);

let warnings = WarningsState::init_state(&ctx);

let interpreter_id = runtime::alloc_interpreter_id();

let runtime_root_id = parent_state.map_or(interpreter_id, |parent| parent.runtime_root_id);

// Process main OS thread identity is process-global; subinterpreters inherit

// it from the parent so `is_main_thread()` stays correct when running on the

// main OS thread under a subinterpreter.

#[cfg(feature = "threading")]

let main_thread_ident = AtomicCell::new(parent_state.map_or(0, |p| p.main_thread_ident.load()));

let feature_flags = interp_config.feature_flags();

let own_gil = interp_config.own_gil();

// Create PyGlobalState (≈ PyInterpreterState)

let global_state = PyRc::new(PyGlobalState {

gc: crate::gc_state::GcInterpreterState::new(&ctx),

interpreter_id,

runtime_root_id,

whence,

is_main,

config,

module_defs: all_module_defs,

frozen,

stacksize: AtomicCell::new(0),

thread_count: AtomicCell::new(0),

hash_secret,

atexit_funcs: PyMutex::default(),

codec_registry,

finalizing: AtomicBool::new(false),

warnings,

override_frozen_modules: AtomicCell::new(0),

before_forkers: PyMutex::default(),

after_forkers_child: PyMutex::default(),

after_forkers_parent: PyMutex::default(),

int_max_str_digits,

switch_interval: AtomicCell::new(0.005),

global_trace_func: PyMutex::default(),

global_profile_func: PyMutex::default(),

type_mutex: PyMutex::default(),

#[cfg(feature = "threading")]

main_thread_ident,

#[cfg(feature = "threading")]

thread_frames: parking_lot::Mutex::new(std::collections::HashMap::new()),

#[cfg(feature = "threading")]

thread_handles: parking_lot::Mutex::new(Vec::new()),

#[cfg(feature = "threading")]

shutdown_handles: parking_lot::Mutex::new(Vec::new()),

monitoring: PyMutex::default(),

monitoring_events: AtomicCell::new(0),

instrumentation_version: AtomicU64::new(0),

#[cfg(feature = "threading")]

stop_the_world: StopTheWorldState::new(),

feature_flags,

own_gil,

running_main: AtomicBool::new(false),

ready: AtomicBool::new(false),

id_refcount: AtomicI64::new(0),

require_idref: AtomicBool::new(false),

});

// Create VM with the global state

// Note: Don't clone here - init_hooks need exclusive access to mutate state

let mut vm = VirtualMachine::new(ctx, global_state);

// Execute initialization hooks (can mutate vm.state)

for hook in init_hooks {

hook(&mut vm);

}

// Call custom init function (can mutate vm.state)

init(&mut vm);

// Register before `initialize()` runs any Python: it allocates GC-tracked

// objects, so a collection on another thread has to be able to stop this

// interpreter while that happens. It cannot be registered earlier — the

// hooks above take `PyRc::get_mut` on the state, which fails once the

// registry holds a weak reference to it.

runtime::register_interpreter(&vm.state);

// `initialize()` runs Python bytecode directly (e.g. importing `codecs`

// and `encodings`) before any `enter_vm` scope exists, so attach this

// thread for the duration so type cache reads see it as ATTACHED.

let vm_guard = thread::VmBootstrapGuard::new(&vm);

vm.initialize();

vm.state.ready.store(true, Ordering::Release);

drop(vm_guard);

// Clone global_state for Interpreter after all initialization is done

let global_state = vm.state.clone();

(vm, global_state)

}

/// Bootstrap a subinterpreter from an already-entered parent VM.

fn create_subinterpreter_from_parent(

parent: &VirtualMachine,

config: runtime::InterpreterConfig,

) -> Result<Interpreter, &'static str> {

config.check()?;

// Suspend the caller's current VM attachment (if any) for the duration

// of subinterpreter initialization. Nested bootstrap would otherwise

// swap `CURRENT_THREAD_SLOT` to the new interpreter while leaving the

// outer interpreter's attach state inconsistent. Always restore, even

// if initialization panics.

#[cfg(feature = "threading")]

let _restore_parent = {

let saved = thread::current_vm_is_set().then(thread::save_current_thread);

scopeguard::guard(saved, |saved| {

if let Some(saved) = saved {

thread::restore_current_thread(saved);

}

})

};

let (vm, global_state) = initialize_vm(

InitializeVmOpts {

// settings unused when parent_state is Some

settings: Settings::default(),

ctx: parent.ctx.clone(),

module_defs: Vec::new(),

frozen_modules: Vec::new(),

init_hooks: Vec::new(),

is_main: false,

whence: InterpreterWhence::Stdlib,

parent_state: Some(&parent.state),

interp_config: config,

|_| {},

);

let interp = Interpreter { global_state, vm };

// Every interpreter has a `__main__` module once it is initialized.

interp.enter(|vm| {

let _ = vm.ensure_main_module();

});

Ok(interp)

}

impl InterpreterBuilder {

/// Create a new interpreter configuration with default settings.

#[must_use]

pub fn new() -> Self {

Self {

settings: Settings::default(),

ctx: Context::genesis().clone(),

module_defs: Vec::new(),

frozen_modules: Vec::new(),

init_hooks: Vec::new(),

}

/// Set custom settings for the interpreter.

///

/// If called multiple times, only the last settings will be used.

#[must_use]

pub fn settings(mut self, settings: Settings) -> Self {

self.settings = settings;

self

}

/// Add a single native module definition.

///

/// # Example

/// ```

/// use rustpython_vm::{Interpreter, builtins::PyModuleDef};

///

/// let builder = Interpreter::builder(Default::default());

/// // Note: In practice, use module_def from your #[pymodule]

/// // let def = mymodule::module_def(&builder.ctx);

/// // let interp = builder.add_native_module(def).build();

/// let interp = builder.build();

/// ```

#[must_use]

pub fn add_native_module(self, def: &'static builtins::PyModuleDef) -> Self {

self.add_native_modules(&[def])

}

/// Add multiple native module definitions.

///

/// # Example

/// ```

/// use rustpython_vm::Interpreter;

///

/// let builder = Interpreter::builder(Default::default());

/// // In practice, use module_defs from rustpython_stdlib:

/// // let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx);

/// // let interp = builder.add_native_modules(&defs).build();

/// let interp = builder.build();

/// ```

#[must_use]

pub fn add_native_modules(mut self, defs: &[&'static builtins::PyModuleDef]) -> Self {

self.module_defs.extend_from_slice(defs);

self

}

/// Add a custom initialization hook.

///

/// Hooks are executed in the order they are added during interpreter creation.

/// This function will be called after modules are registered but before

/// the VM is initialized, allowing for additional customization.

///

/// # Example

/// ```

/// use rustpython_vm::Interpreter;

///

/// let interp = Interpreter::builder(Default::default())

/// .init_hook(|vm| {

/// // Custom initialization

/// })

/// .build();

/// ```

#[must_use]

pub fn init_hook<F>(mut self, init: F) -> Self

where

F: FnOnce(&mut VirtualMachine) + 'static,

{

self.init_hooks.push(Box::new(init));

self

}

/// Add frozen modules to the interpreter.

///

/// Frozen modules are Python modules compiled into the binary.

/// This method accepts any iterator of (name, FrozenModule) pairs.

///

/// # Example

/// ```

/// use rustpython_vm::Interpreter;

///

/// let interp = Interpreter::builder(Default::default())

/// // In practice: .add_frozen_modules(rustpython_pylib::FROZEN_STDLIB)

/// .build();

/// ```

#[must_use]

pub fn add_frozen_modules<I>(mut self, frozen: I) -> Self

where

I: IntoIterator<Item = (&'static str, FrozenModule)>,

{

self.frozen_modules.extend(frozen);

self

}

/// Build the interpreter.

///

/// This consumes the configuration and returns a fully initialized Interpreter.

#[must_use]

pub fn build(self) -> Interpreter {

let (vm, global_state) = initialize_vm(

InitializeVmOpts {

settings: self.settings,

ctx: self.ctx,

module_defs: self.module_defs,

frozen_modules: self.frozen_modules,

init_hooks: self.init_hooks,

is_main: true,

whence: InterpreterWhence::Runtime,

parent_state: None,

interp_config: runtime::InterpreterConfig::MAIN,

|_| {}, // No additional init needed

);

Interpreter { global_state, vm }

}

/// Alias for `build()` for compatibility with the `interpreter()` pattern.

#[must_use]

pub fn interpreter(self) -> Interpreter {

self.build()

}

impl Default for InterpreterBuilder {

fn default() -> Self {

Self::new()

}

/// One isolated Python interpreter in the process (≈ CPython `PyInterpreterState` + main tstate).

///

/// Historically RustPython exposed a single process-level `Interpreter`. For PEP 734

/// (multiple interpreters / subinterpreters) this type is now the owned handle for

/// **one** interpreter. Use [`Interpreter::create_subinterpreter`] to create additional

/// isolated interpreters that share the process-wide type context but not modules or

/// `PyGlobalState`.

///

/// # Examples

/// Runs a simple embedded hello world program.

/// ```

/// use rustpython_vm::Interpreter;

/// use rustpython_vm::compiler::Mode;

/// Interpreter::without_stdlib(Default::default()).enter(|vm| {

/// let scope = vm.new_scope_with_builtins();

/// let source = r#"print("Hello World!")"#;

/// let code_obj = vm.compile(

/// source,

/// Mode::Exec,

/// "<embedded>",

/// ).map_err(|err| err.into_pyexception(vm, Some(source))).unwrap();

/// vm.run_code_obj(code_obj, scope).unwrap();

/// });

/// ```

pub struct Interpreter {

pub global_state: PyRc<PyGlobalState>,

vm: VirtualMachine,

}

impl Interpreter {

/// Create a new interpreter configuration builder.

///

/// # Example

/// ```

/// use rustpython_vm::Interpreter;

///

/// let builder = Interpreter::builder(Default::default());

/// // In practice, add stdlib: builder.add_native_modules(&stdlib_module_defs(&builder.ctx))

/// let interp = builder.build();

/// ```

#[must_use]

pub fn builder(settings: Settings) -> InterpreterBuilder {

InterpreterBuilder::new().settings(settings)

}

/// This is a bare unit to build up an interpreter without the standard library.

/// To create an interpreter with the standard library with the `rustpython` crate, use `rustpython::InterpreterBuilder`.

/// To create an interpreter without the `rustpython` crate, but only with `rustpython-vm`,

/// try to build one from the source code of `InterpreterBuilder`. It will not be a one-liner but it also will not be too hard.

#[must_use]

pub fn without_stdlib(settings: Settings) -> Self {

Self::with_init(settings, |_| {})

}

/// Create with initialize function taking mutable vm reference.

///

/// Note: This is a legacy API. To add stdlib, use `Interpreter::builder()` instead.

pub fn with_init<F>(settings: Settings, init: F) -> Self

where

F: FnOnce(&mut VirtualMachine),

{

let (vm, global_state) = initialize_vm(

InitializeVmOpts {

settings,

ctx: Context::genesis().clone(),

module_defs: Vec::new(),

frozen_modules: Vec::new(),

init_hooks: Vec::new(),

is_main: true,

whence: InterpreterWhence::Runtime,

parent_state: None,

interp_config: runtime::InterpreterConfig::MAIN,

init,

);

Self { global_state, vm }

}

/// Process-global interpreter id (main is [`super::MAIN_INTERPRETER_ID`]).

#[inline]

#[must_use]

pub fn id(&self) -> i64 {

self.global_state.interpreter_id

}

/// Where this interpreter was created.

#[inline]

#[must_use]

pub fn whence(&self) -> InterpreterWhence {

self.global_state.whence

}

/// Whether this is a top-level interpreter rather than a subinterpreter.

///

/// Every top-level interpreter answers `true`; for *the* process main, use

/// [`Interpreter::is_process_main`].

#[inline]

#[must_use]

pub fn is_main(&self) -> bool {

self.global_state.is_main

}

/// Whether this is the PEP 734 process main interpreter (`get_main()`).

///

/// Unlike [`Interpreter::is_main`], which is set for every top-level

/// interpreter, this is true for only the single first-registered main.

#[inline]

#[must_use]

pub fn is_process_main(&self) -> bool {

runtime::main_interpreter_id() == Some(self.id())

}

/// Create a subinterpreter and hand ownership to the runtime, returning its

/// id. The runtime keeps it alive until [`runtime::take_owned_interpreter`].

///

/// This is the shape `_interpreters.create()` uses: Python receives an

/// id, not an owned handle.

#[cfg(feature = "threading")]

#[must_use]

pub fn create_owned_subinterpreter(&self) -> i64 {

self.create_owned_subinterpreter_with_config(runtime::InterpreterConfig::ISOLATED)

.expect("the isolated config is always valid")

}

/// Create a runtime-owned subinterpreter with the given PEP 734 config.

#[cfg(feature = "threading")]

pub fn create_owned_subinterpreter_with_config(

&self,

config: runtime::InterpreterConfig,

) -> Result<i64, &'static str> {

Ok(runtime::store_owned_interpreter(

self.create_subinterpreter_with_config(config)?,

))

}

/// Create an isolated subinterpreter sharing this interpreter's type context

/// (`Context`) and module definitions, but with its own `sys.modules`,

/// builtins module instance, thread registry, and stop-the-world state.

///

/// May be called while the parent is entered (matching CPython, where

/// `_interpreters.create()` runs under the main interpreter). When the

/// calling thread is currently attached to a VM, that attachment is

/// temporarily saved so the subinterpreter can bootstrap as an outermost

/// enter (correct thread-slot / stop-the-world state).

#[must_use]

pub fn create_subinterpreter(&self) -> Self {

self.create_subinterpreter_with_config(runtime::InterpreterConfig::ISOLATED)

.expect("the isolated config is always valid")

}

/// Create a subinterpreter from a parent VM (the currently entered one).

pub fn create_subinterpreter_from_vm(

parent: &VirtualMachine,

config: runtime::InterpreterConfig,

) -> Result<Self, &'static str> {

create_subinterpreter_from_parent(parent, config)

}

/// Create a subinterpreter with an explicit PEP 734 / `PyInterpreterConfig`.

pub fn create_subinterpreter_with_config(

&self,

config: runtime::InterpreterConfig,

) -> Result<Self, &'static str> {

create_subinterpreter_from_parent(&self.vm, config)

}

/// Spawn a new OS-thread VM that shares this interpreter's `sys` / builtins.

#[cfg(feature = "threading")]

pub fn new_thread(&self) -> thread::ThreadedVirtualMachine {

self.vm.new_thread()

}

/// Run a function with the main virtual machine and return a PyResult of the result.

///

/// To enter vm context multiple times or to avoid buffer/exception management, this function is preferred.

/// `enter` is lightweight and it returns a python object in PyResult.

/// You can stop or continue the execution multiple times by calling `enter`.

///

/// To finalize the vm once all desired `enter`s are called, calling `finalize` will be helpful.

///

/// See also [`Interpreter::run`] for managed way to run the interpreter.

pub fn enter<F, R>(&self, f: F) -> R

where

F: FnOnce(&VirtualMachine) -> R,

{

thread::enter_vm(&self.vm, || f(&self.vm))

}

/// Run [`Interpreter::enter`] and call [`VirtualMachine::expect_pyresult`] for the result.

///

/// This function is useful when you want to expect a result from the function,

/// but also print useful panic information when exception raised.

///

/// See also [`Interpreter::enter`] and [`VirtualMachine::expect_pyresult`] for more information.

pub fn enter_and_expect<F, R>(&self, f: F, msg: &str) -> R

where

F: FnOnce(&VirtualMachine) -> PyResult<R>,

{

self.enter(|vm| {

let result = f(vm);

vm.expect_pyresult(result, msg)

})

}

/// Run a function with the main virtual machine and return exit code.

///

/// To enter vm context only once and safely terminate the vm, this function is preferred.

/// Unlike [`Interpreter::enter`], `run` calls finalize and returns exit code.

/// You will not be able to obtain Python exception in this way.

///

/// See [`Interpreter::finalize`] for the finalization steps.

/// See also [`Interpreter::enter`] for pure function call to obtain Python exception.

pub fn run<F>(self, f: F) -> u32

where

F: FnOnce(&VirtualMachine) -> PyResult<()>,

{

let res = self.enter(|vm| f(vm));

self.finalize(res.err())

}

/// Finalize vm and turns an exception to exit code.

///

/// Finalization steps (matching Py_FinalizeEx):

/// 1. Flush stdout and stderr.

/// 1. Handle exit exception and turn it to exit code.

/// 1. Call threading._shutdown() to join non-daemon threads.

/// 1. Run atexit exit functions.

/// 1. Set finalizing flag (suppresses unraisable exceptions from __del__).

/// 1. Forced GC collection pass (collect cycles while builtins are available).

/// 1. Module finalization (finalize_modules).

/// 1. Clear interpreter-owned cross-interpreter data.

/// 1. Final stdout/stderr flush.

///

/// Note that calling `finalize` is not necessary by purpose though.

pub fn finalize(self, exc: Option<PyBaseExceptionRef>) -> u32 {

self.enter(|vm| {

let mut flush_status = vm.flush_std();

// See if any exception leaked out:

let exit_code = if let Some(exc) = exc {

vm.handle_exit_exception(exc)

} else {

};

// Wait for thread shutdown - call threading._shutdown() if available.

// This waits for all non-daemon threads to complete.

// threading module may not be imported, so ignore import errors.

if let Ok(threading) = vm.import("threading", 0)

&& let Ok(shutdown) = threading.get_attr("_shutdown", vm)

&& let Err(e) = shutdown.call((), vm)

{

vm.run_unraisable(

Some("Exception ignored in threading shutdown".to_owned()),

threading,

);

}

// Run atexit handlers before setting finalizing flag.

// This allows unraisable exceptions from atexit handlers to be reported.

atexit::_run_exitfuncs(vm);

// Clean up any lingering subinterpreters. This has to happen before

// the finalizing flag is set, or else threads might get prematurely

// blocked.

#[cfg(feature = "threading")]

finalize_subinterpreters(vm);

// Now suppress unraisable exceptions from daemon threads and __del__

// methods during the rest of shutdown.

vm.state.finalizing.store(true, Ordering::Release);

// GC pass - collect cycles before module cleanup

vm.state.gc.collect_force(2);

// Module finalization: remove modules from sys.modules, GC collect

// (while builtins is still available for __del__), then clear module dicts.

vm.finalize_modules();

// CPython clears low-level cross-interpreter container data from

// _PyAtExit_Fini(), after Python atexit callbacks and module

// finalization. In particular, values sent by an atexit callback

// must also become unbound when this interpreter goes away.

let interpreter_id = vm.state.interpreter_id;

crate::stdlib::_interpchannels::clear_interpreter(interpreter_id);

crate::stdlib::_interpqueues::clear_interpreter(interpreter_id);

if vm.flush_std() < 0 && flush_status == 0 {

flush_status = -1;

}

// Match CPython: if exit_code is 0 and stdout flush failed, exit 120

let exit_code = if exit_code == 0 && flush_status < 0 {

EXITCODE_FLUSH_FAILURE

} else {

exit_code

};

// Daemon threads may still exist, so use the safe `process()`,

// not `drain_all()`.

#[cfg(feature = "threading")]

crate::object::qsbr::QSBR.process();

exit_code

})

}

/// `finalize_subinterpreters`: destroy the subinterpreters the program left

/// behind, after telling the user they are still around.

#[cfg(feature = "threading")]

fn finalize_subinterpreters(vm: &VirtualMachine) {

if !vm.state.is_main {

return;

}

let root_id = vm.state.runtime_root_id;

// Bail out if there are no subinterpreters left.

if runtime::owned_interpreter_ids_for(root_id).is_empty() {

return;

}

// Warn the user if they forgot to clean up subinterpreters.

let message = vm

.ctx

.new_str("remaining subinterpreters; close them with Interpreter.close()");

let _ = crate::warn::warn(

message.into(),

Some(vm.ctx.exceptions.runtime_warning.to_owned()),

None,

vm,

);

// A subinterpreter's finalizers may create another subinterpreter, so

// re-read the owner table after each destruction like CPython does.

while let Some(id) = runtime::owned_interpreter_ids_for(root_id)

.into_iter()

.next()

{

let _ = runtime::destroy_owned_interpreter(id);

}

fn core_frozen_inits() -> impl Iterator<Item = (&'static str, FrozenModule)> {

let iter = core::iter::empty();

macro_rules! ext_modules {

($iter:ident, $($t:tt)*) => {

let $iter = $iter.chain(py_freeze!($($t)*));

};

}

// Python modules that the vm calls into, but are not actually part of the stdlib. They could

// in theory be implemented in Rust, but are easiest to do in Python for one reason or another.

// Includes _importlib_bootstrap and _importlib_bootstrap_external

ext_modules!(

iter,

dir = "../../Lib/python_builtins",

crate_name = "rustpython_compiler_core"

);

// core stdlib Python modules that the vm calls into, but are still used in Python

// application code, e.g. copyreg

// FIXME: Initializing core_modules here results duplicated frozen module generation for core_modules.

// We need a way to initialize this modules for both `Interpreter::without_stdlib()` and `InterpreterBuilder::new().init_stdlib().interpreter()`

// #[cfg(not(feature = "freeze-stdlib"))]

ext_modules!(

iter,

dir = "../../Lib/core_modules",

crate_name = "rustpython_compiler_core"

);

// Collect frozen module entries

let mut entries: Vec<_> = iter.collect();

// Add test module aliases

if let Some(hello_code) = entries

.iter()

.find(|(n, _)| *n == "__hello__")

.map(|(_, m)| m.code)

{

entries.push((

"__hello_alias__",

FrozenModule {

code: hello_code,

package: false,

));

entries.push((

"__phello_alias__",

FrozenModule {

code: hello_code,

package: true,

));

entries.push((

"__phello_alias__.spam",

FrozenModule {

code: hello_code,

package: false,

));

entries.push((

"__hello_only__",

FrozenModule {

code: hello_code,

package: false,

));

}

if let Some(code) = entries

.iter()

.find(|(n, _)| *n == "__phello__")

.map(|(_, m)| m.code)

{

entries.push((

"__phello__.__init__",

FrozenModule {

code,

package: false,

));

}

if let Some(code) = entries

.iter()

.find(|(n, _)| *n == "__phello__.ham")

.map(|(_, m)| m.code)

{

entries.push((

"__phello__.ham.__init__",

FrozenModule {

code,

package: false,

));

}

entries.into_iter()

}

#[cfg(test)]

mod tests {

use super::*;

use crate::{

AsObject, PyObjectRef,

builtins::{PyStr, int},

vm::{MAIN_INTERPRETER_ID, runtime},

};

use malachite_bigint::ToBigInt;

#[test]

fn add_py_integers() {

Interpreter::without_stdlib(Default::default()).enter(|vm| {

let a: PyObjectRef = vm.ctx.new_int(33_i32).into();

let b: PyObjectRef = vm.ctx.new_int(12_i32).into();

let res = vm._add(&a, &b).unwrap();

let value = int::get_value(&res);

assert_eq!(*value, 45_i32.to_bigint().unwrap());

})

}

#[test]

fn multiply_str() {

Interpreter::without_stdlib(Default::default()).enter(|vm| {

let a = vm.new_pyobj(crate::common::ascii!("Hello "));

let b = vm.new_pyobj(4_i32);

let res = vm._mul(&a, &b).unwrap();

let value = res.downcast_ref::<PyStr>().unwrap();

assert_eq!(value.as_wtf8(), "Hello Hello Hello Hello ")

})

}

/// Main interpreter is marked main with Runtime whence and is registered.

#[test]

fn main_interpreter_identity() {

let main = Interpreter::without_stdlib(Default::default());

assert!(main.is_main());

assert_eq!(main.whence(), InterpreterWhence::Runtime);

assert!(

runtime::list_interpreters()

.iter()

.any(|info| info.id == main.id() && info.whence == InterpreterWhence::Runtime)

);

// When this is the sole sequential main in a quiet process, id is 0;

// under parallel tests the id is still unique and registered.

assert!(main.id() >= MAIN_INTERPRETER_ID);

}

/// Subinterpreters get distinct ids, Stdlib whence, and appear in the registry.

#[test]

fn create_subinterpreter_registers_distinct_ids() {

let main = Interpreter::without_stdlib(Default::default());

let sub1 = main.create_subinterpreter();

let sub2 = main.create_subinterpreter();

assert!(main.is_main());

assert!(!sub1.is_main());

assert!(!sub2.is_main());

assert_eq!(sub1.whence(), InterpreterWhence::Stdlib);

assert_eq!(sub2.whence(), InterpreterWhence::Stdlib);

assert_ne!(main.id(), sub1.id());

assert_ne!(main.id(), sub2.id());

assert_ne!(sub1.id(), sub2.id());

let ids: Vec<i64> = runtime::list_interpreters()

.into_iter()

.map(|i| i.id)

.collect();

assert!(ids.contains(&main.id()));

assert!(ids.contains(&sub1.id()));

assert!(ids.contains(&sub2.id()));

}

/// An interpreter stays looked-up-able until nothing holds its state.

///

/// Dropping the handle is not the end of its life: `new_thread()` workers

/// hold their own reference, and a collection in progress holds one for

/// every live interpreter while the world is stopped. So the registry entry

/// goes away eventually rather than at the drop.

fn wait_until_unregistered(id: i64) {

use core::time::Duration;

use std::time::Instant;

let deadline = Instant::now() + Duration::from_secs(30);

while runtime::lookup_interpreter(id).is_some() {

assert!(

Instant::now() < deadline,

"interpreter {id} still registered long after its last reference"

);

std::thread::yield_now();

}

/// A collection snapshots the registry and then reads tracked objects with

/// the interpreters it found parked. An interpreter that registered inside

/// that window would be missing from the snapshot, so nothing would stop it

/// and its bootstrap would run under the scan; registration therefore waits

/// for the stop to end.

#[cfg(feature = "threading")]

#[test]

fn registering_waits_for_an_in_flight_stop() {

use core::time::Duration;

use std::sync::mpsc;

// Stands in for a collector between its snapshot and its restart.

let admission = runtime::lock_admission_for_stop();

let (tx, rx) = mpsc::channel();

let worker = std::thread::spawn(move || {

let interp = Interpreter::without_stdlib(Default::default());

tx.send(interp.id()).expect("receiver is alive");

interp

});

assert!(

matches!(

rx.recv_timeout(Duration::from_millis(200)),

Err(mpsc::RecvTimeoutError::Timeout)

"an interpreter registered while a stop-the-world was in flight"

);

drop(admission);

let id = rx

.recv_timeout(Duration::from_secs(30))

.expect("registration proceeds once the world restarts");

assert!(runtime::lookup_interpreter(id).is_some());

drop(worker.join().expect("worker did not panic"));

wait_until_unregistered(id);

}

/// Dropping a subinterpreter releases it; main remains.

#[test]

fn drop_subinterpreter_unregisters() {

let main = Interpreter::without_stdlib(Default::default());

let sub_id = {

let sub = main.create_subinterpreter();

let id = sub.id();

assert!(runtime::lookup_interpreter(id).is_some());

View remainder of file in raw view

FilesExpand file tree

interpreter.rs

Latest commit

History

interpreter.rs

File metadata and controls