diff --git a/tests/snippets/weakrefs.py b/tests/snippets/weakrefs.py index ffc61f4271b..ee201bb71d4 100644 --- a/tests/snippets/weakrefs.py +++ b/tests/snippets/weakrefs.py @@ -1,13 +1,62 @@ +import sys from _weakref import ref +from testutils import assert_raises + +data_holder = {} + class X: - pass + def __init__(self, param=0): + self.param = param + + def __str__(self): + return f"param: {self.param}" a = X() b = ref(a) + +def callback(weak_ref): + assert weak_ref is c + assert b() is None, 'reference to same object is dead' + assert c() is None, 'reference is dead' + data_holder['first'] = True + + +c = ref(a, callback) + + +def never_callback(_weak_ref): + data_holder['never'] = True + + +# weakref should be cleaned up before object, so callback is never called +ref(a, never_callback) + assert callable(b) assert b() is a +assert 'first' not in data_holder +del a +assert b() is None +assert 'first' in data_holder +assert 'never' not in data_holder + +# TODO proper detection of RustPython if sys.implementation.name == 'RustPython': +if not hasattr(sys, 'implementation'): + # implementation detail that the object isn't dropped straight away + # this tests that when an object is resurrected it still acts as normal + delayed_drop = X(5) + delayed_drop_ref = ref(delayed_drop) + + delayed_drop = None + + assert delayed_drop_ref() is not None + value = delayed_drop_ref() + del delayed_drop # triggers process_deletes + + assert str(value) == "param: 5" + +assert_raises(TypeError, lambda: ref(1), "can't create weak reference to an int") diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index d3aaa6ee059..c0d4635e753 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -16,8 +16,7 @@ use super::obj::objstr; use super::obj::objtype; use super::pyobject::{ - AttributeProtocol, IdProtocol, PyContext, PyFuncArgs, PyObject, PyObjectPayload, PyObjectRef, - PyResult, Scope, TypeProtocol, + AttributeProtocol, IdProtocol, PyContext, PyFuncArgs, PyObjectRef, PyResult, TypeProtocol, }; use super::stdlib::io::io_open; @@ -265,17 +264,7 @@ fn make_scope(vm: &mut VirtualMachine, locals: Option<&PyObjectRef>) -> PyObject }; // TODO: handle optional globals - // Construct new scope: - let scope_inner = Scope { - locals, - parent: None, - }; - - PyObject { - payload: PyObjectPayload::Scope { scope: scope_inner }, - typ: None, - } - .into_ref() + vm.ctx.new_scope_with_locals(None, locals) } fn builtin_format(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { diff --git a/vm/src/frame.rs b/vm/src/frame.rs index d57c9de73d7..a0aee226ad4 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -140,6 +140,8 @@ impl Frame { } }; + PyObjectRef::process_deletes(vm); + vm.current_frame = prev_frame; value } @@ -844,6 +846,10 @@ impl Frame { // Assume here that locals is a dict let name = vm.ctx.new_str(name.to_string()); vm.call_method(&locals, "__delitem__", vec![name])?; + + // process possible delete + PyObjectRef::process_deletes(vm); + Ok(None) } diff --git a/vm/src/obj/mod.rs b/vm/src/obj/mod.rs index f5dff7563cd..ebd3d7961c3 100644 --- a/vm/src/obj/mod.rs +++ b/vm/src/obj/mod.rs @@ -28,4 +28,5 @@ pub mod objstr; pub mod objsuper; pub mod objtuple; pub mod objtype; +pub mod objweakref; pub mod objzip; diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index a0b651f5e7a..185c406060e 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -1,6 +1,6 @@ use super::super::format::{FormatParseError, FormatPart, FormatString}; use super::super::pyobject::{ - PyContext, PyFuncArgs, PyObject, PyObjectPayload, PyObjectRef, PyResult, TypeProtocol, + PyContext, PyFuncArgs, PyObjectPayload, PyObjectRef, PyResult, TypeProtocol, }; use super::super::vm::VirtualMachine; use super::objint; @@ -1126,8 +1126,8 @@ pub fn subscript(vm: &mut VirtualMachine, value: &str, b: PyObjectRef) -> PyResu // help get optional string indices fn get_slice( - start: Option<&std::rc::Rc>>, - end: Option<&std::rc::Rc>>, + start: Option<&PyObjectRef>, + end: Option<&PyObjectRef>, len: usize, ) -> Result<(usize, usize), String> { let start_idx = match start { diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index 95c493a450e..0b3be70a61a 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -232,7 +232,7 @@ pub fn get_attributes(obj: &PyObjectRef) -> PyAttributes { } // Get instance attributes: - if let PyObjectPayload::Instance { dict } = &obj.borrow().payload { + if let PyObjectPayload::Instance { dict, .. } = &obj.borrow().payload { for (name, value) in dict.borrow().iter() { attributes.insert(name.to_string(), value.clone()); } diff --git a/vm/src/obj/objweakref.rs b/vm/src/obj/objweakref.rs new file mode 100644 index 00000000000..7c3eba943ec --- /dev/null +++ b/vm/src/obj/objweakref.rs @@ -0,0 +1,86 @@ +use super::super::pyobject::{ + PyContext, PyFuncArgs, PyObject, PyObjectPayload, PyObjectRef, PyObjectWeakRef, PyResult, + TypeProtocol, +}; +use super::super::vm::VirtualMachine; +use super::objtype; // Required for arg_check! to use isinstance + +fn ref_new(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { + // TODO: check first argument for subclass of `ref`. + arg_check!( + vm, + args, + required = [(cls, Some(vm.ctx.type_type())), (referent, None)], + optional = [(callback, None)] + ); + let weak_referent = PyObjectRef::downgrade(referent); + let weakref = PyObject::new( + PyObjectPayload::WeakRef { + referent: weak_referent, + callback: callback.cloned(), + }, + cls.clone(), + ); + if referent.borrow_mut().add_weakref(&weakref) { + Ok(weakref) + } else { + let referent_repr = vm.to_pystr(&referent.typ())?; + Err(vm.new_type_error(format!( + "cannot create weak reference to '{}' object", + referent_repr + ))) + } +} + +/// Dereference the weakref, and check if we still refer something. +fn ref_call(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(zelf, Some(vm.ctx.weakref_type()))]); + let referent = get_value(zelf); + let py_obj = if let Some(obj) = referent.upgrade() { + obj + } else { + vm.get_none() + }; + Ok(py_obj) +} + +fn get_value(obj: &PyObjectRef) -> PyObjectWeakRef { + if let PyObjectPayload::WeakRef { referent, .. } = &obj.borrow().payload { + referent.clone() + } else { + panic!("Inner error getting weak ref {:?}", obj); + } +} + +fn get_callback(obj: &PyObjectRef) -> Option { + if let PyObjectPayload::WeakRef { callback, .. } = &obj.borrow().payload { + callback.as_ref().cloned() + } else { + panic!("Inner error getting weak ref callback {:?}", obj); + } +} + +pub fn clear_weak_ref(obj: &PyObjectRef) { + if let PyObjectPayload::WeakRef { + ref mut referent, .. + } = &mut obj.borrow_mut().payload + { + referent.clear(); + } else { + panic!("Inner error getting weak ref {:?}", obj); + } +} + +pub fn notify_weak_ref(vm: &mut VirtualMachine, obj: PyObjectRef) -> PyResult { + if let Some(callback) = get_callback(&obj) { + vm.invoke(callback.clone(), PyFuncArgs::new(vec![obj], vec![])) + } else { + Ok(vm.get_none()) + } +} + +pub fn init(context: &PyContext) { + let weakref_type = &context.weakref_type; + context.set_attr(weakref_type, "__new__", context.new_rustfunc(ref_new)); + context.set_attr(weakref_type, "__call__", context.new_rustfunc(ref_call)); +} diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 4ec2058e4da..eeba2635107 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -28,6 +28,7 @@ use super::obj::objstr; use super::obj::objsuper; use super::obj::objtuple; use super::obj::objtype; +use super::obj::objweakref; use super::obj::objzip; use super::vm::VirtualMachine; use num_bigint::BigInt; @@ -35,8 +36,11 @@ use num_bigint::ToBigInt; use num_complex::Complex64; use num_traits::{One, Zero}; use std::cell::RefCell; -use std::collections::HashMap; +use std::collections::{HashMap, VecDeque}; use std::fmt; +use std::mem; +use std::ops::Deref; +use std::ptr; use std::rc::{Rc, Weak}; /* Python objects and references. @@ -54,20 +58,121 @@ Basically reference counting, but then done by rust. */ /* -The PyRef type implements +The PyObjectRef type implements https://doc.rust-lang.org/std/cell/index.html#introducing-mutability-inside-of-something-immutable */ -pub type PyRef = Rc>; /// The `PyObjectRef` is one of the most used types. It is a reference to a /// python object. A single python object can have multiple references, and /// this reference counting is accounted for by this type. Use the `.clone()` /// method to create a new reference and increment the amount of references /// to the python object by 1. -pub type PyObjectRef = PyRef; +#[derive(Debug, Clone)] +pub struct PyObjectRef { + rc: Rc>, +} + +// TODO support multiple virtual machines, how? +thread_local! { + static DELETE_QUEUE: RefCell> = RefCell::new(VecDeque::new()); +} + +impl PyObjectRef { + pub fn downgrade(this: &Self) -> PyObjectWeakRef { + PyObjectWeakRef { + weak: Rc::downgrade(&this.rc), + } + } + + pub fn strong_count(this: &Self) -> usize { + Rc::strong_count(&this.rc) + } + + pub fn process_deletes(vm: &mut VirtualMachine) { + while let Some(obj) = DELETE_QUEUE.with(|q| q.borrow_mut().pop_front()) { + // check is still needs to be deleted + if PyObjectRef::strong_count(&obj) == 1 { + let weak_refs = match obj.borrow().payload { + PyObjectPayload::Instance { ref weak_refs, .. } => weak_refs + .iter() + .flat_map(|x| x.upgrade()) + .collect::>(), + _ => panic!("Non-instance object can't have weakrefs"), + }; + + for weak_ref in weak_refs.iter() { + objweakref::clear_weak_ref(weak_ref); + } + + for weak_ref in weak_refs.into_iter().rev() { + if let Err(err) = objweakref::notify_weak_ref(vm, weak_ref) { + exceptions::print_exception(vm, &err); + } + } + + assert_eq!(PyObjectRef::strong_count(&obj), 1, + "Reference count of {:?} changed during execution of callbacks of weak references", + &obj); + + let mut manually_drop = mem::ManuallyDrop::new(obj); -/// Same as PyObjectRef, except for being a weak reference. -pub type PyObjectWeakRef = Weak>; + // dispose of value without calling normal drop + unsafe { + let rc_ptr = &mut manually_drop.rc as *mut _; + ptr::drop_in_place(rc_ptr); + } + } else { + info!("Object {:?} resurrected", obj); + } + } + } +} + +impl Deref for PyObjectRef { + type Target = RefCell; + + fn deref(&self) -> &Self::Target { + &self.rc + } +} + +impl Drop for PyObjectRef { + fn drop(&mut self) { + if PyObjectRef::strong_count(self) == 1 { + let has_weakrefs = match self.borrow().payload { + PyObjectPayload::Instance { ref weak_refs, .. } => !weak_refs.is_empty(), + _ => false, + }; + + if has_weakrefs { + // The PyObject is going to be dropped, delete it later when we have access to vm. + // This will error when the tls is destroyed, thus meaning objects won't actually + // be properly destroyed. + if DELETE_QUEUE + .try_with(|q| q.borrow_mut().push_back(self.clone())) + .is_err() + { + warn!("Object hasn't been cleaned up {:?}.", self); + } + } + } + } +} + +#[derive(Debug, Clone)] +pub struct PyObjectWeakRef { + weak: Weak>, +} + +impl PyObjectWeakRef { + pub fn upgrade(&self) -> Option { + self.weak.upgrade().map(|x| PyObjectRef { rc: x }) + } + + pub fn clear(&mut self) { + self.weak = Weak::default(); + } +} /// Use this type for function which return a python object or and exception. /// Both the python object and the python exception are `PyObjectRef` types @@ -148,6 +253,7 @@ pub struct PyContext { pub module_type: PyObjectRef, pub bound_method_type: PyObjectRef, pub member_descriptor_type: PyObjectRef, + pub weakref_type: PyObjectRef, pub object: PyObjectRef, pub exceptions: exceptions::ExceptionZoo, } @@ -164,11 +270,7 @@ pub struct Scope { } fn _nothing() -> PyObjectRef { - PyObject { - payload: PyObjectPayload::None, - typ: None, - } - .into_ref() + PyObject::new_no_type(PyObjectPayload::None) } pub fn create_type( @@ -204,6 +306,7 @@ impl PyContext { ); let property_type = create_type("property", &type_type, &object_type, &dict_type); let super_type = create_type("super", &type_type, &object_type, &dict_type); + let weakref_type = create_type("ref", &type_type, &object_type, &dict_type); let generator_type = create_type("generator", &type_type, &object_type, &dict_type); let bound_method_type = create_type("method", &type_type, &object_type, &dict_type); let member_descriptor_type = @@ -289,6 +392,7 @@ impl PyContext { module_type, bound_method_type, member_descriptor_type, + weakref_type, type_type, exceptions, }; @@ -320,6 +424,7 @@ impl PyContext { objbool::init(&context); objcode::init(&context); objframe::init(&context); + objweakref::init(&context); objnone::init(&context); exceptions::init(&context); context @@ -447,6 +552,9 @@ impl PyContext { pub fn member_descriptor_type(&self) -> PyObjectRef { self.member_descriptor_type.clone() } + pub fn weakref_type(&self) -> PyObjectRef { + self.weakref_type.clone() + } pub fn type_type(&self) -> PyObjectRef { self.type_type.clone() } @@ -535,12 +643,16 @@ impl PyContext { pub fn new_scope(&self, parent: Option) -> PyObjectRef { let locals = self.new_dict(); + self.new_scope_with_locals(parent, locals) + } + + pub fn new_scope_with_locals( + &self, + parent: Option, + locals: PyObjectRef, + ) -> PyObjectRef { let scope = Scope { locals, parent }; - PyObject { - payload: PyObjectPayload::Scope { scope }, - typ: None, - } - .into_ref() + PyObject::new_no_type(PyObjectPayload::Scope { scope }) } pub fn new_module(&self, name: &str, scope: PyObjectRef) -> PyObjectRef { @@ -634,6 +746,7 @@ impl PyContext { PyObject::new( PyObjectPayload::Instance { dict: RefCell::new(dict), + weak_refs: Vec::new(), }, class, ) @@ -660,7 +773,8 @@ impl PyContext { pub fn set_attr(&self, obj: &PyObjectRef, attr_name: &str, value: PyObjectRef) { match obj.borrow().payload { PyObjectPayload::Module { ref dict, .. } => self.set_attr(dict, attr_name, value), - PyObjectPayload::Instance { ref dict } | PyObjectPayload::Class { ref dict, .. } => { + PyObjectPayload::Instance { ref dict, .. } + | PyObjectPayload::Class { ref dict, .. } => { dict.borrow_mut().insert(attr_name.to_string(), value); } PyObjectPayload::Scope { ref scope } => { @@ -799,7 +913,7 @@ impl AttributeProtocol for PyObjectRef { } None } - PyObjectPayload::Instance { ref dict } => dict.borrow().get(attr_name).cloned(), + PyObjectPayload::Instance { ref dict, .. } => dict.borrow().get(attr_name).cloned(), _ => None, } } @@ -811,7 +925,7 @@ impl AttributeProtocol for PyObjectRef { PyObjectPayload::Class { ref mro, .. } => { class_has_item(self, attr_name) || mro.iter().any(|d| class_has_item(d, attr_name)) } - PyObjectPayload::Instance { ref dict } => dict.borrow().contains_key(attr_name), + PyObjectPayload::Instance { ref dict, .. } => dict.borrow().contains_key(attr_name), _ => false, } } @@ -1015,9 +1129,11 @@ pub enum PyObjectPayload { }, WeakRef { referent: PyObjectWeakRef, + callback: Option, }, Instance { dict: RefCell, + weak_refs: Vec, }, RustFunction { function: Box PyResult>, @@ -1076,9 +1192,32 @@ impl PyObject { .into_ref() } + fn new_no_type(payload: PyObjectPayload) -> PyObjectRef { + PyObject { + payload, + typ: None, + // dict: HashMap::new(), // dict, + } + .into_ref() + } + // Move this object into a reference object, transferring ownership. - pub fn into_ref(self) -> PyObjectRef { - Rc::new(RefCell::new(self)) + fn into_ref(self) -> PyObjectRef { + PyObjectRef { + rc: Rc::new(RefCell::new(self)), + } + } + + pub fn add_weakref(&mut self, weakref: &PyObjectRef) -> bool { + match self.payload { + PyObjectPayload::Instance { + ref mut weak_refs, .. + } => { + weak_refs.push(PyObjectRef::downgrade(weakref)); + true + } + _ => false, + } } } diff --git a/vm/src/stdlib/weakref.rs b/vm/src/stdlib/weakref.rs index 8f407675bbb..0c542019539 100644 --- a/vm/src/stdlib/weakref.rs +++ b/vm/src/stdlib/weakref.rs @@ -5,51 +5,11 @@ //! - [rust weak struct](https://doc.rust-lang.org/std/rc/struct.Weak.html) //! -use super::super::obj::objtype; -use super::super::pyobject::{ - PyContext, PyFuncArgs, PyObject, PyObjectPayload, PyObjectRef, PyObjectWeakRef, PyResult, - TypeProtocol, -}; -use super::super::VirtualMachine; -use std::rc::Rc; +use super::super::pyobject::{PyContext, PyObjectRef}; pub fn mk_module(ctx: &PyContext) -> PyObjectRef { let py_mod = ctx.new_module("_weakref", ctx.new_scope(None)); - let py_ref_class = ctx.new_class("ref", ctx.object()); - ctx.set_attr(&py_ref_class, "__new__", ctx.new_rustfunc(ref_new)); - ctx.set_attr(&py_ref_class, "__call__", ctx.new_rustfunc(ref_call)); - ctx.set_attr(&py_mod, "ref", py_ref_class); + ctx.set_attr(&py_mod, "ref", ctx.weakref_type()); py_mod } - -fn ref_new(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { - // TODO: check first argument for subclass of `ref`. - arg_check!(vm, args, required = [(cls, None), (referent, None)]); - let referent = Rc::downgrade(referent); - Ok(PyObject::new( - PyObjectPayload::WeakRef { referent }, - cls.clone(), - )) -} - -/// Dereference the weakref, and check if we still refer something. -fn ref_call(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { - // TODO: check first argument for subclass of `ref`. - arg_check!(vm, args, required = [(cls, None)]); - let referent = get_value(cls); - let py_obj = if let Some(obj) = referent.upgrade() { - obj - } else { - vm.get_none() - }; - Ok(py_obj) -} - -fn get_value(obj: &PyObjectRef) -> PyObjectWeakRef { - if let PyObjectPayload::WeakRef { referent } = &obj.borrow().payload { - referent.clone() - } else { - panic!("Inner error getting weak ref {:?}", obj); - } -} diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index e5c701fd6fc..5d2d7a171dd 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -1,6 +1,5 @@ use obj::objtype; use pyobject::{PyContext, PyFuncArgs, PyObjectRef, PyResult, TypeProtocol}; -use std::rc::Rc; use std::{env, mem}; use vm::VirtualMachine; @@ -24,7 +23,7 @@ fn getframe(vm: &mut VirtualMachine, _args: PyFuncArgs) -> PyResult { fn sys_getrefcount(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(object, None)]); - let size = Rc::strong_count(&object); + let size = PyObjectRef::strong_count(object); Ok(vm.ctx.new_int(size)) }