問題描述
泡菜cython類 (pickle cython class)
I have to save and load a cython class instance. My cython class is this plus several methods:
import numpy as np
cimport numpy as np
cimport cython
cdef class Perceptron_avg_my:
cdef int wlen,freePos
cdef np.ndarray w,wtot,wac,wtotc #np.ndarray[np.int32_t]
cdef np.ndarray wmean #np.ndarray[np.float32_t]
cdef public dict fpos
def __cinit__(self,np.int64_t wlen=4*10**7):
self.fpos= dict()
self.freePos=1
self.wlen=wlen
self.w=np.zeros(wlen,np.int32)
self.wtot=np.zeros(wlen,np.int32)
self.wac=np.zeros(wlen,np.int32)
self.wtotc=np.zeros(wlen,np.int32)
self.wmean=np.zeros(wlen,np.float32)
cpdef evaluate_noavg(self,list f):
cdef np.ndarray[np.int32_t] w = self.w
cdef dict fpos = self.fpos
cdef bytes ff
cdef int i
cdef long int score=0
for ff in f:
i=fpos.get(ff,0)
if i != 0:
score += w[i]
return score
I was thinking to use the cPickle module. I understand that I have to implement a __reduce__(self) method but I have some problem to find an example and to understand well the documentation
I tried to add something like this to Perceptron_avg_my but not works:
def rebuild(self,l):
self.fpos=l[0]
self.freePos=l[1]
def __reduce__(self):
#print 'reduce call'
return (Perceptron_avg_my.rebuild,(self.fpos,self.freePos))
any suggestions? Thanks a lot!!!
參考解法
方法 1:
I don't know if you found it, but the official Python documentation has a section on pickling extension types (unfortunately there doesn't seem to be a Python 3 version of this doc, but it works the same in Python 3).
I think you have three problems here. Firstly, the function returned by __reduce__ is supposed to create a new object from scratch and return it, whereas your rebuild function just sets some attributes. Secondly, the tuple returned by __reduce__ must itself be picklable, and as a method, Perceptron_avg_my.rebuild is not picklable (I think this is expected to be fixed in python 3.3 or 3.4). Instead, you could turn it into a module-level function. Finally, the arguments (self.fpos,self.freePos) are passed to rebuild individually - you don't have to unpack the tuple yourself.
The following seems to work for me (though you probably want to store the values of the other attributes too, otherwise they will just have the initial values set by __init__):
#inside the class definition
def __reduce__(self):
return (rebuild, (self.wlen, self.fpos, self.freePos))
#standalone function
def rebuild(wlen, fpos, freePos):
p = Perceptron_avg_my(wlen)
p.fpos = fpos
p.freePos = freePos
return p
方法 2:
As of Cython 0.26 (released July 2017), implementing the pickle protocol is no longer necessary. All cdef classes that do not contain pointers or unions can automatically be pickled. For classes containing structs automatic pickling is disabled by default, due to (among other reasons) high code overhead. Automatic pickling can be enabled for classes with structs by using the @cython.auto_pickle(True) decorator.
More information can be found in the changelog and on the website of Stefan Behnel.
方法 3:
I used this workaround that works but I am not sure that it is the best solution.
I created a new support file to declare the function called by reduce (if I put it in the cython module it not works):
#perceptron_supp.py
from perceptron import Perceptron
def rebuild_perceptron(wlen,freePos,fpos,w,nw_avg,wtot_avg,wsup_avg,wmean_avg,wtot_my,wac_my,wtotc_my,wmean_my):
return Perceptron(wlen,True,freePos,fpos,w,nw_avg,wtot_avg,wsup_avg,wmean_avg,wtot_my,wac_my,wtotc_my,wmean_my)
and then I import this function in cython module:
#perceptron.pyx
import numpy as np
cimport numpy as np
cimport cython
#added
from perceptron_supp import rebuild_perceptron
cdef class Perceptron:
cdef int wlen,freePos
cdef dict fpos
cdef np.ndarray w #np.ndarray[np.int32_t]
cdef int nw_avg
cdef np.ndarray wtot_avg,wsup_avg #np.ndarray[np.int32_t]
cdef np.ndarray wmean_avg #np.ndarray[np.float64_t]
cdef np.ndarray wtot_my,wac_my,wtotc_my #np.ndarray[np.int32_t]
cdef np.ndarray wmean_my #np.ndarray[np.float64_t]
def __cinit__(self,int wlen=4*10**7,setValues=False,freePos=0,fpos=0,w=0,nw_avg=0,wtot_avg=0,wsup_avg=0,wmean_avg=0,wtot_my=0,wac_my=0,wtotc_my=0,wmean_my=0):
if not setValues:
self.wlen=wlen
self.freePos=1
self.fpos= dict()
self.w=np.zeros(wlen,np.int32)
self.nw_avg=1
self.wtot_avg=np.zeros(wlen,np.int32)
self.wsup_avg=np.zeros(wlen,np.int32)
self.wmean_avg=np.zeros(wlen,np.float64)
self.wtot_my=np.zeros(wlen,np.int32)
self.wac_my=np.zeros(wlen,np.int32)
self.wtotc_my=np.zeros(wlen,np.int32)
self.wmean_my=np.zeros(wlen,np.float64)
else:
self.wlen=wlen
self.freePos=freePos
self.fpos=fpos
self.w=w
self.nw_avg=nw_avg
self.wtot_avg=wtot_avg
self.wsup_avg=wsup_avg
self.wmean_avg=wmean_avg
self.wtot_my=wtot_my
self.wac_my=wac_my
self.wtotc_my=wtotc_my
self.wmean_my=wmean_my
def __reduce__(self):
return (rebuild_perceptron,(self.wlen,self.freePos,self.fpos,self.w,self.nw_avg,self.wtot_avg,self.wsup_avg,self.wmean_avg,self.wtot_my,self.wac_my,self.wtotc_my,self.wmean_my))
when I use my perceptron module I have just to do: from perceptron import Perceptron and now I can do cPyckle.dump or cPickle.load when I need.
If somebody has a better solution thanks a lot!!!
(by Francesco、James、m00am、Francesco)