B []o @sndZddlmZmZmZddlmZddlZddlZddlm Z ddl Z ddl m Z ddlmZddlmZmZdd lmZmZmZmZmZdd lmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,dd l-m.Z.dd l/m0Z0m1Z1m2Z2dd l3m4Z4m5Z5ddl6m7Z7ddl8m9Z9m:Z:m;Z;ddlZ>ddl?m@Z@mAZAddlBmCZCmDZEmFZGdaHddZId;ddZJe Kde Kde Kde Kde Kde Kde KdddZLd d!ZMe jNe jNe jOd"ZPeQe d#re jRePd$<d%d&ZSd'd(ZTd.writerzFile b'{}' does not exist) warningswarn FutureWarningpop compressorr- isinstancestropenFileNotFoundErrorformatrgetvalue) path_or_bufr;r<r5moder?r9msgbufr=)r;r<r> to_msgpackbs.     rOutf-8Fc stjdtddt\}}|r,tSfdd}ttrytd }||SQRXWn&tk rd }t|YnXtt rd}zt }||S|dk r| Xnt d rtjr|Std dS) aX Load msgpack pandas object from the specified file path. .. deprecated:: 0.25.0 read_msgpack is deprecated and will be removed in a future version. It is recommended to use pyarrow for on-the-wire transmission of pandas objects. Parameters ---------- path_or_buf : str, path object or file-like object Any valid string path is acceptable. The string could be a URL. Valid URL schemes include http, ftp, s3, and file. For file URLs, a host is expected. If you want to pass in a path object, pandas accepts any ``os.PathLike``. By file-like object, we refer to objects with a ``read()`` method, such as a file handler (e.g. via builtin ``open`` function) or ``StringIO``. encoding : Encoding for decoding msgpack str type iterator : boolean, if True, return an iterator to the unpacker (default is False) Returns ------- obj : same type as object stored in file Notes ----- read_msgpack is only guaranteed to be backwards compatible to pandas 0.20.3. zThe read_msgpack is deprecated and will be removed in a future version. It is recommended to use pyarrow for on-the-wire transmission of pandas objects.r2)r3csVtt|fdi}t|dkr,|dSrRy Wntk rPYnX|S)Nencodingr)listunpacklencloseIOError)r9Z unpacked_obj)rQr<rK should_closer=r>reads  zread_msgpack..readrbNzFile b'{}' does not existrYz7path_or_buf needs to be a string file path or file-like)r@rArBr.IteratorrErFrGrHrIbytesrrVhasattrcallablerY ValueError)rKrQiteratorr<_rYr9rMr=)rQr<rKrXr> read_msgpacks2%     rbzM8[ns]zM8[us]zm8[ns]zm8[us]int64category)zdatetime64[ns]zdatetime64[us]ztimedelta64[ns]ztimedelta64[us]rdcCs|tkrt|Stj||S)z1 return my dtype mapping, whether number or name ) dtype_dictnptypeDictget)tr=r=r> dtype_forsrm)complexZ complex128Z complex64float128Z complex256cCs&t|}tj|||d||S)zO Convert strings to complex number instance with specified numpy type. y?)c2f_dictrirj)riZ ctype_nameZftyper=r=r>c2fsrscCs|j}t|r|St|r&|St|r8|d}|}tdkr~tddd}|t j krf|S| }t d| |Stdkrtddd}|t j kr|S| }t d|j ||jdSt d| S) z$ convert the numpy values to a list i8zlibz(zlib is required when `compress='zlib'`.)extrarbloscz)zlib is required when `compress='blosc'`.)Ztypesize)dtyper r ZraveltolistrviewrDrriZobject_tostringr/r4itemsize)valuesrxvrurwr=r=r>converts.      rc Cst|to|jdk}|r|j}t|r*|St|r@tj|tdSt |j }|sX| d}|r|dkrxt ddd}|j }n$|dkrt ddd}|j }ntd ytjt|||dStk r}z&|jd}t|d krtd tWdd}~XYnXtj||d}|}d |j_|S) Nr)rxlatin1ruz(zlib is required when `compress='zlib'`.)rvrwz)zlib is required when `compress='blosc'`.z)compress must be one of 'zlib' or 'blosc'rRzUcopying data after decompressing; this may mean that decompress is caching its resultT)rEr/codedatar r riarrayobjectrbaseencoder decompressr_Z frombuffer_move_into_mutable_buffer_BadMover;rUr@rArcopyflagsZ writeable) r}rxr4Z as_is_extrurrwerNr=r=r> unconvertEsB      rc Cst|}t|trt|trHd|jjt|dd|jj|jj |jj dSt|t rd|jjt|ddt|dd|j j t|jtdSt|trt|dd}|dk r|j}|d }d |jjt|dd|j j t|jt|dd|td St|ttfrBt|trd }nd }||jjt|ddt|ddt|ddt|dddSt|trvd|jjt|dd|j j t|jtdSd|jjt|dd|j j t|jtdSn8t|trd|jjt|dd|j|j|jtdSt|tr&t|trtdn,d|jjt|dd|j|j j t|jtdSnt |t!rt|t"rHtdn8|j#}|$s`|%}d|jj|j&dd|j'Dd SnVt|t(t)t*j+t,t*j-fs|t.krt|t/r|j0}|dk r|j}|j1}|dk r|j2}d!|j3||d"S|t.krd#d$iSt|t*j-r d%|4d&d'St|t,rBd(|j5|j6|j7fd'St|t*j+r^d)t8|d'St|t(rxd*|9d'St|t)rd+|9d'St:d,j;|d-n4t|t<rd.|j=|j2d/St|t>rd0|j?|j@|jAd1St|tBrd2|jj|jC|jD|jEd3St|tFr*d4|jj|jG|jEd5St|t*jHrVd6|jI|jJ|j j t|td7St|t*jKrt*L|rd8d9|j j t*M|Nt*O|Nd:Sd8|j j |Nd;Sn,t|tPrd9t*M|Nt*O|Nd<S|S)=z Data encoder range_indexnameN)typklassrstartstopstep period_indexfreqstr)rrrfreqrxrr4tzUTCdatetime_index)rrrrxrrrr4interval_indexinterval_arrayleftrightclosed)rrrrrr multi_indexnames)rrrrxrr4index)rrrrxrr4rd)rrrcodes categoriesorderedr4z(msgpack sparse series is not implementedseries)rrrrrxrr4z'msgpack sparse frame is not implemented block_managerc Ss4g|],}|jjt|j|jj|jj|jjt dqS))locsr}shaperxrr4) Zmgr_locsZas_arrayrr}rrxr __class____name__rD).0br=r=r> szencode..)rraxesblocks timestamp)rvaluerrrnat timedelta64rt)rrr datetime64rrz-cannot encode this datetimelike object: {obj})objperiod)rordinalrinterval)rrrr block_index)rrblocsblengthslength int_index)rrindicesrndarray)rrndimrxrr4 np_scalar np_complex)rsub_typrxrealimag)rrxr)rrr)QtyperErrrrgetattrZ_rangerrrrrxrrZasi8rDrzone tz_convertrr#rr}rrrrrr,NotImplementedErrorr issubclassr'r+_dataZis_consolidatedZ consolidaterrrrrirrrrr tzinforrrrzdaysseconds microsecondsrF isoformat ExceptionrIrrrrrrr%rrrr&rrrrnumberZ iscomplexobjr__repr__rrn)rZtobjrrrrr=r=r>rs                                                   rc s|d}|dkr|S|dkrLd|kr.|dn|d}t|d|d|dS|d krXtS|d krtt|d |dd S|d krt|d}t|d||d}t|||ddS|dkrt|d|d|d|ddS|dkr$t|d}t|d||d}dd|D}tj ||ddS|dkrxt|dt j |d}t |d|dd}| dd}tt||f|S|dkrt|dt j |d}t |d|dd}t|f|}|d}|dk r|d |}|S|d!krt|d"j|d#|d$|d%|ddS|d&krJt|d"j}||d'|d(|d)d*S|d+krlt|d#|d$|d%S|d,krt|d}|d } t|d||d}t|| ||dd-S|d.kr|d/fd0d1fd2d|d3D} t|d"t| S|d4krt|dS|d5kr0t t|dS|d6krJt|dS|d7kr`t|dS|d8kr|t t|dS|d9krt|d"|d:|d;|d<S|d=krt|d"|d:|d>S|d?krt|dt j|d|d |d@S|dAkrt|dBdCkr4t!|dD|dE|dSt|d}y ||dSt"t#fk rp|$|dSXn@|dCkrt%|dDdF|dEdGSt&|t t't(fr|S|SdS)Hz5 Decoder for deserializing numpy data types. rNrroffsetrr)rrrrr)rrrrxrr4r)rxrrrrr)rrcSsg|] }t|qSr=)tuple)rxr=r=r>r{szdecode..r)rr)rrrr)rrrrrrrdrrr)rrrrr)rrxrrrcstt|dt|d|d|d}d|kr8|d}nd|d}t|drt|tjsntt ||j dkst|j t ||dd }t |t t|d ||dd S) Nr}rxr4rrritemszM8[ns])rxr)r}r placementrx)r)rrmZ get_indexerr rErirAssertionErrorrrxr"r*rr!)rr}r)rr=r> create_blocks$   zdecode..create_blockcsg|] }|qSr=r=)rr)rr=r>rsrrrrrrrrrrrrrrrrrrr+j))rkr rrrmrrrr from_tuplesrircdictrCrr$rZ tz_localizerglobalsZ from_arrays from_codesrrr(rrrrrintrjZreshapersr_ TypeErrorrrnrErSset) rrrrxrdresultrrrrr=)rrr>decodecs                         "       rstrictrRcCst||||||d|S)z5 Pack an object and return the packed bytes. )defaultrQunicode_errorsuse_single_float autoreset use_bin_type)r0r8)orrQrrrrr=r=r>r8s r8c Cst|||||||||d S)zf Unpack a packed object, return an iterator Note: packed lists will be returned as tuples ) object_hook list_hookuse_listrQrobject_pairs_hookmax_buffer_sizeext_hook)r1) packedrrrrQrrrrr=r=r>rTsrTcs*eZdZedddddffdd ZZS)r0zutf-8rFrRcstj||||||ddS)N)rrQrrrr)super__init__)selfrrQrrrr)rr=r>r!s zPacker.__init__)r __module__ __qualname__rr __classcell__r=r=)rr>r0 s r0c s2eZdZdddedddddef fdd ZZS)r1NrFzutf-8rc s$tj||||||||| | d dS)N) file_like read_sizerrrrrQrrr)rr) rrrrrrrrQrrr)rr=r>r5s zUnpacker.__init__)rrrrr/rrr=r=)rr>r14sr1c@s eZdZdZddZddZdS)r[zF manage the unpacking iteration, close the file on completion cKs||_||_dS)N)pathr<)rrr<r=r=r>rUszIterator.__init__ccsd}zt|jtr\ytj|j}Wntk r<d}YnX|rPt|jd}q~t|j}n"t|jdstt|j}n d}|j}t |}x|D] }|VqWWd|r| XdS)NTFrZrY) rErrFosexistsrrGrr]rTrV)rZ needs_closingZ path_existsr9unpackerrr=r=r>__iter__Ys&      zIterator.__iter__N)rrr__doc__rrr=r=r=r>r[Psr[)rPF)N)[rrrriorrr@Zdateutil.parserrZnumpyriZpandas.compat._optionalrZ pandas.errorsrZpandas.util._mover rr rZpandas.core.dtypes.commonr r r rrZpandasrrrrrrrrrrrrrrrrr Z pandas.corer!Zpandas.core.arraysr"r#r$Zpandas.core.arrays.sparser%r&Zpandas.core.genericr'Zpandas.core.internalsr(r)r*Zpandas.core.sparse.apir+r,Zpandas.io.commonr-r.Zpandas.io.msgpackr/r0Z_Packerr1Z _UnpackerrDrOrbrxrhrmZfloat64Zfloat32rpr]rorsrrrrr8rTr[r=r=r=r>'st    L  : X   - =b