B [] @sdZddlZddlmZmZddlZddlZddlZddlZddlm Z m Z m Z m Z ddl Z ddlZddlmZmZddlmZmZddlmZddlmZdd lmZdd lmZmZm Z m!Z!m"Z"m#Z#m$Z$dd l%m&Z&dd l'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4dd l5m6Z6ddl7m8Z8m9Z9ddl:m;mm?Z?m@Z@ddlAmBZBddlCmDZDmEZEmFZFddlGmHZHddlImJZJmKZKdZLdZMddZNddZOddZPe?ZQddZRGdddeSZTGd d!d!eSZUGd"d#d#eVZWd$ZXGd%d&d&eVZYd'ZZGd(d)d)eVZ[d*Z\d+Z]d,d,d-d-d.Z^d/Z_e.d0e0d1e(d2e/d3iZ`d4d5d5d6d7d8d9d:Zad;dd?d@dAZbe(dgiZcdBZddCZeefdD8ejgdEdFedejhdGejgdHdeeeid,d-dgdGWdQRXdajdFakdIdJZlddKdLZmddNdOZndPdQZoGdRdSdSZpGdTdUdUZqGdVdWdWZrGdXdYdYerZsGdZd[d[erZtGd\d]d]etZuGd^d_d_euZvGd`dadaZwGdbdcdcewZxGdddedeexZyGdfd4d4eyZzGdgd5d5eyZ{Gdhd6d6exZ|GdidjdjexZ}Gdkd7d7e}Z~Gdld9d9e}ZGdmdndnexZGdod8d8eZGdpdqdqewZGdrd@d@eZGdsdtdteZGdudvdveZGdwd>d>eZGdxd<dnz_ensure_term..) scope_levelN)r0listtuplefilterr(appendTermlen)whererBlevelZwlistwr5r5r6 _ensure_termas   rLc@s eZdZdS)PossibleDataLossErrorN)__name__ __module__ __qualname__r5r5r5r6rMysrMc@s eZdZdS)ClosedFileErrorN)rNrOrPr5r5r5r6rQ}srQc@s eZdZdS)IncompatibilityWarningN)rNrOrPr5r5r5r6rRsrRz where criteria is being ignored as this version [%s] is too old (or not-defined), read the file in and write it out to a new file to upgrade (with the copy_to method) c@s eZdZdS)AttributeConflictWarningN)rNrOrPr5r5r5r6rSsrSzu the [%s] attribute of the existing index is [%s] which conflicts with the new [%s], resetting the attribute to None c@s eZdZdS)DuplicateWarningN)rNrOrPr5r5r5r6rTsrTz; duplicate entries in table, taking most recently appended z your performance may suffer as PyTables will pickle object types that it cannot map directly to c-types [inferred_type->%s,key->%s] [items->%s] fixedtable)frUtrVa the table keyword has been deprecated use the format='fixed(f)|table(t)' keyword instead fixed(f) : specifies the Fixed format and is the default for put operations table(t) : specifies the Table format and is the default for append operations series sparse_seriesframe sparse_frameLegacySeriesFixedLegacyFrameFixed SeriesFixedSparseSeriesFixed FrameFixedSparseFrameFixed)rrZ DataMatrixrYrZr[r\ GenericTableAppendableSeriesTableAppendableMultiSeriesTableAppendableFrameTableAppendableMultiFrameTable WORMTable) generic_tableappendable_seriesappendable_multiseriesappendable_frameappendable_multiframewormz; : boolean drop ALL nan rows when appending to a table z~ : format default format writing format, if None, then put will default to 'fixed' and append will default to 'table' zio.hdfZ dropna_tableF) validatordefault_formatcCs>tdkr:ddl}|ay|jjdkaWntk r8YnXtS)Nrstrict) _table_modtablesfileZ_FILE_OPEN_POLICY!_table_file_open_policy_is_strictAttributeError)rsr5r5r6_tablessrwc sj|rfdd}nfdd}t|}t|tr^t||||d} || WdQRXn||dS)z- store this object, close it if we opened it cs|jfS)N)rF)store)keykwargsvaluer5r6r@ rAzto_hdf..cs|jfS)N)put)rx)ryrzr{r5r6r@rA)mode complevelcomplibN)r-r0r;HDFStore) path_or_bufryr{r}r~rrFrzrWrxr5)ryrzr{r6to_hdfs  rrc Ks|dkrtd|d|kr2t|ddd|d<t|trT|jsJtd|}d}ntt|}t|tsnt dyt j |}Wnt tfk rd}YnX|std j|d t|fd |i|}d }yv|d kr*|}t|dkrtd|d}x*|dd D]} t| |stdqW|j}|j|fd|i|Stt tfk ry |Wntk rxYnXYnXd S)aK Read from the store, close it if we opened it. Retrieve pandas object stored in file, optionally based on where criteria Parameters ---------- path_or_buf : str, path object, pandas.HDFStore or file-like object Any valid string path is acceptable. The string could be a URL. Valid URL schemes include http, ftp, s3, and file. For file URLs, a host is expected. A local file could be: ``file://localhost/path/to/table.h5``. If you want to pass in a path object, pandas accepts any ``os.PathLike``. Alternatively, pandas accepts an open :class:`pandas.HDFStore` object. By file-like object, we refer to objects with a ``read()`` method, such as a file handler (e.g. via builtin ``open`` function) or ``StringIO``. .. versionadded:: 0.19.0 support for pathlib, py.path. .. versionadded:: 0.21.0 support for __fspath__ protocol. key : object, optional The group identifier in the store. Can be omitted if the HDF file contains a single pandas object. mode : {'r', 'r+', 'a'}, optional Mode to use when opening the file. Ignored if path_or_buf is a :class:`pandas.HDFStore`. Default is 'r'. where : list, optional A list of Term (or convertible) objects. start : int, optional Row number to start selection. stop : int, optional Row number to stop selection. columns : list, optional A list of columns names to return. iterator : bool, optional Return an iterator object. chunksize : int, optional Number of rows to include in an iteration when using an iterator. errors : str, default 'strict' Specifies how encoding and decoding errors are to be handled. See the errors argument for :func:`open` for a full list of options. **kwargs Additional keyword arguments passed to HDFStore. Returns ------- item : object The selected object. Return type depends on the object stored. See Also -------- DataFrame.to_hdf : Write a HDF file from a DataFrame. HDFStore : Low-level access to HDF files. Examples -------- >>> df = pd.DataFrame([[1, 1.0, 'a']], columns=['x', 'y', 'z']) >>> df.to_hdf('./store.h5', 'data') >>> reread = pd.read_hdf('./store.h5') )rzr+azOmode {0} is not allowed while performing a read. Allowed modes are r, r+ and a.rIr>)rBz&The HDFStore must be open for reading.Fz5Support for generic buffers has not been implemented.zFile {path} does not exist)pathr}TNrzNo dataset in HDF5 file.z?key must be provided when HDF5 file contains multiple datasets. auto_close) ValueErrorformatrLr0ris_openIOErrorr-r;NotImplementedErrorosrexists TypeErrorFileNotFoundErrorgroupsrH_is_metadata_of _v_pathnameselectKeyErrorcloserv) rryr}rzrxrrrZcandidate_only_groupZgroup_to_checkr5r5r6read_hdfsTD        rcCsJ|j|jkrdS|}x0|jdkrD|j}||kr<|jdkrmetaT)Z_v_depthZ _v_parent_v_name)groupZ parent_groupcurrentparentr5r5r6rs   rc@seZdZdZdZddZddZedd Zed d Zd d Z ddZ ddZ ddZ ddZ ddZddZddZddZddZd d!Zd"d#ZeZd[d%d&Zd'd(Zed)d*Zd\d+d,Zd-d.Zd]d/d0Zd^d1d2Zd3d4Zd_d5d6Zd`d7d8Zdad9d:Z dbdd?Z"d@dAZ#dBdCZ$dddEdFZ%dGdHZ&dIdJZ'dedLdMZ(dNdOZ)dPdQZ*dRdSZ+dfdTdUZ,dgdVdWZ-dXdYZ.dS)hraK Dict-like IO interface for storing pandas objects in PyTables either Fixed or Table format. Parameters ---------- path : string File path to HDF5 file mode : {'a', 'w', 'r', 'r+'}, default 'a' ``'r'`` Read-only; no data can be modified. ``'w'`` Write; a new file is created (an existing file with the same name would be deleted). ``'a'`` Append; an existing file is opened for reading and writing, and if the file does not exist it is created. ``'r+'`` It is similar to ``'a'``, but the file must already exist. complevel : int, 0-9, default None Specifies a compression level for data. A value of 0 or None disables compression. complib : {'zlib', 'lzo', 'bzip2', 'blosc'}, default 'zlib' Specifies the compression library to be used. As of v0.20.2 these additional compressors for Blosc are supported (default if no compressor specified: 'blosc:blosclz'): {'blosc:blosclz', 'blosc:lz4', 'blosc:lz4hc', 'blosc:snappy', 'blosc:zlib', 'blosc:zstd'}. Specifying a compression library which is not available issues a ValueError. fletcher32 : bool, default False If applying compression use the fletcher32 checksum Examples -------- >>> bar = pd.DataFrame(np.random.randn(10, 4)) >>> store = pd.HDFStore('test.h5') >>> store['foo'] = bar # write to HDF5 >>> bar = store['foo'] # retrieve >>> store.close() NFcKsd|krtdtd}|dk r@||jjkr@tdj|jjd|dkrX|dk rX|jj}t||_|dkrnd}||_d|_ |r|nd|_ ||_ ||_ d|_ |jfd|i|dS) Nrz-format is not a defined argument for HDFStorersz)complib only supports {libs} compression.)Zlibsrrr})rr filtersZ all_complibsrZdefault_complibr-_path_mode_handle _complevel_complib _fletcher32_filtersopen)selfrr}r~r fletcher32rzrsr5r5r6__init__s& zHDFStore.__init__cCs|jS)N)r)rr5r5r6 __fspath__szHDFStore.__fspath__cCs||jjS)z return the root node )_check_if_openrroot)rr5r5r6rsz HDFStore.rootcCs|jS)N)r)rr5r5r6filenameszHDFStore.filenamecCs ||S)N)get)rryr5r5r6 __getitem__szHDFStore.__getitem__cCs|||dS)N)r|)rryr{r5r5r6 __setitem__ szHDFStore.__setitem__cCs ||S)N)remove)rryr5r5r6 __delitem__szHDFStore.__delitem__c Cs@y ||Sttfk r"YnXtdjt|j|ddS)z& allow attribute access to get stores z+'{object}' object has no attribute '{name}')objectr<N)rrrQrvrtyperN)rr<r5r5r6 __getattr__s zHDFStore.__getattr__cCs8||}|dk r4|j}||ks0|dd|kr4dSdS)z| check for existence of this key can match the exact pathname or the pathnm w/o the leading '/' Nr>TF)get_noder)rrynoder<r5r5r6 __contains__s  zHDFStore.__contains__cCs t|S)N)rHr)rr5r5r6__len__(szHDFStore.__len__cCsdjt|t|jdS)Nz{type} File path: {path} )rr)rrr/r)rr5r5r6__repr__+szHDFStore.__repr__cCs|S)Nr5)rr5r5r6 __enter__0szHDFStore.__enter__cCs |dS)N)r)rexc_type exc_value tracebackr5r5r6__exit__3szHDFStore.__exit__cCsdd|DS)z Return a (potentially unordered) list of the keys corresponding to the objects stored in the HDFStore. These are ABSOLUTE path-names (e.g. have the leading '/' Returns ------- list cSsg|] }|jqSr5)r).0nr5r5r6 @sz!HDFStore.keys..)r)rr5r5r6keys6s z HDFStore.keyscCs t|S)N)iterr)rr5r5r6__iter__BszHDFStore.__iter__ccs"x|D]}|j|fVq WdS)z' iterate on key->group N)rr)rgr5r5r6itemsEszHDFStore.itemsrc Kst}|j|krL|jdkr$|dkr$n"|dkrF|jrFtd|j|j||_|jrZ||jr|jdkrtj|j|j |j d|_ y|j |j|jf||_ Wntk r}z:dt|krtdj|jd |j |jd f||_ nWd d }~XYntk rB}z.d t|kr.td j|j|d}|Wd d }~XYnLtk r}z,|jd krzdt|krztt|Wd d }~XYnXd S)z Open the file in the specified mode Parameters ---------- mode : {'a', 'w', 'r', 'r+'}, default 'a' See HDFStore docstring or tables.open_file for info about modes )rrK)rzr+)rKzGRe-opening the file [{0}] with mode [{1}] will delete the current file!r)rzcan not be writtenz Opening {path} in read-only mode)rrNZFILE_OPEN_POLICYa3PyTables [{version}] no longer supports opening multiple files even in read-only mode on this HDF5 version [{hdf_version}]. You can accept this and not open the same file multiple times at once, upgrade the HDF5 version, or downgrade to PyTables 3.0.0 which allows files to be opened multiple times at once )versionZ hdf_versionzUnable to open/create file)rwrrrMrrrrFiltersrrr open_filerrr;printr __version__Zget_hdf5_version Exception)rr}rzrser5r5r6rNsB    z HDFStore.opencCs|jdk r|jd|_dS)z0 Close the PyTables file handle N)rr)rr5r5r6rs  zHDFStore.closecCs|jdkrdSt|jjS)zF return a boolean indicating whether the file is open NF)rboolZisopen)rr5r5r6rs zHDFStore.is_opencCsF|jdk rB|j|rByt|jWntk r@YnXdS)a Force all buffered modifications to be written to disk. Parameters ---------- fsync : bool (default False) call ``os.fsync()`` on the file handle to force writing to disk. Notes ----- Without ``fsync=True``, flushing may not guarantee that the OS writes to disk. With fsync, the operation will block until the OS claims the file has been written; however, other caching layers may still interfere. N)rflushrfsyncfilenoOSError)rrr5r5r6rs  zHDFStore.flushcCs,||}|dkr"tdj|d||S)z Retrieve pandas object stored in file Parameters ---------- key : object Returns ------- obj : same type as object stored in file Nz!No object named {key} in the file)ry)rrr _read_group)rryrr5r5r6rs z HDFStore.getc st||} | dkr"tdj|dt|dd}|| fdd} t|| |j|||||d } | S) a Retrieve pandas object stored in file, optionally based on where criteria Parameters ---------- key : object where : list of Term (or convertible) objects, optional start : integer (defaults to None), row number to start selection stop : integer (defaults to None), row number to stop selection columns : a list of columns that if not None, will limit the return columns iterator : boolean, return an iterator, default False chunksize : nrows to include in iteration, return an iterator auto_close : boolean, should automatically close the store when finished, default is False Returns ------- The selected object Nz!No object named {key} in the file)ryr>)rBcsj|||dS)N)startstoprIcolumns)read)_start_stop_where)rr4r5r6funcszHDFStore.select..func)rInrowsrriterator chunksizer) rrrrL_create_storer infer_axes TableIteratorr get_result) rryrIrrrrrrrzrritr5)rr4r6rs&!   zHDFStore.selectcKs*t|dd}||jf|||d|S)aA return the selection as an Index Parameters ---------- key : object where : list of Term (or convertible) objects, optional start : integer (defaults to None), row number to start selection stop : integer (defaults to None), row number to stop selection r>)rB)rIrr)rL get_storerread_coordinates)rryrIrrrzr5r5r6select_as_coordinatess  zHDFStore.select_as_coordinatescKs||jfd|i|S)a return a single column from the table. This is generally only useful to select an indexable Parameters ---------- key : object column: the column of interest Raises ------ raises KeyError if the column is not found (or key is not a valid store) raises ValueError if the column can not be extracted individually (it is part of a data block) column)r read_column)rryrrzr5r5r6 select_column szHDFStore.select_columnc  szt|dd}t|ttfr.t|dkr.|d}t|trXjf||||||dSt|ttfsntdt|s~td|dkr|d}fdd |D |} d} xzt | |fgt |D]^\} }| dkrt d j|d | jstd j| jd | dkr| j} q| j| krtdqWtddDdfdd}t| ||| ||||| d }|jddS)a Retrieve pandas objects from multiple tables Parameters ---------- keys : a list of the tables selector : the table to apply the where criteria (defaults to keys[0] if not supplied) columns : the columns I want back start : integer (defaults to None), row number to start selection stop : integer (defaults to None), row number to stop selection iterator : boolean, return an iterator, default False chunksize : nrows to include in iteration, return an iterator Raises ------ raises KeyError if keys or selector is not found or keys is empty raises TypeError if keys is not a list or tuple raises ValueError if the tables are not ALL THE SAME DIMENSIONS r>)rBr)ryrIrrrrrzkeys must be a list/tuplez keys must have a non-zero lengthNcsg|]}|qSr5)r)rk)rr5r6rosz/HDFStore.select_as_multiple..zInvalid table [{key}])ryzKobject [{obj}] is not a table, and cannot be used in all select as multiple)objz,all tables must have exactly the same nrows!cSsh|]}|jddqS)r)non_index_axes)rrXr5r5r6 sz.HDFStore.select_as_multiple..cs,fddD}t|ddS)Nc s&g|]}|jfdqS))rIrrr)r)rrX)rrrrrzr5r6rsz=HDFStore.select_as_multiple..func..F)axisverify_integrity)r! _consolidate)rrrobjs)rrrztbls)rrrr6rsz)HDFStore.select_as_multiple..func)rIrrrrrrT) coordinates)rLr0rCrDrHr;rrrr itertoolschainziprris_tablepathnamerrr)rrrIselectorrrrrrrrzr4rrXrrrr5)rrrzrrr6select_as_multiple4s^"   "    zHDFStore.select_as_multiplecKs<|dkrtdpd}|||}|j||fd|i|dS)ae Store object in HDFStore Parameters ---------- key : object value : {Series, DataFrame} format : 'fixed(f)|table(t)', default is 'fixed' fixed(f) : Fixed format Fast writing/reading. Not-appendable, nor searchable table(t) : Table format Write as a PyTables Table structure which may perform worse but allow more flexible operations like searching / selecting subsets of the data append : boolean, default False This will force Table format, append the input data to the existing. data_columns : list of columns to create as data columns, or True to use all columns. See `here `__. encoding : default None, provide an encoding for strings dropna : boolean, default False, do not write an ALL nan row to the store settable by the option 'io.hdf.dropna_table' Nzio.hdf.default_formatrUrF)r _validate_format_write_to_group)rryr{rrFrzr5r5r6r|s  z HDFStore.putcCst|dd}y||}WnZtk r0YnFtk rt|dk rNtd||}|dk rp|jdddSYnXt|||r|j jddn|j std|j |||dSdS) a= Remove pandas object partially by specifying the where condition Parameters ---------- key : string Node to remove or delete rows from where : list of Term (or convertible) objects, optional start : integer (defaults to None), row number to start selection stop : integer (defaults to None), row number to stop selection Returns ------- number of rows removed (or None if not a Table) Raises ------ raises KeyError if key is not a valid store r>)rBNz5trying to remove a node with a non-None where clause!T) recursivez7can only remove with where on objects written as tables)rIrr) rLrrrrrZ _f_removecom _all_nonerrdelete)rryrIrrr4r5r5r6rs&    zHDFStore.removeTcKs^|dk rtd|dkr td}|dkr4tdp2d}|||}|j||f||d|dS)a Append to Table in file. Node must already exist and be Table format. Parameters ---------- key : object value : {Series, DataFrame} format : 'table' is the default table(t) : table format Write as a PyTables Table structure which may perform worse but allow more flexible operations like searching / selecting subsets of the data append : boolean, default True, append the input data to the existing data_columns : list of columns, or True, default None List of columns to create as indexed data columns for on-disk queries, or True to use all columns. By default only the axes of the object are indexed. See `here `__. min_itemsize : dict of columns that specify minimum string sizes nan_rep : string to use as string nan representation chunksize : size to chunk the writing expectedrows : expected TOTAL row size of this table encoding : default None, provide an encoding for strings dropna : boolean, default False, do not write an ALL nan row to the store settable by the option 'io.hdf.dropna_table' Notes ----- Does *not* check if data being appended overlaps with existing data in the table, so be careful Nz>columns is not a supported keyword in append, try data_columnszio.hdf.dropna_tablezio.hdf.default_formatrV)rFdropna)rr rr)rryr{rrFrrrzr5r5r6rFs$  zHDFStore.appendc s|dk rtdt|ts"td||kr2tdtttjttt d}d} g} x<| D]0\} } | dkr| dk rtd| } qj| | qjW| dk rڈj |} | t| }t| |}| ||| <|dkr||}|r2fdd|D}t|}x|D]}||}qWj|xN| D]B\} } | |krR|nd}j| |d }|j| |fd |i|qksz.HDFStore.append_to_multiple..)r data_columns)rr0dictrrCsetrangendim _AXES_MAPrrextendaxes differencersorted get_indexertakevaluesnext intersectionlocreindexrF)rdr{rrr rrzrZ remain_keyZ remain_valuesrvorderedZorddZidxsZ valid_indexrdcvalr5)r{r6append_to_multiple%sH &   zHDFStore.append_to_multiplecKs:t||}|dkrdS|js*td|jf|dS)z Create a pytables index on the table Parameters ---------- key : object (the node to index) Raises ------ raises if the node is not a table Nz1cannot create table index on a Fixed format store)rwrrr create_index)rryrzr4r5r5r6create_table_indexzs  zHDFStore.create_table_indexcCs"t|dd|jDS)zreturn a list of all the top-level nodes (that are not themselves a pandas storage object) Returns ------- list cSsPg|]H}t|tjjst|jddsHt|ddsHt|tjjr|jdkr|qS) pandas_typeNrV) r0rrlinkLinkgetattr_v_attrsrVTabler)rrr5r5r6rs  z#HDFStore.groups..)rwrr walk_groups)rr5r5r6rszHDFStore.groups/ccst|x|j|D]}t|jdddk r4qg}g}xP|jD]B}t|jdd}|dkr~t|t j j r| |j qH| |j qHW|jd||fVqWdS)a Walk the pytables group hierarchy for pandas objects This generator will yield the group path, subgroups and pandas object names for each group. Any non-pandas PyTables objects that are not a group will be ignored. The `where` group itself is listed first (preorder), then each of its child groups (following an alphanumerical order) is also traversed, following the same procedure. .. versionadded:: 0.24.0 Parameters ---------- where : str, optional Group where to start walking. If not supplied, the root group is used. Yields ------ path : str Full path to a group (without trailing '/') groups : list of str names of the groups contained in `path` leaves : list of str names of the pandas objects contained in `path` rNr&)rwrrr%r"r# _v_childrenrr0rrrGrouprFrrrstrip)rrIrrZleaveschildrr5r5r6walksz HDFStore.walkcCsH|y"|dsd|}|j|j|Stjjk rBdSXdS)z; return the node with the key or None if it does not exist r&N)r startswithrrrrr exceptionsZNoSuchNodeError)rryr5r5r6rs zHDFStore.get_nodecCs8||}|dkr"tdj|d||}||S)z> return the storer object for a key, raise if not in the file Nz!No object named {key} in the file)ry)rrrrr)rryrr4r5r5r6rs   zHDFStore.get_storerrKc  Cst|||||d} |dkr&t|}t|ttfs:|g}x|D]} || } | dk r@| | krl|rl| | || } | jrd} |rdd| j D} | j | | | t | dd| j dq@| j | | | j dq@W| S) a, copy the existing store to a new file, upgrading in place Parameters ---------- propindexes: restore indexes in copied file (defaults to True) keys : list of keys to include in the copy (defaults to all) overwrite : overwrite (remove and replace) existing nodes in the new store (default is True) mode, complib, complevel, fletcher32 same as in HDFStore.__init__ Returns ------- open file handle of the new store )r}rr~rNFcSsg|]}|jr|jqSr5) is_indexedr<)rrr5r5r6rsz!HDFStore.copy..r)rrr9)r9)rrCrr0rDrrrrr rFr"r9r|)rrtr}Z propindexesrrr~r overwriteZ new_storerr4datarr5r5r6copys2       z HDFStore.copyc Csdjt|t|jd}|jrtt|}t|rg}g}x|D]}y<| |}|dk r| t|j pj|| t|p|dWqDt k r}z$| || djt|dWdd}~XYqDXqDW|t d||7}q|d7}n|d 7}|S) z Print detailed information on the store. .. versionadded:: 0.21.0 Returns ------- str z{type} File path: {path} )rrNzinvalid_HDFStore nodez![invalid_HDFStore node: {detail}])detail EmptyzFile is CLOSED)rrr/rrrrCrrHrrFrrr.)routputZlkeysrrrr4r2r5r5r6info$s,    $ z HDFStore.infocCs|jstd|jdS)Nz{0} file is not open!)rrQrr)rr5r5r6rNszHDFStore._check_if_opencCsD|}yt||d<Wn"tk r>td|YnX|S)z5 validate / deprecate formats; return the new kwargs rz'invalid HDFStore format specified [{0}])r1 _FORMAT_MAPlowerrrr)rrrzr5r5r6rRs zHDFStore._validate_formatc  s:fdd}ttjdd}ttjdd}|dkrdkr|ttddshttjjrrd}d}qtd n>yt t }Wnt k r|d YnXdkr|d 7}d|kryt t ||fSt k r|d YnX|dkrdk r|d krPtdd} | dk r| jdkr>d}n| jdkrd}nb|dkrtdd} | dk r| jdkrd}n| jdkrd}n|dkrd}n|dkrd}nJd}y.jjj} t| dkr| ddkrd}Wntk rYnXyt t||fSt k r4|dYnXdS)z$ return a suitable class to operate c s"tdj|tddS)Nzcannot properly create the storer for: [{t}] [group->{group},value->{value},format->{format},append->{append},kwargs->{kwargs}])rXrr{rrFrz)rrr)rX)rFrrrzr{r5r6errorasz&HDFStore._create_storer..errorrN table_typerV frame_tablerizKcannot create a storer if the object is not existing nor a value are passed _TYPE_MAPZ_table _STORER_MAP series_tablerr>rjrkrlrm wide_tableZappendable_panelZ ndim_tableZappendable_ndimZ legacy_panelrr{Z legacy_frame _TABLE_MAP)r7r"r#rwr0rrrVr$rr<rrglobalsr=nlevelsfieldsrH IndexErrorr@) rrrr{rFrzr9ptttrrCr5)rFrrrzr{r6r^sl                zHDFStore._create_storercKs`||} | dk r*|s*|jj| ddd} t|ddrF|dksB|rFdS| dkr|d} d} xX| D]P} t| spqb| } | ds| d7} | | 7} || } | dkr|j| | } | } qbW|j| ||f||d|}|r |j r|j r|dkr|j rt d|j s| n| |j s*|r*t d |j f|||d ||j r\|r\|j|d dS) NT)remptyrVr&)rFr9rUzCan only append to Tablesz0Compression not supported on Fixed format stores)rrFr)r)rr remove_noder"splitrHendswith create_grouprr is_existsrset_object_infowriter)rryr{rrrFrr9rzrpathsrpnew_pathr4r5r5r6rsB       zHDFStore._write_to_groupcKs||}||jf|S)N)rrr)rrrzr4r5r5r6rs zHDFStore._read_group)NNNF)r)F)NNNNFNF)NNN)NNNNNFNF)NF)NNN)NTNN)NNF)r&)rKTNNNFT)NNF)TFNN)/rNrOrP__doc__rrpropertyrrrrrrrrrrrrrr iteritemsrrrrrrrrrr|rrFrrrr+rrr1r6rrrrrr5r5r5r6rs+       J  5  e  5 0 T .   2* c 6rc@s4eZdZdZd ddZddZdd Zd d d ZdS)ra define the iteration interface on a table Parameters ---------- store : the reference store s : the referred storer func : the function to execute the query where : the where of the query nrows : the rows to iterate on start : the passed start value (default is None) stop : the passed stop value (default is None) iterator : boolean, whether to use the default iterator chunksize : the passed chunking value (default is 50000) auto_close : boolean, automatically close the store at the end of iteration, default is False kwargs : the passed kwargs NFc Cs||_||_||_||_|jjrN|dkr,d}|dkr8d}|dkrD|}t||}||_||_||_d|_ |sr| dk r| dkr~d} t | |_ nd|_ | |_ dS)Nri) rxr4rrIrminrrrrintrr) rrxr4rrIrrrrrrr5r5r6rs,    zTableIterator.__init__ccsh|j}xT||jkrZt||j|j}|dd|j||}|}|dkst|sRq|VqW|dS)N)rrrUrrrrHr)rrrr{r5r5r6r?s  zTableIterator.__iter__cCs|jr|jdS)N)rrxr)rr5r5r6rOszTableIterator.closecCsr|jdk r0|jjstd|jj|jd|_|S|rN|jj|j|j|jd}n|j}| |j|j|}| |S)Nz0can only use an iterator or chunksize on a table)rI)rIrr) rr4rrrrIrrrrr)rrrIresultsr5r5r6rSs zTableIterator.get_result)NNFNF)F)rNrOrPrRrrrrr5r5r5r6rs  rc @s8eZdZdZdZdZdddgZdEddZdFd d Zd d Z d dZ ddZ ddZ ddZ ddZeddZddZddZdGddZdd Zed!d"Zed#d$Zed%d&Zed'd(Zd)d*ZdHd+d,Zd-d.Zd/d0Zd1d2ZdId3d4Zd5d6Zd7d8Z d9d:Z!d;d<Z"d=d>Z#d?d@Z$dAdBZ%dCdDZ&dS)JIndexCola3 an index column description class Parameters ---------- axis : axis which I reference values : the ndarray like converted values kind : a string description of this type typ : the pytables type pos : the position in the pytables Tfreqtz index_nameNc Ks||_||_||_||_||_||_||_||_| |_| |_ | |_ | |_ d|_ d|_ d|_|dk rn|||| dk r|| dS)N)rkindtypitemsizer<cname kind_attrrposrYrZr[rVrmetadataset_nameset_pos)rrr\r]r_r^r<rr`rarYrZr[rzr5r5r6rs& zIndexCol.__init__cCs,||_|pdj|d|_|jdkr(||_|S)z set the name of this indexer z {name}_kind)r<N)r<rr`r_)rr<r`r5r5r6rcs  zIndexCol.set_namecCs ||_|S)z! set the axis over which I index )r)rrr5r5r6set_axisszIndexCol.set_axiscCs$||_|dk r |jdk r ||j_|S)z. set the position of this column in the Table N)rar]Z_v_pos)rrar5r5r6rdszIndexCol.set_poscCs ||_|S)N)rV)rrVr5r5r6 set_tableszIndexCol.set_tablec CsFttt|j|j|j|j|jf}dddt dddddg|DS) N,css |]\}}dj||dVqdS)z{key}->{value})ryr{N)r)rryr{r5r5r6rsz$IndexCol.__repr__..r<r_rrar\) rDmapr/r<r_rrar\joinr)rtempr5r5r6rs  zIndexCol.__repr__cstfdddDS)z compare 2 col items c3s&|]}t|dt|dkVqdS)N)r")rr)otherrr5r6rsz"IndexCol.__eq__..)r<r_rra)r)rrkr5)rkrr6__eq__s zIndexCol.__eq__cCs || S)N)rl)rrkr5r5r6__ne__szIndexCol.__ne__cCs,yt|jj|jjStk r&YnXdS)z' return whether I am an indexed column N)r"rVrr_r.rv)rr5r5r6r.szIndexCol.is_indexedcCst|}|S)N)r1)rnew_selfr5r5r6r1s z IndexCol.copycCs.|j}|}||||||S)z@infer this column from the table: create and return a new object)rVr1rfget_attr read_metadata)rhandlerrVrnr5r5r6infers   zIndexCol.infercCs|jjdk r||j}t||j||}t}|jdk rDt|j|d<|jdk r\t|j|d<yt |f||_ Wn2t k rd|krd|d<t |f||_ YnXt |j |j |_ |S)z; set the values from this selection: take = take ownership NrYr<)dtyperCr__maybe_convertr\rrYr7r[rrr_set_tzrZ)rrnan_repr9errorsrrrzr5r5r6converts     zIndexCol.convertcCsd|j|_}|S)z( return the values & release the memory N)r)rrr5r5r6 take_dataszIndexCol.take_datacCs|jjS)N)rVr#)rr5r5r6attrs szIndexCol.attrscCs|jjS)N)rV description)rr5r5r6r{szIndexCol.descriptioncCst|j|jdS)z# return my current col description N)r"r{r_)rr5r5r6colsz IndexCol.colcCs|jS)z return my cython values )r)rr5r5r6cvaluesszIndexCol.cvaluescCs t|jS)N)rr)rr5r5r6rszIndexCol.__iter__cCsPt|jdkrLt|tr$||j}|dk rL|jj|krLtj ||j d|_dS)z maybe set a string col itemsize: min_itemsize can be an integer or a dict with this columns name with an integer size stringN)r^ra) r7r\r0rrr<r]r^rw StringColra)r min_itemsizer5r5r6maybe_set_sizes   zIndexCol.maybe_set_sizecCs |dS)N)validate_names)rrqrFr5r5r6validate+szIndexCol.validatecCsdS)Nr5)rr5r5r6r.szIndexCol.validate_namescCs>||j||||||||dS)N)rfrV validate_col validate_attrvalidate_metadatawrite_metadataset_attr)rrqrFr5r5r6validate_and_set1s     zIndexCol.validate_and_setcCsVt|jdkrR|j}|dk rR|dkr*|j}|j|krLtdj||j|jd|jSdS)z< validate this column: return the compared against itemsize r~NzTrying to store a string with len [{itemsize}] in [{cname}] column but this column has a limit of [{c_itemsize}]! Consider using min_itemsize to preset the sizes on these columns)r^r_Z c_itemsize)r7r\r|r^rrr_)rr^cr5r5r6r9s zIndexCol.validate_colcCs>|r:t|j|jd}|dk r:||jkr:tdj||jddS)Nz3incompatible kind in col [{existing} - {self_kind}])existingZ self_kind)r"rzr`r\rr)rrFZ existing_kindr5r5r6rOs zIndexCol.validate_attrcCsx|jD]}t||d}t||j}||}||kr|dk r||kr|dkrt|||f}tj|tddd||<t ||dqt dj |j|||dq|dk s|dk r|||<qW|S)zs set/update the info for this indexable with the key/value if there is a conflict raise/warn as needed N)rYr[) stacklevelzkinvalid info for [{name}] for [{key}], existing_value [{existing_value}] conflicts with new value [{value}])r<ryexisting_valuer{) _info_fieldsr" _get_infor<rattribute_conflict_docwarningswarnrSsetattrrr)rr6ryr{idxrwsr5r5r6 update_infoYs&      zIndexCol.update_infocCs$||j}|dk r |j|dS)z# set my state from the passed info N)rr<__dict__update)rr6rr5r5r6set_infos zIndexCol.set_infocCst|j|jd|_dS)z set the kind for this column N)r"rzr`r\)rr5r5r6roszIndexCol.get_attrcCst|j|j|jdS)z set the kind for this column N)rrzr`r\)rr5r5r6rszIndexCol.set_attrcCs||j|_dS)z( retrieve the metadata for this columns N)rpr_rb)rrqr5r5r6rpszIndexCol.read_metadatacCsB|jdkr>|j}||j}|dk r>|dk r>t||s>tddS)z< validate that kind=category does not change the categories categoryNzEcannot append a categorical with different categories to the existing)rrbrpr_rr)rrqZ new_metadataZ cur_metadatar5r5r6rs   zIndexCol.validate_metadatacCs|jdk r||j|jdS)z set the meta data N)rbrr_)rrqr5r5r6rs zIndexCol.write_metadata) NNNNNNNNNNNN)N)NN)N)N)'rNrOrPrRis_an_indexableis_data_indexablerrrcrerdrfrrlrmrSr.r1rrrxryrzr{r|r}rrrrrrrrrrorrprrr5r5r5r6rXls\              &rXc@s6eZdZdZeddZd ddZddZd d ZdS) GenericIndexColz< an index which is not represented in the data of the table cCsdS)NFr5)rr5r5r6r.szGenericIndexCol.is_indexedNcCsF|dk r |nd}|dk r&t||jjn|jj}tt|||_|S)a set the values from this selection: take = take ownership Parameters ---------- values : np.ndarray nan_rep : str encoding : str errors : str start : int, optional Table row number: the start of the sub-selection. stop : int, optional Table row number: the end of the sub-selection. Values larger than the underlying table's row count are normalized to that. Nr)rUrVrrr1aranger)rrrvr9rwrrr5r5r6rxszGenericIndexCol.convertcCsdS)Nr5)rr5r5r6roszGenericIndexCol.get_attrcCsdS)Nr5)rr5r5r6rszGenericIndexCol.set_attr)NN) rNrOrPrRrSr.rxrorr5r5r5r6rs   rcs"eZdZdZdZdZddgZed=ddZd>fdd Z d d Z d d Z d?ddZ ddZ ddZddZd@ddZddZddZdAddZdBdd Zd!d"Zd#d$ZdCd%d&Zd'd(ZdDd)d*ZdEd+d,Zd-d.ZdFd/d0Zed1d2Zed3d4Zd5d6Z dGd7d8Z!d9d:Z"d;d<Z#Z$S)HDataColaM a data holding column, by definition this is not indexable Parameters ---------- data : the actual data cname : the column name in the table to hold the data (typically values) meta : a string description of the metadata metadata : the actual metadata FrZrNcKs|dkr|pdj|d}|dkr$|}yL|ddkrn|ddkrn|ddkrntd|}|rnd j|dd }Wntk rYnX|f||d |S) z' return a new datacol with the block i Nzvalues_block_{idx})rrr> zvalues_block_(\d+)zvalues_{group})r)r<r_)rresearchrrD)clsir<r_rrzmr5r5r6create_for_blocks$ zDataCol.create_for_blockc  s`tjf||||d| d|_dj|jd|_||_dj|jd|_||| |dS)N)rr\r]r_z {name}_dtype)r<z {name}_meta) superrrsrr< dtype_attrr meta_attrset_data set_metadata) rrr\r]r_r0rrbblockrz) __class__r5r6rs  zDataCol.__init__c CsFttt|j|j|j|j|jf}dddt dddddg|DS) Nrgcss |]\}}dj||dVqdS)z{key}->{value})ryr{N)r)rryr{r5r5r6rsz#DataCol.__repr__..r<r_rsr\shape) rDrhr/r<r_rsr\rrir)rrjr5r5r6rs zDataCol.__repr__cstfdddDS)z compare 2 col items c3s&|]}t|dt|dkVqdS)N)r")rr)rkrr5r6rsz!DataCol.__eq__..)r<r_rsra)r)rrkr5)rkrr6rls zDataCol.__eq__cCsF||_|dk rB|dk r&||_|n|jdkrB|jj|_|dS)N)r0rsset_kindr<)rr0rsr5r5r6rs   zDataCol.set_datacCsd|j|_}|S)z& return the data & release the memory N)r0)rr0r5r5r6ry%szDataCol.take_datacCs$|dk rtj|dd}||_dS)z record the metadata NF)r1)r1arrayravelrb)rrbr5r5r6r*szDataCol.set_metadatacCs|jdk rt|j}|ds(|dr0d|_n|drBd|_nv|drTd|_nd|dsh|drpd|_nH|drd |_n6|d rd |_n$|d rd |_ntd j||d |jdkrt|j|j d|_dS)Nr~bytesfloatcomplexrVuintintegerrr timedeltarz.cannot interpret dtype of [{dtype}] in [{obj}])rsr) rsr7r,r\AssertionErrorrr]r"r{r_)rrsr5r5r6r0s*         zDataCol.set_kindrqc Cst||_|jr |j|||dS|jr4|j||dS|jrD||S|jrT| |S|j rd| |S|j j } tj|jdd} | dkrtdnV| dkrtdnD| d krtd n2| d ks| d kr||||||||n ||d S)z+ create and setup my atom from the block b )rr6)r6F)skipnarz+[date] is not implemented as a table columnrz>too many timezones in this block, create separate data columnsunicodez.[unicode] is not implemented as a table columnr~rN)rCrZis_categoricalset_atom_categoricalZ is_datetimetzset_atom_datetime64tzZ is_datetimeset_atom_datetime64Z is_timedeltaset_atom_timedelta64Z is_complexset_atom_complexrsr<r infer_dtyperset_atom_string set_atom_data) rr block_items existing_colrrvr6r9rwrs inferred_typer5r5r6set_atomOs<      zDataCol.set_atomcCstj||jddS)Nr)r^r)rwrr)rrr^r5r5r6get_atom_stringszDataCol.get_atom_stringcCs,|j|dd}t|tr |d}|j}tj|dd} | dkrxJt|D]>\} } || } tj| dd} | dkrJt dj | | dqJWt |||} | j }t|t rt||jp|dpd}t|pd|}|dk r||}||kr|}||_ d|_||||_|| jd j |d dd dS) NF)Zdowncastr)rr~zXCannot serialize the column [{item}] because its data contents are [{type}] object dtype)itemrrz|S{size})size)r1)Zfillnar0rCrr rr enumerateigetrr_convert_string_arrayr^rrVrr<maxrr\rr]rastype)rrrrrrvr9rwr0rrrr|Zdata_convertedr^Zecir5r5r6rs8     zDataCol.set_atom_stringcCsL|dkr|j}|jdr0dj|ddd}ndj|d}tt|S)z2 return the PyTables column class for this column Nrz UInt{name}Col)r<z {name}Col)r\r,r capitalizer"rw)rr\Zcol_namer5r5r6get_atom_coltypes  zDataCol.get_atom_coltypecCs|j|d|jddS)N)r\r)r)rr)rrr\r5r5r6 get_atom_dataszDataCol.get_atom_datacCsX|jj|_t|jddd}tj||jdd|_| |j j |jj dddS)Nrr)r^rF)r1) rsr<r\rVrIrwZ ComplexColrr]rrrr)rrr^r5r5r6rs zDataCol.set_atom_complexcCs4|jj|_|||_||jj|jjdddS)NF)r1) rsr<r\rr]rrrr)rrr5r5r6rs  zDataCol.set_atom_datacCs|j}|j}d|_|jj|_|jdkr.tdt|dkrBtd|j|_|j ||jjd|_ | t |d|_ ||jj||dS)Nrr>zonly support 1-d categoricalsz&only support single block categoricals)r\r)rcodesr\rsr<r rrHrrr]rr+rr categoriesr)rrrr6rrr5r5r6rs   zDataCol.set_atom_categoricalcCstj|jddS)Nr)r)rwInt64Colr)rrr5r5r6get_atom_datetime64szDataCol.get_atom_datetime64cCs6d|_|||_|dkr&|jd}||ddS)N datetime64i8)r\rr]rviewr)rrrr5r5r6rs   zDataCol.set_atom_datetime64cCsV|dkr|j}|j|j}t|jj|_||d|_|||_ | |ddS)Nr) rasi8reshaper_get_tzrZrr\rr]r)rrr6rr5r5r6rs  zDataCol.set_atom_datetime64tzcCstj|jddS)Nr)r)rwrr)rrr5r5r6get_atom_timedelta64 szDataCol.get_atom_timedelta64cCs6d|_|||_|dkr&|jd}||ddS)N timedelta64r)r\rr]rrr)rrrr5r5r6r s   zDataCol.set_atom_timedelta64cCst|jddS)Nr)r"r0)rr5r5r6r sz DataCol.shapecCs|jS)z return my cython values )r0)rr5r5r6r} szDataCol.cvaluescCs`|r\t|j|jd}|dk r2|t|jkr2tdt|j|jd}|dk r\||jkr\tddS)zAvalidate that we have the same order as the existing & same dtypeNz4appended items do not match existing items in table!z@appended items dtype do not match existing items dtype in table!)r"rzr`rCrrrrs)rrFZexisting_fieldsZexisting_dtyper5r5r6r! szDataCol.validate_attrc Cs|jjdk r||j}||t|j}|jdk rt|j}|dkr`t|j|jdd|_nh|dkr~t j |jdd|_nJ|dkry t j d d |jDt d|_Wn0t k rt j d d |jDt d|_YnXn|d krt j d d |jDt d|_n|dkr|j } |j} | dkr6tgt jd} n.cSsg|]}t|qSr5)r fromtimestamp)rrr5r5r6rR srcSsg|]}t|qSr5)rr)rrr5r5r6rV srr)rrF)r1Or~)rvr9rw)rsrCr_rr7rrur0rZr1asarrayrrrbrrfloat64r"anyrrVZcumsumrr$Z from_codesrrr\_unconvert_string_array) rrrvr9rwrrrrsrrmaskr5r5r6rx1 sN              zDataCol.convertcCsBt|j|jd|_t|j|jd|_t|j|jd|_|dS)z get the data for this column N) r"rzr`rrrsrrr)rr5r5r6ro szDataCol.get_attrcCsDt|j|j|jt|j|j|j|jdk r@t|j|j|jdS)z set the data for this column N)rrzr`rrrrsr)rr5r5r6r s zDataCol.set_attr)NNNN)NNNNNNNN)N)Nrq)N)N)NN)N)N)N)NN)%rNrOrPrRrrr classmethodrrrrlrryrrrrrrrrrrrrrrrrSrr}rrxror __classcell__r5r5)rr6rsP     ' 34        Prc@s>eZdZdZdZddZddZddd Zd d Zd d Z dS)DataIndexableColz- represent a data column that can be indexed TcCst|jstddS)Nz-cannot have non-object label DataIndexableCol)rrZ is_objectr)rr5r5r6r szDataIndexableCol.validate_namescCstj|dS)N)r^)rwr)rrr^r5r5r6r sz DataIndexableCol.get_atom_stringNcCs|j|dS)N)r\)r)rrr\r5r5r6r szDataIndexableCol.get_atom_datacCs tS)N)rwr)rrr5r5r6r sz$DataIndexableCol.get_atom_datetime64cCs tS)N)rwr)rrr5r5r6r sz%DataIndexableCol.get_atom_timedelta64)N) rNrOrPrRrrrrrrr5r5r5r6r s rc@seZdZdZddZdS)GenericDataIndexableColz* represent a generic pytables data column cCsdS)Nr5)rr5r5r6ro sz GenericDataIndexableCol.get_attrN)rNrOrPrRror5r5r5r6r src@sBeZdZdZdZdZdZdZd=ddZe ddZ d d Z e d d Z e d dZ ddZddZddZe ddZe ddZe ddZe ddZe ddZe dd Ze d!d"Ze d#d$Ze d%d&Zd'd(Zd)d*Ze d+d,Ze d-d.Ze d/d0Zd1d2Zd>d3d4Z d5d6Z!d7d8Z"d9d:Z#d?d;d<Z$dS)@Fixeda represent an object in my store facilitate read/write of various types of objects this is an abstract base class Parameters ---------- parent : my parent HDFStore group : the group node where the table resides NFrqcKs(||_||_t||_||_|dS)N)rrr:r9rw set_version)rrrr9rwrzr5r5r6r s  zFixed.__init__cCs*|jddko(|jddko(|jddkS)Nrr>rr)r)rr5r5r6is_old_version szFixed.is_old_versioncCsltt|jjdd}y8tdd|dD|_t|jdkrJ|jd|_Wntk rfd|_YnXdS) z compute and set our version pandas_versionNcss|]}t|VqdS)N)rV)rr?r5r5r6r sz$Fixed.set_version...r)r)rrr) r7r"rr#rDrIrrHrv)rrr5r5r6r szFixed.set_versioncCstt|jjddS)Nr)r7r"rr#)rr5r5r6r szFixed.pandas_typecCsdS)NrUr5)rr5r5r6 format_type szFixed.format_typecCsV||j}|dk rPt|ttfr@djddd|Dd}dj|j|dS|jS) z* return a pretty representation of myself Nz [{shape}]rgcss|]}t|VqdS)N)r/)rr?r5r5r6r sz!Fixed.__repr__..)rz{type:12.12} (shape->{shape}))rr)rrr0rCrDrrir)rr4r5r5r6r s zFixed.__repr__cCs&t|j|j_tt|j_|dS)z set my pandas type & version N)r; pandas_kindrzr_versionrr)rr5r5r6rM s zFixed.set_object_infocCst|}|S)N)r1)rrnr5r5r6r1 s z Fixed.copycCs|jS)N)obj_type)rr5r5r6storage_obj_type szFixed.storage_obj_typecCs|jS)N)r)rr5r5r6r sz Fixed.shapecCs|jjS)N)rr)rr5r5r6r szFixed.pathnamecCs|jjS)N)rr)rr5r5r6r sz Fixed._handlecCs|jjS)N)rr)rr5r5r6r szFixed._filterscCs|jjS)N)rr)rr5r5r6r szFixed._complevelcCs|jjS)N)rr)rr5r5r6r szFixed._fletcher32cCs|jjS)N)rr)rr5r5r6r szFixed._complibcCs|jjS)N)rr#)rr5r5r6rz sz Fixed.attrscCsdS)z set our object attributes Nr5)rr5r5r6 set_attrs szFixed.set_attrscCsdS)z get our object attributes Nr5)rr5r5r6 get_attrs szFixed.get_attrscCs|jS)z return my storable )r)rr5r5r6storable szFixed.storablecCsdS)NFr5)rr5r5r6rL% szFixed.is_existscCst|jddS)Nr)r"r)rr5r5r6r) sz Fixed.nrowscCs|dkr dSdS)z' validate against an existing storable NTr5)rrkr5r5r6r- szFixed.validatecCsdS)z- are we trying to operate on an old version? Tr5)rrIr5r5r6validate_version3 szFixed.validate_versioncCs|j}|dkrdS|dS)zh infer the axes of my storer return a boolean indicating if we have a valid storer or not NFT)rr)rr4r5r5r6r7 s zFixed.infer_axescKs tddS)Nz>cannot read on an abstract storer: subclasses should implement)r)rrzr5r5r6rA sz Fixed.readcKs tddS)Nz?cannot write on an abstract storer: sublcasses should implement)r)rrzr5r5r6rNF sz Fixed.writecKs0t|||r$|jj|jdddStddS)zs support fully deleting the node in its entirety (only) - where specification must be None T)rNz#cannot delete on an abstract storer)rrrrHrr)rrIrrrzr5r5r6rK sz Fixed.delete)Nrq)N)NNN)%rNrOrPrRrrr rrrSrrrrrrMr1rrrrrrrrrzrrrrLrrrrrrNrr5r5r5r6r sB                    rc@seZdZdZedediZddeDZgZ ddZ dd Z d d Z d d Z eddZddZddZddZd1ddZddZddZddZdd Zd!d"Zd#d$Zd%d&Zd'd(Zd2d)d*Zd+d,Zd-d.Zd3d/d0ZdS)4 GenericFixedz a generified fixed version rZperiodcCsi|]\}}||qSr5r5)rrrr5r5r6 \ szGenericFixed.cCs|j|dS)N)_index_type_mapr)rrr5r5r6_class_to_alias` szGenericFixed._class_to_aliascCst|tr|S|j|tS)N)r0r_reverse_index_maprr)raliasr5r5r6_alias_to_classc s zGenericFixed._alias_to_classcCs0|tkrddd}|S|tkr,ddd}|S|S)NcSs.tj|jd|d}|dk r*|d|}|S)N)r<rYUTC)r _simple_newr tz_localize tz_convert)rrYrZresultr5r5r6rWl sz*GenericFixed._get_index_factory..fcSstj|d|dS)N)r<rY)rr)rrYrZr5r5r6rWv s)NN)NN)rr)rklassrWr5r5r6_get_index_factoryi s  zGenericFixed._get_index_factorycCsFt|}|dd}|dk r&td|dd}|dk rBtd|S)zz remove table keywords from kwargs and return raise if any keywords are passed which are not-None rNzqcannot pass a column specification when reading a Fixed format store. this store must be selected in its entiretyrIzucannot pass a where specification when reading from a Fixed format store. this store must be selected in its entirety)r1popr)rrzrrIr5r5r6 validate_read} s   zGenericFixed.validate_readcCsdS)NTr5)rr5r5r6rL szGenericFixed.is_existscCs|j|j_|j|j_dS)z set our object attributes N)r9rzrw)rr5r5r6r s zGenericFixed.set_attrsc CsVtt|jdd|_tt|jdd|_x(|jD]}t||tt|j|dq0WdS)z retrieve our attributes r9Nrwrq)r:r"rzr9r7rw attributesr)rrr5r5r6r s zGenericFixed.get_attrscKs |dS)N)r)rrrzr5r5r6rN szGenericFixed.writeNc Csddl}t|j|}|j}t|dd}t||jrD|d||}nrt|dd} t|dd} | dk rttj| | d}n |||}| dkrt|t|d dd d }n| d krtj |d d}|r|j S|SdS)z4 read an array for the specified node (off of group rN transposedF value_typer)rsrrZT)rrzm8[ns]) rsr"rr#r0ZVLArrayr1rGrurT) rryrrrsrrzrretrsrr5r5r6 read_array s$      zGenericFixed.read_arraycKstt|jdj|d}|dkr.|j|f|S|dkrD|j|f|S|dkrZ|j|f|S|dkr|jt|j|f|\}}|St dj|ddS) Nz {key}_variety)rymultir sparseintregularz%unrecognized index variety: {variety})variety) r7r"rzrread_multi_indexread_block_indexread_sparse_intindexread_index_noderr)rryrzr_rr5r5r6 read_index szGenericFixed.read_indexcCsLt|tr0t|jdj|dd|||nt|tr^t|jdj|dd|||nt|trt|jdj|dd| ||nt|jdj|ddt ||j |j |j d}|||jt|j|}|j|j_|j|j_t|ttfr |t||j_t|dr"|j|j_t|d rH|jdk rHt|j|j_dS) Nz {key}_variety)ryrrrrrrYrZ)r0rrrzrwrite_multi_indexr%write_block_indexr&write_sparse_intindex_convert_indexr9rwrrc write_arrayrr"rr\r#r<rrrr index_classhasattrrYrZr)rryr convertedrr5r5r6 write_index s.        zGenericFixed.write_indexcCsH|dj|d|j|dj|d|jt|jdj|d|jdS)Nz {key}_blocs)ryz{key}_blengthsz {key}_length)r#rblocsblengthsrrzlength)rryrr5r5r6r  szGenericFixed.write_block_indexcKsLt|jdj|d}|jdj|df|}|jdj|df|}t|||S)Nz {key}_length)ryz {key}_blocsz{key}_blengths)r"rzrrr%)rryrzr*r(r)r5r5r6r szGenericFixed.read_block_indexcCs2|dj|d|jt|jdj|d|jdS)Nz {key}_indices)ryz {key}_length)r#rindicesrrzr*)rryrr5r5r6r! sz"GenericFixed.write_sparse_intindexcKs4t|jdj|d}|jdj|df|}t||S)Nz {key}_length)ryz {key}_indices)r"rzrrr&)rryrzr*r+r5r5r6r sz!GenericFixed.read_sparse_intindexc Cst|jdj|d|jxtt|j|j|jD]\}\}}}t |rNt ddj||d}t ||j |j |j|}|||jt|j|} |j| j_|| j_t| jdj||d|dj||d} || |q0WdS) Nz {key}_nlevels)ryz=Saving a MultiIndex with an extension dtype is not supported.z{key}_level{idx})ryrz{key}_name{name})ryr<z{key}_label{idx})rrzrrBrrlevelsrnamesrrr"r9rwrrcr#rr"rr\r#r<) rryrrlev level_codesr< level_keyZ conv_levelr label_keyr5r5r6r s""  zGenericFixed.write_multi_indexc Kst|jdj|d}g}g}g}xrt|D]f}dj||d}|jt|j|f|\} } || || dj||d} |j| f|} || q*Wt|||ddS)Nz {key}_nlevels)ryz{key}_level{idx})ryrz{key}_label{idx}T)r,rr-r) r"rzrr rrrFrr) rryrzrBr,rr-rr0r<r.r1r/r5r5r6r( s  zGenericFixed.read_multi_indexc Cs"|||}d|jkrF|t|jdrFtjt|jdt|jdd}t|jj}d}d|jkrtt|jj}t|}| tt|jdd}| |}i} d|jkr|jd| d<d|jkr|jd| d<|d kr|t |||j |j d fd ti| } n|t |||j |j d f| } || _|| fS) Nrr)rsr<r$rrYrZ)rr)r9rwrs)r#_is_empty_arrayr"r1rGr7r\r=r<rr _unconvert_indexr9rwr) rrrrr0r\r<r$factoryrzrr5r5r6r< s>         zGenericFixed.read_index_nodecCsNtd|j}|j|j||t|jt|j|j _ |j t|j|j _ dS)z write a 0-len array )r>N) r1rGr r create_arrayrr;rsr"r#rr)rryr{Zarrr5r5r6write_array_emptyn szGenericFixed.write_array_emptycCstdd|DS)z(Returns true if any axis is zero length.css|]}|dkVqdS)rNr5)rr?r5r5r6ry sz/GenericFixed._is_empty_array..)r)rrr5r5r6r2w szGenericFixed._is_empty_arrayc Csf||jkr|j|j|||j}d}t|r:td|sRt|drR|j}d}|j dk rd}yt j |j }Wntk rYnX|dk r|s|jj|j|||j|j d}||dd<|t|j|j_n |||dS|j jtjkr|tj|dd}|r nN|dkrnBy t|}Wntk r8YnXt|||f} tj| tdd |j |j|t !} | "|n|r|||nt#|j r|j$|j||%d d t|j|j_&nt'|j r |j$|j||j(t|j|} t)|j*| j_*d | j_&nHt+|j r@|j$|j||%d d t|j|j_&n|j$|j|||t|j|j_dS) NFz]Cannot store a category dtype in a HDF5 dataset that uses format="fixed". Use format="table".rT)r)rr~)rrrr),rrrHr2rrrr%rrrwZAtomZ from_dtypersrZ create_carrayr"r#rr6rr1Zobject_r rrrCrperformance_docrrrZcreate_vlarray ObjectAtomrFrr5rrrrrrZr) rryr{rZ empty_arrayratomcarrZvlarrrr5r5r6r#{ sl              zGenericFixed.write_array)NN)NN)N) rNrOrPrRrrrrrrrrr rrSrLrrrNrrr'r rr!rrrrr6r2r#r5r5r5r6rW s2   " 2 rc@seZdZdddZdS) LegacyFixedNcCs4t|j|}|||}|jj}t|||j|jdS)N)r9rw)r"rr#r\_unconvert_index_legacyr9rw)rryrrrr0r\r5r5r6read_index_legacy s   zLegacyFixed.read_index_legacy)NN)rNrOrPr>r5r5r5r6r< sr<c@seZdZddZdS)r]cKs*||}|d}|d}t||dS)Nrr)r)rr>rr)rrzrrr5r5r6r s   zLegacySeriesFixed.readN)rNrOrPrr5r5r5r6r] sc@seZdZddZdS)r^cKs6||}|d}|d}|d}t|||dS)Nrrr)rr)rr>rr)rrzrrrr5r5r6r s     zLegacyFrameFixed.readN)rNrOrPrr5r5r5r6r^ scs:eZdZdZdgZeddZddZfddZZ S) r_rYr<c Cs0ytt|jdfSttfk r*dSXdS)Nr)rHr"rrrv)rr5r5r6r szSeriesFixed.shapecKs2||}|jd|}|jd|}t|||jdS)Nrr)rr<)r)r)rrrrr<)rrzrrr5r5r6r s   zSeriesFixed.readc s:tj|f||d|j|d|j|j|j_dS)Nrr)rrNr'rr#rr<rz)rrrz)rr5r6rN szSeriesFixed.write) rNrOrPrrrSrrrNrr5r5)rr6r_ s  cseZdZfddZZS) SparseFixedcs(t|}d|ksd|kr$td|S)z= we don't support start, stop kwds in Sparse rrz;start and/or stop are not supported in fixed Sparse reading)rrr)rrz)rr5r6r s  zSparseFixed.validate_read)rNrOrPrrr5r5)rr6r? sr?cs2eZdZdZdddgZddZfddZZS) r`rZr< fill_valuer\cKsF||}|d}|d}|d}t||||jp8d|j|jdS)Nr sp_valuessp_indexr)rZ sparse_indexr\r@r<)rrrrr\r@r<)rrzrrArBr5r5r6r s    zSparseSeriesFixed.readc s\tj|f||d|j|d|j|d|j|j|j_|j |j_ |j |j_ dS)NrrBrA) rrNr'rrBr#rAr<rzr@r\)rrrz)rr5r6rN" s  zSparseSeriesFixed.write)rNrOrPrrrrNrr5r5)rr6r` s cs0eZdZdZddgZddZfddZZS)rbr\ default_kinddefault_fill_valuecKsn||}|d}i}x@|D]8}dj|d}t|jt|j|}||||<qWt |||j |j dS)Nrzsparse_series_{columns})r)rrCrD) rrrr`rr"rrrrrCrD)rrzrZsdictrryr4r5r5r6r0 s    zSparseFrameFixed.readc stj|f|x`|D]T\}}dj|d}||jjkrL|j|j|}n t|j|}t |j |}||qW|j |j _ |j |j _ |d|jdS)z6 write it as a collection of individual sparse series zsparse_series_{name})r<rN)rrNrrrr'rrKr"r`rrDrzrCr'r)rrrzr<ssryrr4)rr5r6rN@ s      zSparseFrameFixed.write)rNrOrPrrrrNrr5r5)rr6rb, scs>eZdZddgZdZeddZd ddZfd d ZZ S) BlockManagerFixedr nblocksFcCsy|j}d}xFt|jD]8}t|jdj|d}t|dd}|dk r||d7}qWt|jd}t|dd}|dk rt|d|d}ng}|||jr|ddd}|St k rdSXdS)Nrzblock{idx}_items)rrZ block0_valuesr>r) r r rGr"rrrCrFis_shape_reversedrv)rr rrrrr5r5r6rT s&    zBlockManagerFixed.shapeNcKs||}|d}g}xLt|jD]>}||kr<||fnd\}}|jdj|d||d} || q(W|d} g} xXt|jD]J}|dj|d} |j dj|d||d} t | | | d}| |qW|t | |S) Nr)NNz axis{idx})r)rrzblock{idx}_itemszblock{idx}_values) placement) rrZ_get_block_manager_axisr r rrrFrGrr,rr*)rrrrzZ select_axisr rrraxrblocks blk_itemsrblkr5r5r6rs s  zBlockManagerFixed.readc stj|f||j}|s&|}|j|j_x@t|jD]2\}}|dkrZ|j sZt d| dj |d|qZ%dWd?d@Z&dAdBZ'e dCdDZ(dXdEdFZ)dGdHZ*dIdJZ+dKdLZ,dYdMdNZ-dZdOdPZ.d[dQdRZ/d\dSdTZ0d]dUdVZ1Z2S)^r$a represent a table: facilitate read/write of various types of tables Attrs in Table Node ------------------- These are attributes that are store in the main table node, they are necessary to recreate these tables when read back in. index_axes : a list of tuples of the (original indexing axis and index column) non_index_axes: a list of tuples of the (original index axis and columns on a non-indexing axis) values_axes : a list of the columns which comprise the data of this table data_columns : a list of the columns that we are allowing indexing (these become single columns in values_axes), or True to force all columns nan_rep : the string to use for nan representations for string objects levels : the names of levels metadata : the names of the metadata columns r?Nr>TFcsDtj||g|_g|_g|_g|_g|_t|_d|_ d|_ dS)N) rr index_axesr values_axesrrbrr6rv selection)rargsrz)rr5r6r szTable.__init__cCs|jddS)Nrr)r:rI)rr5r5r6table_type_short szTable.table_type_shortcCsdS)NrVr5)rr5r5r6r szTable.format_typec Cs|djt|jr"d|jndd}d}|jrRdjddd|jDd }d j|j||j|j |j dd d|j D|d S) z* return a pretty representation of myself z,dc->[{columns}]rgr)rz [{version}]rcss|]}t|VqdS)N)r;)rr?r5r5r6r sz!Table.__repr__..)rzg{pandas_type:12.12}{ver} (typ->{table_type},nrows->{nrows},ncols->{ncols},indexers->[{index_axes}]{dc})css|] }|jVqdS)N)r<)rrr5r5r6r s)rverr:rncolsrPr) rrrHrrirrrrTrrVrP)rrrUr5r5r6r szTable.__repr__cCs"x|jD]}||jkr|SqWdS)z return the axis for c N)r r<)rrrr5r5r6r s  zTable.__getitem__cCs|dkr dS|j|jkr.tdj|j|jdxzdD]r}t||d}t||d}||kr4x8t|D],\}}||}||krbtdj|||dqbWtdj|||dq4WdS) z$ validate against an existing table Nz8incompatible table_type with existing [{other} - {self}])rkr)rPrrQzMinvalid combinate of [{c}] on appending data [{sax}] vs current table [{oax}])rsaxoaxzKinvalid combinate of [{c}] on appending data [{sv}] vs current table [{ov}])rsvov)r:rrr"rrr)rrkrrYrZrrWrXr5r5r6r s&    zTable.validatecCs t|jtS)z@the levels attribute is 1 or a list in the case of a multi-index)r0r,rC)rr5r5r6is_multi_index szTable.is_multi_indexcCsdd|jD|_dS)z create / validate metadata cSsg|]}|jdk r|jqS)N)rbr<)rrr5r5r6r& sz+Table.validate_metadata..N)rQrb)rrr5r5r6r$ szTable.validate_metadatacCsDddt|jjD}y ||fStk r>tdYnXdS)z\validate that we can store the multi-index; reset and return the new object cSs&g|]\}}|dk r|nd|qS)Nz level_{0})r)rrlr5r5r6r- sz-Table.validate_multiindex..zBduplicate names/columns in the multi-index when storing as a tableN)rrr-Z reset_indexr)rrr,r5r5r6validate_multiindex( s zTable.validate_multiindexcCstdd|jDS)z/ based on our axes, compute the expected nrows cSsg|]}|jjdqS)r)r}r)rrr5r5r6r: sz(Table.nrows_expected..)r1prodrP)rr5r5r6nrows_expected7 szTable.nrows_expectedcCs d|jkS)z has this table been created rV)r)rr5r5r6rL< szTable.is_existscCst|jddS)NrV)r"r)rr5r5r6rA szTable.storablecCs|jS)z. return the table group (this is my storable) )r)rr5r5r6rVE sz Table.tablecCs|jjS)N)rVrs)rr5r5r6rsJ sz Table.dtypecCs|jjS)N)rVr{)rr5r5r6r{N szTable.descriptioncCst|j|jS)N)rrrPrQ)rr5r5r6r R sz Table.axescCstdd|jDS)z0 the number of total columns in the values axes css|]}t|jVqdS)N)rHr)rrr5r5r6rY szTable.ncols..)sumrQ)rr5r5r6rVV sz Table.ncolscCsdS)NFr5)rr5r5r6 is_transposed[ szTable.is_transposedcCs(ttdd|jDdd|jDS)z@return a tuple of my permutated axes, non_indexable at the frontcSsg|]}t|dqS)r)rV)rrr5r5r6rd sz*Table.data_orientation..cSsg|]}t|jqSr5)rVr)rrr5r5r6re s)rDrrrrP)rr5r5r6data_orientation_ szTable.data_orientationcs<tddjDfddjDfddjDS)z> return a dict of the kinds allowable columns for this object cSsg|]}|j|fqSr5)r_)rrr5r5r6rn sz$Table.queryables..cs g|]\}}jj|dfqS)N)r _AXIS_NAMES)rrr)rr5r6rp scs&g|]}|jtjkr|j|fqSr5)r<rrr_)rr)rr5r6rt s)rrPrrQ)rr5)rr6 queryablesi s" zTable.queryablescCsdd|jDS)z return a list of my index cols cSsg|]}|j|jfqSr5)rr_)rrr5r5r6r| sz$Table.index_cols..)rP)rr5r5r6 index_colsz szTable.index_colscCsdd|jDS)z! return a list of my values cols cSsg|] }|jqSr5)r_)rrr5r5r6r sz%Table.values_cols..)rQ)rr5r5r6 values_cols~ szTable.values_colscCsdj|jj|dS)z+ return the metadata pathname for this key z{group}/meta/{key}/meta)rry)rrr)rryr5r5r6_get_metadata_path szTable._get_metadata_pathcCs0t|}|jj|||d|j|j|jddS)z write out a meta data array to the key as a fixed-format Series Parameters ---------- key : string values : ndarray rV)rr9rwrvN)rrr|rgr9rwrv)rryrr5r5r6r s zTable.write_metadatacCs0tt|jdd|ddk r,|j||SdS)z) return the meta data array for this key rN)r"rrrrg)rryr5r5r6rp szTable.read_metadatacCs|j|j_dS)z update our table index info N)r6rz)rr5r5r6r szTable.set_infocCsxt|j|j_||j_||j_|j|j_|j|j_|j|j_|j|j_|j |j_ |j |j_ |j |j_ | dS)z! set our table type & indexables N) r;r:rzrerfrrrvr9rwr,rbr)rr5r5r6r s         zTable.set_attrscstjddpg_tjddp$g_tjddp:t_tjdd_ttjdd_t tjdd_ tjd dpg_ fd d j D_ fd d j D_tjd dpg_dS)z retrieve our attributes rNrr6rvr9rwrqr,csg|]}|jr|qSr5)rrr)rr)rr5r6r sz#Table.get_attrs..csg|]}|js|qSr5)rrr)rr)rr5r6r srb)r"rzrrrr6rvr:r9r7rwr, indexablesrPrQrb)rr5)rr6r szTable.get_attrscCs\|dk rX|jddkrX|jddkrX|jddkrXtddd|jD}t|tdS) z- are we trying to operate on an old version? Nrr>rrrcSsg|] }t|qSr5)r;)rr?r5r5r6r sz*Table.validate_version..)rincompatibility_docrirrrR)rrIrr5r5r6r s*zTable.validate_versioncCs^|dkr dSt|tsdS|}x6|D]*\}}|dkr>q,||kr,tdj|dq,WdS)zvalidate the min_itemsize doesn't contain items that are not in the axes this needs data_columns to be defined NrzDmin_itemsize has the key [{key}] which is not an axis or data_column)ry)r0rrdrrr)rrqrrr5r5r6validate_min_itemsize s zTable.validate_min_itemsizecszjdkrtg_jddtjjDtjtjfddjfddtjjDjS)z1 create/cache the indexables if they don't exist NcSs"g|]\}\}}t|||dqS))r<rra)rX)rrrr<r5r5r6r sz$Table.indexables..cs(t}|krt}|j|||jdS)N)rr<rar)rrrr)rrr )base_posrrr5r6rW s zTable.indexables..fcsg|]\}}||qSr5r5)rrr)rWr5r6r s) _indexablesr rrzrerrrHrf)rr5)rlrrWrr6rh s   zTable.indexablesc Cs$|s dS|dkrdS|dks(|dkr8dd|jD}t|ttfsL|g}t}|dk rb||d<|dk rr||d<|j}x|D]}t|j|d}|dk r~|j r|j }|j } |j } |dk r| |kr| n| |d<|dk r| |kr| n| |d<|j s~|jdrtd |jf|q~WdS) a  Create a pytables index on the specified columns note: cannot index Time64Col() or ComplexCol currently; PyTables must be >= 3.0 Parameters ---------- columns : False (don't create an index), True (create all columns index), None or list_like (the indexers to index) optlevel: optimization level (defaults to 6) kind : kind of index (defaults to 'medium') Raises ------ raises if the node is not a table NFTcSsg|]}|jr|jqSr5)rr_)rrr5r5r6rsz&Table.create_index..optlevelr\rzColumns containing complex values can be stored but cannot be indexed when using table format. Either use fixed format, set index=False, or do not include the columns containing complex values to data_columns when initializing the table.)rr r0rDrCrrVr"rr.rrnr\Z remove_indexrr,rr) rrrnr\kwrVrrrZ cur_optlevelZcur_kindr5r5r6r s@   zTable.create_indexc Ks~|||sdSt|fd|i||_|j}xB|jD]8}||j|j||j |j |j | d| ddq>WdS)z^create and return the axes sniffed from the table: return boolean for success FrIrr)rvr9rwrrT) rr SelectionrRrr rr6rxrvr9rwr)rrIrzrrr5r5r6 read_axesDs    zTable.read_axescCs|S)z return the data for this obj r5)rrr5r5r6 get_objectdszTable.get_objectcst|jsgS|jd\}|j|t}|ddkrN|rNtd|||dkr`t}n |dkrlg}t|trt || fdd| Dfd d|DS) z[take the input data_columns and min_itemize and create a data columns spec rrrz.csg|]}|kr|qSr5r5)rr) axis_labelsr5r6rs) rHrr6rrrrrCr0rr r)rrrrr6r5)rtrsr6validate_data_columnshs$     zTable.validate_data_columnsc% sf|dkrHytt}Wn.tk rFtdjjjtdYnXfdd|D}r}|dd|j D}|j }|j }|j _ |j _ t|j_nd}t|jdkrtdg_g_ |dkrd }|_ txtjD]\} } | |kr>j| } t| j j j| | | <qt| } |dk rtj} |j| d}tt| t|sttt| tt|r|} t j| }t| j!|d <| j"j#|d <j$| | fqWfd dt|D_ tj }xjD]} | j%|d q Wx$jD]} t&| d| dq(Wdd}'(}|j)j*}||j)|}tjrjd\}}+||}t|r|j,t-|.t-||dj)}t|j*}|||}x>|D]6}|j,|g|dj)}|/|j*|/|||j*qW|dk rddt0||D}g}g}x||j1D]r}t2|j3}y&|4|\}}|$||$|Wn:t5tfk rtdjd6dd|DdYnXq@W|}|}g_1xvtt0||D]b\} \}}t7} d} |r&t|dkr&|d|kr&t8} |d} j $| |dk rv|rvy|j1| }!Wn.t5tfk rrtdj||j1dYnXnd}!yL| j9| | j:d}"|"j;|||!||j j jd|"<|j1$|"Wnht=ttfk r}#z|#Wdd}#~#XYn<t>k r.}$zt>dj|j?j@||$dWdd}$~$XYnX|d7}qWA|B||rbC|dS)a create and return the axes legacy tables create an indexable column, indexable index, non-indexable fields Parameters ---------- axes: a list of the axes in order to create (names or numbers of the axes) obj : the object to create axes on validate: validate the obj against an existing object already written min_itemsize: a dict of the min size for a column in bytes nan_rep : a values to use for string column nan_rep encoding : the encoding for string values data_columns : a list of columns that we want to create separate to allow indexing (or True will force all columns) NzFcannot properly create the storer for: [group->{group},value->{value}])rr{csg|]}|qSr5)_get_axis_number)rr)rr5r6rsz%Table.create_axes..cSsg|] }|jqSr5)r)rrr5r5r6rsr>z.get_blk_items..r5)rzrKr5)rzr6 get_blk_itemssz(Table.create_axes..get_blk_items)rcSs"i|]\}}||ft|qSr5)rDtolist)rbb_itemsr5r5r6r sz%Table.create_axes..zEcannot match existing table structure for [{items}] on appending datargcss|]}t|VqdS)N)r/)rrr5r5r6r0sz$Table.create_axes..)rzCIncompatible appended table [{blocks}]with existing table [{table}])rKrV)rr<r)rrrrrvr9rwr6zNcannot find the correct atom type -> [dtype->{name},items->{items}] {detail!s})r<rr2)Dr rrrrrrrr1rPrrvr9rwr6rHr rrrrr rcr"rrcrerCrr1rrrr-rrNrFr _reindex_axisrrrrNrKrurrrr rrQrDrr rDrirrrrrrdrrrsr<rkrr)%rr rrrvrrrzZexisting_tablerrr<Z append_axisZindexerZ exist_axisr6rxr{Z block_objrKrLrrtrzrZby_itemsZ new_blocksZ new_blk_itemsZearr}r~r rr|rr2r5)ryrrr6 create_axess                       & "   $  zTable.create_axesc s|dk rt|}|dk rBjrBx"jD]}||kr&|d|q&Wx jD]\}}t|||qJWjjdk rx4jjD]$\}}fdd}|||q~WS)z process axes filters NrcsxjD]}|}|}|dk s0t||krhjrN|tj}||}j j ||dS||kr t t |j}t |}t trd|}||}j j ||dSq Wtdj|ddS)N)rr>z.cannot find the field [{field}] for filtering!)field)rcrrv _get_axisrr[unionrr,rZ _getitem_axisr)r"r0rrr)rfiltZ axis_nameZ axis_numberZ axis_valuesZtakersr)roprr5r6process_filters&      z*Table.process_axes..process_filter) rCr[r,insertrrrRrEr) rrrrrlabelsrrrr5)rrrr6 process_axesws  $zTable.process_axescCs|dkrt|jd}td|d}dd|jD|d<|rl|dkrJ|jpHd}tj|||p\|jd }||d <n|jdk r|j|d <|S) z< create the description of the table from the axes & values Ni'rV)r< expectedrowscSsi|]}|j|jqSr5)r]r_)rrr5r5r6rsz,Table.create_description..r{ )r~rrr) rr_rr rrwrrr)rrr~rrrrr5r5r6create_descriptions      zTable.create_descriptionc Ks|||sdSt|f|||d||_|j}|jjdk rxT|jjD]D\}}}|j||| dd} ||| j |||j }qTWt |S)z]select coordinates (row numbers) from a table; return the coordinates object F)rIrrNr>)rr) rrrprR select_coordsrErrrUrilocrr) rrIrrrzZcoordsrrrr0r5r5r6rs   "zTable.read_coordinatesc Cs||sdS|dk r$tdx~|jD]t}||jkr,|jsPtdj|dt|j j |}| |j t t|j||||j|j|jd|jd|dSq,Wtd j|ddS) zareturn a single column from the table, generally only indexables are interesting FNz4read_column does not currently accept a where clausezMcolumn [{column}] can not be extracted individually; it is not data indexable)r)rvr9rwT)r<z(column [{column}] not found in the table)rrrr r<rrrr"rVrrr6rrurxrvr9rwryrZr)rrrIrrrrr5r5r6rs0       zTable.read_column)N)NNN)TNNN)N)NNFN)NNN)NNN)3rNrOrPrRrr:r,rrHrrSrTrrrrr[rr]r_rLrrVrsr{r rVrarbrdrerfrgrrprrrrrkrhrrqrrrurrrrrrr5r5)rr6r$ sd  #             ! F * c >  r$c@s$eZdZdZdZddZddZdS)rhz a write-once read-many table: this format DOES NOT ALLOW appending to a table. writing is a one-time operation the data are stored in a format that allows for searching the data on disk rncKs tddS)zS read the indices and the indexing array, calculate offset rows and return z!WORMTable needs to implement readN)r)rrzr5r5r6rszWORMTable.readcKs tddS)z write in a format that we can search later on (but cannot append to): write out the indices and the values using _write_array (e.g. a CArray) create an indexing table so that we can search z"WORKTable needs to implement writeN)r)rrzr5r5r6rN#szWORMTable.writeN)rNrOrPrRr:rrNr5r5r5r6rhsc@sVeZdZdZeddddeddddd ed d d dd gZd ZdZddZ dddZ dS) LegacyTablea an appendable table: allow append/query/delete operations to a (possibly) already existing appendable table this table ALLOWS append (but doesn't require them), and stores the data in a format that can be easily searched rr>r)r<rrarrZ columns_kind)r<rraZ index_kindrCr)r<r_r`ralegacycKs tddS)Nz2write operations are not allowed on legacy tables!)r)rrzr5r5r6rN<szLegacyTable.writeNcKs|jfd|i|sdSdS)zSwe have n indexable columns, with an arbitrary number of data axes rIN)rq)rrIrrzr5r5r6r?szLegacyTable.read)NN) rNrOrPrRrXrrmr:r rNrr5r5r5r6r+s rc @s>eZdZdZdZdZd ddZdddZd d Zdd d Z dS)AppendableTablez* support the new appendable table formats NZ appendableFc Ks|s|jr|j|jd|jf||||d| x|jD]} | ||qNcSsg|] }|jqSr5)r})rrr5r5r6rsz.AppendableTable.write_data..csg|]}|jdqS)r)r)rbi)indexesr5r6rscsg|]}|jdqS)r)r)rr)rr5r6rscSsg|] }|qSr5)ry)rrr5r5r6rsc Ss,g|]$}|tt|j|jdqS)r>)Z transposer1Zrollrr )rrr5r5r6rsi)rscsg|]}|qSr5r5)rr)end_istart_ir5r6rscsg|]}|qSr5r5)rr)rrr5r6rs)rrr)rsr-r_rQr"r0rr0r1ndarrayrFrrHrrPrr^r ZtilerepeatrrrGrUrVwrite_data_chunk)rrrr-rmasksrrrnindexesZbindexesrrZrepeaterrZbvaluesrZ new_shaperowschunksr5)rrrr6rs\        zAppendableTable.write_datac Csdx|D]}t|jsdSqWy|djd}|t|krJtj||jd}|jj}t|}x t|D]\} } | ||| <qdWx$t|D]\} }|||| |<qW|dk r|j t dd} | s|| }Wn4t k r} zt dj | dWdd} ~ XYnXy$t|r(|j||jWn4t k r^} ztdj | dWdd} ~ XYnXdS) z Parameters ---------- rows : an empty memory space where we are putting the chunk indexes : an array of the indexes mask : an array of the masks values : an array of the values Nr)rsF)r1z"cannot create row-data -> {detail})r2z)tables cannot write this data -> {detail})r1r^rrHrGrsr-rrrrrrrrVrFrr) rrrrrrrr-rrrrr2r5r5r6rs4    "  z AppendableTable.write_data_chunkcKsr|dkst|sf|dkr:|dkr:|j}|jj|jddn(|dkrH|j}|jj||d}|j|S|srdS|j}t ||f||d||_ |j }t | }t|} | rn|} t| | dkj} t| sdg} | d| kr| | | ddkr| dd| } xLt| D]@} |t| | }|j||jd||jddd| } q W|j| S)NT)r)rrr>rr)rHrrrHrrVZ remove_rowsrrrprRrrZ sort_valuesdiffrCrrFrr reversedrr )rrIrrrzrrVrZ sorted_serieslnrrZpgrrr5r5r6rsB      "  zAppendableTable.delete) NFNNNNNNF)F)NNN) rNrOrPrRrmr:rNrrrr5r5r5r6rHs , K0rc@s>eZdZdZdZdZdZeZe ddZ ddZ d d d Z d S) rfz* support the new appendable table formats r;rlrcCs|jdjdkS)Nrr>)rPr)rr5r5r6raEsz"AppendableFrameTable.is_transposedcCs|jr |j}|S)z these are written transposed )rar)rrr5r5r6rrIszAppendableFrameTable.get_objectNcKs|jfd|i|sdSt|jr<|j|jddtnt}|jdj}g}x|jD]}|ddkrzt |j}n t |j}|d} | dk r|j | dd|j r|j} |} t |t|ddd } n |jj} t |t|ddd } |} | jd krt| tjr| d | jdf} t| tt| d } t| g| | g}|t|qZWt|d krh|d}n t|d d }|j||d }|S)NrIrrrr-T)inplacer<)r<r>)rI)r)r)rqrHrr6rrrPrrQr from_tuplesr set_namesrar}r"rr r0r1rrrr,rr*rFrr!r)rrIrrzr6rframesrrr-rZindex_Zcols_rrzdfr5r5r6rOs<$      zAppendableFrameTable.read)NN) rNrOrPrRrr:r rrrSrarrrr5r5r5r6rf=s csXeZdZdZdZdZdZeZe Z e ddZ ddZ dfd d Zdfd d ZZS)rdz* support the new appendable table formats r>rjrcCsdS)NFr5)rr5r5r6rasz#AppendableSeriesTable.is_transposedcCs|S)Nr5)rrr5r5r6rrsz AppendableSeriesTable.get_objectNc sLt|ts.|jpd}t||i|jd}|g|_tjf||jd|S)z- we are going to write this as a frame table r)r)rr)r0rr<rrrrNr|)rrrrzr<)rr5r6rNs   zAppendableSeriesTable.writec s|j}|dk r6|r6x"|jD]}||kr|d|qWtjfd|i|}|r`|j|jdd|jdddf}|jdkrd|_|S)NrrT)rr)r[r,rrr set_indexrr<)rrrzr[rr4)rr5r6rs   zAppendableSeriesTable.read)N)N)rNrOrPrRrr:r rrrrrSrarrrNrrr5r5)rr6rds cs(eZdZdZdZdZfddZZS)rez* support the new appendable table formats r>rkc sJ|jpd}||\}|_t|j}||||_tjfd|i|S)z- we are going to write this as a frame table rr)r<r]r,rCrFrrrN)rrrzr<r)rr5r6rNs    z AppendableMultiSeriesTable.write)rNrOrPrRrr:rNrr5r5)rr6resc@sTeZdZdZdZdZdZeZe ddZ e ddZ d d Z e d d Z d dZdS)rcz< a table that read/writes the generic pytables table format r;rircCs|jS)N)r)rr5r5r6rszGenericTable.pandas_typecCst|jddp|jS)NrV)r"r)rr5r5r6rszGenericTable.storablecsTg_d_g_fddjD_fddjD_ddjD_dS)z retrieve our attributes Ncsg|]}|jr|qSr5)rrr)rr)rr5r6rsz*GenericTable.get_attrs..csg|]}|js|qSr5)rrr)rr)rr5r6rscSsg|] }|jqSr5)r<)rrr5r5r6rs)rrvr,rhrPrQr)rr5)rr6rs zGenericTable.get_attrscCs^|jdkrX|j}tdddg|_x6t|jD](\}}t|||g|jd}|j|q,W|jS)z2 create the indexables from the table description Nrr)r<r)r<rarr)rmr{rrZ_v_namesrrrF)rrrrrr5r5r6rhs zGenericTable.indexablescKs tddS)Nz cannot write on an generic table)r)rrzr5r5r6rNszGenericTable.writeN)rNrOrPrRrr:r rrrSrrrrhrNr5r5r5r6rcs   csPeZdZdZdZeZdZe dZ e ddZ d fdd Z fd d ZZS) rgz a frame with a multi-index rmrz ^level_\d+$cCsdS)NZappendable_multir5)rr5r5r6rTsz*AppendableMultiFrameTable.table_type_shortNc sl|dkrg}n|dkr |j}||\}|_x"|jD]}||kr8|d|q8Wtjf||d|S)NTr)rr)rr|r]r,rrrN)rrrrzr)rr5r6rNs  zAppendableMultiFrameTable.writec s>tjf|}|j}|jfdd|jjD|_|S)Ncs g|]}j|rdn|qS)N) _re_levelsr)rr\)rr5r6rsz2AppendableMultiFrameTable.read..)rrrr,rrr-)rrzr)r)rr6r s  zAppendableMultiFrameTable.read)N)rNrOrPrRr:rrr rcompilerrSrTrNrrr5r5)rr6rgs   cCs||}t|}|dk r"t|}|dks4||rB||rB|St|}|dk rlt|j|dd}||stddg|j}|||<|jt|}|S)NF)sort) rr)equalsuniquerslicer rrD)rrrrkrJZslicerr5r5r6rs   rcCs4y ||}Wn"tk r.t}||<YnX|S)z# get/create the info for this name )rr)r6r<rr5r5r6r+s  rcCs"t|}|dkr|}|S)z- for a tz-aware type, return an encoded zone N)r get_timezone utcoffset total_seconds)rZzoner5r5r6r7s  rcCs||dk rft|dd}|}tt|}t||d}|jdkrP|d|}|rx|dkrxt |}n|rxt j |dd}|S)aH coerce the values to a DatetimeIndex if tz is set preserve the input shape if possible Parameters ---------- values : ndarray tz : string/pickled tz object preserve_UTC : boolean, preserve the UTC of the result coerce : if we do not have a passed timezone, coerce to M8[ns] ndarray Nr<)r<rzM8[ns])rs) r"rr rr7rrZrr rCr1r)rrZZ preserve_UTCrr<r5r5r6ru?s     rurqc Cst|dd}t|trF|j}t|dtt|ddt|dd|dSt|trv|j}t|dtt|dd|dSt|tt frt}t|j d|t|dd|dSt|t rt d t j|d d }t|}|dkr|d }t|dtt|ddt|dd|dS|dkrF|d }t|dtt|dd|dS|d krtjdd|Dtjd}t|d t|dS|dkrtjdd|Dtjd}t|dt|dS|dkrt|||}|jj} t|dt| | |dS|dkr:|dkr*t}ttj|ddd||dSt d|n|dkrjt}ttj|tjdd||dS|dkrt}ttj|tjdd||dSt}ttj|ddd||dSdS)Nr<rrYrZ)rYrZr[r)rYr[rzMultiIndex not supported here!F)rrrcSs$g|]}t||jdqS)g.A)timemktime timetuple microsecond)rrr5r5r6rsz"_convert_index..)rs)r[rcSsg|] }|qSr5) toordinal)rrr5r5r6rsr~)r^r[rrUrrz?[unicode] is not supported as a in index type for [{0}] formatsZfloatingr)r"r0rrrXrwrr rrZ_ndarray_valuesrrr rr1rrrZ Time64ColZint32Z Time32Colrrsr^rr9rZint64Z Float64Col) rr9rwrr[r&r:rrr^r5r5r6r"\s                                 r"cCst|}|dkrt|}n|dkr,t|}n|dkrNtjdd|Dtd}n|dkrytjdd|Dtd}Wqtk rtjd d|Dtd}YqXnV|d krt|}nB|d krt|d||d }n(|d krt|d}ntdj|d|S)NrrrcSsg|]}t|qSr5)rr)rrr5r5r6rsz$_unconvert_index..)rsrcSsg|]}t|qSr5)rr)rrr5r5r6rscSsg|]}t|qSr5)rr)rrr5r5r6rs)rrr~)rvr9rwrrzunrecognized index type {kind})r\) r7rr r1rrrrr)r0r\r9rwrr5r5r6r3s*    r3cCs`t|}|dkrt|}nB|dkr2tj|td}n*|dkrLt|d||d}ntdj|d|S)Nrr)rsr~)rvr9rwzunrecognized index type {kind})r\)r7r#r1rrrrr)r0r\rr9rwrr5r5r6r=s r=cCsn|dk r0t|r0t|j||j|j}|dkrTt|}t dt |}t j |dj|dd}|S)a we take a string-like that is object dtype and coerce to a fixed size string type Parameters ---------- data : a numpy array of object dtype encoding : None or string-encoding errors : handler for encoding errors itemsize : integer, optional, defaults to the max length of the strings Returns ------- data in a fixed-length string dtype, encoded to bytes if needed Nr>zS{size})r)rs)rHrrr;encoderrrrr libwritersmax_len_string_arrayr1rr)r0r9rwr^ensuredr5r5r6rs  rcCs|j}tj|td}t|}|dk rt|rtt |}d |}t |dt rnt |jj||dj}n|j|ddjtdd}|dkrd}t||}||S) aT inverse of _convert_string_array Parameters ---------- data : fixed length string dtyped array nan_rep : the storage repr of NaN, optional encoding : the encoding of the data, optional errors : handler for encoding errors, default 'strict' Returns ------- an object array of the decoded data )rsNzU{0}r)rwF)r1rw)rr1rrrr:rHrrrrr0rrr;r3rrZ!string_array_replace_from_nan_repr)r0rvr9rwrr^rsr5r5r6rs  rcCs t|rt|||}||}|S)N) _need_convert_get_converter)rZval_kindr9rwconvr5r5r6rt=s rtcsRt|}|dkrddS|dkr(ddS|dkr>fddStdj|d dS) NrcSstj|ddS)NzM8[ns])rs)r1r)r?r5r5r6r@HrAz _get_converter..rcSst|ddS)NT)cache)r#Z to_pydatetime)r?r5r5r6r@JrAr~cst|dS)N)r9rw)r)r?)r9rwr5r6r@LrAzinvalid kind {kind})r\)r7rr)r\r9rwr5)r9rwr6rEsrcCst|}|dkrdSdS)N)rrr~TF)r7)r\r5r5r6rQsrc@s2eZdZdZd ddZddZddZd d ZdS) rpz Carries out a selection operation on a tables.Table object. Parameters ---------- table : a Table object where : list of Terms (or convertible to) start, stop: indices to start and/or stop selection NcCsP||_||_||_||_d|_d|_d|_d|_t|ryt j |dd}|dksZ|dkrt |}|j t jkr|j|j}}|dkrd}|dkr|jj}t ||||_nNt|j jt jr|jdk r||jks|jdk r||jkrtd||_Wntk rYnX|jdkrL|||_|jdk rL|j\|_|_dS)NF)rrbooleanrz3where must have index locations >= start and < stop)rVrIrr conditionrEZtermsrrr rr1rrsZbool_rr issubclassrrrrgenerateevaluate)rrVrIrrinferredr5r5r6res@       zSelection.__init__c Cs\|dkr dS|j}yt|||jjdStk rVtd|d|YnXdS)z) where can be a : dict,list,tuple,string N)rdr9aThe passed where expression: {0} contains an invalid variable reference all of the variable references must be a reference to an axis (e.g. 'index' or 'columns'), or a data_column The currently defined references are: {1} rg) rVrdr'r9 NameErrorrrrir)rrIrjr5r5r6rs zSelection.generatecCsX|jdk r(|jjj|j|j|jdS|jdk rB|jj|jS|jjj|j|jdS)z( generate the selection N)rr) rrVZ read_whererrrrrr)rr5r5r6rs   zSelection.selectcCs|j|j}}|jj}|dkr$d}n|dkr4||7}|jdkrD|}n|dkrT||7}|jdk rz|jjj|j||ddS|jdk r|jSt ||S)z( generate the selection NrT)rrr) rrrVrrZget_where_listrrr1r)rrrrr5r5r6rs    zSelection.select_coords)NNN)rNrOrPrRrrrrr5r5r5r6rpXs   , rp)NNNN)Nr)N)FF)NrqN)Nrq)FNrq)N)NNrq)rRr1rrrrrrtypingrrrrrZnumpyr1Zpandas._configrr Z pandas._libsr r rZpandas._libs.tslibsr Zpandas.compat._optionalr Z pandas.errorsrZpandas.core.dtypes.commonrrrrrrrZpandas.core.dtypes.missingrZpandasrrrrrrrrrr r!r"r#Zpandas.core.arrays.categoricalr$Zpandas.core.arrays.sparser%r&Zpandas.core.commoncorecommonrZ pandas.core.computation.pytablesr'r(Zpandas.core.indexr)Zpandas.core.internalsr*r+r,Zpandas.io.commonr-Zpandas.io.formats.printingr.r/rr8r7r:r=rGrLrrMrQWarningrRrirSrrTZ duplicate_docr8r7Zformat_deprecate_docr<r=r@r Z dropna_docZ format_docZ config_prefixZregister_optionZis_boolZis_one_of_factoryrrrurwrrrrrrXrrrrrrr<r]r^r_r?r`rbrFrar$rhrrrfrdrercrgrrrrur"r3r=rrrtrrrpr5r5r5r6s   $ <         Xl<&G){   $UqvF+3%    q   '