U C^@sHddlmZddlmZddlmZGdddeZGdddeZd S) )AbstractFileSysteminfer_storage_options)HadoopFileSystemcs^eZdZdZdddZdd d Zd d ZdddZeddZ e ddZ fddZ Z S) PyArrowHDFSzAdapted version of Arrow's HadoopFileSystem This is a very simple wrapper over pa.hdfs.HadoopFileSystem, which passes on all calls to the underlying class. defaultrNlibhdfscKsD|jr dStj|f|||||||f|_t||||||d|_dS)a7 Parameters ---------- host: str Hostname, IP or "default" to try to read from Hadoop config port: int Port to connect on, or default from Hadoop config if 0 user: str or None If given, connect as this username kerb_ticket: str or None If given, use this ticket for authentication driver: 'libhdfs' or 'libhdfs3' Binary driver; libhdfs if the JNI library and default extra_conf: None or dict Passed on to HadoopFileSystem N)hostportuser kerb_ticketdriver extra_conf)_cachedr__init__parsrpahdfs)selfr r r r rrkwargsr?/tmp/pip-install-6_kvzl1k/fsspec/fsspec/implementations/hdfs.pyr szPyArrowHDFS.__init__rbTcKst|||f|||d|S)a Parameters ---------- path: str Location of file; should start with '/' mode: str block_size: int Hadoop block size, e.g., 2**26 autocommit: True Transactions are not yet implemented for HDFS; errors if not True kwargs: dict or None Hadoop config parameters Returns ------- HDFSFile file-like instance ) block_size autocommit cache_options)HDFSFile)rpathmoderrrrrrr_open5szPyArrowHDFS._opencCs t|jfSN)rrrprotocolrrr __reduce_ex__[szPyArrowHDFS.__reduce_ex__csRj||}|r<|D]"}|d|d<|d|d<qnfdd|D}|S)Nkindtypenamecsg|]}|qSr)_strip_protocol).0prrr esz"PyArrowHDFS.ls..)rlsr')rrZdetailoutr)rr*rr,^s zPyArrowHDFS.lscCsXt|}i}|ddr$|d|d<|ddr<|d|d<|ddrT|d|d<|S)Nr usernamer r )rget)pathsopsr-rrr_get_kwargs_from_urlshs      z!PyArrowHDFS._get_kwargs_from_urlscCst|}|dS)Nrr)clsrr1rrrr'tszPyArrowHDFS._strip_protocolcsddkrfddSdkr"tStd}|dd}|dk rTdkrTt|StSdS)N)rr__getattribute__r#openr,makedirscsttf||Sr )getattrr)argskwitemrrrz.PyArrowHDFS.__getattribute__.. __class____dict__r)chmodchownr df disk_usagedownloadrexistsr get_capacityget_space_usedr Zis_openr Zstrip_protocolmkdirmvr rFrGrBr@rArCrDuploadr2Z read_parquetZrmstatrJ)robjectr4r/r7super)rr;drr>r:rr4ys   ! zPyArrowHDFS.__getattribute__)rrNNr N)rNTN)T)__name__ __module__ __qualname____doc__rrr#r, staticmethodr2 classmethodr'r4 __classcell__rrrOrrs( + &   rc@sDeZdZdZdddZddd Zd d Zd d ZddZddZ dS)rz`Wrapper around arrow's HdfsFile Allows seek beyond EOF and (eventually) commit/discard T readaheadNc KsR|s td||_||_||_||_|jj|||f||_|jrN| |_ dS)Nz2HDFSFile cannot be opened with 'autocommit=False'.) NotImplementedErrorfsrrrrr5fhreadablesize seek_size) rrYrrrrZ cache_typerrrrrrs  zHDFSFile.__init__rcCs*|dkr|rt||j}|j||S)Nr)r[minr]rZseek)rlocwhencerrrr_s z HDFSFile.seekcCs t|j|Sr )r7rZ)rr;rrr __getattr__szHDFSFile.__getattr__cCst|j|j|j|jffSr )rrYrrrr!rrrr#szHDFSFile.__reduce_ex__cCs|Sr rr*rrr __enter__szHDFSFile.__enter__cCs |dSr )close)rexc_typeexc_valexc_tbrrr__exit__szHDFSFile.__exit__)TrWN)r) rPrQrRrSrr_rbr#rcrhrrrrrs   rN) specrutilsrZ pyarrow.hdfsrrrLrrrrrs   ,