// Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. #pragma once #include #include #include #include "arrow/filesystem/filesystem.h" #include "arrow/io/hdfs.h" #include "arrow/util/uri.h" namespace arrow { namespace fs { /// Options for the HDFS implementation. struct ARROW_EXPORT HdfsOptions { HdfsOptions() = default; ~HdfsOptions() = default; /// Hdfs configuration options, contains host, port, driver io::HdfsConnectionConfig connection_config; /// Used by Hdfs OpenWritable Interface. int32_t buffer_size = 0; int16_t replication = 3; int64_t default_block_size = 0; void ConfigureEndPoint(std::string host, int port); void ConfigureReplication(int16_t replication); void ConfigureUser(std::string user_name); void ConfigureBufferSize(int32_t buffer_size); void ConfigureBlockSize(int64_t default_block_size); void ConfigureKerberosTicketCachePath(std::string path); void ConfigureExtraConf(std::string key, std::string val); bool Equals(const HdfsOptions& other) const; static Result FromUri(const ::arrow::internal::Uri& uri); static Result FromUri(const std::string& uri); }; /// HDFS-backed FileSystem implementation. /// /// implementation notes: /// - This is a wrapper of arrow/io/hdfs, so we can use FileSystem API to handle hdfs. class ARROW_EXPORT HadoopFileSystem : public FileSystem { public: ~HadoopFileSystem() override; std::string type_name() const override { return "hdfs"; } HdfsOptions options() const; bool Equals(const FileSystem& other) const override; /// \cond FALSE using FileSystem::GetFileInfo; /// \endcond Result GetFileInfo(const std::string& path) override; Result> GetFileInfo(const FileSelector& select) override; Status CreateDir(const std::string& path, bool recursive = true) override; Status DeleteDir(const std::string& path) override; Status DeleteDirContents(const std::string& path) override; Status DeleteRootDirContents() override; Status DeleteFile(const std::string& path) override; Status Move(const std::string& src, const std::string& dest) override; Status CopyFile(const std::string& src, const std::string& dest) override; Result> OpenInputStream( const std::string& path) override; Result> OpenInputFile( const std::string& path) override; Result> OpenOutputStream( const std::string& path) override; Result> OpenAppendStream( const std::string& path) override; /// Create a HdfsFileSystem instance from the given options. static Result> Make(const HdfsOptions& options); protected: explicit HadoopFileSystem(const HdfsOptions& options); class Impl; std::unique_ptr impl_; }; } // namespace fs } // namespace arrow