// Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. #pragma once #include #include #include #include #include #include #include #include "arrow/testing/visibility.h" #include "arrow/type.h" namespace arrow { class Array; namespace random { using SeedType = int32_t; constexpr SeedType kSeedMax = std::numeric_limits::max(); class ARROW_TESTING_EXPORT RandomArrayGenerator { public: explicit RandomArrayGenerator(SeedType seed) : seed_distribution_(static_cast(1), kSeedMax), seed_rng_(seed) {} /// \brief Generate a null bitmap /// /// \param[in] size the size of the bitmap to generate /// \param[in] null_probability the probability of a bit being zero /// /// \return a generated Buffer std::shared_ptr NullBitmap(int64_t size, double null_probability = 0); /// \brief Generate a random BooleanArray /// /// \param[in] size the size of the array to generate /// \param[in] true_probability the probability of a value being 1 / bit-set /// \param[in] null_probability the probability of a row being null /// /// \return a generated Array std::shared_ptr Boolean(int64_t size, double true_probability, double null_probability = 0); /// \brief Generate a random UInt8Array /// /// \param[in] size the size of the array to generate /// \param[in] min the lower bound of the uniform distribution /// \param[in] max the upper bound of the uniform distribution /// \param[in] null_probability the probability of a row being null /// /// \return a generated Array std::shared_ptr UInt8(int64_t size, uint8_t min, uint8_t max, double null_probability = 0); /// \brief Generate a random Int8Array /// /// \param[in] size the size of the array to generate /// \param[in] min the lower bound of the uniform distribution /// \param[in] max the upper bound of the uniform distribution /// \param[in] null_probability the probability of a row being null /// /// \return a generated Array std::shared_ptr Int8(int64_t size, int8_t min, int8_t max, double null_probability = 0); /// \brief Generate a random UInt16Array /// /// \param[in] size the size of the array to generate /// \param[in] min the lower bound of the uniform distribution /// \param[in] max the upper bound of the uniform distribution /// \param[in] null_probability the probability of a row being null /// /// \return a generated Array std::shared_ptr UInt16(int64_t size, uint16_t min, uint16_t max, double null_probability = 0); /// \brief Generate a random Int16Array /// /// \param[in] size the size of the array to generate /// \param[in] min the lower bound of the uniform distribution /// \param[in] max the upper bound of the uniform distribution /// \param[in] null_probability the probability of a row being null /// /// \return a generated Array std::shared_ptr Int16(int64_t size, int16_t min, int16_t max, double null_probability = 0); /// \brief Generate a random UInt32Array /// /// \param[in] size the size of the array to generate /// \param[in] min the lower bound of the uniform distribution /// \param[in] max the upper bound of the uniform distribution /// \param[in] null_probability the probability of a row being null /// /// \return a generated Array std::shared_ptr UInt32(int64_t size, uint32_t min, uint32_t max, double null_probability = 0); /// \brief Generate a random Int32Array /// /// \param[in] size the size of the array to generate /// \param[in] min the lower bound of the uniform distribution /// \param[in] max the upper bound of the uniform distribution /// \param[in] null_probability the probability of a row being null /// /// \return a generated Array std::shared_ptr Int32(int64_t size, int32_t min, int32_t max, double null_probability = 0); /// \brief Generate a random UInt64Array /// /// \param[in] size the size of the array to generate /// \param[in] min the lower bound of the uniform distribution /// \param[in] max the upper bound of the uniform distribution /// \param[in] null_probability the probability of a row being null /// /// \return a generated Array std::shared_ptr UInt64(int64_t size, uint64_t min, uint64_t max, double null_probability = 0); /// \brief Generate a random Int64Array /// /// \param[in] size the size of the array to generate /// \param[in] min the lower bound of the uniform distribution /// \param[in] max the upper bound of the uniform distribution /// \param[in] null_probability the probability of a row being null /// /// \return a generated Array std::shared_ptr Int64(int64_t size, int64_t min, int64_t max, double null_probability = 0); /// \brief Generate a random HalfFloatArray /// /// \param[in] size the size of the array to generate /// \param[in] min the lower bound of the distribution /// \param[in] max the upper bound of the distribution /// \param[in] null_probability the probability of a row being null /// /// \return a generated Array std::shared_ptr Float16(int64_t size, int16_t min, int16_t max, double null_probability = 0); /// \brief Generate a random FloatArray /// /// \param[in] size the size of the array to generate /// \param[in] min the lower bound of the uniform distribution /// \param[in] max the upper bound of the uniform distribution /// \param[in] null_probability the probability of a row being null /// \param[in] nan_probability the probability of a row being NaN /// /// \return a generated Array std::shared_ptr Float32(int64_t size, float min, float max, double null_probability = 0, double nan_probability = 0); /// \brief Generate a random DoubleArray /// /// \param[in] size the size of the array to generate /// \param[in] min the lower bound of the uniform distribution /// \param[in] max the upper bound of the uniform distribution /// \param[in] null_probability the probability of a row being null /// \param[in] nan_probability the probability of a row being NaN /// /// \return a generated Array std::shared_ptr Float64(int64_t size, double min, double max, double null_probability = 0, double nan_probability = 0); template std::shared_ptr Numeric(int64_t size, CType min, CType max, double null_probability = 0) { switch (ArrowType::type_id) { case Type::UINT8: return UInt8(size, static_cast(min), static_cast(max), null_probability); case Type::INT8: return Int8(size, static_cast(min), static_cast(max), null_probability); case Type::UINT16: return UInt16(size, static_cast(min), static_cast(max), null_probability); case Type::INT16: return Int16(size, static_cast(min), static_cast(max), null_probability); case Type::UINT32: return UInt32(size, static_cast(min), static_cast(max), null_probability); case Type::INT32: return Int32(size, static_cast(min), static_cast(max), null_probability); case Type::UINT64: return UInt64(size, static_cast(min), static_cast(max), null_probability); case Type::INT64: return Int64(size, static_cast(min), static_cast(max), null_probability); case Type::HALF_FLOAT: return Float16(size, static_cast(min), static_cast(max), null_probability); case Type::FLOAT: return Float32(size, static_cast(min), static_cast(max), null_probability); case Type::DOUBLE: return Float64(size, static_cast(min), static_cast(max), null_probability); default: return nullptr; } } /// \brief Generate an array of offsets (for use in e.g. ListArray::FromArrays) /// /// \param[in] size the size of the array to generate /// \param[in] first_offset the first offset value (usually 0) /// \param[in] last_offset the last offset value (usually the size of the child array) /// \param[in] null_probability the probability of an offset being null /// \param[in] force_empty_nulls if true, null offsets must have 0 "length" /// /// \return a generated Array std::shared_ptr Offsets(int64_t size, int32_t first_offset, int32_t last_offset, double null_probability = 0, bool force_empty_nulls = false); /// \brief Generate a random StringArray /// /// \param[in] size the size of the array to generate /// \param[in] min_length the lower bound of the string length /// determined by the uniform distribution /// \param[in] max_length the upper bound of the string length /// determined by the uniform distribution /// \param[in] null_probability the probability of a row being null /// /// \return a generated Array std::shared_ptr String(int64_t size, int32_t min_length, int32_t max_length, double null_probability = 0); /// \brief Generate a random LargeStringArray /// /// \param[in] size the size of the array to generate /// \param[in] min_length the lower bound of the string length /// determined by the uniform distribution /// \param[in] max_length the upper bound of the string length /// determined by the uniform distribution /// \param[in] null_probability the probability of a row being null /// /// \return a generated Array std::shared_ptr LargeString(int64_t size, int32_t min_length, int32_t max_length, double null_probability = 0); /// \brief Generate a random StringArray with repeated values /// /// \param[in] size the size of the array to generate /// \param[in] unique the number of unique string values used /// to populate the array /// \param[in] min_length the lower bound of the string length /// determined by the uniform distribution /// \param[in] max_length the upper bound of the string length /// determined by the uniform distribution /// \param[in] null_probability the probability of a row being null /// /// \return a generated Array std::shared_ptr StringWithRepeats(int64_t size, int64_t unique, int32_t min_length, int32_t max_length, double null_probability = 0); /// \brief Like StringWithRepeats but return BinaryArray std::shared_ptr BinaryWithRepeats(int64_t size, int64_t unique, int32_t min_length, int32_t max_length, double null_probability = 0); /// \brief Generate a random FixedSizeBinaryArray /// /// \param[in] size the size of the array to generate /// \param[in] byte_width the byte width of fixed-size binary items /// \param[in] null_probability the probability of a row being null /// /// \return a generated Array std::shared_ptr FixedSizeBinary(int64_t size, int32_t byte_width, double null_probability = 0); /// \brief Generate a random ListArray /// /// \param[in] values The underlying values array /// \param[in] size The size of the generated list array /// \param[in] null_probability the probability of a list value being null /// \param[in] force_empty_nulls if true, null list entries must have 0 length /// /// \return a generated Array std::shared_ptr List(const Array& values, int64_t size, double null_probability, bool force_empty_nulls = false); /// \brief Generate a random SparseUnionArray /// /// The type ids are chosen randomly, according to a uniform distribution, /// amongst the given child fields. /// /// \param[in] fields Vector of Arrays containing the data for each union field /// \param[in] size The size of the generated sparse union array std::shared_ptr SparseUnion(const ArrayVector& fields, int64_t size); /// \brief Generate a random DenseUnionArray /// /// The type ids are chosen randomly, according to a uniform distribution, /// amongst the given child fields. The offsets are incremented along /// each child field. /// /// \param[in] fields Vector of Arrays containing the data for each union field /// \param[in] size The size of the generated sparse union array std::shared_ptr DenseUnion(const ArrayVector& fields, int64_t size); /// \brief Generate a random Array of the specified type, size, and null_probability. /// /// Generation parameters other than size and null_probability are determined based on /// the type of Array to be generated. /// If boolean the probabilities of true,false values are 0.25,0.75 respectively. /// If numeric min,max will be the least and greatest representable values. /// If string min_length,max_length will be 0,sqrt(size) respectively. /// /// \param[in] type the type of Array to generate /// \param[in] size the size of the Array to generate /// \param[in] null_probability the probability of a slot being null /// \return a generated Array std::shared_ptr ArrayOf(std::shared_ptr type, int64_t size, double null_probability); SeedType seed() { return seed_distribution_(seed_rng_); } private: std::uniform_int_distribution seed_distribution_; std::default_random_engine seed_rng_; }; } // namespace random // // Assorted functions // template void randint(int64_t N, T lower, T upper, std::vector* out) { const int random_seed = 0; std::default_random_engine gen(random_seed); std::uniform_int_distribution d(lower, upper); out->resize(N, static_cast(0)); std::generate(out->begin(), out->end(), [&d, &gen] { return static_cast(d(gen)); }); } template void random_real(int64_t n, uint32_t seed, T min_value, T max_value, std::vector* out) { std::default_random_engine gen(seed); std::uniform_real_distribution d(min_value, max_value); out->resize(n, static_cast(0)); std::generate(out->begin(), out->end(), [&d, &gen] { return static_cast(d(gen)); }); } template void rand_uniform_int(int64_t n, uint32_t seed, T min_value, T max_value, U* out) { assert(out || (n == 0)); std::default_random_engine gen(seed); std::uniform_int_distribution d(min_value, max_value); std::generate(out, out + n, [&d, &gen] { return static_cast(d(gen)); }); } } // namespace arrow