#!/usr/bin/env bash set -ex FILENAME="awswrangler-layer-${1}.zip" NINJA=${2} pushd /aws-sdk-pandas rm -rf python dist/pyarrow_files "dist/${FILENAME}" "${FILENAME}" popd rm -rf dist arrow export ARROW_HOME=$(pwd)/dist export LD_LIBRARY_PATH=$(pwd)/dist/lib:$LD_LIBRARY_PATH git clone \ --depth 1 \ --branch apache-arrow-10.0.0 \ --single-branch \ https://github.com/apache/arrow.git mkdir $ARROW_HOME mkdir arrow/cpp/build pushd arrow/cpp/build cmake \ -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ -DCMAKE_INSTALL_LIBDIR=lib \ -DARROW_PYTHON=ON \ -DARROW_PARQUET=ON \ -DARROW_DATASET=ON \ -DARROW_WITH_SNAPPY=ON \ -DARROW_WITH_ZLIB=ON \ -DARROW_FLIGHT=OFF \ -DARROW_GANDIVA=OFF \ -DARROW_ORC=OFF \ -DARROW_CSV=ON \ -DARROW_JSON=ON \ -DARROW_COMPUTE=ON \ -DARROW_FILESYSTEM=ON \ -DARROW_PLASMA=OFF \ -DARROW_WITH_BZ2=OFF \ -DARROW_WITH_ZSTD=OFF \ -DARROW_WITH_LZ4=OFF \ -DARROW_WITH_BROTLI=OFF \ -DARROW_BUILD_TESTS=OFF \ -GNinja \ .. eval $NINJA eval "${NINJA} install" popd pushd arrow/python export CMAKE_PREFIX_PATH=${ARROW_HOME}${CMAKE_PREFIX_PATH:+:${CMAKE_PREFIX_PATH}} export ARROW_PRE_0_15_IPC_FORMAT=0 export PYARROW_WITH_HDFS=0 export PYARROW_WITH_FLIGHT=0 export PYARROW_WITH_GANDIVA=0 export PYARROW_WITH_ORC=0 export PYARROW_WITH_CUDA=0 export PYARROW_WITH_PLASMA=0 export PYARROW_WITH_PARQUET=1 export PYARROW_WITH_DATASET=1 export PYARROW_WITH_FILESYSTEM=1 export PYARROW_WITH_CSV=1 export PYARROW_WITH_JSON=1 export PYARROW_WITH_COMPUTE=1 python3 setup.py build_ext \ --build-type=release \ --bundle-arrow-cpp \ bdist_wheel pip3 install dist/pyarrow-*.whl -t /aws-sdk-pandas/dist/pyarrow_files popd pushd /aws-sdk-pandas pip3 install . -t ./python ".[redshift,mysql,postgres,gremlin,opensearch,openpyxl]" rm -rf python/pyarrow* rm -rf python/boto* rm -f /aws-sdk-pandas/dist/pyarrow_files/pyarrow/libarrow.so rm -f /aws-sdk-pandas/dist/pyarrow_files/pyarrow/libparquet.so rm -f /aws-sdk-pandas/dist/pyarrow_files/pyarrow/libarrow_python.so cp -r /aws-sdk-pandas/dist/pyarrow_files/pyarrow* python/ # Removing nonessential files find python -name '*.so' -type f -exec strip "{}" \; find python -wholename "*/tests/*" -type f -delete find python -regex '^.*\(__pycache__\|\.py[co]\)$' -delete zip -r9 "${FILENAME}" ./python mv "${FILENAME}" dist/ rm -rf python dist/pyarrow_files "${FILENAME}" popd rm -rf dist arrow