This notebook automates the (re)build, install, zip, upload to S3, and aws cli command to publish new version to an AWS layer.
from my_utils.os_utils import subprocess_execute
import shutil
# list of packages to build
packages_todo = [
# list of tuples: ('package_name','from_pip')
('pandas',True),
('my_utils',False),
('my_dev',False),
]
package_base_dir = '/home/jovyan/packages'
work_dir = '/home/jovyan/work'
s3_path = 's3://andrewm4894-python-packages/latest'
s3_region = 'us-west-2'
s3_bucket = s3_path.split('s3://')[1].split('/')[0]
s3_url = f"https://s3.console.aws.amazon.com/s3/buckets/{s3_path.replace('s3://','')}?region={s3_region}&tab=overview"
aws_layer_name = 'my-aws-python-packages'
# build and install locally each package
for package_name, from_pip in packages_todo:
cmd = ''
# run pip install within the python dir
if from_pip:
# if from pip then just use name
cmd = cmd + f'pip install --upgrade --force-reinstall "{package_name}" -t {work_dir}/python/python/.'
else:
# cd into the package dir
cmd = f'cd {package_base_dir}/{package_name}'
# run the build commands to create local egg file to be zipped up into an AWS layer
cmd = cmd + f' && python setup.py bdist_egg'
# then do a local install
cmd = cmd + f' && pip install --upgrade "{package_base_dir}/{package_name}" -t {work_dir}/python/python/.'
# kick off big command to do the stuff
result = subprocess_execute(cmd)
# look at results
print(result)
... cmd to execute: pip install --upgrade --force-reinstall "pandas" -t /home/jovyan/work/python/python/. ... result: Collecting pandas Downloading https://files.pythonhosted.org/packages/73/9b/52e228545d14f14bb2a1622e225f38463c8726645165e1cb7dde95bfe6d4/pandas-0.25.1-cp36-cp36m-manylinux1_x86_64.whl (10.5MB) Collecting pytz>=2017.2 (from pandas) Downloading https://files.pythonhosted.org/packages/87/76/46d697698a143e05f77bec5a526bf4e56a0be61d63425b68f4ba553b51f2/pytz-2019.2-py2.py3-none-any.whl (508kB) Collecting numpy>=1.13.3 (from pandas) Downloading https://files.pythonhosted.org/packages/75/92/57179ed45307ec6179e344231c47da7f3f3da9e2eee5c8ab506bd279ce4e/numpy-1.17.1-cp36-cp36m-manylinux1_x86_64.whl (20.4MB) Collecting python-dateutil>=2.6.1 (from pandas) Downloading https://files.pythonhosted.org/packages/41/17/c62faccbfbd163c7f57f3844689e3a78bae1f403648a6afb1d0866d87fbb/python_dateutil-2.8.0-py2.py3-none-any.whl (226kB) Collecting six>=1.5 (from python-dateutil>=2.6.1->pandas) Downloading https://files.pythonhosted.org/packages/73/fb/00a976f728d0d1fecfe898238ce23f502a721c0ac0ecfedb80e0d88c64e9/six-1.12.0-py2.py3-none-any.whl Installing collected packages: pytz, numpy, six, python-dateutil, pandas Successfully installed numpy-1.17.1 pandas-0.25.1 python-dateutil-2.8.0 pytz-2019.2 six-1.12.0 ... cmd to execute: cd /home/jovyan/packages/my_utils && python setup.py bdist_egg && pip install --upgrade "/home/jovyan/packages/my_utils" -t /home/jovyan/work/python/python/. ... result: running bdist_egg running egg_info writing my_utils.egg-info/PKG-INFO writing dependency_links to my_utils.egg-info/dependency_links.txt writing top-level names to my_utils.egg-info/top_level.txt reading manifest file 'my_utils.egg-info/SOURCES.txt' writing manifest file 'my_utils.egg-info/SOURCES.txt' installing library code to build/bdist.linux-x86_64/egg running install_lib running build_py creating build/bdist.linux-x86_64/egg creating build/bdist.linux-x86_64/egg/my_utils copying build/lib/my_utils/os_utils.py -> build/bdist.linux-x86_64/egg/my_utils copying build/lib/my_utils/__init__.py -> build/bdist.linux-x86_64/egg/my_utils byte-compiling build/bdist.linux-x86_64/egg/my_utils/os_utils.py to os_utils.cpython-36.pyc byte-compiling build/bdist.linux-x86_64/egg/my_utils/__init__.py to __init__.cpython-36.pyc creating build/bdist.linux-x86_64/egg/EGG-INFO copying my_utils.egg-info/PKG-INFO -> build/bdist.linux-x86_64/egg/EGG-INFO copying my_utils.egg-info/SOURCES.txt -> build/bdist.linux-x86_64/egg/EGG-INFO copying my_utils.egg-info/dependency_links.txt -> build/bdist.linux-x86_64/egg/EGG-INFO copying my_utils.egg-info/not-zip-safe -> build/bdist.linux-x86_64/egg/EGG-INFO copying my_utils.egg-info/top_level.txt -> build/bdist.linux-x86_64/egg/EGG-INFO creating 'dist/my_utils-0.1-py3.6.egg' and adding 'build/bdist.linux-x86_64/egg' to it removing 'build/bdist.linux-x86_64/egg' (and everything under it) Processing /home/jovyan/packages/my_utils Installing collected packages: my-utils Running setup.py install for my-utils: started Running setup.py install for my-utils: finished with status 'done' Successfully installed my-utils-0.1 ... cmd to execute: cd /home/jovyan/packages/my_dev && python setup.py bdist_egg && pip install --upgrade "/home/jovyan/packages/my_dev" -t /home/jovyan/work/python/python/. ... result: running bdist_egg running egg_info writing my_dev.egg-info/PKG-INFO writing dependency_links to my_dev.egg-info/dependency_links.txt writing top-level names to my_dev.egg-info/top_level.txt reading manifest file 'my_dev.egg-info/SOURCES.txt' writing manifest file 'my_dev.egg-info/SOURCES.txt' installing library code to build/bdist.linux-x86_64/egg running install_lib running build_py creating build/bdist.linux-x86_64/egg creating build/bdist.linux-x86_64/egg/my_dev copying build/lib/my_dev/dev.py -> build/bdist.linux-x86_64/egg/my_dev copying build/lib/my_dev/__init__.py -> build/bdist.linux-x86_64/egg/my_dev byte-compiling build/bdist.linux-x86_64/egg/my_dev/dev.py to dev.cpython-36.pyc byte-compiling build/bdist.linux-x86_64/egg/my_dev/__init__.py to __init__.cpython-36.pyc creating build/bdist.linux-x86_64/egg/EGG-INFO copying my_dev.egg-info/PKG-INFO -> build/bdist.linux-x86_64/egg/EGG-INFO copying my_dev.egg-info/SOURCES.txt -> build/bdist.linux-x86_64/egg/EGG-INFO copying my_dev.egg-info/dependency_links.txt -> build/bdist.linux-x86_64/egg/EGG-INFO copying my_dev.egg-info/not-zip-safe -> build/bdist.linux-x86_64/egg/EGG-INFO copying my_dev.egg-info/top_level.txt -> build/bdist.linux-x86_64/egg/EGG-INFO creating 'dist/my_dev-0.1-py3.6.egg' and adding 'build/bdist.linux-x86_64/egg' to it removing 'build/bdist.linux-x86_64/egg' (and everything under it) Processing /home/jovyan/packages/my_dev Installing collected packages: my-dev Running setup.py install for my-dev: started Running setup.py install for my-dev: finished with status 'done' Successfully installed my-dev-0.1
# make zip
shutil.make_archive(f'{work_dir}/python', 'zip', f'{work_dir}/python')
'/home/jovyan/work/python.zip'
# upload zip to s3 using aws cli
cmd = f'aws s3 cp --region {s3_region} {work_dir}/python.zip {s3_path}/'
subprocess_execute(cmd)
print(f'... zip should be updated at: {s3_url} ...')
... cmd to execute: aws s3 cp --region us-west-2 /home/jovyan/work/python.zip s3://andrewm4894-python-packages/latest/ ... result: ... zip should be updated at: https://s3.console.aws.amazon.com/s3/buckets/andrewm4894-python-packages/latest?region=us-west-2&tab=overview ...
# publish zip file from s3 location to aws layer
cmd = f"aws lambda publish-layer-version"
cmd += f" --layer-name {aws_layer_name}"
cmd += f" --content S3Bucket={s3_bucket},S3Key={s3_path.split('/')[-1]}/python.zip"
cmd += f" --region {s3_region}"
subprocess_execute(cmd)
... cmd to execute: aws lambda publish-layer-version --layer-name my-aws-python-packages --content S3Bucket=andrewm4894-python-packages,S3Key=latest/python.zip --region us-west-2 ... result:
'{\n "Content": {\n "Location": "https://awslambda-us-west-2-layers.s3.us-west-2.amazonaws.com/snapshots/908566381001/my-aws-python-packages-a2132736-f0ed-4bf2-8ca8-a3237a60254f?versionId=NyejNbE2dp8dUKR3Ln05R_x1QgJ5KBNB&X-Amz-Security-Token=12345&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20190902T221416Z&X-Amz-SignedHeaders=host&X-Amz-Expires=600&X-Amz-Credential=1234&X-Amz-Signature=1234",\n "CodeSha256": "1234=",\n "CodeSize": 38572755\n },\n "LayerArn": "arn:aws:lambda:us-west-2:1234:layer:my-aws-python-packages",\n "LayerVersionArn": "arn:aws:lambda:us-west-2:1234:layer:my-aws-python-packages:1",\n "Description": "",\n "CreatedDate": "2019-09-02T22:14:32.712+0000",\n "Version": 1\n}\n'