Module continuous_delivery_scripts.utils.aws_helpers
Helpers for interacting with AWS services such as S3.
Based on the Python SDK called BOTO https://aws.amazon.com/sdk-for-python/.
Expand source code
#
# Copyright (C) 2020-2025 Arm Limited or its affiliates and Contributors. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
"""Helpers for interacting with AWS services such as S3.
Based on the Python SDK called BOTO https://aws.amazon.com/sdk-for-python/.
"""
import boto3
import logging
import os
from pathlib import Path
import mimetypes
from continuous_delivery_scripts.utils.configuration import ConfigurationVariable, configuration
from typing import List, Optional, Tuple
logger = logging.getLogger(__name__)
def _get_aws_config() -> Tuple[str, dict]:
s3_region = configuration.get_value("AWS_DEFAULT_REGION")
s3_config = {
"aws_access_key_id": configuration.get_value("AWS_ACCESS_KEY_ID"),
"aws_secret_access_key": configuration.get_value("AWS_SECRET_ACCESS_KEY"),
}
return s3_region, s3_config
def upload_file(file: Path, bucket_dir: Optional[str], bucket_name: str) -> None:
"""Uploads a file onto AWS S3.
Args:
file: path to the file to upload
bucket_dir: name of the folder where to put the file in S3 bucket
bucket_name: name of the bucket to target
"""
if not bucket_name:
bucket_name = str(configuration.get_value(ConfigurationVariable.AWS_BUCKET))
logger.info(f"Uploading {file} to AWS")
if not file.exists():
raise FileNotFoundError(file)
s3_region, s3_config = _get_aws_config()
client = boto3.client("s3", **s3_config)
dest_filename = file.name
key = f"{bucket_dir}/{dest_filename}"
extension = "".join(file.suffixes)
bucket = bucket_name
client.upload_file(
str(file),
bucket,
key,
ExtraArgs={"ContentType": mimetypes.types_map.get(extension, "application/octet-stream")} if extension else {},
)
# Ensures the file is publicly available and reachable
# by anyone having access to the bucket.
client.put_object_acl(ACL="public-read", Bucket=bucket, Key=key)
def _upload_directory_file_contents(bucket_name: str, files: List[str], bucket_dir: str, root: str) -> None:
for name in files:
upload_file(Path(root).joinpath(name), bucket_dir, bucket_name)
def _upload_directory_directories(bucket_name: str, dirs: List[str], bucket_dir: str, root: str) -> None:
for name in dirs:
upload_directory(Path(root).joinpath(name), bucket_dir, bucket_name)
def _determine_destination(bucket_dir: str, real_dir_path: Path, current_folder: Path) -> str:
if current_folder != real_dir_path:
bucket_dest = str(Path(bucket_dir).joinpath(current_folder.relative_to(real_dir_path)))
else:
bucket_dest = bucket_dir
return bucket_dest.replace(os.sep, "/")
def upload_directory(dir: Path, bucket_dir: str, bucket_name: str) -> None:
"""Uploads the contents of a directory (recursively) onto AWS S3.
Args:
dir: folder to upload
bucket_dir: name of the folder where to put the directory contents in S3 bucket
bucket_name: name of the bucket to target
"""
if not bucket_name:
bucket_name = str(configuration.get_value(ConfigurationVariable.AWS_BUCKET))
logger.info(f"Uploading {dir} to AWS")
if not dir.exists():
raise FileNotFoundError(dir)
if dir.is_file():
upload_file(dir, bucket_dir, bucket_name)
return
def onerror(exception: Exception) -> None:
logger.error(exception)
real_dir_path = dir.resolve()
for root, dirs, files in os.walk(str(real_dir_path), followlinks=True, onerror=onerror):
new_bucket_dir = _determine_destination(bucket_dir, real_dir_path, Path(root))
_upload_directory_directories(bucket_name, dirs, new_bucket_dir, root)
_upload_directory_file_contents(bucket_name, files, new_bucket_dir, root)
Functions
def upload_directory(dir: pathlib.Path, bucket_dir: str, bucket_name: str) ‑> None
-
Uploads the contents of a directory (recursively) onto AWS S3.
Args
dir
- folder to upload
bucket_dir
- name of the folder where to put the directory contents in S3 bucket
bucket_name
- name of the bucket to target
Expand source code
def upload_directory(dir: Path, bucket_dir: str, bucket_name: str) -> None: """Uploads the contents of a directory (recursively) onto AWS S3. Args: dir: folder to upload bucket_dir: name of the folder where to put the directory contents in S3 bucket bucket_name: name of the bucket to target """ if not bucket_name: bucket_name = str(configuration.get_value(ConfigurationVariable.AWS_BUCKET)) logger.info(f"Uploading {dir} to AWS") if not dir.exists(): raise FileNotFoundError(dir) if dir.is_file(): upload_file(dir, bucket_dir, bucket_name) return def onerror(exception: Exception) -> None: logger.error(exception) real_dir_path = dir.resolve() for root, dirs, files in os.walk(str(real_dir_path), followlinks=True, onerror=onerror): new_bucket_dir = _determine_destination(bucket_dir, real_dir_path, Path(root)) _upload_directory_directories(bucket_name, dirs, new_bucket_dir, root) _upload_directory_file_contents(bucket_name, files, new_bucket_dir, root)
def upload_file(file: pathlib.Path, bucket_dir: Optional[str], bucket_name: str) ‑> None
-
Uploads a file onto AWS S3.
Args
file
- path to the file to upload
bucket_dir
- name of the folder where to put the file in S3 bucket
bucket_name
- name of the bucket to target
Expand source code
def upload_file(file: Path, bucket_dir: Optional[str], bucket_name: str) -> None: """Uploads a file onto AWS S3. Args: file: path to the file to upload bucket_dir: name of the folder where to put the file in S3 bucket bucket_name: name of the bucket to target """ if not bucket_name: bucket_name = str(configuration.get_value(ConfigurationVariable.AWS_BUCKET)) logger.info(f"Uploading {file} to AWS") if not file.exists(): raise FileNotFoundError(file) s3_region, s3_config = _get_aws_config() client = boto3.client("s3", **s3_config) dest_filename = file.name key = f"{bucket_dir}/{dest_filename}" extension = "".join(file.suffixes) bucket = bucket_name client.upload_file( str(file), bucket, key, ExtraArgs={"ContentType": mimetypes.types_map.get(extension, "application/octet-stream")} if extension else {}, ) # Ensures the file is publicly available and reachable # by anyone having access to the bucket. client.put_object_acl(ACL="public-read", Bucket=bucket, Key=key)