[PYTHON] List the AMIs used by AWS Data Pipeline

This is the script used to find out the Image ID of the AMI used by AWS Data Pipeline.

import boto3
from time import sleep

session = boto3.Session(profile_name="{Profile name}")
client = session.client("datapipeline")


def main():
    for x in list_all_pipelines():
        for i in get_ec2_resources(x)
            print(x, i)


def list_all_pipelines():
    """If you do not specify a marker, it will end in the middle
    """
    result = client.list_pipelines()
    pipelines = [x["id"] for x in result["pipelineIdList"]]
    marker = result["marker"]
    while marker is not None:
        y = client.list_pipelines(marker=marker)
        pipelines += [x["id"] for x in y["pipelineIdList"]]
        marker = y.get("marker", None)
    return pipelines


def get_ec2_resources(pipeline_id):
    resources = []
    result = client.get_pipeline_definition(pipelineId=pipeline_id)
    for x in result["pipelineObjects"]:
        if x["name"] == "Ec2":
            for y in x["fields"]:
                if y["key"] == "imageId":
                    resources.append(y["stringValue"])
    return resources


if __name__ == "__main__":
    main()

Recommended Posts

List the AMIs used by AWS Data Pipeline
Gzip the data by streaming
Get the column list & data list of CASTable
Periodically execute Python Script on AWS Data Pipeline
Search by the value of the instance in the list
List AWS ami
Authentication information used by Boto3 (AWS SDK for Python)
List of Python code used in big data analysis
How to check the Java version used by Maven
Let's visualize the rainfall data released by Shimane Prefecture
List the classes that can be referenced by ObjCClass
How to unprefix the DB name used by pytest-django