Prefect workflow --- outperform airflow by creating flow in python host process

Prefect workflow

https://docs.prefect.io/v3/get-started/quickstart

Convert a Python script into your first Prefect workflow

 

Prefect makes it easy to deploy Python scripts, run them on a schedule, make them resilient to failure, and observe them in a UI.

To do this, you need to perform the following tasks:

1. Install Prefect
2. Connect to a Prefect API (self-hosted or Prefect Cloud)
3. Add decorators to the functions in the script

import httpx

from prefect import flow, task # Prefect flow and task decorators


@flow(log_prints=True)
def show_stars(github_repos: list[str]):
    """Flow: Show the number of stars that GitHub repos have"""
    for repo in github_repos:
        # Call Task 1
        repo_stats = fetch_stats(repo)

        # Call Task 2
        stars = get_stars(repo_stats)

        # Print the result
        print(f"{repo}: {stars} stars")


@task
def fetch_stats(github_repo: str):
    """Task 1: Fetch the statistics for a GitHub repo"""
    return httpx.get(f"https://api.github.com/repos/{github_repo}").json()


@task
def get_stars(repo_stats: dict):
    """Task 2: Get the number of stars from GitHub repo statistics"""
    return repo_stats['stargazers_count']


# Run the flow
if __name__ == "__main__":
    show_stars([
        "PrefectHQ/prefect",
        "pydantic/pydantic",
        "huggingface/transformers"
    ])