From 58a6cb40fb0d26804b031c5bbd4bd07c2de73bb4 Mon Sep 17 00:00:00 2001 From: ByronHsu Date: Tue, 12 Dec 2023 10:59:16 -0800 Subject: [PATCH] Create FlyIn Plugin Documentation (#1314) Signed-off-by: byhsu Signed-off-by: Future Outlier Signed-off-by: byronhsu Signed-off-by: ByronHsu Co-authored-by: byhsu Co-authored-by: Future Outlier Co-authored-by: Kevin Su Co-authored-by: Nikki Everett --- dev-requirements.in | 1 + dev-requirements.txt | 112 ++-- docs-requirements.txt | 168 +++-- docs/index.md | 1 + docs/integrations.md | 2 + examples/flyin_plugin/Dockerfile | 22 + examples/flyin_plugin/README.md | 34 + .../flyin_plugin/flyin_plugin/__init__.py | 0 examples/flyin_plugin/flyin_plugin/vscode.py | 169 +++++ examples/flyin_plugin/requirements.in | 1 + examples/flyin_plugin/requirements.txt | 609 ++++++++++++++++++ 11 files changed, 964 insertions(+), 155 deletions(-) create mode 100644 examples/flyin_plugin/Dockerfile create mode 100644 examples/flyin_plugin/README.md create mode 100644 examples/flyin_plugin/flyin_plugin/__init__.py create mode 100644 examples/flyin_plugin/flyin_plugin/vscode.py create mode 100644 examples/flyin_plugin/requirements.in create mode 100644 examples/flyin_plugin/requirements.txt diff --git a/dev-requirements.in b/dev-requirements.in index 996558ba2..d6048dfcc 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -1,3 +1,4 @@ +autoflake flytekit black coverage diff --git a/dev-requirements.txt b/dev-requirements.txt index 60a8e9d5f..25af42015 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile dev-requirements.in +# pip-compile ./dev-requirements.in # adlfs==2023.10.0 # via flytekit @@ -20,8 +20,12 @@ aiosignal==1.3.1 # via aiohttp arrow==1.3.0 # via cookiecutter +async-timeout==4.0.3 + # via aiohttp attrs==23.1.0 # via aiohttp +autoflake==2.2.1 + # via -r ./dev-requirements.in azure-core==1.29.5 # via # adlfs @@ -37,7 +41,7 @@ binaryornot==0.4.4 # via cookiecutter black==23.11.0 # via - # -r dev-requirements.in + # -r ./dev-requirements.in # flake8-black botocore==1.31.17 # via aiobotocore @@ -66,51 +70,48 @@ click==8.1.7 cloudpickle==3.0.0 # via flytekit codespell==2.2.6 - # via -r dev-requirements.in + # via -r ./dev-requirements.in cookiecutter==2.5.0 # via flytekit coverage==7.3.2 - # via -r dev-requirements.in + # via -r ./dev-requirements.in croniter==2.0.1 # via flytekit -cryptography==41.0.5 +cryptography==41.0.7 # via # azure-identity # azure-storage-blob # msal # pyjwt - # pyopenssl dataclasses-json==0.5.9 # via flytekit decorator==5.1.1 # via gcsfs -deprecated==1.2.14 - # via flytekit diskcache==5.6.3 # via flytekit distlib==0.3.7 # via virtualenv docker==6.1.3 # via flytekit -docker-image-py==0.1.12 - # via flytekit docstring-parser==0.15 # via flytekit +exceptiongroup==1.2.0 + # via pytest filelock==3.13.1 # via virtualenv flake8==6.1.0 # via - # -r dev-requirements.in + # -r ./dev-requirements.in # flake8-black # flake8-isort flake8-black==0.3.6 - # via -r dev-requirements.in + # via -r ./dev-requirements.in flake8-isort==6.1.1 - # via -r dev-requirements.in -flyteidl==1.10.0 + # via -r ./dev-requirements.in +flyteidl==1.10.6 # via flytekit -flytekit==1.10.1 - # via -r dev-requirements.in +flytekit==1.10.2 + # via -r ./dev-requirements.in frozenlist==1.4.0 # via # aiohttp @@ -123,15 +124,11 @@ fsspec==2023.9.2 # s3fs gcsfs==2023.9.2 # via flytekit -gitdb==4.0.11 - # via gitpython -gitpython==3.1.40 - # via flytekit -google-api-core==2.14.0 +google-api-core==2.15.0 # via # google-cloud-core # google-cloud-storage -google-auth==2.23.4 +google-auth==2.25.1 # via # gcsfs # google-api-core @@ -141,7 +138,7 @@ google-auth==2.23.4 # kubernetes google-auth-oauthlib==1.1.0 # via gcsfs -google-cloud-core==2.3.3 +google-cloud-core==2.4.1 # via google-cloud-storage google-cloud-storage==2.13.0 # via gcsfs @@ -151,25 +148,25 @@ google-crc32c==1.5.0 # google-resumable-media google-resumable-media==2.6.0 # via google-cloud-storage -googleapis-common-protos==1.61.0 +googleapis-common-protos==1.62.0 # via # flyteidl # flytekit # google-api-core # grpcio-status -grpcio==1.59.3 +grpcio==1.60.0 # via # flytekit # grpcio-status -grpcio-status==1.59.3 +grpcio-status==1.60.0 # via flytekit -identify==2.5.32 +identify==2.5.33 # via pre-commit idna==3.6 # via # requests # yarl -importlib-metadata==6.8.0 +importlib-metadata==7.0.0 # via # flytekit # keyring @@ -179,7 +176,7 @@ isodate==0.6.1 # via azure-storage-blob isort==5.12.0 # via - # -r dev-requirements.in + # -r ./dev-requirements.in # flake8-isort jaraco-classes==3.3.0 # via keyring @@ -212,17 +209,17 @@ marshmallow-jsonschema==0.13.0 # via flytekit mashumaro==3.11 # via - # -r dev-requirements.in + # -r ./dev-requirements.in # flytekit mccabe==0.7.0 # via flake8 mdurl==0.1.2 # via markdown-it-py mock==5.1.0 - # via -r dev-requirements.in + # via -r ./dev-requirements.in more-itertools==10.1.0 # via jaraco-classes -msal==1.25.0 +msal==1.26.0 # via # azure-datalake-store # azure-identity @@ -234,14 +231,12 @@ multidict==6.0.4 # aiohttp # yarl mypy==1.7.1 - # via -r dev-requirements.in + # via -r ./dev-requirements.in mypy-extensions==1.0.0 # via # black # mypy # typing-inspect -natsort==8.4.0 - # via flytekit nodeenv==1.8.0 # via pre-commit numpy==1.26.2 @@ -263,7 +258,7 @@ pandas==1.5.3 # via flytekit pathspec==0.11.2 # via black -platformdirs==4.0.0 +platformdirs==4.1.0 # via # black # virtualenv @@ -272,7 +267,7 @@ pluggy==1.3.0 portalocker==2.8.2 # via msal-extensions pre-commit==3.5.0 - # via -r dev-requirements.in + # via -r ./dev-requirements.in protobuf==4.24.4 # via # flyteidl @@ -283,7 +278,7 @@ protobuf==4.24.4 # protoc-gen-swagger protoc-gen-swagger==0.1.0 # via flyteidl -pyarrow==10.0.1 +pyarrow==14.0.1 # via flytekit pyasn1==0.5.1 # via @@ -296,23 +291,22 @@ pycodestyle==2.11.1 pycparser==2.21 # via cffi pyflakes==3.1.0 - # via flake8 + # via + # autoflake + # flake8 pygments==2.17.2 # via rich pyjwt[crypto]==2.8.0 # via # msal # pyjwt -pyopenssl==23.3.0 - # via flytekit pytest==7.4.3 - # via -r dev-requirements.in + # via -r ./dev-requirements.in python-dateutil==2.8.2 # via # arrow # botocore # croniter - # flytekit # kubernetes # pandas python-json-logger==2.0.7 @@ -324,7 +318,6 @@ pytimeparse==1.1.8 pytz==2023.3.post1 # via # croniter - # flytekit # pandas pyyaml==6.0.1 # via @@ -332,8 +325,6 @@ pyyaml==6.0.1 # flytekit # kubernetes # pre-commit -regex==2023.10.3 - # via docker-image-py requests==2.31.0 # via # azure-core @@ -356,7 +347,7 @@ rich==13.7.0 # cookiecutter # flytekit # rich-click -rich-click==1.7.1 +rich-click==1.7.2 # via flytekit rsa==4.9 # via google-auth @@ -368,20 +359,24 @@ six==1.16.0 # isodate # kubernetes # python-dateutil -smmap==5.0.1 - # via gitdb -sortedcontainers==2.4.0 - # via flytekit statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify +tomli==2.0.1 + # via + # autoflake + # black + # flake8-black + # mypy + # pytest types-python-dateutil==2.8.19.14 # via arrow typing-extensions==4.8.0 # via # azure-core # azure-storage-blob + # black # flytekit # mashumaro # mypy @@ -396,20 +391,15 @@ urllib3==1.26.18 # flytekit # kubernetes # requests -virtualenv==20.24.7 +virtualenv==20.25.0 # via pre-commit -websocket-client==1.6.4 +websocket-client==1.7.0 # via # docker # kubernetes -wheel==0.42.0 - # via flytekit wrapt==1.16.0 - # via - # aiobotocore - # deprecated - # flytekit -yarl==1.9.3 + # via aiobotocore +yarl==1.9.4 # via aiohttp zipp==3.17.0 # via importlib-metadata diff --git a/docs-requirements.txt b/docs-requirements.txt index 76b919250..2ad265e8a 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile docs-requirements.in +# pip-compile ./docs-requirements.in # adlfs==2023.10.0 # via flytekit @@ -28,10 +28,12 @@ arrow==1.3.0 # via cookiecutter astroid==3.0.1 # via - # -r docs-requirements.in + # -r ./docs-requirements.in # sphinx-autoapi asttokens==2.4.1 # via stack-data +async-timeout==4.0.3 + # via aiohttp attrs==23.1.0 # via # aiohttp @@ -96,13 +98,12 @@ cookiecutter==2.5.0 # via flytekit croniter==2.0.1 # via flytekit -cryptography==41.0.5 +cryptography==41.0.7 # via # azure-identity # azure-storage-blob # msal # pyjwt - # pyopenssl cycler==0.12.1 # via matplotlib dacite==1.8.1 @@ -115,16 +116,12 @@ decorator==5.1.1 # via # gcsfs # ipython -deprecated==1.2.14 - # via flytekit diskcache==5.6.3 # via flytekit distlib==0.3.7 # via virtualenv docker==6.1.3 # via flytekit -docker-image-py==0.1.12 - # via flytekit docstring-parser==0.15 # via flytekit docutils==0.17.1 @@ -134,6 +131,8 @@ docutils==0.17.1 # sphinx-panels # sphinx-rtd-theme # sphinx-tabs +exceptiongroup==1.2.0 + # via ipython executing==2.0.1 # via stack-data fastjsonschema==2.19.0 @@ -142,23 +141,23 @@ filelock==3.13.1 # via # torch # virtualenv -flyteidl==1.10.0 +flyteidl==1.10.6 # via # flytekit # flytekitplugins-kfpytorch -flytekit==1.10.1 +flytekit==1.10.2 # via - # -r docs-requirements.in + # -r ./docs-requirements.in # flytekitplugins-deck-standard # flytekitplugins-kfpytorch # flytekitplugins-sqlalchemy -flytekitplugins-deck-standard==1.10.1 - # via -r docs-requirements.in -flytekitplugins-kfpytorch==1.10.1 - # via -r docs-requirements.in -flytekitplugins-sqlalchemy==1.10.1 - # via -r docs-requirements.in -fonttools==4.45.1 +flytekitplugins-deck-standard==1.10.2 + # via -r ./docs-requirements.in +flytekitplugins-kfpytorch==1.10.2 + # via -r ./docs-requirements.in +flytekitplugins-sqlalchemy==1.10.2 + # via -r ./docs-requirements.in +fonttools==4.46.0 # via matplotlib frozenlist==1.4.0 # via @@ -166,25 +165,21 @@ frozenlist==1.4.0 # aiosignal fsspec==2023.9.2 # via - # -r docs-requirements.in + # -r ./docs-requirements.in # adlfs # flytekit # gcsfs # s3fs # torch furo @ git+https://github.com/flyteorg/furo@main - # via -r docs-requirements.in + # via -r ./docs-requirements.in gcsfs==2023.9.2 # via flytekit -gitdb==4.0.11 - # via gitpython -gitpython==3.1.40 - # via flytekit -google-api-core==2.14.0 +google-api-core==2.15.0 # via # google-cloud-core # google-cloud-storage -google-auth==2.23.4 +google-auth==2.25.1 # via # gcsfs # google-api-core @@ -194,7 +189,7 @@ google-auth==2.23.4 # kubernetes google-auth-oauthlib==1.1.0 # via gcsfs -google-cloud-core==2.3.3 +google-cloud-core==2.4.1 # via google-cloud-storage google-cloud-storage==2.13.0 # via gcsfs @@ -204,24 +199,24 @@ google-crc32c==1.5.0 # google-resumable-media google-resumable-media==2.6.0 # via google-cloud-storage -googleapis-common-protos==1.61.0 +googleapis-common-protos==1.62.0 # via # flyteidl # flytekit # google-api-core # grpcio-status -grpcio==1.59.3 +grpcio==1.60.0 # via - # -r docs-requirements.in + # -r ./docs-requirements.in # flytekit # grpcio-status -grpcio-status==1.59.3 +grpcio-status==1.60.0 # via - # -r docs-requirements.in + # -r ./docs-requirements.in # flytekit htmlmin==0.1.12 # via ydata-profiling -identify==2.5.32 +identify==2.5.33 # via pre-commit idna==3.6 # via @@ -233,17 +228,17 @@ imagehash==4.3.1 # ydata-profiling imagesize==1.4.1 # via sphinx -importlib-metadata==6.8.0 +importlib-metadata==7.0.0 # via # flytekit # jupyter-cache # keyring # myst-nb -ipykernel==6.26.0 +ipykernel==6.27.1 # via myst-nb -ipython==8.18.0 +ipython==8.18.1 # via - # -r docs-requirements.in + # -r ./docs-requirements.in # ipykernel # ipywidgets # myst-nb @@ -275,7 +270,7 @@ jsonpickle==3.0.2 # via flytekit jsonschema==4.20.0 # via nbformat -jsonschema-specifications==2023.11.1 +jsonschema-specifications==2023.11.2 # via jsonschema jupyter-cache==0.6.1 # via myst-nb @@ -291,8 +286,8 @@ jupyter-core==5.5.0 # nbformat jupyterlab-widgets==3.0.9 # via ipywidgets -jupytext==1.15.2 - # via -r docs-requirements.in +jupytext==1.16.0 + # via -r ./docs-requirements.in keyring==24.3.0 # via flytekit kiwisolver==1.4.5 @@ -324,9 +319,9 @@ marshmallow-jsonschema==0.13.0 # via flytekit mashumaro==3.11 # via - # -r docs-requirements.in + # -r ./docs-requirements.in # flytekit -matplotlib==3.7.3 +matplotlib==3.8.2 # via # phik # seaborn @@ -346,7 +341,7 @@ more-itertools==10.1.0 # via jaraco-classes mpmath==1.3.0 # via sympy -msal==1.25.0 +msal==1.26.0 # via # azure-datalake-store # azure-identity @@ -364,11 +359,9 @@ multimethod==1.10 mypy-extensions==1.0.0 # via typing-inspect myst-nb==0.17.2 - # via -r docs-requirements.in + # via -r ./docs-requirements.in myst-parser==0.18.1 # via myst-nb -natsort==8.4.0 - # via flytekit nbclient==0.7.4 # via # jupyter-cache @@ -416,6 +409,7 @@ packaging==23.2 # via # docker # ipykernel + # jupytext # marshmallow # matplotlib # plotly @@ -431,7 +425,7 @@ pandas==1.5.3 # ydata-profiling parso==0.8.3 # via jedi -patsy==0.5.3 +patsy==0.5.4 # via statsmodels pexpect==4.9.0 # via ipython @@ -439,13 +433,13 @@ phik==0.12.3 # via ydata-profiling pillow==10.1.0 # via - # -r docs-requirements.in + # -r ./docs-requirements.in # imagehash # matplotlib # sphinx-gallery # visions # wordcloud -platformdirs==4.0.0 +platformdirs==4.1.0 # via # jupyter-core # virtualenv @@ -470,12 +464,12 @@ protoc-gen-swagger==0.1.0 psutil==5.9.6 # via ipykernel psycopg2-binary==2.9.9 - # via -r docs-requirements.in + # via -r ./docs-requirements.in ptyprocess==0.7.0 # via pexpect pure-eval==0.2.2 # via stack-data -pyarrow==10.0.1 +pyarrow==14.0.1 # via flytekit pyasn1==0.5.1 # via @@ -501,8 +495,6 @@ pyjwt[crypto]==2.8.0 # via # msal # pyjwt -pyopenssl==23.3.0 - # via flytekit pyparsing==3.1.1 # via matplotlib python-dateutil==2.8.2 @@ -510,7 +502,6 @@ python-dateutil==2.8.2 # arrow # botocore # croniter - # flytekit # jupyter-client # kubernetes # matplotlib @@ -524,7 +515,6 @@ pytimeparse==1.1.8 pytz==2023.3.post1 # via # croniter - # flytekit # pandas pywavelets==1.5.0 # via imagehash @@ -540,16 +530,14 @@ pyyaml==6.0.1 # pre-commit # sphinx-autoapi # ydata-profiling -pyzmq==25.1.1 +pyzmq==25.1.2 # via # ipykernel # jupyter-client -referencing==0.31.0 +referencing==0.32.0 # via # jsonschema # jsonschema-specifications -regex==2023.10.3 - # via docker-image-py requests==2.31.0 # via # azure-core @@ -575,9 +563,9 @@ rich==13.7.0 # cookiecutter # flytekit # rich-click -rich-click==1.7.1 +rich-click==1.7.2 # via flytekit -rpds-py==0.13.1 +rpds-py==0.13.2 # via # jsonschema # referencing @@ -586,7 +574,7 @@ rsa==4.9 s3fs==2023.9.2 # via flytekit scikit-learn==1.3.2 - # via -r docs-requirements.in + # via -r ./docs-requirements.in scipy==1.11.4 # via # imagehash @@ -606,17 +594,13 @@ six==1.16.0 # python-dateutil # sphinx-code-include # sphinxext-remoteliteralinclude -smmap==5.0.1 - # via gitdb snowballstemmer==2.2.0 # via sphinx -sortedcontainers==2.4.0 - # via flytekit soupsieve==2.5 # via beautifulsoup4 sphinx==4.5.0 # via - # -r docs-requirements.in + # -r ./docs-requirements.in # furo # myst-nb # myst-parser @@ -635,27 +619,27 @@ sphinx==4.5.0 # sphinxcontrib-youtube # sphinxext-remoteliteralinclude sphinx-autoapi==2.0.1 - # via -r docs-requirements.in + # via -r ./docs-requirements.in sphinx-basic-ng==1.0.0b2 # via furo sphinx-code-include==1.1.1 - # via -r docs-requirements.in + # via -r ./docs-requirements.in sphinx-copybutton==0.5.2 - # via -r docs-requirements.in + # via -r ./docs-requirements.in sphinx-fontawesome==0.0.6 - # via -r docs-requirements.in + # via -r ./docs-requirements.in sphinx-gallery==0.15.0 - # via -r docs-requirements.in + # via -r ./docs-requirements.in sphinx-panels==0.6.0 - # via -r docs-requirements.in + # via -r ./docs-requirements.in sphinx-prompt==1.5.0 - # via -r docs-requirements.in + # via -r ./docs-requirements.in sphinx-rtd-theme==1.3.0 - # via -r docs-requirements.in + # via -r ./docs-requirements.in sphinx-tabs==3.4.0 - # via -r docs-requirements.in + # via -r ./docs-requirements.in sphinx-tags==0.2.1 - # via -r docs-requirements.in + # via -r ./docs-requirements.in sphinxcontrib-applehelp==1.0.4 # via sphinx sphinxcontrib-devhelp==1.0.2 @@ -667,15 +651,15 @@ sphinxcontrib-jquery==4.1 sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-mermaid==0.9.2 - # via -r docs-requirements.in + # via -r ./docs-requirements.in sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxcontrib-youtube==1.3.0 - # via -r docs-requirements.in + # via -r ./docs-requirements.in sphinxext-remoteliteralinclude==0.4.0 - # via -r docs-requirements.in + # via -r ./docs-requirements.in sqlalchemy==2.0.23 # via # flytekitplugins-sqlalchemy @@ -701,14 +685,14 @@ threadpoolctl==3.2.0 toml==0.10.2 # via jupytext torch==2.1.1 - # via -r docs-requirements.in -tornado==6.3.3 + # via -r ./docs-requirements.in +tornado==6.4 # via # ipykernel # jupyter-client tqdm==4.66.1 # via ydata-profiling -traitlets==5.13.0 +traitlets==5.14.0 # via # comm # ipykernel @@ -725,6 +709,7 @@ types-python-dateutil==2.8.19.14 # via arrow typing-extensions==4.8.0 # via + # astroid # azure-core # azure-storage-blob # flytekit @@ -749,7 +734,7 @@ urllib3==1.26.18 # flytekit # kubernetes # requests -virtualenv==20.24.7 +virtualenv==20.25.0 # via pre-commit visions[type-image-path]==0.7.5 # via @@ -757,24 +742,19 @@ visions[type-image-path]==0.7.5 # ydata-profiling wcwidth==0.2.12 # via prompt-toolkit -websocket-client==1.6.4 +websocket-client==1.7.0 # via # docker # kubernetes -wheel==0.42.0 - # via flytekit widgetsnbextension==4.0.9 # via ipywidgets wordcloud==1.9.2 # via ydata-profiling wrapt==1.16.0 - # via - # aiobotocore - # deprecated - # flytekit -yarl==1.9.3 + # via aiobotocore +yarl==1.9.4 # via aiohttp -ydata-profiling==4.6.2 +ydata-profiling==4.6.3 # via flytekitplugins-deck-standard zipp==3.17.0 # via importlib-metadata diff --git a/docs/index.md b/docs/index.md index d94148a3f..782b06c06 100644 --- a/docs/index.md +++ b/docs/index.md @@ -444,6 +444,7 @@ auto_examples/databricks_plugin/index auto_examples/dbt_plugin/index auto_examples/dolt_plugin/index auto_examples/duckdb_plugin/index +auto_examples/flyin_plugin/index auto_examples/greatexpectations_plugin/index auto_examples/hive_plugin/index auto_examples/k8s_pod_plugin/index diff --git a/docs/integrations.md b/docs/integrations.md index de04dded8..f05ea330c 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -102,6 +102,8 @@ the Flyte task that use the respective plugin. - Execute queries using AWS Athena * - {doc}`AWS Batch ` - Running tasks and workflows on AWS batch service +* - {doc}`Flyte Interactive ` + - Execute tasks using Flyte Interactive to debug. * - {doc}`Hive ` - Run Hive jobs in your workflows. * - {doc}`MMCloud ` diff --git a/examples/flyin_plugin/Dockerfile b/examples/flyin_plugin/Dockerfile new file mode 100644 index 000000000..e0058d424 --- /dev/null +++ b/examples/flyin_plugin/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.10-slim-buster +LABEL org.opencontainers.image.source https://github.com/flyteorg/flytesnacks + +WORKDIR /root +ENV PYTHONPATH /root + +RUN apt-get update && apt-get install build-essential -y \ + && apt-get clean autoclean \ + && apt-get autoremove --yes \ + && rm -rf /var/lib/{apt,dpkg,cache,log}/ \ + && useradd -u 1000 flytekit \ + && chown flytekit: /root \ + && chown flytekit: /home \ + && : + +# Install Python dependencies +COPY requirements.txt /root +RUN pip install -r /root/requirements.txt +RUN pip freeze + +# Copy the actual code +COPY . /root diff --git a/examples/flyin_plugin/README.md b/examples/flyin_plugin/README.md new file mode 100644 index 000000000..c0692d4d0 --- /dev/null +++ b/examples/flyin_plugin/README.md @@ -0,0 +1,34 @@ +(flyte-interactive)= + +# FlyIn + +```{eval-rst} +.. tags:: Advanced +``` + + +FlyIn provides interactive task development in a remote environment. This allows developers to leverage remote environment capabilities while accessing features like debugging, code inspection, and Jupyter Notebook, traditionally available in local IDEs. + + +Flyte tasks, designed as one-off jobs, require users to wait until completion to view results. These tasks are developed locally in a virtual environment before being deployed remotely. However, differences in data access, GPU availability, and dependencies between local and remote environments often lead to discrepancies, making local success an unreliable indicator of remote success. This results in frequent, tedious debugging cycles. + + + +## Installation + +To use the Flyte interactive plugin, run the following command: + +```{eval-rst} +.. prompt:: bash + + pip install flytekitplugins-flyin +``` + + +## Acknowledgement + +This feature was created at LinkedIn and later donated to Flyte. + +```{auto-examples-toc} +vscode +``` diff --git a/examples/flyin_plugin/flyin_plugin/__init__.py b/examples/flyin_plugin/flyin_plugin/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/flyin_plugin/flyin_plugin/vscode.py b/examples/flyin_plugin/flyin_plugin/vscode.py new file mode 100644 index 000000000..763a648bf --- /dev/null +++ b/examples/flyin_plugin/flyin_plugin/vscode.py @@ -0,0 +1,169 @@ +# %% [markdown] +# # FlyIn Vscode Decorator +# +# `@vscode` converts a Python task into a Visual Studio Code server with just one line change, +# enabling connection and debugging in remote environments. + +# %% +from flytekit import task, workflow +from flytekitplugins.flyin import vscode + +# %% [markdown] +# ## Usage +# ### Add `@vscode` decorator to task function definition + +# %% +@task +@vscode +def train(): + print("forward") + print("backward") + + +@workflow +def wf_train(): + train() + + +# %% [markdown] +# The `@vscode` decorator, when applied, converts a task into a Visual Studio Code server during runtime. This process overrides the standard execution of the task's function body, initiating a command to start a Visual Studio Code server instead. +# ### 2. Connect to the VSCode server +# You can connect in two ways: +# 1. **(Recommended) Expose a URL on the Flyte console.** Set up ingress on the Flyte backend to expose a URL on the Flyte console. Details are to be determined (TBD). + +# 2. **Use port forwarding.** To use port forwarding, execute the following command: +# ``` +# $ kubectl port-forward +# ``` +# Then, open a browser and navigate to `localhost:`, replacing `` with the port number configured above. You should be presented with the interface shown in the image below. + +# ### 3. Interactively debug the task +# +# To run the task in VSCode, select "Run and debug" from the left panel and execute the "interactive debugging" configuration. +# +# This will run your task with inputs from the previous task. It's important to note that the task runs entirely within VSCode and does not write the output to Flyte storage. +# +# For inspecting intermediate states, set breakpoints in the Python code and use the debugger for tracing. +# +# The `launch.json` file generated by FlyIn offers a convenient method to run the task, but you can still use VSCode as you would locally. For instance, you can run a Python script directly from the embedded terminal using `python hello.py`. +# +# + + +# ### 4. Resume your task with updated code +# After you finish debugging, you can resume your task with updated code by executing the "resume task" configuration. +# This will **terminate the code server**, run the task with inputs from the previous task, and write the output to Flyte storage. +# +# ** Note: Remember to persist your code (e.g. to GitHub) before resuming the task, since you will lose the connection to the VSCode server afterwards.** +# + +# ## Advanced usage +# ### Install extensions +# Like local VSCode, you can install a variety of extensions to assist development. Available extensions differ from official VSCode for legal reasons and are hosted on [Open VSX Registry](https://open-vsx.org/). +# +# Python and Jupyter extensions are installed by default. Additional extensions can be added as shown below: +# +# %% +from flytekit import task, workflow +from flytekitplugins.flyin import COPILOT_EXTENSION, VscodeConfig, vscode + +config = VscodeConfig() +config.add_extensions(COPILOT_EXTENSION) # Use predefined URL +config.add_extensions( + "https://open-vsx.org/api/vscodevim/vim/1.27.0/file/vscodevim.vim-1.27.0.vsix" +) # Copy raw URL from Open VSX + + +@task(container_image="localhost:30000/flytekit-flyin:0.0.0") +@vscode(config=config) +def t_ext(): + pass + + +@workflow +def wf_ext(): + t_ext() + + +# %% [markdown] +# ### Manage resources +# To manage resources, FlyIn can terminate pods after a period of idleness (no active HTTP connections). Idleness is monitored via a heartbeat file. +# %% +from flytekit import task, workflow +from flytekitplugins.flyin import vscode + + +@task +@vscode(max_idle_seconds=60000) # 60000 seconds +def task_with_max_idle(): + pass + + +@workflow +def wf_idle(): + task_with_max_idle() + + +# %% [markdown] +# ### Pre/Post hooks +# +# FlyIn allows execution of functions before and after VSCode starts. This can be used for tasks requiring setup or cleanup. +# +# %% +from flytekit import task, workflow +from flytekitplugins.flyin import vscode + + +def set_up_proxy(): + print("set up") + + +def push_code(): + print("push code") + + +@task +@vscode(pre_execute=set_up_proxy, post_execute=push_code) +def t_hook(): + pass + + +@workflow +def wf_hook(): + task_with_max_idle() + + +# %% [markdown] +# ### Run FlyIn alongside tasks to initiate VSCode after failure +# FlyIn can initiate VSCode after task failure, preventing task termination and enabling inspection. +# %% +from flytekit import task, workflow +from flytekitplugins.flyin import vscode + + +@task +@vscode(run_task_first=True) +def t_exception(): + return 1 // 0 # causes exception + + +@workflow +def wf_exception(): + task_with_max_idle() + + +# %% [markdown] +# ### Prebuild a Docker image with VSCode +# To skip downloading VSCode and extensions at runtime, +# they can be prebuilt into a Docker image, accelerating setup. +# +# ``` +# # Include this line if `curl` isn't installed in the image. +# RUN apt-get -y install curl +# Download and extract VSCode. +# RUN mkdir -p /tmp/code-server +# RUN curl -kfL -o /tmp/code-server/code-server-4.18.0-linux-amd64.tar.gz https://github.com/coder/code-server/releases/download/v4.18.0/code-server-4.18.0-linux-amd64.tar.gz +# RUN tar -xzf /tmp/code-server/code-server-4.18.0-linux-amd64.tar.gz -C /tmp/code-server/ +# ENV PATH="/tmp/code-server/code-server-4.18.0-linux-amd64/bin:${PATH}" +# # TODO: download and install extensions +# ``` diff --git a/examples/flyin_plugin/requirements.in b/examples/flyin_plugin/requirements.in new file mode 100644 index 000000000..ae7e4b7ff --- /dev/null +++ b/examples/flyin_plugin/requirements.in @@ -0,0 +1 @@ +flytekitplugins-flyin diff --git a/examples/flyin_plugin/requirements.txt b/examples/flyin_plugin/requirements.txt new file mode 100644 index 000000000..f0e017fbd --- /dev/null +++ b/examples/flyin_plugin/requirements.txt @@ -0,0 +1,609 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile requirements.in +# +adlfs==2023.10.0 + # via flytekit +aiobotocore==2.5.4 + # via s3fs +aiohttp==3.9.1 + # via + # adlfs + # aiobotocore + # gcsfs + # s3fs +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp +anyio==4.1.0 + # via jupyter-server +argon2-cffi==23.1.0 + # via jupyter-server +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +arrow==1.3.0 + # via + # cookiecutter + # isoduration +asttokens==2.4.1 + # via stack-data +async-lru==2.0.4 + # via jupyterlab +async-timeout==4.0.3 + # via aiohttp +attrs==23.1.0 + # via + # aiohttp + # jsonschema + # referencing +azure-core==1.29.5 + # via + # adlfs + # azure-identity + # azure-storage-blob +azure-datalake-store==0.0.53 + # via adlfs +azure-identity==1.15.0 + # via adlfs +azure-storage-blob==12.19.0 + # via adlfs +babel==2.13.1 + # via jupyterlab-server +beautifulsoup4==4.12.2 + # via nbconvert +binaryornot==0.4.4 + # via cookiecutter +bleach==6.1.0 + # via nbconvert +botocore==1.31.17 + # via aiobotocore +cachetools==5.3.2 + # via google-auth +certifi==2023.11.17 + # via + # kubernetes + # requests +cffi==1.16.0 + # via + # argon2-cffi-bindings + # azure-datalake-store + # cryptography +chardet==5.2.0 + # via binaryornot +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # cookiecutter + # flytekit + # rich-click +cloudpickle==3.0.0 + # via flytekit +comm==0.2.0 + # via + # ipykernel + # ipywidgets +cookiecutter==2.5.0 + # via flytekit +croniter==2.0.1 + # via flytekit +cryptography==41.0.7 + # via + # azure-identity + # azure-storage-blob + # msal + # pyjwt + # secretstorage +dataclasses-json==0.5.9 + # via flytekit +debugpy==1.8.0 + # via ipykernel +decorator==5.1.1 + # via + # gcsfs + # ipython +defusedxml==0.7.1 + # via nbconvert +diskcache==5.6.3 + # via flytekit +docker==6.1.3 + # via flytekit +docstring-parser==0.15 + # via flytekit +exceptiongroup==1.2.0 + # via + # anyio + # ipython +executing==2.0.1 + # via stack-data +fastjsonschema==2.19.0 + # via nbformat +flyteidl==1.10.6 + # via flytekit +flytekit==1.10.2 + # via flytekitplugins-flyin +flytekitplugins-flyin==1.10.2 + # via -r requirements.in +fqdn==1.5.1 + # via jsonschema +frozenlist==1.4.0 + # via + # aiohttp + # aiosignal +fsspec==2023.9.2 + # via + # adlfs + # flytekit + # gcsfs + # s3fs +gcsfs==2023.9.2 + # via flytekit +google-api-core==2.15.0 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.25.1 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # kubernetes +google-auth-oauthlib==1.1.0 + # via gcsfs +google-cloud-core==2.4.1 + # via google-cloud-storage +google-cloud-storage==2.13.0 + # via gcsfs +google-crc32c==1.5.0 + # via + # google-cloud-storage + # google-resumable-media +google-resumable-media==2.6.0 + # via google-cloud-storage +googleapis-common-protos==1.62.0 + # via + # flyteidl + # flytekit + # google-api-core + # grpcio-status +grpcio==1.60.0 + # via + # flytekit + # grpcio-status +grpcio-status==1.60.0 + # via flytekit +idna==3.6 + # via + # anyio + # jsonschema + # requests + # yarl +importlib-metadata==7.0.0 + # via + # flytekit + # keyring +ipykernel==6.27.1 + # via + # jupyter + # jupyter-console + # jupyterlab + # qtconsole +ipython==8.18.1 + # via + # ipykernel + # ipywidgets + # jupyter-console +ipywidgets==8.1.1 + # via jupyter +isodate==0.6.1 + # via azure-storage-blob +isoduration==20.11.0 + # via jsonschema +jaraco-classes==3.3.0 + # via keyring +jedi==0.19.1 + # via ipython +jeepney==0.8.0 + # via + # keyring + # secretstorage +jinja2==3.1.2 + # via + # cookiecutter + # jupyter-server + # jupyterlab + # jupyterlab-server + # nbconvert +jmespath==1.0.1 + # via botocore +joblib==1.3.2 + # via flytekit +json5==0.9.14 + # via jupyterlab-server +jsonpickle==3.0.2 + # via flytekit +jsonpointer==2.4 + # via jsonschema +jsonschema[format-nongpl]==4.20.0 + # via + # jupyter-events + # jupyterlab-server + # nbformat +jsonschema-specifications==2023.11.2 + # via jsonschema +jupyter==1.0.0 + # via flytekitplugins-flyin +jupyter-client==8.6.0 + # via + # ipykernel + # jupyter-console + # jupyter-server + # nbclient + # qtconsole +jupyter-console==6.6.3 + # via jupyter +jupyter-core==5.5.0 + # via + # ipykernel + # jupyter-client + # jupyter-console + # jupyter-server + # jupyterlab + # nbclient + # nbconvert + # nbformat + # qtconsole +jupyter-events==0.9.0 + # via jupyter-server +jupyter-lsp==2.2.1 + # via jupyterlab +jupyter-server==2.12.1 + # via + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # notebook + # notebook-shim +jupyter-server-terminals==0.4.4 + # via jupyter-server +jupyterlab==4.0.9 + # via notebook +jupyterlab-pygments==0.3.0 + # via nbconvert +jupyterlab-server==2.25.2 + # via + # jupyterlab + # notebook +jupyterlab-widgets==3.0.9 + # via ipywidgets +keyring==24.3.0 + # via flytekit +kubernetes==28.1.0 + # via flytekit +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.3 + # via + # jinja2 + # nbconvert +marshmallow==3.20.1 + # via + # dataclasses-json + # marshmallow-enum + # marshmallow-jsonschema +marshmallow-enum==1.5.1 + # via + # dataclasses-json + # flytekit +marshmallow-jsonschema==0.13.0 + # via flytekit +mashumaro==3.11 + # via flytekit +matplotlib-inline==0.1.6 + # via + # ipykernel + # ipython +mdurl==0.1.2 + # via markdown-it-py +mistune==3.0.2 + # via nbconvert +more-itertools==10.1.0 + # via jaraco-classes +msal==1.26.0 + # via + # azure-datalake-store + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity +multidict==6.0.4 + # via + # aiohttp + # yarl +mypy-extensions==1.0.0 + # via typing-inspect +nbclient==0.9.0 + # via nbconvert +nbconvert==7.12.0 + # via + # jupyter + # jupyter-server +nbformat==5.9.2 + # via + # jupyter-server + # nbclient + # nbconvert +nest-asyncio==1.5.8 + # via ipykernel +notebook==7.0.6 + # via jupyter +notebook-shim==0.2.3 + # via + # jupyterlab + # notebook +numpy==1.26.2 + # via + # flytekit + # pandas + # pyarrow +oauthlib==3.2.2 + # via + # kubernetes + # requests-oauthlib +overrides==7.4.0 + # via jupyter-server +packaging==23.2 + # via + # docker + # ipykernel + # jupyter-server + # jupyterlab + # jupyterlab-server + # marshmallow + # nbconvert + # qtconsole + # qtpy +pandas==1.5.3 + # via flytekit +pandocfilters==1.5.0 + # via nbconvert +parso==0.8.3 + # via jedi +pexpect==4.9.0 + # via ipython +platformdirs==4.1.0 + # via jupyter-core +portalocker==2.8.2 + # via msal-extensions +prometheus-client==0.19.0 + # via jupyter-server +prompt-toolkit==3.0.41 + # via + # ipython + # jupyter-console +protobuf==4.24.4 + # via + # flyteidl + # flytekit + # google-api-core + # googleapis-common-protos + # grpcio-status + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl +psutil==5.9.6 + # via ipykernel +ptyprocess==0.7.0 + # via + # pexpect + # terminado +pure-eval==0.2.2 + # via stack-data +pyarrow==14.0.1 + # via flytekit +pyasn1==0.5.1 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth +pycparser==2.21 + # via cffi +pygments==2.17.2 + # via + # ipython + # jupyter-console + # nbconvert + # qtconsole + # rich +pyjwt[crypto]==2.8.0 + # via + # msal + # pyjwt +python-dateutil==2.8.2 + # via + # arrow + # botocore + # croniter + # jupyter-client + # kubernetes + # pandas +python-json-logger==2.0.7 + # via + # flytekit + # jupyter-events +python-slugify==8.0.1 + # via cookiecutter +pytimeparse==1.1.8 + # via flytekit +pytz==2023.3.post1 + # via + # croniter + # pandas +pyyaml==6.0.1 + # via + # cookiecutter + # flytekit + # jupyter-events + # kubernetes +pyzmq==25.1.2 + # via + # ipykernel + # jupyter-client + # jupyter-console + # jupyter-server + # qtconsole +qtconsole==5.5.1 + # via jupyter +qtpy==2.4.1 + # via qtconsole +referencing==0.32.0 + # via + # jsonschema + # jsonschema-specifications + # jupyter-events +requests==2.31.0 + # via + # azure-core + # azure-datalake-store + # cookiecutter + # docker + # flytekit + # gcsfs + # google-api-core + # google-cloud-storage + # jupyterlab-server + # kubernetes + # msal + # requests-oauthlib +requests-oauthlib==1.3.1 + # via + # google-auth-oauthlib + # kubernetes +rfc3339-validator==0.1.4 + # via + # jsonschema + # jupyter-events +rfc3986-validator==0.1.1 + # via + # jsonschema + # jupyter-events +rich==13.7.0 + # via + # cookiecutter + # flytekit + # rich-click +rich-click==1.7.2 + # via flytekit +rpds-py==0.13.2 + # via + # jsonschema + # referencing +rsa==4.9 + # via google-auth +s3fs==2023.9.2 + # via flytekit +secretstorage==3.3.3 + # via keyring +send2trash==1.8.2 + # via jupyter-server +six==1.16.0 + # via + # asttokens + # azure-core + # bleach + # isodate + # kubernetes + # python-dateutil + # rfc3339-validator +sniffio==1.3.0 + # via anyio +soupsieve==2.5 + # via beautifulsoup4 +stack-data==0.6.3 + # via ipython +statsd==3.3.0 + # via flytekit +terminado==0.18.0 + # via + # jupyter-server + # jupyter-server-terminals +text-unidecode==1.3 + # via python-slugify +tinycss2==1.2.1 + # via nbconvert +tomli==2.0.1 + # via jupyterlab +tornado==6.4 + # via + # ipykernel + # jupyter-client + # jupyter-server + # jupyterlab + # notebook + # terminado +traitlets==5.14.0 + # via + # comm + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-console + # jupyter-core + # jupyter-events + # jupyter-server + # jupyterlab + # matplotlib-inline + # nbclient + # nbconvert + # nbformat + # qtconsole +types-python-dateutil==2.8.19.14 + # via arrow +typing-extensions==4.8.0 + # via + # async-lru + # azure-core + # azure-storage-blob + # flytekit + # mashumaro + # rich-click + # typing-inspect +typing-inspect==0.9.0 + # via dataclasses-json +uri-template==1.3.0 + # via jsonschema +urllib3==1.26.18 + # via + # botocore + # docker + # flytekit + # kubernetes + # requests +wcwidth==0.2.12 + # via prompt-toolkit +webcolors==1.13 + # via jsonschema +webencodings==0.5.1 + # via + # bleach + # tinycss2 +websocket-client==1.7.0 + # via + # docker + # jupyter-server + # kubernetes +widgetsnbextension==4.0.9 + # via ipywidgets +wrapt==1.16.0 + # via aiobotocore +yarl==1.9.4 + # via aiohttp +zipp==3.17.0 + # via importlib-metadata