[DO NOT MERGE] [SNOW-1491199] merge phase0 server side

.github/workflows/daily_jupyter_nb_test.yml

@@ -46,7 +46,7 @@ jobs:
               run: python -m pip wheel -v -w dist --no-deps .
             - name: Show wheels generated
               run: ls -lh dist
-            - uses: actions/upload-artifact@v1
+            - uses: actions/upload-artifact@v4
               with:
                 name: wheel
                 path: dist/
@@ -83,7 +83,7 @@ jobs:
                   PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
                   CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
             - name: Download wheel(s)
-              uses: actions/download-artifact@v3
+              uses: actions/download-artifact@v4
               with:
                 name: wheel
                 path: dist

.github/workflows/daily_modin_precommit.yml

@@ -52,6 +52,7 @@ jobs:
         run: ls -lh dist
       - uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: wheel
           path: dist/
 
@@ -166,6 +167,7 @@ jobs:
           SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
       - uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
           path: |
             .tox/.coverage
@@ -234,6 +236,7 @@ jobs:
           SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
       - uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-disable-sql-simplifier
           path: |
             .tox/.coverage
@@ -302,6 +305,7 @@ jobs:
           SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
       - uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-enable-cte-optimization
           path: |
             .tox/.coverage
@@ -348,11 +352,13 @@ jobs:
       - name: Publish html coverage
         uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: overall_cov_html
           path: .tox/htmlcov
       - name: Publish xml coverage
         uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: overall_cov_xml
           path: .tox/coverage.xml
       - uses: codecov/codecov-action@v1
@@ -385,5 +391,6 @@ jobs:
       - name: Upload html files
         uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: docs
           path: docs/build/html

.github/workflows/daily_precommit.yml

@@ -71,6 +71,7 @@ jobs:
         run: ls -lh dist
       - uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: wheel
           path: dist/
 
@@ -143,6 +144,7 @@ jobs:
           SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
       - uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
           path: |
             .tox/.coverage
@@ -183,6 +185,7 @@ jobs:
         shell: bash
       - uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: coverage_linux-fips-3.8-${{ matrix.cloud-provider }}
           path: |
             .coverage
@@ -251,6 +254,7 @@ jobs:
           SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
       - uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-disable-sql-simplifier
           path: |
             .tox/.coverage
@@ -312,6 +316,7 @@ jobs:
           SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
       - uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: coverage_nopandas
           path: |
             .tox/.coverage
@@ -372,6 +377,7 @@ jobs:
           SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
       - uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-local-testing
           path: |
             .tox/.coverage
@@ -440,6 +446,7 @@ jobs:
           SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
       - uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-enable-cte-optimization
           path: |
             .tox/.coverage
@@ -483,11 +490,13 @@ jobs:
       - name: Publish html coverage
         uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: overall_cov_html
           path: .tox/htmlcov
       - name: Publish xml coverage
         uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: overall_cov_xml
           path: .tox/coverage.xml
       - uses: codecov/codecov-action@v1
@@ -520,5 +529,6 @@ jobs:
       - name: Upload html files
         uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: docs
           path: docs/build/html

.github/workflows/precommit.yml

@@ -73,6 +73,7 @@ jobs:
         run: ls -lh dist
       - uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: wheel
           path: dist/
 
@@ -85,7 +86,7 @@ jobs:
       matrix:
         os: [macos-latest, windows-latest-64-cores, ubuntu-latest-64-cores]
         python-version: ["3.9", "3.10", "3.11"]
-        cloud-provider: [aws, azure, gcp]
+        cloud-provider: [aws, gcp, azure]
         exclude:
           # only run macos with aws py3.9 for doctest
           - os: macos-latest
@@ -180,6 +181,7 @@ jobs:
           SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
       - uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
           path: |
             .tox/.coverage
@@ -238,11 +240,158 @@ jobs:
           SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
       - uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-local-testing
           path: |
             .tox/.coverage
             .tox/coverage.xml
 
+  test-ast-encoding:
+    name: Test AST expectations (phase0) py-${{ matrix.os }}-${{ matrix.python-version }}
+    needs: build
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, windows-latest, ubuntu-latest]
+        # Protobuf encoding differs per python version, use 3.8 by default.
+        python-version: [ "3.8"]
+        cloud-provider: [aws]
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Display Python version
+        run: python -c "import sys; print(sys.version)"
+      - name: Decrypt parameters.py
+        shell: bash
+        run: .github/scripts/decrypt_parameters.sh
+        env:
+          PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
+          CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
+      - name: Download wheel(s)
+        uses: actions/download-artifact@v4
+        with:
+          name: wheel
+          path: dist
+      - name: Show wheels downloaded
+        run: ls -lh dist
+        shell: bash
+      - name: Upgrade setuptools, pip and wheel
+        run: python -m pip install -U setuptools pip wheel
+      - name: Install tox
+        run: python -m pip install tox
+      - name: Set up windows environment
+        if: runner.os == 'Windows'
+        run: |
+            "TZ=EST" >> $env:GITHUB_ENV
+            echo "TZ=EST" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+      - name: Run tests
+        run: TZ=EST python -m tox -e "py${PYTHON_VERSION/\./}-ast"
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          cloud_provider: ${{ matrix.cloud-provider }}
+          PYTEST_ADDOPTS: --color=yes --tb=short
+          TOX_PARALLEL_NO_SPINNER: 1
+          SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
+          TZ: EST
+        shell: bash
+
+  doctest-ast-phase0:
+    name: Doctest AST (phase0) py-${{ matrix.os }}-${{ matrix.python-version }}
+    needs: build
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest ]
+        python-version: [ "3.9"]
+        cloud-provider: [ aws ]
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Display Python version
+        run: python -c "import sys; print(sys.version)"
+      - name: Decrypt parameters.py
+        shell: bash
+        run: .github/scripts/decrypt_parameters.sh
+        env:
+          PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
+          CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
+      - name: Download wheel(s)
+        uses: actions/download-artifact@v4
+        with:
+          name: wheel
+          path: dist
+      - name: Show wheels downloaded
+        run: ls -lh dist
+        shell: bash
+      - name: Upgrade setuptools, pip and wheel
+        run: python -m pip install -U setuptools pip wheel
+      - name: Install tox
+        run: python -m pip install tox
+      - name: Run doctests for AST (I/II)
+        run: python -m tox -e "py${PYTHON_VERSION}-doctest-notudf-ci"
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          cloud_provider: ${{ matrix.cloud-provider }}
+          PYTEST_ADDOPTS: --color=yes --tb=short --enable-ast
+          TOX_PARALLEL_NO_SPINNER: 1
+        shell: bash
+
+  test-ast-phase0:
+    name: Test AST enabled (phase0) py-${{ matrix.os }}-${{ matrix.python-version }}
+    needs: build
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest ]
+        python-version: [ "3.9", "3.11" ]
+        cloud-provider: [ aws ]
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Display Python version
+        run: python -c "import sys; print(sys.version)"
+      - name: Decrypt parameters.py
+        shell: bash
+        run: .github/scripts/decrypt_parameters.sh
+        env:
+          PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
+          CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
+      - name: Download wheel(s)
+        uses: actions/download-artifact@v4
+        with:
+          name: wheel
+          path: dist
+      - name: Show wheels downloaded
+        run: ls -lh dist
+        shell: bash
+      - name: Upgrade setuptools, pip and wheel
+        run: python -m pip install -U setuptools pip wheel
+      - name: Install tox
+        run: python -m pip install tox
+      - name: Run tests for AST (II/II)
+        run: python -m tox -e "py${PYTHON_VERSION/\./}-notdoctest-ci"
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          cloud_provider: ${{ matrix.cloud-provider }}
+          PYTEST_ADDOPTS: --color=yes --tb=short --enable-ast
+          TOX_PARALLEL_NO_SPINNER: 1
+        shell: bash
+
   test-enable-cte-optimization:
     name: Test Enable CTE Optimization py-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
     needs: build
@@ -295,6 +444,7 @@ jobs:
           SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
       - uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-enable-cte-optimization
           path: |
             .tox/.coverage
@@ -309,7 +459,7 @@ jobs:
       matrix:
         os: [macos-latest, windows-latest-64-cores, ubuntu-latest-64-cores]
         python-version: [ "3.9", "3.10", "3.11" ]
-        cloud-provider: [aws, azure, gcp]
+        cloud-provider: [aws, gcp, azure]
         exclude:
           # only run macos with aws py3.9 for doctest
           - os: macos-latest
@@ -404,6 +554,7 @@ jobs:
           SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
       - uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-snowpark-pandas-testing
           path: |
             .tox/.coverage
@@ -461,6 +612,7 @@ jobs:
         shell: bash
       - uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: coverage_modin_extra_without_pandas_extra
           path: |
             .tox/.coverage
@@ -512,11 +664,13 @@ jobs:
       - name: Publish html coverage
         uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: overall_cov_html
           path: .tox/htmlcov
       - name: Publish xml coverage
         uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: overall_cov_xml
           path: .tox/coverage.xml
       - uses: codecov/codecov-action@v4
@@ -556,5 +710,6 @@ jobs:
       - name: Upload html files
         uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: docs
           path: docs/build/html