Merge pull request sonic-net#2856 from theasianpianist/master-dash-merge

Merge DASH branch into master
Pterosaur · Aug 16, 2023 · bb99f41 · bb99f41
2 parents ca72820 + 84b32af
commit bb99f41
Show file tree

Hide file tree

Showing 41 changed files with 5,921 additions and 143 deletions.
diff --git a/.azure-pipelines/build-docker-sonic-vs-template.yml b/.azure-pipelines/build-docker-sonic-vs-template.yml
@@ -14,15 +14,51 @@ parameters:
 - name: swss_artifact_name
   type: string
 
-- name: sairedis_artifact_name
+- name: swss_common_artifact_name
   type: string
 
-- name: swss_common_artifact_name
+- name: swss_common_branch
   type: string
+  default: '$(BUILD_BRANCH)'
 
 - name: artifact_name
   type: string
 
+- name: buildimage_artifact_name
+  type: string
+  default: 'sonic-buildimage.vs'
+
+- name: buildimage_artifact_project
+  type: string
+  default: 'build'
+
+- name: buildimage_artifact_pipeline
+  type: string
+  default: 'Azure.sonic-buildimage.official.vs'
+
+- name: buildimage_artifact_branch
+  type: string
+  default: '$(BUILD_BRANCH)'
+
+- name: sairedis_artifact_name
+  type: string
+
+- name: sairedis_artifact_project
+  type: string
+  default: 'build'
+
+- name: sairedis_artifact_pipeline
+  type: string
+  default: 'Azure.sonic-sairedis'
+
+- name: sairedis_artifact_branch
+  type: string
+  default: '$(BUILD_BRANCH)'
+
+- name: sairedis_artifact_pattern
+  type: string
+  default: '**'
+
 - name: asan
   type: boolean
   default: false
@@ -43,20 +79,26 @@ jobs:
       pipeline: Azure.sonic-swss-common
       artifact: ${{ parameters.swss_common_artifact_name }}
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      runBranch: 'refs/heads/${{ parameters.swss_common_branch }}'
       path: $(Build.ArtifactStagingDirectory)/download
-      allowPartiallySucceededBuilds: true
     displayName: "Download sonic swss common deb packages"
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
-      project: build
-      pipeline: Azure.sonic-sairedis
+      project: ${{ parameters.sairedis_artifact_project }}
+      pipeline: ${{ parameters.sairedis_artifact_pipeline }}
       artifact: ${{ parameters.sairedis_artifact_name }}
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/$(BUILD_BRANCH)'
-      path: $(Build.ArtifactStagingDirectory)/download
-      allowPartiallySucceededBuilds: true
+      runBranch: 'refs/heads/${{ parameters.sairedis_artifact_branch }}'
+      path: $(Build.ArtifactStagingDirectory)/download/sairedis
+      patterns: |
+        ${{ parameters.sairedis_artifact_pattern }}/libsaivs_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsaivs-dev_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsairedis_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsairedis-dev_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsaimetadata_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsaimetadata-dev_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/syncd-vs_*.deb
     displayName: "Download sonic sairedis deb packages"
   - task: DownloadPipelineArtifact@2
     inputs:
@@ -66,11 +108,11 @@ jobs:
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
-      project: build
-      pipeline: Azure.sonic-buildimage.official.vs
-      artifact: sonic-buildimage.vs
+      project: ${{ parameters.buildimage_artifact_project }}
+      pipeline: ${{ parameters.buildimage_artifact_pipeline }}
+      artifact: ${{ parameters.buildimage_artifact_name }}
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      runBranch: 'refs/heads/${{ parameters.buildimage_artifact_branch }}'
       path: $(Build.ArtifactStagingDirectory)/download
       patterns: '**/target/${{ parameters.artifact_name }}.gz'
     displayName: "Download sonic-buildimage ${{ parameters.artifact_name }}"
@@ -82,6 +124,7 @@ jobs:
 
       mkdir -p .azure-pipelines/docker-sonic-vs/debs
 
+      find $(Build.ArtifactStagingDirectory)/download/sairedis -name '*.deb' -exec cp "{}" .azure-pipelines/docker-sonic-vs/debs \;
       cp -v $(Build.ArtifactStagingDirectory)/download/*.deb .azure-pipelines/docker-sonic-vs/debs
 
       pushd .azure-pipelines

diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml
@@ -26,9 +26,29 @@ parameters:
 - name: sairedis_artifact_name
   type: string
 
+- name: sairedis_artifact_project
+  type: string
+  default: 'build'
+
+- name: sairedis_artifact_pipeline
+  type: string
+  default: 'Azure.sonic-sairedis'
+
+- name: sairedis_artifact_branch
+  type: string
+  default: '$(BUILD_BRANCH)'
+
+- name: sairedis_artifact_pattern
+  type: string
+  default: '**'
+
 - name: swss_common_artifact_name
   type: string
 
+- name: swss_common_branch
+  type: string
+  default: '$(BUILD_BRANCH)'
+
 - name: artifact_name
   type: string
 
@@ -43,6 +63,18 @@ parameters:
 - name: common_lib_artifact_name
   type: string
 
+- name: common_lib_artifact_project
+  type: string
+  default: 'build'
+
+- name: common_lib_artifact_pipeline
+  type: string
+  default: 'Azure.sonic-buildimage.common_libs'
+
+- name: common_lib_artifact_branch
+  type: string
+  default: '$(BUILD_BRANCH)'
+
 - name: asan
   type: boolean
   default: false
@@ -81,41 +113,39 @@ jobs:
       pipeline: Azure.sonic-swss-common
       artifact: ${{ parameters.swss_common_artifact_name }}
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/$(BUILD_BRANCH)'
-      path: $(Build.ArtifactStagingDirectory)/download
-      allowPartiallySucceededBuilds: true
+      runBranch: 'refs/heads/${{ parameters.swss_common_branch }}'
+      path: $(Build.ArtifactStagingDirectory)/download/swsscommon
       patterns: |
         libswsscommon_1.0.0_${{ parameters.arch }}.deb
         libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb
     displayName: "Download sonic swss common deb packages"
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
-      project: build
-      pipeline: Azure.sonic-sairedis
+      project: ${{ parameters.sairedis_artifact_project }}
+      pipeline: ${{ parameters.sairedis_artifact_pipeline }}
       artifact: ${{ parameters.sairedis_artifact_name }}
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/$(BUILD_BRANCH)'
-      path: $(Build.ArtifactStagingDirectory)/download
-      allowPartiallySucceededBuilds: true
+      runBranch: 'refs/heads/${{ parameters.sairedis_artifact_branch }}'
+      path: $(Build.ArtifactStagingDirectory)/download/sairedis
       patterns: |
-        libsaivs_*.deb
-        libsaivs-dev_*.deb
-        libsairedis_*.deb
-        libsairedis-dev_*.deb
-        libsaimetadata_*.deb
-        libsaimetadata-dev_*.deb
-        syncd-vs_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsaivs_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsaivs-dev_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsairedis_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsairedis-dev_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsaimetadata_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsaimetadata-dev_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/syncd-vs_*.deb
     displayName: "Download sonic sairedis deb packages"
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
-      project: build
-      pipeline: Azure.sonic-buildimage.common_libs
+      project: ${{ parameters.common_lib_artifact_project }}
+      pipeline: ${{ parameters.common_lib_artifact_pipeline }}
       artifact: ${{ parameters.common_lib_artifact_name }}
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/$(BUILD_BRANCH)'
-      path: $(Build.ArtifactStagingDirectory)/download
+      runBranch: 'refs/heads/${{ parameters.common_lib_artifact_branch }}'
+      path: $(Build.ArtifactStagingDirectory)/download/common
       patterns: |
         target/debs/bullseye/libnl-3-200_*.deb
         target/debs/bullseye/libnl-3-dev_*.deb
@@ -126,12 +156,17 @@ jobs:
         target/debs/bullseye/libnl-nf-3-200_*.deb
         target/debs/bullseye/libnl-nf-3-dev_*.deb
         target/debs/bullseye/libyang_*.deb
+        target/debs/bullseye/libprotobuf*.deb
+        target/debs/bullseye/libprotoc*.deb
+        target/debs/bullseye/protobuf-compiler*.deb
+        target/debs/bullseye/libdashapi*.deb
     displayName: "Download common libs"
   - script: |
       set -ex
       cd download
-      sudo dpkg -i $(find target/debs/bullseye -type f)
-      sudo dpkg -i $(ls *.deb)
+      sudo dpkg -i $(find common -type f -name '*.deb')
+      sudo dpkg -i $(find swsscommon -type f -name '*.deb')
+      sudo dpkg -i $(find sairedis -type f -name '*.deb')
       cd ..
       rm -rf download
     workingDirectory: $(Build.ArtifactStagingDirectory)

diff --git a/.azure-pipelines/test-docker-sonic-vs-template.yml b/.azure-pipelines/test-docker-sonic-vs-template.yml
@@ -20,10 +20,26 @@ parameters:
   type: string
   default: docker-sonic-vs
 
+- name: swss_common_branch
+  type: string
+  default: '$(BUILD_BRANCH)'
+
+- name: sonic_buildimage_ubuntu20_04
+  type: string
+  default: 'master'
+
 - name: asan
   type: boolean
   default: false
 
+- name: num_ports
+  type: number
+  default: 0
+
+- name: run_tests_pattern
+  type: string
+  default: ""
+
 jobs:
 - job:
   displayName: vstest
@@ -48,15 +64,27 @@ jobs:
       pipeline: Azure.sonic-swss-common
       artifact: sonic-swss-common.amd64.ubuntu20_04
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      runBranch: 'refs/heads/${{ parameters.swss_common_branch }}'
       path: $(Build.ArtifactStagingDirectory)/download
     displayName: "Download sonic swss common deb packages"
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      source: specific
+      project: build
+      pipeline: sonic-net.sonic-buildimage-ubuntu20.04
+      artifact: sonic-buildimage.amd64.ubuntu20_04
+      runVersion: 'latestFromBranch'
+      runBranch: 'refs/heads/${{ parameters.sonic_buildimage_ubuntu20_04 }}'
+      path: $(Build.ArtifactStagingDirectory)/download
+    displayName: "Download sonic buildimage ubuntu20.04 deb packages"
 
   - script: |
       set -ex
       sudo .azure-pipelines/build_and_install_module.sh
 
       sudo apt-get install -y libhiredis0.14 libyang0.16
+      sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/libprotobuf*_amd64.deb $(Build.ArtifactStagingDirectory)/download/libprotobuf-lite*_amd64.deb $(Build.ArtifactStagingDirectory)/download/python3-protobuf*_amd64.deb 
+      sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/libdashapi*.deb
       sudo dpkg -i --force-confask,confnew $(Build.ArtifactStagingDirectory)/download/libswsscommon_1.0.0_amd64.deb || apt-get install -f
       sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/python3-swsscommon_1.0.0_amd64.deb
 
@@ -76,16 +104,24 @@ jobs:
       sudo /sbin/ip link del Vrf1 type vrf table 1001
       pushd tests
 
-      params=''
+      params=""
       if [ '${{ parameters.archive_gcov }}' == True ]; then
-        params='--keeptb'
+        params=" ${params} --keeptb "
       fi
       if [ '${{ parameters.asan }}' == True ]; then
-        params='--graceful-stop'
+        params=" ${params} --graceful-stop "
+      fi
+      if [ ${{ parameters.num_ports }} -gt 0 ]; then
+        params=" ${params} --num-ports=${{ parameters.num_ports }} "
       fi
 
-      all_tests=$(ls test_*.py)
+      all_tests=$(ls test_*.py | grep -v test_dash)
       all_tests="${all_tests} p4rt"
+
+      if [ -n '${{ parameters.run_tests_pattern }}' ]; then
+        all_tests=" $(ls ${{ parameters.run_tests_pattern }}) "
+      fi
+
       test_set=()
       # Run 20 tests as a set.
       for test in ${all_tests}; do