name: CI
on:
  push:
    branches:
      - 'master'
      - 'develop'
  pull_request:
    types: [ready_for_review, opened, synchronize, reopened]
    paths-ignore:
      - 'site/**'
      - '**/*.md'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  CYPRESS_VERIFY_TIMEOUT: 180000 # https://docs.cypress.io/guides/guides/command-line#cypress-verify
  CVAT_VERSION: "local"

jobs:
  search_cache:
    if: |
      github.event.pull_request.draft == false &&
      !startsWith(github.event.pull_request.title, '[WIP]') &&
      !startsWith(github.event.pull_request.title, '[Dependent]')
    uses: ./.github/workflows/search-cache.yml

  build:
    needs: search_cache
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7

      - name: Verify version consistency
        run: ./dev/update_version.py --verify-current

      - name: Check changelog fragments
        run: ./dev/check_changelog_fragments.py

      - name: CVAT server. Getting cache from the default branch
        uses: actions/cache@v6
        with:
          path: /tmp/cvat_cache_server
          key: ${{ runner.os }}-build-server-${{ needs.search_cache.outputs.sha }}

      - name: CVAT UI. Getting cache from the default branch
        uses: actions/cache@v6
        with:
          path: /tmp/cvat_cache_ui
          key: ${{ runner.os }}-build-ui-${{ needs.search_cache.outputs.sha }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4

      - name: Create artifact directories
        run: |
          mkdir /tmp/cvat_server
          mkdir /tmp/cvat_ui
          mkdir /tmp/cvat_sdk

      - name: CVAT server. Build and push
        uses: docker/build-push-action@v7
        with:
          build-args: |
              "COVERAGE_PROCESS_START=.coveragerc"
          cache-from: type=local,src=/tmp/cvat_cache_server
          context: .
          file: Dockerfile
          tags: cvat/server:${{ env.CVAT_VERSION }}
          outputs: type=docker,dest=/tmp/cvat_server/image.tar

      - name: Instrumentation of the code then rebuilding the CVAT UI
        run: |
          corepack enable yarn
          yarn --immutable
          yarn run coverage

      - name: CVAT UI. Build and push
        uses: docker/build-push-action@v7
        with:
          cache-from: type=local,src=/tmp/cvat_cache_ui
          context: .
          file: Dockerfile.ui
          tags: cvat/ui:${{ env.CVAT_VERSION }}
          outputs: type=docker,dest=/tmp/cvat_ui/image.tar

      - name: CVAT SDK. Build
        run: |
          pip3 install --user -r cvat-sdk/gen/requirements.txt
          ./cvat-sdk/gen/generate.sh

          cp -r cvat-sdk/* /tmp/cvat_sdk/

      - name: Verify API schema
        id: verify_schema
        run: |
          docker load --input /tmp/cvat_server/image.tar
          docker run --rm "cvat/server:${CVAT_VERSION}" bash \
            -c 'python manage.py spectacular' > cvat/schema-expected.yml

          if ! git diff --no-index cvat/schema.yml cvat/schema-expected.yml; then
            echo
            echo 'API schema has changed! Please update cvat/schema.yml:'
            echo
            echo '  docker run --rm cvat/server:dev bash \'
            echo "    -c 'python manage.py spectacular' > cvat/schema.yml"
            exit 1
          fi

      - name: Verify migrations
        run: |
          docker run --rm "cvat/server:${CVAT_VERSION}" bash \
            -c 'python manage.py makemigrations --check'

      - name: Verify redis migration filenames
        run: |
          docker run --rm "cvat/server:${CVAT_VERSION}" bash \
            -c 'python manage.py checkredismigrations'

      - name: Upload CVAT server artifact
        uses: actions/upload-artifact@v7
        with:
          name: cvat_server
          path: /tmp/cvat_server/image.tar

      - name: Upload CVAT UI artifact
        uses: actions/upload-artifact@v7
        with:
          name: cvat_ui
          path: /tmp/cvat_ui/image.tar

      - name: Upload CVAT SDK artifact
        uses: actions/upload-artifact@v7
        with:
          name: cvat_sdk
          path: /tmp/cvat_sdk/

  rest_api_testing:
    needs: build
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - pattern: "test_task_data"
            allow_cache: 'true'
          - pattern: "test_task_data"
            allow_cache: 'no'
          - pattern: "not test_task_data"
            allow_cache: 'no'
          - pattern: "not test_task_data"
            allow_cache: 'true'
    name: rest_api_testing [${{ matrix.pattern }}, cache-${{ matrix.allow_cache }}]
    steps:
      - uses: actions/checkout@v7

      - uses: actions/setup-python@v6
        with:
          python-version: '3.10'

      - name: Download CVAT server image
        uses: actions/download-artifact@v8
        with:
          name: cvat_server
          path: /tmp/cvat_server/

      - name: Download CVAT UI images
        uses: actions/download-artifact@v8
        with:
          name: cvat_ui
          path: /tmp/cvat_ui/

      - name: Load Docker images
        run: |
          docker load --input /tmp/cvat_server/image.tar
          docker load --input /tmp/cvat_ui/image.tar
          docker image ls -a

      - name: Generate SDK
        run: |
          pip3 install -r cvat-sdk/gen/requirements.txt
          ./cvat-sdk/gen/generate.sh

      - name: Install SDK
        run: |
          pip3 install -r ./tests/python/requirements.txt \
            -e './cvat-sdk[masks,pytorch]' -e ./cvat-cli \
            --extra-index-url https://download.pytorch.org/whl/cpu

      - name: Run REST API and SDK tests
        id: run_tests
        env:
          COVERAGE_PROCESS_START: ".coveragerc"
          CVAT_ALLOW_STATIC_CACHE: ${{ matrix.allow_cache }}
        run: |
          pytest tests/python/ -k "${{ matrix.pattern }}" --cov --cov-report=json --alluredir=tests/python/allure-results
          for COVERAGE_FILE in `find -name "coverage*.json" -type f -printf "%f\n"`; do mv ${COVERAGE_FILE} "${COVERAGE_FILE%%.*}_0.json"; done


      - name: Uploading code coverage results as an artifact
        uses: actions/upload-artifact@v7
        with:
          name: ${{format(
            'coverage_results_rest_api-{0}-cache-{1}',
            matrix.pattern, matrix.allow_cache
            )}}
          path: |
            coverage*.json

      - name: Creating a log file from cvat containers
        if: failure() && steps.run_tests.conclusion == 'failure'
        env:
          LOGS_DIR: "${{ github.workspace }}/rest_api_testing"
        run: |
            mkdir $LOGS_DIR
            docker logs test_cvat_server_1 > $LOGS_DIR/cvat_server.log
            docker logs test_cvat_worker_export_1 > $LOGS_DIR/cvat_worker_export.log
            docker logs test_cvat_worker_import_1 > $LOGS_DIR/cvat_worker_import.log
            docker logs test_cvat_worker_annotation_1 > $LOGS_DIR/cvat_worker_annotation.log
            docker logs test_cvat_opa_1 2> $LOGS_DIR/cvat_opa.log
            # all remaining workers (any one can fail)
            docker logs test_cvat_worker_utils_1 > $LOGS_DIR/cvat_worker_utils.log
            docker logs test_cvat_worker_annotation_1 > $LOGS_DIR/cvat_worker_annotation.log
            docker logs test_cvat_worker_quality_reports_1 > $LOGS_DIR/cvat_worker_quality_reports.log
            docker logs test_cvat_worker_consensus_1 > $LOGS_DIR/cvat_worker_consensus.log
            docker logs test_cvat_worker_webhooks_1 > $LOGS_DIR/cvat_worker_webhooks.log
            docker logs test_cvat_worker_export_1 > $LOGS_DIR/cvat_worker_export.log
            docker logs test_cvat_redis_inmem_1 > $LOGS_DIR/cvat_redis_inmem.log
            docker logs test_cvat_redis_ondisk_1 > $LOGS_DIR/cvat_redis_ondisk.log



      - name: Creating a log file from cvat containers
        if: failure() && steps.run_tests.conclusion == 'failure'
        uses: actions/upload-artifact@v7
        with:
          name: ${{format(
              'rest_api_container_logs-{0}-cache-{1}',
              matrix.pattern, matrix.allow_cache
            )}}
          path: "${{ github.workspace }}/rest_api_testing"

      - name: Upload allure results
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: ${{format(
              'allure-results-tests-api-{0}-cache-{1}',
              matrix.pattern,matrix.allow_cache
            )}}
          path: tests/python/allure-results

  unit_testing:
    needs: build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7

      - name: Download CVAT server image
        uses: actions/download-artifact@v8
        with:
          name: cvat_server
          path: /tmp/cvat_server/

      - name: Load Docker server image
        run: |
          docker load --input /tmp/cvat_server/image.tar
          docker image ls -a

      - name: Running OPA tests
        run: |
          python cvat/apps/iam/rules/tests/generate_tests.py

          docker compose run --rm -v "$PWD:/mnt/src:ro" -w /mnt/src \
            cvat_opa test cvat/apps/*/rules

      - name: Running unit tests
        env:
          HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}
          CONTAINER_COVERAGE_DATA_DIR: "/coverage_data"
        run: |
          docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa cvat_server cvat_db

          max_tries=12
          while [[  $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done

          docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml \
            run --env PYTHONDEVMODE=1 cvat_ci /bin/bash \
            -c 'coverage run -a manage.py test -v 2 cvat/apps && coverage json && mv coverage.json ${CONTAINER_COVERAGE_DATA_DIR}/unit_tests_coverage.json'

      - name: Uploading code coverage results as an artifact
        uses: actions/upload-artifact@v7
        with:
          name: coverage_results_unit_tests
          path: |
            ${{ github.workspace }}/coverage-final.json
            ${{ github.workspace }}/unit_tests_coverage.json

      - name: Creating a log file from cvat containers
        if: failure()
        env:
          LOGS_DIR: "${{ github.workspace }}/unit_testing"
        run: |
            mkdir $LOGS_DIR
            docker logs cvat_server > $LOGS_DIR/cvat_server.log
            docker logs cvat_opa 2> $LOGS_DIR/cvat_opa.log

      - name: Uploading "cvat" container logs as an artifact
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: unit_tests_container_logs
          path: "${{ github.workspace }}/unit_testing"

  e2e_testing:
    needs: build
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        specs: ['actions_tasks', 'actions_tasks2', 'actions_tasks3', 'actions_tasks4',
                'actions_objects', 'actions_objects2', 'actions_users',
                'actions_projects_models', 'canvas3d_functionality',
                'issues_prs', 'issues_prs2', 'features', 'features2',
                'audio_workspace']
    steps:
      - uses: actions/checkout@v7

      - uses: actions/setup-node@v6
        with:
            node-version: '22.x'
            # NOTE: corepack is not bundled after Node 25
            # https://nodejs.org/docs/v22.18.0/api/corepack.html#corepack

      - name: Download CVAT server image
        uses: actions/download-artifact@v8
        with:
          name: cvat_server
          path: /tmp/cvat_server/

      - name: Download CVAT UI image
        uses: actions/download-artifact@v8
        with:
          name: cvat_ui
          path: /tmp/cvat_ui/

      - name: Load Docker images
        run: |
          docker load --input /tmp/cvat_server/image.tar
          docker load --input /tmp/cvat_ui/image.tar
          docker image ls -a

      - name: Run CVAT instance
        run: |
          docker compose \
            -f docker-compose.yml \
            -f docker-compose.dev.yml \
            -f components/serverless/docker-compose.serverless.yml \
            -f tests/docker-compose.minio.yml \
            -f tests/docker-compose.file_share.yml up -d

      - name: Waiting for server
        env:
          API_ABOUT_PAGE: "localhost:8080/api/server/about"
        run: |
          max_tries=60
          status_code=$(curl -s -o /tmp/server_response -w "%{http_code}" ${API_ABOUT_PAGE})
          while [[  $status_code != "200" && max_tries -gt 0 ]]
          do
            echo Number of attempts left: $max_tries
            echo Status code of response: $status_code

            sleep 5
            status_code=$(curl -s -o /tmp/server_response -w "%{http_code}" ${API_ABOUT_PAGE})
            (( max_tries-- ))
          done

      - name: Run E2E tests
        env:
          DJANGO_SU_NAME: 'admin'
          DJANGO_SU_EMAIL: 'admin@localhost.company'
          DJANGO_SU_PASSWORD: '12qwaszx'

        run: |
          docker exec -i cvat_server /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell"
          cd ./tests
          corepack enable yarn
          yarn --immutable

          if [[ ${{ matrix.specs }} == canvas3d_* ]]; then
            npx cypress run \
              --headed \
              --browser chrome \
              --config-file cypress_canvas3d.config.js \
              --spec 'cypress/e2e/setup/setup_canvas3d.js,cypress/e2e/canvas3d_*/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js'
          elif [[ ${{ matrix.specs }} == audio_* ]]; then
            npx cypress run \
              --headed \
              --browser chrome \
              --config-file cypress_audio.config.js \
              --spec 'cypress/e2e/setup/setup_audio.js,cypress/e2e/audio_workspace/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js'
          else
            npx cypress run \
              --browser chrome \
              --spec 'cypress/e2e/setup/setup.js,cypress/e2e/setup/setup_project.js,cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js'
          fi
          mv coverage/coverage-final.json coverage/${{ matrix.specs }}_coverage.json

      - name: Uploading code coverage results as an artifact
        uses: actions/upload-artifact@v7
        with:
          name: coverage_results_e2e_${{ matrix.specs }}
          path: |
            tests/coverage/${{ matrix.specs }}_coverage.json

      - name: Creating a log file from "cvat" container logs
        if: failure()
        run: |
            docker logs cvat_server > ${{ github.workspace }}/tests/cvat_${{ matrix.specs }}.log
            docker logs cvat_worker_export > ${{ github.workspace }}/tests/cvat_worker_export_${{ matrix.specs }}.log
            docker logs cvat_worker_import > ${{ github.workspace }}/tests/cvat_worker_import_${{ matrix.specs }}.log

      - name: Uploading "cvat" container logs as an artifact
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: e2e_container_logs_${{ matrix.specs }}
          path: ${{ github.workspace }}/tests/cvat_${{ matrix.specs }}.log

      - name: Uploading cypress screenshots as an artifact
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: cypress_screenshots_${{ matrix.specs }}
          path: ${{ github.workspace }}/tests/cypress/screenshots

      - name: Uploading cypress videos as an artifact
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: cypress_videos_${{ matrix.specs }}
          path: ${{ github.workspace }}/tests/cypress/videos

      - name: Upload Allure results as an artifact
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: allure-results-e2e-${{ matrix.specs }}
          path: tests/allure-results-e2e

  helm_rest_api_testing:
    needs: build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7

      - name: Start minikube
        uses: medyagh/setup-minikube@latest
        with:
          cpus: max
          memory: max

      - name: Try the cluster
        run: kubectl get pods -A

      - name: Download CVAT server image
        uses: actions/download-artifact@v8
        with:
          name: cvat_server
          path: /tmp/cvat_server/

      - name: Download CVAT UI images
        uses: actions/download-artifact@v8
        with:
          name: cvat_ui
          path: /tmp/cvat_ui/

      - name: Load images
        run: |
          eval $(minikube -p minikube docker-env)
          docker load --input /tmp/cvat_server/image.tar
          docker load --input /tmp/cvat_ui/image.tar
          docker image ls -a

      - uses: azure/setup-helm@v5

      - name: Update Helm chart dependencies
        working-directory: helm-chart
        run: |
          helm dependency update

      - name: Deploy to minikube
        run: |
          printf "  service:\n    externalIPs:\n      - $(minikube ip)\n" >> helm-chart/test.values.yaml
          helm upgrade release-${{ github.run_id }}-${{ github.run_attempt }} --install helm-chart \
            -f helm-chart/cvat.values.yaml \
            -f helm-chart/test.values.yaml \
            --set cvat.backend.tag=${{ env.CVAT_VERSION }} \
            --set cvat.frontend.tag=${{ env.CVAT_VERSION }}

      - name: Update test config
        run: |
          sed -i -e 's$http://localhost:8080$http://cvat.local:80$g' tests/python/shared/utils/config.py
          find tests/python/shared/assets/ -type f -name '*.json' | xargs sed -i -e 's$http://localhost:8080$http://cvat.local$g'
          echo "$(minikube ip) cvat.local" | sudo tee -a /etc/hosts

      - name: Generate SDK
        run: |
          pip3 install --user -r cvat-sdk/gen/requirements.txt
          ./cvat-sdk/gen/generate.sh

      - name: Install test requirements
        run: |
          pip3 install --user cvat-sdk/ cvat-cli/ -r tests/python/requirements.txt

      - name: Wait for CVAT to be ready
        run: |
          max_tries=60
          while [[ $(kubectl get pods -l component=server -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" && max_tries -gt 0 ]]; do echo "waiting for CVAT pod" && (( max_tries-- )) && sleep 5; done
          while [[ $(kubectl get pods -l app.kubernetes.io/name=postgresql -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" && max_tries -gt 0 ]]; do echo "waiting for DB pod" && (( max_tries-- )) && sleep 5; done
          while [[ $(curl -s -o /tmp/server_response -w "%{http_code}" cvat.local/api/server/about) != "200" && max_tries -gt 0 ]]; do echo "waiting for CVAT" && (( max_tries-- )) && sleep 5; done
          kubectl get pods
          kubectl logs $(kubectl get pods -l component=server -o jsonpath='{.items[0].metadata.name}')

      - name: REST API and SDK tests
        # We don't have external services in Helm tests, so we ignore corresponding cases
        # They are still tested without Helm
        run: |
          kubectl cp tests/mounted_file_share $(kubectl get pods -l component=server -o jsonpath='{.items[0].metadata.name}'):/home/django/share/
          kubectl exec $(kubectl get pods -l component=server -o jsonpath='{.items[0].metadata.name}') -- bash -c "mv /home/django/share/mounted_file_share/* /home/django/share/ && rm -r /home/django/share/mounted_file_share"
          pytest --timeout 30 --platform=kube -m "not with_external_services" tests/python --log-level INFO

      - name: Creating a log file from "cvat" container logs
        if: failure()
        env:
            LOGS_DIR: "${{ github.workspace }}/rest_api_testing"
        run: |
          mkdir ${LOGS_DIR}
          for backend_pod in $(kubectl get pods -l tier=backend -o 'jsonpath={.items[*].metadata.name}'); do
            kubectl logs ${backend_pod} >${LOGS_DIR}/${backend_pod}.log
          done
          kubectl logs $(kubectl get pods -l app.kubernetes.io/name=traefik -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/traefik.log

      - name: Uploading "cvat" container logs as an artifact
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: helm_rest_api_container_logs
          path: "${{ github.workspace }}/rest_api_testing"

  publish_dev_images:
    if: github.ref == 'refs/heads/develop'
    needs: [rest_api_testing, unit_testing, e2e_testing]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7

      - name: Download CVAT server images
        uses: actions/download-artifact@v8
        with:
          name: cvat_server
          path: /tmp/cvat_server/

      - name: Download CVAT UI images
        uses: actions/download-artifact@v8
        with:
          name: cvat_ui
          path: /tmp/cvat_ui/

      - name: Load Docker images
        run: |
          docker load --input /tmp/cvat_server/image.tar
          docker load --input /tmp/cvat_ui/image.tar

      - name: Login to Docker Hub
        uses: docker/login-action@v4
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Push to Docker Hub
        env:
          SERVER_IMAGE_REPO: ${{ secrets.DOCKERHUB_WORKSPACE }}/server
          UI_IMAGE_REPO: ${{ secrets.DOCKERHUB_WORKSPACE }}/ui
        run: |
          docker tag "cvat/server:${CVAT_VERSION}" "${SERVER_IMAGE_REPO}:dev"
          docker push "${SERVER_IMAGE_REPO}:dev"

          docker tag "cvat/ui:${CVAT_VERSION}" "${UI_IMAGE_REPO}:dev"
          docker push "${UI_IMAGE_REPO}:dev"

  generate_report:
    name: Generate Allure Report and Upload to S3
    needs: [rest_api_testing, e2e_testing]
    if: github.event_name == 'push' && github.ref == 'refs/heads/develop' && github.repository == 'cvat.ai/cvat'
    permissions:
      id-token: write
      contents: read
    uses: ./.github/workflows/generate-allure-report.yml
    secrets:
      AWS_ALLURE_REPORTS_ROLE: ${{ secrets.AWS_ALLURE_REPORTS_ROLE }}

  codecov:
    runs-on: ubuntu-latest
    needs: [unit_testing, e2e_testing, rest_api_testing]
    steps:
      - uses: actions/checkout@v7

      - name: Merge coverage artifacts
        uses: actions/upload-artifact/merge@v7
        with:
          name: coverage_results
          pattern: coverage_results_*
          delete-merged: true

      - name: Downloading coverage results
        uses: actions/download-artifact@v8
        with:
          name: coverage_results

      - name: Upload coverage reports to Codecov with GitHub Action
        uses: codecov/codecov-action@v7
        with:
          token: ${{ secrets.CODECOV_TOKEN }}