Merge branch 'demisto:master' into gem_test

.circleci/config.yml

@@ -106,7 +106,7 @@ references:
           neo4j_conf_file="/etc/neo4j/neo4j.conf"
           sudo echo "dbms.security.procedures.unrestricted=apoc.*" >> $neo4j_conf_file
           sudo echo "dbms.security.procedures.allowlist=apoc.*" >> $neo4j_conf_file
-          sudo echo "dbms.memory.transaction.total.max=600m" >> $neo4j_conf_file
+          sudo echo "dbms.memory.transaction.total.max=2000m" >> $neo4j_conf_file
 
           apoc_conf_file="/etc/neo4j/apoc.conf"
           sudo echo "apoc.export.file.enabled=true" > $apoc_conf_file

.github/CODEOWNERS

@@ -82,18 +82,19 @@
 .gitlab/ci/* @yucohen
 .gitlab/* @yucohen
 .gitlab-ci.yml @yucohen
-/Tests/scripts/wait_in_line_for_cloud_env.sh @daryakoval
+/Tests/scripts/wait_in_line_for_cloud_env.sh @yucohen
 .gitlab/ci/.gitlab-ci.staging.yml @ilaner
-/Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh @daryakoval
-/Tests/Marketplace/search_and_uninstall_pack.py @daryakoval
-/Tests/scripts/install_content_and_test_integrations.sh @daryakoval
-/Tests/configure_and_test_integration_instances.py @daryakoval
-/Tests/scripts/print_cloud_machine_details.sh @daryakoval
-/Tests/scripts/run_tests.sh @daryakoval
-/Tests/scripts/download_demisto_conf.sh @daryakoval
-Tests/scripts/test_modeling_rules.sh @daryakoval
-Tests/scripts/lock_cloud_machines.py @daryakoval
+/Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh @yucohen
+/Tests/Marketplace/search_and_uninstall_pack.py @yucohen
+/Tests/scripts/install_content_and_test_integrations.sh @yucohen
+/Tests/configure_and_test_integration_instances.py @yucohen
+/Tests/scripts/print_cloud_machine_details.sh @yucohen
+/Tests/scripts/run_tests.sh @yucohen
+/Tests/scripts/download_demisto_conf.sh @yucohen
+Tests/scripts/test_modeling_rules.sh @AradCarmi
+Tests/scripts/lock_cloud_machines.py @yucohen
 Tests/Marketplace/server_content_items.json @dantavori
+validation_config.toml @YuvHayun @JudahSchwartz @GuyAfik @anara123
 
 # SDK Related
 .gitlab/ci/.gitlab-ci.sdk-nightly.yml @dorschw
@@ -113,7 +114,7 @@ poetry.lock @ilaner @dorschw
 .devcontainer/* @ilaner
 
 # Demisto Class
-Packs/ApiModules/Scripts/DemistoClassApiModule/DemistoClassApiModule.py @daryakoval
+Packs/ApiModules/Scripts/DemistoClassApiModule/DemistoClassApiModule.py @dantavori
 
 # TIM Related
 /Packs/TAXIIServer/Integrations/* @MLainer1
@@ -139,4 +140,4 @@ Packs/ApiModules/Scripts/DemistoClassApiModule/DemistoClassApiModule.py @daryako
 /Packs/Base/Scripts/DBotFindSimilarIncidentsByIndicators/ @jlevypaloalto
 /Packs/Base/Scripts/DBotSuggestClassifierMapping/ @jlevypaloalto
 /Packs/Base/Scripts/GetMLModelEvaluation/ @jlevypaloalto
-/Packs/Base/Scripts/DBotMLFetchData/ @jlevypaloalto
+/Packs/Base/Scripts/DBotMLFetchData/ @jlevypaloalto

.github/workflows/check-nightly-ok-label.yml

@@ -0,0 +1,51 @@
+name: Check nightly-ok label
+
+on:
+  pull_request:
+    types: [opened, synchronize, labeled, unlabeled]
+
+jobs:
+  check_label:
+    runs-on: ubuntu-latest
+    if: github.repository == 'demisto/content' && github.event.pull_request.head.repo.fork == false
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Check if files under .gitlab directory are changed
+        id: check-changes
+        run: |
+          CHANGED_FILES=$(git diff --name-only origin/master origin/${{ github.head_ref || github.ref_name }})
+          echo "All changed files:"
+          echo "${CHANGED_FILES}"
+          GITLAB_CHANGED_FILES=$( [[ $CHANGED_FILES == *".gitlab/ci"* ]] && echo true || echo false)
+          echo "Files in the .gitlab folder have changed: ${GITLAB_CHANGED_FILES}"
+          echo "gitlab_changed_files=$GITLAB_CHANGED_FILES" >> $GITHUB_OUTPUT
+          if [[ $GITLAB_CHANGED_FILES == true ]]; then
+            echo 'Files under .gitlab folder has changed, Will check if the PR has the `nightly-ok` label.'
+          else
+            echo 'Files in the .gitlab folder have not been changed.'
+          fi
+
+      - name: Check if PR has the nightly-ok label
+        uses: actions/github-script@v7
+        id: check-label
+        with:
+          script: |
+            const gitlabChangedFiles = ${{ steps.check-changes.outputs.gitlab_changed_files }};
+            if(gitlabChangedFiles) {
+              console.log('Files under .gitlab folder has changed, Will check if the PR has the `nightly-ok` label.');
+              const labels = context.payload.pull_request.labels.map(label => label.name);
+              const hasLabel = labels.includes('nightly-ok');
+              if (hasLabel) {
+                console.log('All good, the PR has the `nightly-ok` label.');
+              } else {
+                console.log('PR does not have the `nightly-ok` label. It is required when changing files under the `.gitlab` directory. Please run nightly using the Utils/gitlab_triggers/trigger_content_nightly_build.sh script, check that succeeded, and add the `nightly-ok` label');
+                process.exit(1); // Exit with failure status if label is missing
+              }
+            } else {
+              console.log('Files in the .gitlab folder have not been changed.');
+            }

.github/workflows/create-internal-pr-from-external.yml

@@ -19,11 +19,11 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Setup Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
           python-version: '3.10'
       - name: Setup Poetry
-        uses: Gr1N/setup-poetry@v8
+        uses: Gr1N/setup-poetry@v9
       - name: Print Context
         run: |
           echo "$GITHUB_CONTEXT"

.github/workflows/handle-new-external-pr.yml

@@ -17,11 +17,11 @@ jobs:
           fetch-depth: 2
 
       - name: Setup Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
           python-version: '3.10'
       - name: Setup Poetry
-        uses: Gr1N/setup-poetry@v8
+        uses: Gr1N/setup-poetry@v9
       - name: Print Context
         run: |
           echo "$GITHUB_CONTEXT"

.github/workflows/pre-commit-reuse.yml

@@ -19,6 +19,9 @@ jobs:
       with:
         fetch-depth: 0
 
+    - name: Set PYTHONPATH
+      run: echo "PYTHONPATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
+
     - name: Setup python
       uses: actions/setup-python@v4
       with:
@@ -45,13 +48,44 @@ jobs:
 
     - name: "Check coverage.xml exists"
       if: always()
-      id: check_files
+      id: check-coverage-xml-exists
       uses: andstor/file-existence-action@v2
       with:
         files: "coverage_report/coverage.xml"
 
+    - name: "Check pytest report exists"
+      if: always()
+      id: check-pytest-junit-exists
+      uses: andstor/file-existence-action@v2
+      with:
+        files: ".report_pytest.xml"
+
+    - name: Create pack-wise pytest report
+      run: poetry run python Utils/github_workflow_scripts/parse_junit_per_pack.py
+      if: |
+        always() && 
+        steps.check-pytest-junit-exists.outputs.files_exists == 'true' && 
+        github.event.pull_request.head.repo.fork == false
+
+    - name: Upload junit & pack-wise pytest report
+      uses: actions/upload-artifact@v4
+      if: |
+        always() && 
+        steps.check-pytest-junit-exists.outputs.files_exists == 'true' && 
+        github.event.pull_request.head.repo.fork == false
+      with:
+        name: pytest
+        path: |
+          packwise_pytest_time.csv
+          .report_pytest.xml
+        if-no-files-found: error
+
     - name: Pytest coverage comment
-      if: always() && steps.check_files.outputs.files_exists == 'true' && github.event.pull_request.head.repo.fork == false
+      if: |
+        always() && 
+        steps.check-coverage-xml-exists.outputs.files_exists == 'true' && 
+        steps.check-pytest-junit-exists.outputs.files_exists == false && 
+        github.event.pull_request.head.repo.fork == false
       uses: MishaKav/pytest-coverage-comment@main
       with:
         pytest-xml-coverage-path: coverage_report/coverage.xml

.github/workflows/trigger-contribution-build.yml

@@ -24,7 +24,7 @@ jobs:
           PR_NUMBER: ${{ github.event.pull_request.number }}
           BASE_BRANCH: ${{ github.event.pull_request.base.ref }}
           CONTRIB_BRANCH: ${{ github.event.pull_request.head.label }}
-          CONTRIB_REPO: ${{ github.event.repository.name }}
+          CONTRIB_REPO: ${{ github.event.pull_request.head.repo.name }}
           USERNAME: ${{ secrets.SECRET_CHECK_USER_NG }}
           PASSWORD: ${{ secrets.SECRET_CHECK_PASS_NG }}
           GOLD_SERVER_URL: ${{ secrets.GOLD_SERVER_URL_NG }}

.github/workflows/update-demisto-sdk-version.yml

@@ -54,6 +54,7 @@ jobs:
       run: |
         poetry add --group dev demisto-sdk@${{inputs.release_version}}
         poetry lock --no-update
+        git add .
         source .venv/bin/activate
         demisto-sdk pre-commit --mode=ci
         git add .

.gitlab/ci/.gitlab-ci.bucket-upload.yml

@@ -54,6 +54,11 @@ run-validations-upload-flow:
     - .run-validations
     - .bucket-upload-rule
 
+run-validations-upload-flow-new-validate-flow:
+  extends:
+    - .run-validations-new-validate-flow
+    - .bucket-upload-rule
+  allow_failure: true
 
 run-unittests-and-lint-upload-flow:
   cache:

.gitlab/ci/.gitlab-ci.global.yml

@@ -201,7 +201,7 @@
 .get_last_upload_commit: &get_last_upload_commit
   - section_start "Getting last bucket upload commit" --collapsed
   - gcloud auth activate-service-account --key-file="$GCS_MARKET_KEY" >> "${ARTIFACTS_FOLDER}/logs/gcloud_auth.log" 2>&1
-  - gsutil cp "gs://$GCS_MARKET_BUCKET/content/packs/index.json" "${ARTIFACTS_FOLDER_SERVER_TYPE}/previous_index.json"
+  - gsutil cp "gs://$GCS_PRODUCTION_BUCKET/content/packs/index.json" "${ARTIFACTS_FOLDER_SERVER_TYPE}/previous_index.json"
   - export LAST_UPLOAD_COMMIT=$(cat "${ARTIFACTS_FOLDER_SERVER_TYPE}/previous_index.json" | jq -r ".\"commit\"")
   - section_end "Getting last bucket upload commit"
 
@@ -501,8 +501,6 @@
   stage: unittests-and-validations
   extends:
     - .default-job-settings
-  variables:
-    KUBERNETES_CPU_REQUEST: 1000m
   artifacts:
     expire_in: 30 days
     paths:
@@ -545,6 +543,25 @@
     - !reference [ .validate_content_test_conf_branch_merged ]  # This section should be the last one in the script, do not move it.
     - job-done
 
+.run-validations-new-validate-flow:
+  stage: unittests-and-validations
+  extends:
+    - .default-job-settings
+  artifacts:
+    expire_in: 30 days
+    paths:
+      - ${CI_PROJECT_DIR}/artifacts/*
+      - ${CI_PROJECT_DIR}/pipeline_jobs_folder/*
+    when: always
+  script:
+    - section_start "Validate Files and Yaml"
+    - |
+      ./Tests/scripts/linters_runner.sh
+      ./Tests/scripts/new_validate.sh
+    - section_end "Validate Files and Yaml"
+    - !reference [ .validate_content_test_conf_branch_merged ]  # This section should be the last one in the script, do not move it.
+    - job-done
+
 .jobs-done-check:
   stage: are-jobs-really-done
   extends:

.gitlab/ci/.gitlab-ci.on-push.yml

@@ -107,6 +107,16 @@ run-validations:
     - if: '$NIGHTLY'
     - if: '$CI_PIPELINE_SOURCE =~ /^(push|contrib)$/'
 
+# runs in gitlab for the on-push flow (except for contributors)
+run-validations-new-validate-flow:
+  extends:
+    - .run-validations-new-validate-flow
+  rules:
+    - if: '$NIGHTLY'
+    - if: '$CI_PIPELINE_SOURCE =~ /^(push|contrib)$/'
+  allow_failure: true
+
+
 # runs in gitlab for the on-push flow, on every new commit pushed to the branch.
 validate-content-conf:
   tags:
@@ -132,14 +142,18 @@ validate-content-conf:
   cache:
     policy: pull-push
   variables:
-    KUBERNETES_CPU_REQUEST: 2000m
     EXTRACT_PRIVATE_TESTDATA: "true"
   stage: prepare-testing-bucket
   script:
     - !reference [.download-demisto-conf]
     - !reference [.create-release-notes-and-common-docs]
     - !reference [.secrets-fetch]
     - section_start "Create or update content graph" --collapsed
+    - |
+        echo "set DEMISTO_SDK_GRAPH_FORCE_CREATE to true to create graph from scratch"
+        export DEMISTO_SDK_GRAPH_FORCE_CREATE=true
+        echo "DEMISTO_SDK_GRAPH_FORCE_CREATE was set to true to create graph from scratch"
+        echo $DEMISTO_SDK_GRAPH_FORCE_CREATE
     - echo "Staging the repo to include the private packs in the graph"
     - git add Packs
     - echo "Updating the content graph"
@@ -210,6 +224,10 @@ validate-content-conf:
     - ./Tests/scripts/prepare_content_packs_for_testing.sh "$MARKETPLACE_BUCKET" "$STORAGE_BASE_PATH" "$MARKETPLACE_VERSION"
     - section_end "Prepare Content Packs for Testing"
 
+    - section_start "Override and upload core packs versions"
+    - ./Tests/Marketplace/upload_versions_core_files.sh "$MARKETPLACE_BUCKET" "$STORAGE_BASE_PATH" "$MARKETPLACE_VERSION" "$LAST_UPLOAD_COMMIT"
+    - section_end "Override and upload core packs versions"
+
     - section_start "Create Instances for XSOAR"
     - |
       if [[ ${MARKETPLACE_VERSION} = "xsoar" ]]; then
@@ -376,9 +394,6 @@ tests_xsoar_server:
     - !reference [.filter-non-nightly-docker-updates-rule, rules]
     - if: '$CI_PIPELINE_SOURCE =~ /^(push|contrib)$/'
     - if: '$NIGHTLY'
-      when: always
-      variables:
-        KUBERNETES_CPU_REQUEST: 2000m
   parallel:
     matrix:
       - INSTANCE_ROLE: