Merge pull request #1092 from jinsheng200303/dev

update superset to 4.1.1

apps/superset/.env

@@ -5,15 +5,15 @@
 W9_DIST=community
 W9_REPO=apache/superset
 # latest is not stable version
-W9_VERSION=2.1.0
+W9_VERSION=4.1.1
 W9_POWER_PASSWORD=uQ8E1wVTzG8SOk3!
 
 #### --  Not allowed to edit below environments when recreate app based on existing data  -- ####
 W9_ID=superset
 W9_HTTP_PORT=8088
 W9_HTTP_PORT_SET=9001
 W9_LOGIN_USER=admin
-W9_LOGIN_PASSWORD=$W9_POWER_PASSWORD
+W9_LOGIN_PASSWORD=admin
 W9_URL_REPLACE=false
 W9_URL=appname.example.com
 W9_DB_EXPOSE="postgresql"

apps/superset/src/docker/.env

@@ -14,17 +14,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+
+
 COMPOSE_PROJECT_NAME=superset
 
 # database configurations (do not modify)
 DATABASE_DB=superset
 DATABASE_HOST=db
+# Make sure you set this to a unique secure random value on production
 DATABASE_PASSWORD=superset
 DATABASE_USER=superset
 
 EXAMPLES_DB=examples
 EXAMPLES_HOST=db
 EXAMPLES_USER=examples
+# Make sure you set this to a unique secure random value on production
 EXAMPLES_PASSWORD=examples
 EXAMPLES_PORT=5432
 
@@ -34,6 +38,7 @@ DATABASE_PORT=5432
 DATABASE_DIALECT=postgresql
 POSTGRES_DB=superset
 POSTGRES_USER=superset
+# Make sure you set this to a unique secure random value on production
 POSTGRES_PASSWORD=superset
 #MYSQL_DATABASE=superset
 #MYSQL_USER=superset
@@ -51,3 +56,10 @@ SUPERSET_LOAD_EXAMPLES=yes
 CYPRESS_CONFIG=false
 SUPERSET_PORT=8088
 MAPBOX_API_KEY=''
+
+# Make sure you set this to a unique secure random value on production
+SUPERSET_SECRET_KEY=TEST_NON_DEV_SECRET
+
+ENABLE_PLAYWRIGHT=false
+PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
+BUILD_SUPERSET_FRONTEND_IN_DOCKER=true

apps/superset/src/docker/apt-install.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -euo pipefail
+
+# Ensure this script is run as root
+if [[ $EUID -ne 0 ]]; then
+  echo "This script must be run as root" >&2
+  exit 1
+fi
+
+# Check for required arguments
+if [[ $# -lt 1 ]]; then
+  echo "Usage: $0 <package1> [<package2> ...]" >&2
+  exit 1
+fi
+
+# Colors for better logging (optional)
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+RESET='\033[0m'
+
+# Install packages with clean-up
+echo -e "${GREEN}Updating package lists...${RESET}"
+apt-get update -qq
+
+echo -e "${GREEN}Installing packages: $@${RESET}"
+apt-get install -yqq --no-install-recommends "$@"
+
+echo -e "${GREEN}Autoremoving unnecessary packages...${RESET}"
+apt-get autoremove -y
+
+echo -e "${GREEN}Cleaning up package cache and metadata...${RESET}"
+apt-get clean
+rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* /tmp/* /var/tmp/*
+
+echo -e "${GREEN}Installation and cleanup complete.${RESET}"

apps/superset/src/docker/docker-bootstrap.sh

@@ -17,7 +17,7 @@
 #
 
 set -eo pipefail
-
+pip install sqlalchemy-drill psycopg2-binary
 REQUIREMENTS_LOCAL="/app/docker/requirements-local.txt"
 # If Cypress run – overwrite the password for admin and export env variables
 if [ "$CYPRESS_CONFIG" == "true" ]; then
@@ -35,18 +35,11 @@ else
   echo "Skipping local overrides"
 fi
 
-#
-# playwright is an optional package - run only if it is installed
-#
-if command -v playwright > /dev/null 2>&1; then
-  playwright install-deps
-  playwright install chromium
-fi
-
 case "${1}" in
   worker)
     echo "Starting Celery worker..."
-    celery --app=superset.tasks.celery_app:app worker -O fair -l INFO
+    # setting up only 2 workers by default to contain memory usage in dev environments
+    celery --app=superset.tasks.celery_app:app worker -O fair -l INFO --concurrency=${CELERYD_CONCURRENCY:-2}
     ;;
   beat)
     echo "Starting Celery beat..."

apps/superset/src/docker/docker-entrypoint-initdb.d/examples-init.sh

@@ -1,3 +1,22 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 # ------------------------------------------------------------------------
 # Creates the examples database and respective user. This database location
 # and access credentials are defined on the environment variables

apps/superset/src/docker/docker-frontend.sh

@@ -18,12 +18,27 @@
 set -e
 
 # Packages needed for puppeteer:
-apt update
-apt install -y chromium
+if [ "$PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" = "false" ]; then
+    apt update
+    apt install -y chromium
+fi
 
-cd /app/superset-frontend
-npm install -f --no-optional --global webpack webpack-cli
-npm install -f --no-optional
+if [ "$BUILD_SUPERSET_FRONTEND_IN_DOCKER" = "true" ]; then
+    echo "Building Superset frontend in dev mode inside docker container"
+    cd /app/superset-frontend
 
-echo "Running frontend"
-npm run dev
+    if [ "$NPM_RUN_PRUNE" = "true" ]; then
+        echo "Running `npm run prune`"
+        npm run prune
+    fi
+
+    echo "Running `npm install`"
+    npm install
+
+    echo "Running frontend"
+    npm run dev
+
+else
+    echo "Skipping frontend build steps - YOU NEED TO RUN IT MANUALLY ON THE HOST!"
+    echo "https://superset.apache.org/docs/contributing/development/#webpack-dev-server"
+fi

apps/superset/src/docker/docker-init.sh

@@ -22,7 +22,11 @@ set -e
 #
 /app/docker/docker-bootstrap.sh
 
-STEP_CNT=4
+if [ "$SUPERSET_LOAD_EXAMPLES" = "yes" ]; then
+    STEP_CNT=4
+else
+    STEP_CNT=3
+fi
 
 echo_step() {
 cat <<EOF
@@ -57,7 +61,7 @@ superset fab create-admin \
               --firstname Superset \
               --lastname Admin \
               --email admin@superset.com \
-              --password $ADMIN_PASSWORD
+              --password "$ADMIN_PASSWORD"
 echo_step "2" "Complete" "Setting up admin user"
 # Create default roles and permissions
 echo_step "3" "Starting" "Setting up roles and perms"

apps/superset/src/docker/pip-install.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -euo pipefail
+
+# Default flag
+REQUIRES_BUILD_ESSENTIAL=false
+USE_CACHE=true
+
+# Filter arguments
+ARGS=()
+for arg in "$@"; do
+  case "$arg" in
+    --requires-build-essential)
+      REQUIRES_BUILD_ESSENTIAL=true
+      ;;
+    --no-cache)
+      USE_CACHE=false
+      ;;
+    *)
+      ARGS+=("$arg")
+      ;;
+  esac
+done
+
+# Install build-essential if required
+if $REQUIRES_BUILD_ESSENTIAL; then
+  echo "Installing build-essential for package builds..."
+  apt-get update -qq \
+    && apt-get install -yqq --no-install-recommends build-essential
+fi
+
+# Choose whether to use pip cache
+if $USE_CACHE; then
+  echo "Using pip cache..."
+  uv pip install --system "${ARGS[@]}"
+else
+  echo "Disabling pip cache..."
+  uv pip install --system --no-cache-dir "${ARGS[@]}"
+fi
+
+# Remove build-essential if it was installed
+if $REQUIRES_BUILD_ESSENTIAL; then
+  echo "Removing build-essential to keep the image lean..."
+  apt-get autoremove -yqq --purge build-essential \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
+fi
+
+echo "Python packages installed successfully."

apps/superset/src/docker/pythonpath_dev/superset_config.py

@@ -28,13 +28,6 @@
 
 logger = logging.getLogger()
 
-# refer to: https://superset.apache.org/docs/contributing/translations/#enabling-language-selection 
-LANGUAGES = {
-    'en': {'flag': 'us', 'name': 'English'},
-    'fr': {'flag': 'fr', 'name': 'French'},
-    'zh': {'flag': 'cn', 'name': 'Chinese'},
-}
-
 DATABASE_DIALECT = os.getenv("DATABASE_DIALECT")
 DATABASE_USER = os.getenv("DATABASE_USER")
 DATABASE_PASSWORD = os.getenv("DATABASE_PASSWORD")
@@ -81,7 +74,12 @@
 
 class CeleryConfig:
     broker_url = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}"
-    imports = ("superset.sql_lab",)
+    imports = (
+        "superset.sql_lab",
+        "superset.tasks.scheduler",
+        "superset.tasks.thumbnails",
+        "superset.tasks.cache",
+    )
     result_backend = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_RESULTS_DB}"
     worker_prefetch_multiplier = 1
     task_acks_late = False
@@ -101,10 +99,9 @@ class CeleryConfig:
 
 FEATURE_FLAGS = {"ALERT_REPORTS": True}
 ALERT_REPORTS_NOTIFICATION_DRY_RUN = True
-WEBDRIVER_BASEURL = "http://superset:8088/"
+WEBDRIVER_BASEURL = "http://superset:8088/"  # When using docker compose baseurl should be http://superset_app:8088/
 # The base URL for the email report hyperlinks.
 WEBDRIVER_BASEURL_USER_FRIENDLY = WEBDRIVER_BASEURL
-
 SQLLAB_CTAS_NO_LIMIT = True
 
 #

apps/superset/src/docker/pythonpath_dev/superset_config_local.example

@@ -18,7 +18,7 @@
 #
 # This is an example "local" configuration file. In order to set/override config
 # options that ONLY apply to your local environment, simply copy/rename this file
-# to docker/pythonpath/superset_config_docker.py
+# to docker/pythonpath_dev/superset_config_docker.py
 # It ends up being imported by docker/superset_config.py which is loaded by
 # superset/config.py
 #

apps/superset/src/docker/run-server.sh

@@ -26,6 +26,7 @@ gunicorn \
     --workers ${SERVER_WORKER_AMOUNT:-1} \
     --worker-class ${SERVER_WORKER_CLASS:-gthread} \
     --threads ${SERVER_THREADS_AMOUNT:-20} \
+    --log-level "${GUNICORN_LOGLEVEL:info}" \
     --timeout ${GUNICORN_TIMEOUT:-60} \
     --keep-alive ${GUNICORN_KEEPALIVE:-2} \
     --max-requests ${WORKER_MAX_REQUESTS:-0} \