From 0f64f3222d2d366b377401c2707312564e9da090 Mon Sep 17 00:00:00 2001
From: Daniel Standish <15932138+dstandish@users.noreply.github.com>
Date: Fri, 20 Sep 2024 12:05:02 -0700
Subject: [PATCH] Remove redundant dag_id index on log table (#42376)
This index is not needed because there's another index on the table that leads with dag_id.
---
.../0033_3_0_0_remove_redundant_index.py | 49 ++++
airflow/models/log.py | 1 -
airflow/utils/db.py | 2 +-
docs/apache-airflow/img/airflow_erd.sha256 | 2 +-
docs/apache-airflow/img/airflow_erd.svg | 245 +++++++++---------
docs/apache-airflow/migrations-ref.rst | 4 +-
6 files changed, 182 insertions(+), 121 deletions(-)
create mode 100644 airflow/migrations/versions/0033_3_0_0_remove_redundant_index.py
diff --git a/airflow/migrations/versions/0033_3_0_0_remove_redundant_index.py b/airflow/migrations/versions/0033_3_0_0_remove_redundant_index.py
new file mode 100644
index 000000000000..fff72e4b0e9f
--- /dev/null
+++ b/airflow/migrations/versions/0033_3_0_0_remove_redundant_index.py
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Remove redundant index.
+
+Revision ID: 16cbcb1c8c36
+Revises: 522625f6d606
+Create Date: 2024-09-20 09:32:48.707545
+
+"""
+
+from __future__ import annotations
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "16cbcb1c8c36"
+down_revision = "522625f6d606"
+branch_labels = None
+depends_on = None
+airflow_version = "3.0.0"
+
+
+def upgrade():
+ """Apply Remove redundant index."""
+ with op.batch_alter_table("log", schema=None) as batch_op:
+ batch_op.drop_index("idx_log_dag")
+
+
+def downgrade():
+ """Unapply Remove redundant index."""
+ with op.batch_alter_table("log", schema=None) as batch_op:
+ batch_op.create_index("idx_log_dag", ["dag_id"], unique=False)
diff --git a/airflow/models/log.py b/airflow/models/log.py
index 145f659b338d..e01cc29b45e0 100644
--- a/airflow/models/log.py
+++ b/airflow/models/log.py
@@ -49,7 +49,6 @@ class Log(Base):
try_number = Column(Integer)
__table_args__ = (
- Index("idx_log_dag", dag_id),
Index("idx_log_dttm", dttm),
Index("idx_log_event", event),
Index("idx_log_task_instance", dag_id, task_id, run_id, map_index, try_number),
diff --git a/airflow/utils/db.py b/airflow/utils/db.py
index 1dc7275610df..b11e744198dc 100644
--- a/airflow/utils/db.py
+++ b/airflow/utils/db.py
@@ -96,7 +96,7 @@ class MappedClassProtocol(Protocol):
"2.9.0": "1949afb29106",
"2.9.2": "686269002441",
"2.10.0": "22ed7efa9da2",
- "3.0.0": "522625f6d606",
+ "3.0.0": "16cbcb1c8c36",
}
diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256
index 94ef213472b8..237c598ec1dc 100644
--- a/docs/apache-airflow/img/airflow_erd.sha256
+++ b/docs/apache-airflow/img/airflow_erd.sha256
@@ -1 +1 @@
-d58a76011bd215ac7a0c42146986ad0eead5f4a9c8ffd7932b1aa1eb61e6e4af
\ No newline at end of file
+f4379048d3f13f35aaba824c00450c17ad4deea9af82b5498d755a12f8a85a37
\ No newline at end of file
diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg
index 3eaf0f0e52ab..65f94c58ad24 100644
--- a/docs/apache-airflow/img/airflow_erd.svg
+++ b/docs/apache-airflow/img/airflow_erd.svg
@@ -64,131 +64,131 @@
[INTEGER]
-
-
-dag_priority_parsing_request
-
-dag_priority_parsing_request
-
-id
-
- [VARCHAR(32)]
- NOT NULL
-
-fileloc
-
- [VARCHAR(2000)]
- NOT NULL
-
-
+
job
-
-job
-
-id
-
- [INTEGER]
- NOT NULL
-
-dag_id
-
- [VARCHAR(250)]
-
-end_date
-
- [TIMESTAMP]
-
-executor_class
-
- [VARCHAR(500)]
-
-hostname
-
- [VARCHAR(500)]
-
-job_type
-
- [VARCHAR(30)]
-
-latest_heartbeat
-
- [TIMESTAMP]
-
-start_date
-
- [TIMESTAMP]
-
-state
-
- [VARCHAR(20)]
-
-unixname
-
- [VARCHAR(1000)]
+
+job
+
+id
+
+ [INTEGER]
+ NOT NULL
+
+dag_id
+
+ [VARCHAR(250)]
+
+end_date
+
+ [TIMESTAMP]
+
+executor_class
+
+ [VARCHAR(500)]
+
+hostname
+
+ [VARCHAR(500)]
+
+job_type
+
+ [VARCHAR(30)]
+
+latest_heartbeat
+
+ [TIMESTAMP]
+
+start_date
+
+ [TIMESTAMP]
+
+state
+
+ [VARCHAR(20)]
+
+unixname
+
+ [VARCHAR(1000)]
-
+
slot_pool
-
-slot_pool
-
-id
-
- [INTEGER]
- NOT NULL
-
-description
-
- [TEXT]
-
-include_deferred
-
- [BOOLEAN]
- NOT NULL
-
-pool
-
- [VARCHAR(256)]
-
-slots
-
- [INTEGER]
+
+slot_pool
+
+id
+
+ [INTEGER]
+ NOT NULL
+
+description
+
+ [TEXT]
+
+include_deferred
+
+ [BOOLEAN]
+ NOT NULL
+
+pool
+
+ [VARCHAR(256)]
+
+slots
+
+ [INTEGER]
-
+
callback_request
-
-callback_request
-
-id
-
- [INTEGER]
- NOT NULL
-
-callback_data
-
- [JSON]
- NOT NULL
-
-callback_type
-
- [VARCHAR(20)]
- NOT NULL
-
-created_at
-
- [TIMESTAMP]
- NOT NULL
-
-priority_weight
-
- [INTEGER]
- NOT NULL
-
-processor_subdir
-
- [VARCHAR(2000)]
+
+callback_request
+
+id
+
+ [INTEGER]
+ NOT NULL
+
+callback_data
+
+ [JSON]
+ NOT NULL
+
+callback_type
+
+ [VARCHAR(20)]
+ NOT NULL
+
+created_at
+
+ [TIMESTAMP]
+ NOT NULL
+
+priority_weight
+
+ [INTEGER]
+ NOT NULL
+
+processor_subdir
+
+ [VARCHAR(2000)]
+
+
+
+dag_priority_parsing_request
+
+dag_priority_parsing_request
+
+id
+
+ [VARCHAR(32)]
+ NOT NULL
+
+fileloc
+
+ [VARCHAR(2000)]
+ NOT NULL
@@ -2349,5 +2349,16 @@
0..N
{0,1}
+
+
+alembic_version_fab
+
+alembic_version_fab
+
+version_num
+
+ [VARCHAR(32)]
+ NOT NULL
+
diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst
index 65a07aaa4393..ded2b290b5a4 100644
--- a/docs/apache-airflow/migrations-ref.rst
+++ b/docs/apache-airflow/migrations-ref.rst
@@ -39,7 +39,9 @@ Here's the list of all the Database Migrations that are executed via when you ru
+-------------------------+------------------+-------------------+--------------------------------------------------------------+
| Revision ID | Revises ID | Airflow Version | Description |
+=========================+==================+===================+==============================================================+
-| ``522625f6d606`` (head) | ``1cdc775ca98f`` | ``3.0.0`` | Add tables for backfill. |
+| ``16cbcb1c8c36`` (head) | ``522625f6d606`` | ``3.0.0`` | Remove redundant index. |
++-------------------------+------------------+-------------------+--------------------------------------------------------------+
+| ``522625f6d606`` | ``1cdc775ca98f`` | ``3.0.0`` | Add tables for backfill. |
+-------------------------+------------------+-------------------+--------------------------------------------------------------+
| ``1cdc775ca98f`` | ``a2c32e6c7729`` | ``3.0.0`` | Drop ``execution_date`` unique constraint on DagRun. |
+-------------------------+------------------+-------------------+--------------------------------------------------------------+