fix: minor comments

datahub-project · Oct 28, 2024 · 8efa349 · 8efa349
1 parent 102b55f
commit 8efa349
Show file tree

Hide file tree

Showing 3 changed files with 1 addition and 50 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_aspects.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_aspects.py
@@ -176,12 +176,6 @@ def get_container_urn(
         container_key = self.get_container_key(name, path)
         return container_key.as_urn()
 
-    def get_container_space_urn(self) -> str:
-        return self.get_container_urn(name="Spaces", path=[])
-
-    def get_container_source_urn(self) -> str:
-        return self.get_container_urn(name="Sources", path=[])
-
     def create_domain_aspect(self) -> Optional[_Aspect]:
         if self.domain:
             if self.domain.startswith("urn:li:domain:"):
@@ -225,16 +219,6 @@ def populate_container_mcp(
             )
             yield mcp.as_workunit()
 
-        # Container Class for Spaces and Sources
-        if not container.path:
-            container_class = self._create_container_class_containers(container)
-            if container_class:
-                mcp = MetadataChangeProposalWrapper(
-                    entityUrn=container_urn,
-                    aspect=container_class,
-                )
-                yield mcp.as_workunit()
-
         # Data Platform Instance
         data_platform_instance = self._create_data_platform_instance()
         if data_platform_instance:
@@ -409,16 +393,6 @@ def _create_container_class(
             return ContainerClass(container=self.get_container_urn(path=entity.path))
         return None
 
-    def _create_container_class_containers(
-        self, entity: DremioContainer
-    ) -> Optional[ContainerClass]:
-        if entity.subclass == "Dremio Space":
-            return ContainerClass(container=self.get_container_space_urn())
-        elif entity.subclass == "Dremio Source":
-            return ContainerClass(container=self.get_container_source_urn())
-
-        return None
-
     def _create_data_platform_instance(self) -> DataPlatformInstanceClass:
         return DataPlatformInstanceClass(
             platform=f"urn:li:dataPlatform:{self.platform}",
@@ -516,12 +490,6 @@ def _create_schema_field(self, column: DremioDatasetColumn) -> SchemaFieldClass:
             nullable=column.is_nullable == "YES",
         )
 
-    def _get_profile_data(self, dataset: DremioDataset) -> Dict:
-        return self.profiler.profile_table(
-            f"{'.'.join(dataset.path)}.{dataset.resource_name}",
-            [(col.name, col.data_type) for col in dataset.columns],
-        )
-
     def _create_view_properties(
         self, dataset: DremioDataset
     ) -> Optional[ViewPropertiesClass]:

diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_config.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_config.py
@@ -2,7 +2,7 @@
 from typing import List, Literal, Optional
 
 import certifi
-from pydantic import Field, root_validator, validator
+from pydantic import Field, validator
 
 from datahub.configuration.common import AllowDenyPattern, ConfigModel
 from datahub.configuration.source_common import (
@@ -119,15 +119,6 @@ class DremioSourceMapping(EnvConfigMixin, PlatformInstanceConfigMixin, ConfigMod
         description="Alias of platform in Dremio connection",
     )
 
-    @root_validator
-    def check_both_fields_present(cls, values):
-        platform, source_name = values.get("platform"), values.get("source_name")
-        if not platform or not source_name:
-            raise ValueError(
-                "Both 'platform' and 'source_name' must be provided in source_mappings."
-            )
-        return values
-
 
 class DremioSourceConfig(
     DremioConnectionConfig,

diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py
@@ -276,14 +276,6 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
 
         self.source_map = self._build_source_map()
 
-        space_urn = self.dremio_aspects.get_container_space_urn()
-        container = DremioContainer("Spaces", "", [], self.dremio_catalog.dremio_api)
-        yield from self.dremio_aspects.populate_container_mcp(space_urn, container)
-
-        source_urn = self.dremio_aspects.get_container_source_urn()
-        container = DremioContainer("Sources", "", [], self.dremio_catalog.dremio_api)
-        yield from self.dremio_aspects.populate_container_mcp(source_urn, container)
-
         # Process Containers
         containers = self.dremio_catalog.get_containers()
         for container in containers: