fix(log): reduce log volume for ingestion and consumers

datahub-project · Oct 25, 2024 · 984c725 · 984c725
1 parent 87fa5b8
commit 984c725
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 14 deletions.
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java
@@ -164,6 +164,8 @@ public class EntityServiceImpl implements EntityService<ChangeItemImpl> {
   private final Integer ebeanMaxTransactionRetry;
   private final boolean enableBrowseV2;
 
+  private static final long DB_TIMER_LOG_THRESHOLD_MS = 50;
+
   @Getter
   private final Map<Set<ThrottleType>, ThrottleEvent> throttleEvents = new ConcurrentHashMap<>();
 
@@ -997,10 +999,10 @@ private List<UpdateAspectResult> ingestAspectsToLocalDB(
               if (txContext != null) {
                 txContext.commitAndContinue();
               }
-              long took = ingestToLocalDBTimer.stop();
-              log.info(
-                  "Ingestion of aspects batch to database took {} ms",
-                  TimeUnit.NANOSECONDS.toMillis(took));
+              long took = TimeUnit.NANOSECONDS.toMillis(ingestToLocalDBTimer.stop());
+              if (took > DB_TIMER_LOG_THRESHOLD_MS) {
+                log.info("Ingestion of aspects batch to database took {} ms", took);
+              }
 
               // Retention optimization and tx
               if (retentionService != null) {

diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListener.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListener.java
@@ -95,7 +95,7 @@ public void consume(final ConsumerRecord<String, GenericRecord> consumerRecord)
 
       // Here - plug in additional "custom processor hooks"
       for (MetadataChangeLogHook hook : this.hooks) {
-        log.info(
+        log.debug(
             "Invoking MCL hook {} for urn: {}",
             hook.getClass().getSimpleName(),
             event.getEntityUrn());

diff --git a/...tli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java b/...tli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java
@@ -243,13 +243,6 @@ public Task<String> ingestProposal(
       @ActionParam(PARAM_PROPOSAL) @Nonnull MetadataChangeProposal metadataChangeProposal,
       @ActionParam(PARAM_ASYNC) @Optional(UNSET) String async)
       throws URISyntaxException {
-
-      String urn = metadataChangeProposal.getEntityUrn() != null ? metadataChangeProposal.getEntityUrn().toString() :
-        java.util.Optional.ofNullable(metadataChangeProposal.getEntityKeyAspect()).orElse(new GenericAspect())
-            .getValue().asString(StandardCharsets.UTF_8);
-    String proposedValue = java.util.Optional.ofNullable(metadataChangeProposal.getAspect()).orElse(new GenericAspect())
-        .getValue().asString(StandardCharsets.UTF_8);
-
     final boolean asyncBool;
     if (UNSET.equals(async)) {
       asyncBool = Boolean.parseBoolean(System.getenv(ASYNC_INGEST_DEFAULT_NAME));
@@ -266,8 +259,6 @@ public Task<String> ingestProposalBatch(
             @ActionParam(PARAM_PROPOSALS) @Nonnull MetadataChangeProposal[] metadataChangeProposals,
             @ActionParam(PARAM_ASYNC) @Optional(UNSET) String async)
             throws URISyntaxException {
-        log.info("INGEST PROPOSAL BATCH proposals: {}", Arrays.asList(metadataChangeProposals));
-
         final boolean asyncBool;
         if (UNSET.equals(async)) {
             asyncBool = Boolean.parseBoolean(System.getenv(ASYNC_INGEST_DEFAULT_NAME));