Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(log): reduce log volume for ingestion and consumers #11714

Merged
merged 1 commit into from
Oct 26, 2024
Merged

fix(log): reduce log volume for ingestion and consumers #11714

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,8 @@ public class EntityServiceImpl implements EntityService<ChangeItemImpl> {
private final Integer ebeanMaxTransactionRetry;
private final boolean enableBrowseV2;

private static final long DB_TIMER_LOG_THRESHOLD_MS = 50;

@Getter
private final Map<Set<ThrottleType>, ThrottleEvent> throttleEvents = new ConcurrentHashMap<>();

Expand Down Expand Up @@ -997,10 +999,10 @@ private List<UpdateAspectResult> ingestAspectsToLocalDB(
if (txContext != null) {
txContext.commitAndContinue();
}
long took = ingestToLocalDBTimer.stop();
log.info(
"Ingestion of aspects batch to database took {} ms",
TimeUnit.NANOSECONDS.toMillis(took));
long took = TimeUnit.NANOSECONDS.toMillis(ingestToLocalDBTimer.stop());
if (took > DB_TIMER_LOG_THRESHOLD_MS) {
log.info("Ingestion of aspects batch to database took {} ms", took);
}

// Retention optimization and tx
if (retentionService != null) {
Expand Down
2 changes: 1 addition & 1 deletion ...ata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ public void appendRunId(
@Nullable String runId) {
final String docId = indexBuilders.getIndexConvention().getEntityDocumentId(urn);

log.info(
log.debug(
"Appending run id for entity name: {}, doc id: {}, run id: {}", entityName, docId, runId);
esWriteDAO.applyScriptUpdate(
opContext,
Expand Down
2 changes: 1 addition & 1 deletion metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListener.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ public void consume(final ConsumerRecord<String, GenericRecord> consumerRecord)

// Here - plug in additional "custom processor hooks"
for (MetadataChangeLogHook hook : this.hooks) {
log.info(
log.debug(
"Invoking MCL hook {} for urn: {}",
hook.getClass().getSimpleName(),
event.getEntityUrn());
Expand Down
9 changes: 0 additions & 9 deletions ...tli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,13 +243,6 @@ public Task<String> ingestProposal(
@ActionParam(PARAM_PROPOSAL) @Nonnull MetadataChangeProposal metadataChangeProposal,
@ActionParam(PARAM_ASYNC) @Optional(UNSET) String async)
throws URISyntaxException {

String urn = metadataChangeProposal.getEntityUrn() != null ? metadataChangeProposal.getEntityUrn().toString() :
java.util.Optional.ofNullable(metadataChangeProposal.getEntityKeyAspect()).orElse(new GenericAspect())
.getValue().asString(StandardCharsets.UTF_8);
String proposedValue = java.util.Optional.ofNullable(metadataChangeProposal.getAspect()).orElse(new GenericAspect())
.getValue().asString(StandardCharsets.UTF_8);

final boolean asyncBool;
if (UNSET.equals(async)) {
asyncBool = Boolean.parseBoolean(System.getenv(ASYNC_INGEST_DEFAULT_NAME));
Expand All @@ -266,8 +259,6 @@ public Task<String> ingestProposalBatch(
@ActionParam(PARAM_PROPOSALS) @Nonnull MetadataChangeProposal[] metadataChangeProposals,
@ActionParam(PARAM_ASYNC) @Optional(UNSET) String async)
throws URISyntaxException {
log.info("INGEST PROPOSAL BATCH proposals: {}", Arrays.asList(metadataChangeProposals));

final boolean asyncBool;
if (UNSET.equals(async)) {
asyncBool = Boolean.parseBoolean(System.getenv(ASYNC_INGEST_DEFAULT_NAME));
Expand Down
Loading