Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added support for reindexing of dataExpiration #2211

Merged
merged 6 commits into from
Feb 16, 2023
Merged

Added support for reindexing of dataExpiration #2211

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ea62561
Added support for reindexing of dataExpiration
Madhu1029 Jan 11, 2023
ee64582
Merge remote-tracking branch 'upstream/master' into issue2160_dataExp…
Madhu1029 Jan 12, 2023
3526c0a
Merge branch 'master' into issue2160_dataExpiration
Madhu1029 Feb 6, 2023
1458daf
Fixed for suggestions
Madhu1029 Feb 9, 2023
ffafa96
Updated CHANGES_NEXT_RELEASE
Madhu1029 Feb 9, 2023
b6045a0
Updated for dataExpiration
Madhu1029 Feb 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGES_NEXT_RELEASE
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[cygnus-common] Added support for reindexing of dataExpiration (#2160, partially)
[cygnus-ngsi] Upgrade Debian version from 11.3 to 11.6 in Dockerfile
[cygnus-ngsi] Update NameMapping compile log mensage from WARN to ERROR
[cygnus-common] MongoDB indexes are created depending on DM (#2204)
Expand Down
147 changes: 76 additions & 71 deletions cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,41 +101,33 @@ public void createCollection(String dbName, String collectionName, long dataExpi
// check STH indexes documentation at https://github.com/telefonicaid/fiware-sth-comet/blob/master/doc/manuals/db_indexes.md
BasicDBObject keys;
IndexOptions options;
try {
keys = new BasicDBObject();
switch(dataModel) {
case DMBYSERVICEPATH:
keys.append("_id.entityId", 1)
.append("_id.entityType", 1)
.append("_id.attrName", 1)
.append("_id.resolution", 1)
.append("_id.origin", 1);
break;
case DMBYENTITY:
keys.append("_id.attrName", 1)
.append("_id.resolution", 1)
.append("_id.origin", 1);
break;
case DMBYATTRIBUTE:
keys.append("_id.resolution", 1)
.append("_id.origin", 1);
break;
default:
}
options = new IndexOptions().name("cyg_agg_opt");
db.getCollection(collectionName).createIndex(keys, options);
} catch (Exception e) {
LOGGER.warn("Error in collection " + collectionName + " creating index ex=" + e.getMessage());
} // try catch
try {
if (dataExpiration != 0) {
keys = new BasicDBObject().append("_id.origin", 1);
options = new IndexOptions().name("cyg_agg_exp").expireAfter(dataExpiration, TimeUnit.SECONDS);
db.getCollection(collectionName).createIndex(keys, options);
} // if
} catch (Exception e) {
LOGGER.warn("Error in collection " + collectionName + " creating index ex=" + e.getMessage());
} // try catch
keys = new BasicDBObject();
switch(dataModel) {
case DMBYSERVICEPATH:
keys.append("_id.entityId", 1)
.append("_id.entityType", 1)
.append("_id.attrName", 1)
.append("_id.resolution", 1)
.append("_id.origin", 1);
break;
case DMBYENTITY:
keys.append("_id.attrName", 1)
.append("_id.resolution", 1)
.append("_id.origin", 1);
break;
case DMBYATTRIBUTE:
keys.append("_id.resolution", 1)
.append("_id.origin", 1);
break;
default:
}
options = new IndexOptions().name("cyg_agg_opt");
createIndex(db, collectionName, keys, options);
if (dataExpiration != 0) {
keys = new BasicDBObject().append("_id.origin", 1);
options = new IndexOptions().name("cyg_agg_exp").expireAfter(dataExpiration, TimeUnit.SECONDS);
createIndex(db, collectionName, keys, options);
} // if
} // createCollection

/**
Expand Down Expand Up @@ -179,45 +171,58 @@ public void createCollection(String dbName, String collectionName, long collecti
//Index creation based on data model
BasicDBObject keys;
IndexOptions options;
keys = new BasicDBObject();
switch(dataModel) {
case DMBYSERVICEPATH:
keys.append("recvTime", 1)
.append("entityId", 1)
.append("entityType", 1)
.append("attrName", 1)
.append("attrType",1)
.append("attrValue",1);
break;
case DMBYENTITY:
keys.append("recvTime", 1)
.append("attrName", 1)
.append("attrType",1)
.append("attrValue",1);
break;
case DMBYATTRIBUTE:
keys.append("recvTime", 1)
.append("attrType",1)
.append("attrValue",1);
break;
default:
}
options = new IndexOptions().name("cyg_raw_opt");
createIndex(db, collectionName, keys, options);
if (dataExpiration != 0) {
keys = new BasicDBObject().append("recvTime", 1);
options = new IndexOptions().name("cyg_raw_exp").expireAfter(dataExpiration, TimeUnit.SECONDS);
createIndex(db, collectionName, keys, options);
} // if
} // createCollection

/**
* Create an index for dataExpiration in the given raw collection within the given database.
* @param db
* @param collectionName
* @param keys
* @param options
*/
public void createIndex(MongoDatabase db, String collectionName, BasicDBObject keys,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is throws MongoException correct?

Given you capture the exception with the try/catch block, I would say that createIndex() is not able to throw any Exception.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 1458daf

IndexOptions options) {
try {
keys = new BasicDBObject();
switch(dataModel) {
case DMBYSERVICEPATH:
keys.append("recvTime", 1)
.append("entityId", 1)
.append("entityType", 1)
.append("attrName", 1)
.append("attrType",1)
.append("attrValue",1);
break;
case DMBYENTITY:
keys.append("recvTime", 1)
.append("attrName", 1)
.append("attrType",1)
.append("attrValue",1);
break;
case DMBYATTRIBUTE:
keys.append("recvTime", 1)
.append("attrType",1)
.append("attrValue",1);
break;
default:
}
options = new IndexOptions().name("cyg_raw_opt");
db.getCollection(collectionName).createIndex(keys, options);
} catch (Exception e) {
LOGGER.warn("Error in collection " + collectionName + " creating index ex=" + e.getMessage());
} // try catch
try {
if (dataExpiration != 0) {
keys = new BasicDBObject().append("recvTime", 1);
options = new IndexOptions().name("cyg_raw_exp").expireAfter(dataExpiration, TimeUnit.SECONDS);
db.getCollection(collectionName).createIndex(keys, options);
} // if
} catch (Exception e) {
LOGGER.warn("Error in collection " + collectionName + " creating index ex=" + e.getMessage());
} catch(Exception e) {
if (e.getMessage().contains("IndexOptionsConflict")) {
db.getCollection(collectionName).dropIndex(options.getName());
createIndex(db, collectionName, keys, options);
} else {
LOGGER.warn("Error in collection " + collectionName + " creating index ex=" + e.getMessage());
}
} // try catch
} // createCollection
} // createIndex

/**
* Inserts a new document in the given raw collection within the given database (row-like mode).
Expand Down