Merge pull request #25 from /issues/24

Cope with $in / $nin unknown queries

README.md

@@ -75,6 +75,12 @@ Use docker compose to start:
 docker-compose up -d --build
 ```
 
+## Adding new tags
+
+Support for new tags can be done by simply adding a further entity definition in the product-tags.ts file.
+
+The tag won't be picked up for queries until a full import is done (when it will be added to the loaded_tag table).
+
 # Deployment vs Development
 
 The main docker-compose.yml creates the openfoodfacts-query service and associated Postres database and expects MongoDB to already exist.
@@ -101,6 +107,4 @@ There is also an importfromfile endpoint which will import from a file called op
 
 The "count" and "aggregate" POST endpoints accept a MongoDB style filter and aggregate pipeline respectively. Syntax support is only basic and is limted to what Product Opener currently uses. See the tests for some examples of what is supported.
 
-# TODO
 
-- Configure production deployment

src/domain/entities/product-tag-map.ts

@@ -0,0 +1,7 @@
+export class ProductTagMap {
+    static MAPPED_TAGS: { [tag: string] : any; } = {};
+    static mapTag(name, entityClass) {
+        this.MAPPED_TAGS[name] = entityClass;
+    }
+}
+

src/domain/entities/product-tag.ts

@@ -0,0 +1,9 @@
+import { Entity } from '@mikro-orm/core';
+import { ProductTagMap } from './product-tag-map';
+
+export function ProductTag(name: string) {
+  return function (target) {
+    ProductTagMap.mapTag(name, target);
+    Entity()(target);
+  };
+}

src/domain/entities/product-tags.ts

@@ -1,75 +1,76 @@
-import { Entity } from '@mikro-orm/core';
 import { BaseProductTag } from './base-product-tag';
+import { ProductTag } from './product-tag';
 
-@Entity()
+@ProductTag('countries_tags')
 export class ProductCountriesTag extends BaseProductTag {}
-@Entity()
+@ProductTag('nutrition_grades_tags')
 export class ProductNutritionGradesTag extends BaseProductTag {}
-@Entity()
+@ProductTag('nova_groups_tags')
 export class ProductNovaGroupsTag extends BaseProductTag {}
-@Entity()
+@ProductTag('ecoscore_tags')
 export class ProductEcoscoreTag extends BaseProductTag {}
-@Entity()
+@ProductTag('brands_tags')
 export class ProductBrandsTag extends BaseProductTag {}
-@Entity()
+@ProductTag('categories_tags')
 export class ProductCategoriesTag extends BaseProductTag {}
-@Entity()
+@ProductTag('labels_tags')
 export class ProductLabelsTag extends BaseProductTag {}
-@Entity()
+@ProductTag('packaging_tags')
 export class ProductPackagingTag extends BaseProductTag {}
-@Entity()
+@ProductTag('origins_tags')
 export class ProductOriginsTag extends BaseProductTag {}
-@Entity()
+@ProductTag('manufacturing_places_tags')
 export class ProductManufacturingPlacesTag extends BaseProductTag {}
-@Entity()
+@ProductTag('emb_codes_tags')
 export class ProductEmbCodesTag extends BaseProductTag {}
-@Entity()
+@ProductTag('ingredients_tags')
 export class ProductIngredientsTag extends BaseProductTag {}
-@Entity()
+@ProductTag('additives_tags')
 export class ProductAdditivesTag extends BaseProductTag {}
-@Entity()
+@ProductTag('vitamins_tags')
 export class ProductVitaminsTag extends BaseProductTag {}
-@Entity()
+@ProductTag('minerals_tags')
 export class ProductMineralsTag extends BaseProductTag {}
-@Entity()
+@ProductTag('amino_acids_tags')
 export class ProductAminoAcidsTag extends BaseProductTag {}
-@Entity()
+@ProductTag('nucleotides_tags')
 export class ProductNucleotidesTag extends BaseProductTag {}
-@Entity()
+@ProductTag('other_nutritional_substances_tags')
 export class ProductOtherNutritionalSubstancesTag extends BaseProductTag {}
-@Entity()
+@ProductTag('allergens_tags')
 export class ProductAllergensTag extends BaseProductTag {}
-@Entity()
+@ProductTag('traces_tags')
 export class ProductTracesTag extends BaseProductTag {}
-@Entity()
+@ProductTag('misc_tags')
 export class ProductMiscTag extends BaseProductTag {}
-@Entity()
+@ProductTag('languages_tags')
 export class ProductLanguagesTag extends BaseProductTag {}
-@Entity()
+@ProductTag('states_tags')
 export class ProductStatesTag extends BaseProductTag {}
-@Entity()
+@ProductTag('data_sources_tags')
 export class ProductDataSourcesTag extends BaseProductTag {}
-@Entity()
+@ProductTag('entry_dates_tags')
 export class ProductEntryDatesTag extends BaseProductTag {}
-@Entity()
+@ProductTag('last_edit_dates_tags')
 export class ProductLastEditDatesTag extends BaseProductTag {}
-@Entity()
+@ProductTag('last_check_dates_tags')
 export class ProductLastCheckDatesTag extends BaseProductTag {}
-@Entity()
+// Don't use Teams for query tests as we delete the loaded tag in the Import tests
+@ProductTag('teams_tags')
 export class ProductTeamsTag extends BaseProductTag {}
-@Entity()
+@ProductTag('_keywords')
 export class ProductKeywordsTag extends BaseProductTag {}
-@Entity()
+@ProductTag('codes_tags')
 export class ProductCodesTag extends BaseProductTag {}
-@Entity()
+@ProductTag('data_quality_tags')
 export class ProductDataQualityErrorsTag extends BaseProductTag {}
-@Entity()
+@ProductTag('data_quality_errors_tags')
 export class ProductDataQualityTag extends BaseProductTag {}
-@Entity()
+@ProductTag('editors_tags')
 export class ProductEditorsTag extends BaseProductTag {}
-@Entity()
+@ProductTag('stores_tags')
 export class ProductStoresTag extends BaseProductTag {}
-@Entity()
+@ProductTag('ingredients_original_tags')
 export class ProductIngredientsOriginalTag extends BaseProductTag {}
 
 /* From Config_off.pm
@@ -106,43 +107,3 @@ export class ProductIngredientsOriginalTag extends BaseProductTag {}
   teams
 );
 */
-
-export const MAPPED_TAGS = {
-  countries_tags: ProductCountriesTag,
-  nutrition_grades_tags: ProductNutritionGradesTag,
-  nova_groups_tags: ProductNovaGroupsTag,
-  ecoscore_tags: ProductEcoscoreTag,
-  brands_tags: ProductBrandsTag,
-  categories_tags: ProductCategoriesTag,
-  labels_tags: ProductLabelsTag,
-  packaging_tags: ProductPackagingTag,
-  origins_tags: ProductOriginsTag,
-  manufacturing_places_tags: ProductManufacturingPlacesTag,
-  emb_codes_tags: ProductEmbCodesTag,
-  ingredients_tags: ProductIngredientsTag,
-  additives_tags: ProductAdditivesTag,
-  vitamins_tags: ProductVitaminsTag,
-  minerals_tags: ProductMineralsTag,
-  amino_acids_tags: ProductAminoAcidsTag,
-  nucleotides_tags: ProductNucleotidesTag,
-  other_nutritional_substances_tags: ProductOtherNutritionalSubstancesTag,
-  allergens_tags: ProductAllergensTag,
-  traces_tags: ProductTracesTag,
-  misc_tags: ProductMiscTag,
-  languages_tags: ProductLanguagesTag,
-  states_tags: ProductStatesTag,
-  data_sources_tags: ProductDataSourcesTag,
-  entry_dates_tags: ProductEntryDatesTag,
-  last_edit_dates_tags: ProductLastEditDatesTag,
-  last_check_dates_tags: ProductLastCheckDatesTag,
-  // Note do not use the teams_tags in test data as it is deleted in on eof the tests
-  teams_tags: ProductTeamsTag,
-  // Added later
-  _keywords: ProductKeywordsTag,
-  codes_tags: ProductCodesTag,
-  data_quality_tags: ProductDataQualityTag,
-  data_quality_errors_tags: ProductDataQualityErrorsTag,
-  editors_tags: ProductEditorsTag,
-  stores_tags: ProductStoresTag,
-  ingredients_original_tags: ProductIngredientsOriginalTag,
-};

src/domain/services/import.service.spec.ts

@@ -2,10 +2,11 @@ import { DomainModule } from '../domain.module';
 import { ImportService } from './import.service';
 import { EntityManager } from '@mikro-orm/core';
 import { Product } from '../entities/product';
-import { MAPPED_TAGS, ProductIngredientsTag } from '../entities/product-tags';
+import { ProductIngredientsTag } from '../entities/product-tags';
 import { createTestingModule, randomCode } from '../../../test/test.helper';
 import { TagService } from './tag.service';
 import { LoadedTag } from '../entities/loaded-tag';
+import { ProductTagMap } from '../entities/product-tag-map';
 
 let index = 0;
 const productIdNew = randomCode();
@@ -126,7 +127,7 @@ describe('importFromMongo', () => {
       expect(foundOldProduct.lastUpdateId).not.toBe(updateId);
 
       const loadedTags = await app.get(TagService).getLoadedTags();
-      expect(loadedTags).toHaveLength(Object.keys(MAPPED_TAGS).length);
+      expect(loadedTags).toHaveLength(Object.keys(ProductTagMap.MAPPED_TAGS).length);
     });
   });
 
@@ -173,3 +174,11 @@ describe('importFromMongo', () => {
     });
   });
 });
+
+describe('ProductTag', () => {
+    it('should add class to tag array', async () => {
+        await createTestingModule([DomainModule], async (app) => {
+            expect(ProductTagMap.MAPPED_TAGS['categories_tags']).toBeTruthy();
+        });
+    });
+});

src/domain/services/import.service.ts

@@ -3,10 +3,10 @@ import { MAPPED_FIELDS, Product } from '../entities/product';
 import { Ulid } from 'id128';
 import { MongoClient } from 'mongodb';
 import { EntityManager } from '@mikro-orm/postgresql';
-import { MAPPED_TAGS } from '../entities/product-tags';
 import * as fs from 'fs';
 import * as readline from 'readline';
 import { TagService } from './tag.service';
+import { ProductTagMap } from '../entities/product-tag-map';
 
 @Injectable()
 export class ImportService {
@@ -21,7 +21,7 @@ export class ImportService {
   importBatchSize = 100;
   importLogInterval = 1000;
 
-  private tags = Object.keys(MAPPED_TAGS);
+  private tags = Object.keys(ProductTagMap.MAPPED_TAGS);
 
   /** Import Products from MongoDB */
   async importFromMongo(from?: string, skip?: number) {
@@ -163,7 +163,7 @@ export class ImportService {
    */
   private async updateTags(update: boolean, updateId: string) {
     const connection = this.em.getConnection();
-    for (const [tag, entity] of Object.entries(MAPPED_TAGS)) {
+    for (const [tag, entity] of Object.entries(ProductTagMap.MAPPED_TAGS)) {
       let logText = `Updated ${tag}`;
       // Get the underlying table name for the entity
       const tableName = this.em.getMetadata(entity).tableName;

src/domain/services/query.service.spec.ts

@@ -205,6 +205,81 @@ describe('count', () => {
       expect(response).toBe(3);
     });
   });
+
+  it('should throw an unprocessable exception if an $in contains an array', async () => {
+    await createTestingModule([DomainModule], async (app) => {
+      try {
+        await app
+          .get(QueryService)
+          .count({ origins_tags: { $in: ['a', ['b', 'c']] } });
+        fail('should not get here');
+      } catch (e) {
+        expect(e).toBeInstanceOf(UnprocessableEntityException);
+      }
+    });
+  });
+
+  it('should cope with an $in unknown value', async () => {
+    await createTestingModule([DomainModule], async (app) => {
+      const { originValue } = await createTestTags(app);
+      const queryService = app.get(QueryService);
+      const response = await queryService.count({
+        origins_tags: originValue,
+        nucleotides_tags: { $in: [null, []] },
+      });
+      expect(response).toBe(1);
+    });
+  });
+
+  it('should cope with an $in unknown value on a product field', async () => {
+    await createTestingModule([DomainModule], async (app) => {
+      const { originValue } = await createTestTags(app);
+      const queryService = app.get(QueryService);
+      const response = await queryService.count({
+        origins_tags: originValue,
+        creator: { $in: [null, []] },
+      });
+      expect(response).toBe(1);
+    });
+  });
+
+  it('should cope with $nin', async () => {
+    await createTestingModule([DomainModule], async (app) => {
+      const { originValue, aminoValue, aminoValue2 } = await createTestTags(
+        app,
+      );
+      const queryService = app.get(QueryService);
+      const response = await queryService.count({
+        origins_tags: originValue,
+        amino_acids_tags: { $nin: [aminoValue, aminoValue2] },
+      });
+      expect(response).toBe(0);
+    });
+  });
+
+  it('should cope with $nin unknown', async () => {
+    await createTestingModule([DomainModule], async (app) => {
+      const { originValue } = await createTestTags(app);
+      const queryService = app.get(QueryService);
+      const response = await queryService.count({
+        origins_tags: originValue,
+        nucleotides_tags: { $nin: [null, []] },
+      });
+      expect(response).toBe(2);
+    });
+  });
+
+  it('should cope with $nin unknown value on a product field', async () => {
+    await createTestingModule([DomainModule], async (app) => {
+      const { originValue } = await createTestTags(app);
+      const queryService = app.get(QueryService);
+      const response = await queryService.count({
+        origins_tags: originValue,
+        creator: { $nin: [null, []] },
+      });
+      expect(response).toBe(2);
+    });
+  });
 });
 
 describe('aggregate', () => {
@@ -359,6 +434,21 @@ describe('select', () => {
       expect(p4).toBeTruthy();
     });
   });
+
+  it('should cope with $nin', async () => {
+    await createTestingModule([DomainModule], async (app) => {
+      const { originValue, aminoValue, product3 } = await createTestTags(
+        app,
+      );
+      const queryService = app.get(QueryService);
+      const response = await queryService.select({
+        origins_tags: originValue,
+        amino_acids_tags: { $nin: [aminoValue] },
+      });
+      expect(response).toHaveLength(1);
+      expect(response[0].code).toBe(product3.code);
+    });
+  });
 });
 
 async function createTestTags(app) {