Merge pull request bpatrik#902 from grasdk/bugfix/offset-or-ignore

Bugfix/offset or ignore

src/backend/model/database/SearchManager.ts

@@ -364,7 +364,11 @@ export class SearchManager {
     for (const sort of sortings) {
       switch (sort.method) {
         case SortByTypes.Date:
-          query.addOrderBy('media.metadata.creationDate', sort.ascending ? 'ASC' : 'DESC'); //If media.metadata.creationDateOffset is defined, it is an offset of minutes (+/-). If taken into account, it will alter the sort order. Probably should not be done.
+          if (Config.Gallery.ignoreTimestampOffset === true) {
+            query.addOrderBy('media.metadata.creationDate + (media.metadata.creationDateOffset * 60000)', sort.ascending ? 'ASC' : 'DESC');
+          } else {
+            query.addOrderBy('media.metadata.creationDate', sort.ascending ? 'ASC' : 'DESC'); 
+          }
           break;
         case SortByTypes.Rating:
           query.addOrderBy('media.metadata.rating', sort.ascending ? 'ASC' : 'DESC');
@@ -562,15 +566,17 @@ export class SearchManager {
 
           const textParam: { [key: string]: unknown } = {};
           textParam['from' + queryId] = (query as FromDateSearch).value;
-          q.where(
-            `media.metadata.creationDate ${relation} :from${queryId}`, //TODO: If media.metadata.creationDateOffset is defined, it is an offset of minutes (+/-). 
-                                                                       //Example: -600 means in the database UTC-10:00. The time 20:00 in the evening in the UTC-10 timezone, is actually 06:00 the next morning 
-                                                                       //in UTC+00:00. To make search take that into account, one can subtract the offset from the creationDate to "pretend" the photo is taken
-                                                                       //in UTC time. Subtracting -600 minutes (because it's the -10:00 timezone), corresponds to adding 10 hours to the photo's timestamp, thus
-                                                                       //bringing it into the next day as if it was taken at UTC+00:00. Similarly subtracting a positive timezone from a timestamp will "pretend"
-                                                                       //the photo is taken earlier in time (e.g. subtracting 300 from the UTC+05:00 timezone).
-            textParam
-          );
+          if (Config.Gallery.ignoreTimestampOffset === true) {
+            q.where(
+              `(media.metadata.creationDate + (media.metadata.creationDateOffset * 60000)) ${relation} :from${queryId}`,
+              textParam
+            );
+          } else {
+            q.where(
+              `media.metadata.creationDate ${relation} :from${queryId}`,
+              textParam
+            );
+          }
 
           return q;
         });
@@ -589,10 +595,18 @@ export class SearchManager {
 
           const textParam: { [key: string]: unknown } = {};
           textParam['to' + queryId] = (query as ToDateSearch).value;
-          q.where(
-            `media.metadata.creationDate ${relation} :to${queryId}`, //TODO: If media.metadata.creationDateOffset is defined, it is an offset of minutes (+/-). See explanation above.
-            textParam 
-          );
+          if (Config.Gallery.ignoreTimestampOffset === true) {
+            q.where(
+              `(media.metadata.creationDate + (media.metadata.creationDateOffset * 60000)) ${relation} :to${queryId}`,
+              textParam 
+            );
+          } else {
+            q.where(
+              `media.metadata.creationDate ${relation} :to${queryId}`,
+              textParam 
+            );
+
+          }
 
           return q;
         });
@@ -793,18 +807,34 @@ export class SearchManager {
             textParam['to' + queryId] = to.getTime();
             textParam['from' + queryId] = from.getTime();
             if (tq.negate) {
+              if (Config.Gallery.ignoreTimestampOffset === true) {
+                q.where(
+                  `(media.metadata.creationDate + (media.metadata.creationDateOffset * 60000)) >= :to${queryId}`,
+                  textParam
+                ).orWhere(`(media.metadata.creationDate + (media.metadata.creationDateOffset * 60000)) < :from${queryId}`,
+                  textParam);
+              } else {
+                q.where(
+                  `media.metadata.creationDate >= :to${queryId}`,
+                  textParam
+                ).orWhere(`media.metadata.creationDate < :from${queryId}`,
+                  textParam);
 
-              q.where(
-                `media.metadata.creationDate >= :to${queryId}`,            //TODO: If media.metadata.creationDateOffset is defined, it is an offset of minutes (+/-). See explanation above.
-                textParam
-              ).orWhere(`media.metadata.creationDate < :from${queryId}`,   //TODO: If media.metadata.creationDateOffset is defined, it is an offset of minutes (+/-). See explanation above.
-                textParam);
+              }
             } else {
-              q.where(
-                `media.metadata.creationDate < :to${queryId}`,             //TODO: If media.metadata.creationDateOffset is defined, it is an offset of minutes (+/-). See explanation above.
-                textParam
-              ).andWhere(`media.metadata.creationDate >= :from${queryId}`, //TODO: If media.metadata.creationDateOffset is defined, it is an offset of minutes (+/-). See explanation above.
-                textParam);
+              if (Config.Gallery.ignoreTimestampOffset === true) {
+                q.where(
+                  `(media.metadata.creationDate + (media.metadata.creationDateOffset * 60000)) < :to${queryId}`,
+                  textParam
+                ).andWhere(`(media.metadata.creationDate + (media.metadata.creationDateOffset * 60000)) >= :from${queryId}`,
+                  textParam);
+              } else {
+                q.where(
+                  `media.metadata.creationDate < :to${queryId}`,
+                  textParam
+                ).andWhere(`media.metadata.creationDate >= :from${queryId}`,
+                  textParam);
+              }
             }
 
           } else {
@@ -825,29 +855,51 @@ export class SearchManager {
 
               if (Config.Database.type === DatabaseType.sqlite) {
                 if (tq.daysLength == 0) {
-                  q.where(
-                    //TODO: If media.metadata.creationDateOffset is defined, it is an offset of minutes (+/-). See explanation above.
-                    `CAST(strftime('${duration}',media.metadataCreationDate/1000, 'unixepoch') AS INTEGER) ${relationEql} CAST(strftime('${duration}','now') AS INTEGER)`
-                  );
+                  if (Config.Gallery.ignoreTimestampOffset === true) {
+                    q.where(
+                      `CAST(strftime('${duration}',(media.metadataCreationDate + (media.metadataCreationDateOffset * 60000))/1000, 'unixepoch') AS INTEGER) ${relationEql} CAST(strftime('${duration}','now') AS INTEGER)`
+                    );
+                  } else {
+                    q.where(
+                      `CAST(strftime('${duration}',media.metadataCreationDate/1000, 'unixepoch') AS INTEGER) ${relationEql} CAST(strftime('${duration}','now') AS INTEGER)`
+                    );
+                  }
                 } else {
-                  q.where(
-                    //TODO: If media.metadata.creationDateOffset is defined, it is an offset of minutes (+/-). See explanation above.
-                    `CAST(strftime('${duration}',media.metadataCreationDate/1000, 'unixepoch') AS INTEGER) ${relationTop} CAST(strftime('${duration}','now') AS INTEGER)`
-                  )[whereFN](`CAST(strftime('${duration}',media.metadataCreationDate/1000, 'unixepoch') AS INTEGER) ${relationBottom} CAST(strftime('${duration}','now','-:diff${queryId} day') AS INTEGER)`,
-                    textParam);
+                  if (Config.Gallery.ignoreTimestampOffset === true) {
+                    q.where(
+                      `CAST(strftime('${duration}',(media.metadataCreationDate + (media.metadataCreationDateOffset * 60000))/1000, 'unixepoch') AS INTEGER) ${relationTop} CAST(strftime('${duration}','now') AS INTEGER)`
+                    )[whereFN](`CAST(strftime('${duration}',(media.metadataCreationDate + (media.metadataCreationDateOffset * 60000))/1000, 'unixepoch') AS INTEGER) ${relationBottom} CAST(strftime('${duration}','now','-:diff${queryId} day') AS INTEGER)`,
+                      textParam);
+                  } else {
+                    q.where(
+                      `CAST(strftime('${duration}',media.metadataCreationDate/1000, 'unixepoch') AS INTEGER) ${relationTop} CAST(strftime('${duration}','now') AS INTEGER)`
+                    )[whereFN](`CAST(strftime('${duration}',media.metadataCreationDate/1000, 'unixepoch') AS INTEGER) ${relationBottom} CAST(strftime('${duration}','now','-:diff${queryId} day') AS INTEGER)`,
+                      textParam);
+                  }
                 }
               } else {
                 if (tq.daysLength == 0) {
-                  q.where(
-                    //TODO: If media.metadata.creationDateOffset is defined, it is an offset of minutes (+/-). See explanation above.
-                    `CAST(FROM_UNIXTIME(media.metadataCreationDate/1000, '${duration}') AS SIGNED) ${relationEql} CAST(DATE_FORMAT(CURDATE(),'${duration}') AS SIGNED)`
-                  );
+                  if (Config.Gallery.ignoreTimestampOffset === true) {
+                    q.where(
+                      `CAST(FROM_UNIXTIME((media.metadataCreationDate + (media.metadataCreationDateOffset * 60000))/1000, '${duration}') AS SIGNED) ${relationEql} CAST(DATE_FORMAT(CURDATE(),'${duration}') AS SIGNED)`
+                    );
+                  } else {
+                    q.where(
+                      `CAST(FROM_UNIXTIME(media.metadataCreationDate/1000, '${duration}') AS SIGNED) ${relationEql} CAST(DATE_FORMAT(CURDATE(),'${duration}') AS SIGNED)`
+                    );
+                  }
                 } else {
-                  q.where(
-                    //TODO: If media.metadata.creationDateOffset is defined, it is an offset of minutes (+/-). See explanation above.
-                    `CAST(FROM_UNIXTIME(media.metadataCreationDate/1000, '${duration}') AS SIGNED) ${relationTop} CAST(DATE_FORMAT(CURDATE(),'${duration}') AS SIGNED)`
-                  )[whereFN](`CAST(FROM_UNIXTIME(media.metadataCreationDate/1000, '${duration}') AS SIGNED) ${relationBottom} CAST(DATE_FORMAT((DATE_ADD(curdate(), INTERVAL -:diff${queryId} DAY)),'${duration}') AS SIGNED)`,
-                    textParam);
+                  if (Config.Gallery.ignoreTimestampOffset === true) {
+                    q.where(
+                      `CAST(FROM_UNIXTIME((media.metadataCreationDate + (media.metadataCreationDateOffset * 60000))/1000, '${duration}') AS SIGNED) ${relationTop} CAST(DATE_FORMAT(CURDATE(),'${duration}') AS SIGNED)`
+                    )[whereFN](`CAST(FROM_UNIXTIME((media.metadataCreationDate + (media.metadataCreationDateOffset * 60000))/1000, '${duration}') AS SIGNED) ${relationBottom} CAST(DATE_FORMAT((DATE_ADD(curdate(), INTERVAL -:diff${queryId} DAY)),'${duration}') AS SIGNED)`,
+                      textParam);
+                  } else {
+                    q.where(
+                      `CAST(FROM_UNIXTIME(media.metadataCreationDate/1000, '${duration}') AS SIGNED) ${relationTop} CAST(DATE_FORMAT(CURDATE(),'${duration}') AS SIGNED)`
+                    )[whereFN](`CAST(FROM_UNIXTIME(media.metadataCreationDate/1000, '${duration}') AS SIGNED) ${relationBottom} CAST(DATE_FORMAT((DATE_ADD(curdate(), INTERVAL -:diff${queryId} DAY)),'${duration}') AS SIGNED)`,
+                      textParam);
+                  }
                 }
               }
             };

src/backend/model/fileaccess/DiskManager.ts

@@ -277,8 +277,8 @@ export class DiskManager {
       directory.oldestMedia = Number.MIN_SAFE_INTEGER;
 
       directory.media.forEach((m) => {
-          directory.youngestMedia = Math.min(m.metadata.creationDate, directory.youngestMedia);
-          directory.oldestMedia = Math.max(m.metadata.creationDate, directory.oldestMedia);
+          directory.youngestMedia = Math.min(Utils.getTimeMS(m.metadata.creationDate, m.metadata.creationDateOffset, Config.Gallery.ignoreTimestampOffset), directory.youngestMedia);
+          directory.oldestMedia = Math.max(Utils.getTimeMS(m.metadata.creationDate, m.metadata.creationDateOffset, Config.Gallery.ignoreTimestampOffset), directory.oldestMedia);
         }
       );
 

src/backend/model/messenger/EmailMessenger.ts

@@ -70,7 +70,7 @@ export class EmailMessenger extends Messenger<{
         (media[i].metadata as PhotoMetadata).positionData?.country :
         ((media[i].metadata as PhotoMetadata).positionData?.city ?
           (media[i].metadata as PhotoMetadata).positionData?.city : '');
-      const caption = Utils.getFullYear(media[i].metadata.creationDate, media[i].metadata.creationDateOffset) + (location ? ', ' + location : '');
+      const caption = Utils.getFullYear(Utils.getTimeMS(media[i].metadata.creationDate, media[i].metadata.creationDateOffset, Config.Gallery.ignoreTimestampOffset), undefined) + (location ? ', ' + location : '');
       attachments.push({
         filename: media[i].name,
         path: media[i].thumbnailPath,

src/common/Utils.ts

@@ -123,10 +123,11 @@ export class Utils {
     }
   }
 
-  static makeUTCMidnight(d: number | Date) {
+  static makeUTCMidnight(d: number | Date, offset: string) {
     if (!(d instanceof Date)) {
       d = new Date(d);
     }
+    d = new Date(new Date(d).toISOString().substring(0,19) + (offset ? offset : '+00:00'))
     d.setUTCHours(0);
     d.setUTCMinutes(0);
     d.setUTCSeconds(0);
@@ -139,7 +140,7 @@ export class Utils {
     if (!(d instanceof Date)) {
       d = new Date(d);
     }
-    return new Date(new Date(d).toISOString().substring(0,19) + (offset ? offset : '')).getUTCFullYear();
+    return new Date(new Date(d).toISOString().substring(0,19) + (offset ? offset : '+00:00')).getUTCFullYear();
   }
 
   static getFullYear(d: number | Date, offset: string) {
@@ -225,10 +226,21 @@ export class Utils {
     }
   }
 
-    static getLocalTimeMS(creationDate: number, creationDateOffset: string) {
+  //Get the MS of the creationDate, adjusted for  the offset. Effectively getting the MS value as if the photo did not contain an offset.
+  //One can consider this "Local" time of the photo. Starting point is UTC, as MetadataLoader loads timestamps with unknown timestamps as UTC.
+  static getLocalTimeMS(creationDate: number, creationDateOffset: string) {
     const offsetMinutes = Utils.getOffsetMinutes(creationDateOffset);
     return creationDate + (offsetMinutes ? (offsetMinutes * 60000) : 0);
   }
+
+  //Like getLocalTimeMS, but only if localTime is true, otherwise just returns creationDate (global time)
+  static getTimeMS(creationDate: number, creationDateOffset: string, localTime: boolean) {
+    if (localTime) {
+      return Utils.getLocalTimeMS(creationDate, creationDateOffset);
+    } else {
+      return creationDate;
+    }
+  }
 
   static isLeapYear(year: number) {
     return (0 == year % 4) && (0 != year % 100) || (0 == year % 400)

src/common/config/public/ClientConfig.ts

@@ -1082,6 +1082,18 @@ export class ClientGalleryConfig {
   })
   enableDirectorySortingByDate: boolean = false;
 
+  @ConfigProperty({
+    tags: {
+      name: $localize`Ignore timestamp offsets`,
+      priority: ConfigPriority.advanced,
+    },
+    description: $localize`If enabled, timestamp offsets are ignored, meaning that the local times of pictures are used for searching, sorting and grouping. If disabled, global time is used and pictures with no timestamp are assumed to be in UTC (offset +00:00).`
+  })
+  //DEVELOPER NOTE: The Database model stores the timestamp (creationDate) as milliseconds since 1970-01-01 UTC (global time). And stores and offset (creationDateOffset) as minutes.
+  //Ignoring timestamp for the user is the opposite for the database. If the user wants to ignore the offset, we have to add the offset to the creationDate to give the user the right experience.
+  ignoreTimestampOffset: boolean = true; 
+
+
   @ConfigProperty({
     tags: {
       name: $localize`On scroll thumbnail prioritising`,

src/frontend/app/ui/gallery/blog/blog.service.ts

@@ -6,6 +6,7 @@ import {ContentService} from '../content.service';
 import {mergeMap, Observable, shareReplay} from 'rxjs';
 import {MDFilesFilterPipe} from '../../../pipes/MDFilesFilterPipe';
 import {MDFileDTO} from '../../../../../common/entities/MDFileDTO';
+import {Config} from '../../../../../common/config/public/Config';
 
 @Injectable()
 export class BlogService {
@@ -28,7 +29,7 @@ export class BlogService {
         let firstMedia = Number.MAX_SAFE_INTEGER;
         if (content.mediaGroups.length > 0) {
           firstMedia = content.mediaGroups[0].media.reduce((p, m) =>
-            Math.min(m.metadata.creationDate, p), Number.MAX_SAFE_INTEGER);
+            Math.min(Utils.getTimeMS(m.metadata.creationDate, m.metadata.creationDateOffset, Config.Gallery.ignoreTimestampOffset), p), Number.MAX_SAFE_INTEGER);
         }
 
         const files = this.mdFilesFilterPipe.transform(content.metaFile)
@@ -65,7 +66,7 @@ export class BlogService {
 
     const getDateGroup = (date: Date) => {
       // get UTC midnight date
-      const dateNum = Utils.makeUTCMidnight(date).getTime();
+      const dateNum = Utils.makeUTCMidnight(date, undefined).getTime();
       let groupDate = dates.find((d, i) => i > dates.length - 1 ? false : dates[i + 1] > dateNum);   //dates are sorted
 
       // cant find the date. put to the last group (as it was later)