Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improve folder enumeration speed #4922

Merged
merged 9 commits into from
May 23, 2021
Merged

Improve folder enumeration speed #4922

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
0a04999
Removed cache feature
gave92 May 1, 2021
cbc6277
Removed related strings and settings
gave92 May 1, 2021
a9b6907
Reduce calls to StorageFolder methods
gave92 May 1, 2021
6ae2564
Fix icons not loading on phone
gave92 May 1, 2021
95eecb3
Fix merge conflicts
gave92 May 15, 2021
4066fb3
Merge remote-tracking branch 'origin/main' into remove_cache
gave92 May 16, 2021
c2fbcd2
Merge remote-tracking branch 'origin/main' into remove_cache
gave92 May 16, 2021
03fe656
Merge remote-tracking branch 'origin/main' into remove_cache
gave92 May 23, 2021
a3ee462
Merge remote-tracking branch 'origin/main' into remove_cache
gave92 May 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion Files/Files.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,6 @@
<Compile Include="Helpers\CollectionDebugView.cs" />
<Compile Include="Helpers\DynamicDialogFactory.cs" />
<Compile Include="Helpers\Extension.cs" />
<Compile Include="Helpers\FileListCache\CacheEntry.cs" />
<Compile Include="Helpers\FileListCache\FileListCacheController.cs" />
<Compile Include="Helpers\FileListCache\IFileListCache.cs" />
<Compile Include="Helpers\FileListCache\PersistentSQLiteCacheAdapter.cs" />
Expand Down
19 changes: 2 additions & 17 deletions Files/Filesystem/StorageEnumerators/UniversalStorageEnumerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ public static async Task<List<ListedItem>> ListEntries(
string returnformat,
Type sourcePageType,
CancellationToken cancellationToken,
List<string> skipItems,
int countLimit,
Func<List<ListedItem>, Task> intermediateAction
)
Expand Down Expand Up @@ -74,14 +73,7 @@ ex is UnauthorizedAccessException
var folder = await AddFolderAsync(item as StorageFolder, currentStorageFolder, returnformat, cancellationToken);
if (folder != null)
{
if (skipItems?.Contains(folder.ItemPath) ?? false)
{
skipItems.Remove(folder.ItemPath);
}
else
{
tempList.Add(folder);
}
tempList.Add(folder);
}
}
else
Expand All @@ -90,14 +82,7 @@ ex is UnauthorizedAccessException
var fileEntry = await AddFileAsync(file, currentStorageFolder, returnformat, true, sourcePageType, cancellationToken);
if (fileEntry != null)
{
if (skipItems?.Contains(fileEntry.ItemPath) ?? false)
{
skipItems.Remove(fileEntry.ItemPath);
}
else
{
tempList.Add(fileEntry);
}
tempList.Add(fileEntry);
}
}
if (cancellationToken.IsCancellationRequested)
Expand Down
19 changes: 2 additions & 17 deletions Files/Filesystem/StorageEnumerators/Win32StorageEnumerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ public static async Task<List<ListedItem>> ListEntries(
WIN32_FIND_DATA findData,
NamedPipeAsAppServiceConnection connection,
CancellationToken cancellationToken,
List<string> skipItems,
int countLimit,
Func<List<ListedItem>, Task> intermediateAction
)
Expand All @@ -49,14 +48,7 @@ Func<List<ListedItem>, Task> intermediateAction
var file = await GetFile(findData, path, returnformat, connection, cancellationToken);
if (file != null)
{
if (skipItems?.Contains(file.ItemPath) ?? false)
{
skipItems.Remove(file.ItemPath);
}
else
{
tempList.Add(file);
}
tempList.Add(file);
++count;
}
}
Expand All @@ -67,14 +59,7 @@ Func<List<ListedItem>, Task> intermediateAction
var folder = await GetFolder(findData, path, returnformat, cancellationToken);
if (folder != null)
{
if (skipItems?.Contains(folder.ItemPath) ?? false)
{
skipItems.Remove(folder.ItemPath);
}
else
{
tempList.Add(folder);
}
tempList.Add(folder);
++count;
}
}
Expand Down
11 changes: 0 additions & 11 deletions Files/Helpers/FileListCache/CacheEntry.cs
View file
Open in desktop

This file was deleted.

48 changes: 0 additions & 48 deletions Files/Helpers/FileListCache/FileListCacheController.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,59 +20,11 @@ private FileListCacheController()
persistentAdapter = new PersistentSQLiteCacheAdapter();
}

private readonly IMemoryCache filesCache = new MemoryCache(new MemoryCacheOptions
{
SizeLimit = 1_000_000
});

private readonly IMemoryCache fileNamesCache = new MemoryCache(new MemoryCacheOptions
{
SizeLimit = 1_000_000
});

public Task SaveFileListToCache(string path, CacheEntry cacheEntry)
{
if (!App.AppSettings.UseFileListCache)
{
return Task.CompletedTask;
}

if (cacheEntry == null)
{
filesCache.Remove(path);
return persistentAdapter.SaveFileListToCache(path, cacheEntry);
}
filesCache.Set(path, cacheEntry, new MemoryCacheEntryOptions
{
Size = cacheEntry.FileList.Count
});

// save entry to persistent cache in background
return persistentAdapter.SaveFileListToCache(path, cacheEntry);
}

public async Task<CacheEntry> ReadFileListFromCache(string path, CancellationToken cancellationToken)
{
if (!App.AppSettings.UseFileListCache)
{
return null;
}

var entry = filesCache.Get<CacheEntry>(path);
if (entry == null)
{
entry = await persistentAdapter.ReadFileListFromCache(path, cancellationToken);
if (entry?.FileList != null)
{
filesCache.Set(path, entry, new MemoryCacheEntryOptions
{
Size = entry.FileList.Count
});
}
}
return entry;
}

public async Task<string> ReadFileDisplayNameFromCache(string path, CancellationToken cancellationToken)
{
var displayName = fileNamesCache.Get<string>(path);
Expand Down
4 changes: 0 additions & 4 deletions Files/Helpers/FileListCache/IFileListCache.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,6 @@ namespace Files.Helpers.FileListCache
{
internal interface IFileListCache
{
public Task<CacheEntry> ReadFileListFromCache(string path, CancellationToken cancellationToken);

public Task SaveFileListToCache(string path, CacheEntry cacheEntry);

public Task<string> ReadFileDisplayNameFromCache(string path, CancellationToken cancellationToken);

public Task SaveFileDisplayNameToCache(string path, string displayName);
Expand Down
123 changes: 0 additions & 123 deletions Files/Helpers/FileListCache/PersistentSQLiteCacheAdapter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,98 +15,6 @@ internal class PersistentSQLiteCacheAdapter : IFileListCache, IDisposable
private SqliteConnection connection;
private bool disposedValue;

public async Task SaveFileListToCache(string path, CacheEntry cacheEntry)
{
if (!await InitializeIfNeeded())
{
return;
}
const int maxCachedEntries = 128;
try
{
if (cacheEntry == null)
{
using var deleteCommand = new SqliteCommand("DELETE FROM FileListCache WHERE Id = @Id", connection);
deleteCommand.Parameters.Add("@Id", SqliteType.Text).Value = path;
await deleteCommand.ExecuteNonQueryAsync();
return;
}

if (cacheEntry.FileList.Count > maxCachedEntries)
{
cacheEntry.FileList = cacheEntry.FileList.Take(maxCachedEntries).ToList();
}

using var cmd = new SqliteCommand("SELECT Id FROM FileListCache WHERE Id = @Id", connection);
cmd.Parameters.Add("@Id", SqliteType.Text).Value = path;
using var reader = await cmd.ExecuteReaderAsync();
if (reader.HasRows)
{
// need to update entry
using var updateCommand = new SqliteCommand("UPDATE FileListCache SET Timestamp = @Timestamp, Entry = @Entry WHERE Id = @Id", connection);
updateCommand.Parameters.Add("@Id", SqliteType.Text).Value = path;
updateCommand.Parameters.Add("@Timestamp", SqliteType.Integer).Value = GetTimestamp(DateTime.UtcNow);
var settings = new JsonSerializerSettings
{
TypeNameHandling = TypeNameHandling.Auto
};
updateCommand.Parameters.Add("@Entry", SqliteType.Text).Value = JsonConvert.SerializeObject(cacheEntry, settings);
await updateCommand.ExecuteNonQueryAsync();
}
else
{
// need to insert entry
using var insertCommand = new SqliteCommand("INSERT INTO FileListCache (Id, Timestamp, Entry) VALUES (@Id, @Timestamp, @Entry)", connection);
insertCommand.Parameters.Add("@Id", SqliteType.Text).Value = path;
insertCommand.Parameters.Add("@Timestamp", SqliteType.Integer).Value = GetTimestamp(DateTime.UtcNow);
var settings = new JsonSerializerSettings
{
TypeNameHandling = TypeNameHandling.Auto
};
insertCommand.Parameters.Add("@Entry", SqliteType.Text).Value = JsonConvert.SerializeObject(cacheEntry, settings);
await insertCommand.ExecuteNonQueryAsync();
}
}
catch (Exception ex)
{
NLog.LogManager.GetCurrentClassLogger().Warn(ex, ex.Message);
}
}

public async Task<CacheEntry> ReadFileListFromCache(string path, CancellationToken cancellationToken)
{
if (!await InitializeIfNeeded())
{
return null;
}
try
{
using var cmd = new SqliteCommand("SELECT Timestamp, Entry FROM FileListCache WHERE Id = @Id", connection);
cmd.Parameters.Add("@Id", SqliteType.Text).Value = path;

using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
if (!await reader.ReadAsync())
{
return null;
}
var timestamp = reader.GetInt64(0);
var entryAsJson = reader.GetString(1);
var settings = new JsonSerializerSettings
{
TypeNameHandling = TypeNameHandling.Auto
};
var entry = JsonConvert.DeserializeObject<CacheEntry>(entryAsJson, settings);
entry.CurrentFolder.ItemPropertiesInitialized = false;
entry.FileList.ForEach((item) => item.ItemPropertiesInitialized = false);
return entry;
}
catch (Exception ex)
{
NLog.LogManager.GetCurrentClassLogger().Warn(ex, ex.Message);
return null;
}
}

public async Task SaveFileDisplayNameToCache(string path, string displayName)
{
if (!await InitializeIfNeeded())
Expand Down Expand Up @@ -174,11 +82,6 @@ public async Task<string> ReadFileDisplayNameFromCache(string path, Cancellation
}
}

private long GetTimestamp(DateTime dateTime)
{
return new DateTimeOffset(dateTime).ToUnixTimeSeconds();
}

public void Dispose()
{
if (!disposedValue)
Expand All @@ -190,23 +93,6 @@ public void Dispose()

private void RunCleanupRoutine()
{
Task.Run(async () =>
{
try
{
// remove entries that are 1 month old (timestamp is updated every time the cache is set)
var limitTimestamp = GetTimestamp(DateTime.Now.AddMonths(-1));
using var cmd = new SqliteCommand("DELETE FROM FileListCache WHERE Timestamp < @Timestamp", connection);
cmd.Parameters.Add("@Timestamp", SqliteType.Integer).Value = limitTimestamp;

var count = await cmd.ExecuteNonQueryAsync().ConfigureAwait(false);
Debug.WriteLine($"Removed {count} old entries from cache database");
}
catch (Exception ex)
{
NLog.LogManager.GetCurrentClassLogger().Warn(ex, ex.Message);
}
});
}

private async Task<bool> InitializeIfNeeded()
Expand All @@ -227,15 +113,6 @@ private async Task<bool> InitializeIfNeeded()
connection.Open();

// create db schema
var createFileListCacheTable = @"CREATE TABLE IF NOT EXISTS ""FileListCache"" (
""Id"" VARCHAR(5000) NOT NULL,
""Timestamp"" INTEGER NOT NULL,
""Entry"" TEXT NOT NULL,
PRIMARY KEY(""Id"")
)";
using var cmdFileListCacheTable = new SqliteCommand(createFileListCacheTable, connection);
cmdFileListCacheTable.ExecuteNonQuery();

var createFileDisplayNameCacheTable = @"CREATE TABLE IF NOT EXISTS ""FileDisplayNameCache"" (
""Id"" VARCHAR(5000) NOT NULL,
""DisplayName"" TEXT NOT NULL,
Expand Down
12 changes: 0 additions & 12 deletions Files/MultilingualResources/Files.ar.xlf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1844,10 +1844,6 @@
<source>Original path</source>
<target state="translated">المسار الأصلي</target>
</trans-unit>
<trans-unit id="SettingsUseFileListCache.Header" translate="yes" xml:space="preserve">
<source>Cache files and folders for better performance</source>
<target state="translated">ملفات ومجلدات ذاكرة التخزين المؤقت لأداء أفضل</target>
</trans-unit>
<trans-unit id="AppBarButtonAdd.Label" translate="yes" xml:space="preserve">
<source>Add</source>
<target state="translated" state-qualifier="tm-suggestion">إضافة</target>
Expand Down Expand Up @@ -2243,14 +2239,6 @@
<source>Disconnect</source>
<target state="needs-review-translation" state-qualifier="tm-suggestion">قطع الاتصال</target>
</trans-unit>
<trans-unit id="PreemptiveCacheParallelLimit.Header" translate="yes" xml:space="preserve">
<source>Preemptive cache parallel limit (smaller numbers should work better for hard drives)</source>
<target state="new">Preemptive cache parallel limit (smaller numbers should work better for hard drives)</target>
</trans-unit>
<trans-unit id="SettingsUsePreemptiveCache.Header" translate="yes" xml:space="preserve">
<source>Use preemptive cache (preload entries in child directories on navigation)</source>
<target state="new">Use preemptive cache (preload entries in child directories on navigation)</target>
</trans-unit>
<trans-unit id="BundlesWidgetMoreOptionsButton.AutomationProperties.Name" translate="yes" xml:space="preserve">
<source>More bundles options</source>
<target state="new">More bundles options</target>
Expand Down
12 changes: 0 additions & 12 deletions Files/MultilingualResources/Files.cs-CZ.xlf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1859,10 +1859,6 @@
<source>Original path</source>
<target state="translated">Umístění</target>
</trans-unit>
<trans-unit id="SettingsUseFileListCache.Header" translate="yes" xml:space="preserve">
<source>Cache files and folders for better performance</source>
<target state="translated">Indexovat soubory a složky pro lepší výkon</target>
</trans-unit>
<trans-unit id="AppBarButtonAdd.Label" translate="yes" xml:space="preserve">
<source>Add</source>
<target state="translated">Přidat</target>
Expand Down Expand Up @@ -2260,14 +2256,6 @@
<source>Disconnect</source>
<target state="needs-review-translation" state-qualifier="tm-suggestion">Odpojit</target>
</trans-unit>
<trans-unit id="PreemptiveCacheParallelLimit.Header" translate="yes" xml:space="preserve">
<source>Preemptive cache parallel limit (smaller numbers should work better for hard drives)</source>
<target state="new">Preemptive cache parallel limit (smaller numbers should work better for hard drives)</target>
</trans-unit>
<trans-unit id="SettingsUsePreemptiveCache.Header" translate="yes" xml:space="preserve">
<source>Use preemptive cache (preload entries in child directories on navigation)</source>
<target state="new">Use preemptive cache (preload entries in child directories on navigation)</target>
</trans-unit>
<trans-unit id="BundlesWidgetMoreOptionsButton.AutomationProperties.Name" translate="yes" xml:space="preserve">
<source>More bundles options</source>
<target state="new">More bundles options</target>
Expand Down
Loading