From 767dce18d249f21c8b596cb494cf224aff5b2a07 Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Sun, 29 Jan 2023 15:29:31 -0500 Subject: [PATCH] debug migration --- plantit/plantit/celery_tasks.py | 21 +++++++++++---------- plantit/plantit/migration.py | 13 +++++++------ 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/plantit/plantit/celery_tasks.py b/plantit/plantit/celery_tasks.py index bc0db631..07c53778 100644 --- a/plantit/plantit/celery_tasks.py +++ b/plantit/plantit/celery_tasks.py @@ -1133,7 +1133,7 @@ def start_dirt_migration(self, username: str): for file in image_files: # get file entity ID given root image file ID file_entity_id = mig.get_file_entity_id(file.id) - file.entity_id = file_entity_id + file._replace(entity_id=file_entity_id) # if no corresponding file entity for this managed file, skip it if file_entity_id is None: @@ -1145,9 +1145,10 @@ def start_dirt_migration(self, username: str): # if no corresponding marked collection for this image, use an orphan folder named by date (as stored on the DIRT server NFS) if collection_entity_id is None: - logger.warning(f"DIRT root image collection with entity ID {file_entity_id} not found") + logger.warning(f"DIRT root image collection with entity ID {collection_entity_id} not found") # create the collection if we need to + collection_name = file.folder collection_path = join(migration_collection_path, 'collections', file.folder) if file.folder not in collections_created: # mark this collection as seen @@ -1156,10 +1157,10 @@ def start_dirt_migration(self, username: str): # create the collection in the data store logger.info(f"Creating DIRT migration subcollection {collection_path}") client.mkdir(collection_path) - - # otherwise we have a corresponding marked collection, get its title - collection_name, collection_created, collection_changed = mig.get_marked_collection(collection_entity_id) - collection_path = join(migration_collection_path, 'collections', collection_name) + else: + # otherwise we have a corresponding marked collection, get its title + collection_name, collection_created, collection_changed = mig.get_marked_collection(collection_entity_id) + collection_path = join(migration_collection_path, 'collections', collection_name) if collection_name not in collections_created: collections_created.add(collection_name) @@ -1171,7 +1172,7 @@ def start_dirt_migration(self, username: str): # get CyVerse ID of newly created collection stat = client.stat(collection_path) id = stat['id'] - file.collection_datastore_id = id + file._replace(collection_datastore_id=id) # get metadata and environmental data and attach to file metadata, lat, lon, planting, harvest, soil_group, soil_moist, soil_n, soil_p, soil_k, pesticides = mig.get_marked_collection_info( @@ -1195,8 +1196,8 @@ def start_dirt_migration(self, username: str): client.set_metadata(id, props, []) # persist collection information on managed file record - file.collection = collection_name - file.collection_entity_id = collection_entity_id + file._replace(collection=collection_name) + file._replace(collection_entity_id=collection_entity_id) # create managed file record file_rec = ManagedFile.objects.create(migration=migration, @@ -1207,7 +1208,7 @@ def start_dirt_migration(self, username: str): orphan=file.orphan, missing=file.missing, uploaded=file.uploaded, - entity_id=file.file_entity_id, + entity_id=file.entity_id, collection_entity_id=file.collection_entity_id, nfs_path=join(rootnfs_dir, 'root-images', file.folder, file.name), staging_path=join(staging_dir, file.name)) diff --git a/plantit/plantit/migration.py b/plantit/plantit/migration.py index 67d2d858..f5c8b4cb 100644 --- a/plantit/plantit/migration.py +++ b/plantit/plantit/migration.py @@ -64,15 +64,16 @@ async def push_migration_event(user: User, migration: Migration, file: plantit.u class ManagedFile(NamedTuple): id: str name: str - nfs_path: str path: str type: str folder: str orphan: bool missing: bool uploaded: Optional[str] - entity_id: Optional[str] - collection_entity_id: Optional[str] + nfs_path: Optional[str] = None + entity_id: Optional[str] = None + collection: Optional[str] = None + collection_entity_id: Optional[str] = None def row_to_managed_file(row): @@ -94,7 +95,7 @@ def row_to_managed_file(row): return ManagedFile( id=fid, name=name, - nfs_path=path.replace('public://', ''), + path=path.replace('public://', ''), type='metadata', folder=path.rpartition('metadata-files')[2].replace(name, '').replace('/', ''), orphan=False, @@ -105,7 +106,7 @@ def row_to_managed_file(row): return ManagedFile( id=fid, name=name, - nfs_path=path.replace('public://', ''), + path=path.replace('public://', ''), type='output', folder=folder, orphan=False, @@ -115,7 +116,7 @@ def row_to_managed_file(row): return ManagedFile( id=fid, name=name, - nfs_path=path.replace('public://', ''), + path=path.replace('public://', ''), type='logs', folder=path.rpartition('output-logs')[2].replace(name, '').replace('/', ''), orphan=False,