From 895daf1594ff18491c137e6f1e57f98b06d9da06 Mon Sep 17 00:00:00 2001
From: Paul Tomasula <31142705+ptomasula@users.noreply.github.com>
Date: Mon, 17 Jan 2022 11:11:59 -0500
Subject: [PATCH 01/39] Convert app to run on single database
the first step for our next phase of work migrated the dataloader database into the public schema of the ODM2 database. This first step rewires the existing data models to work with a single database, and thereby removes the need for a db_router.
---
src/WebSDL/db_routers.py | 25 ----------------------
src/WebSDL/settings/base.py | 2 --
src/WebSDL/settings/settings_template.json | 18 ----------------
src/dataloader/models.py | 2 +-
src/dataloaderinterface/ajax.py | 2 +-
src/dataloaderservices/views.py | 19 ++++++----------
6 files changed, 9 insertions(+), 59 deletions(-)
delete mode 100644 src/WebSDL/db_routers.py
diff --git a/src/WebSDL/db_routers.py b/src/WebSDL/db_routers.py
deleted file mode 100644
index d0baad8c..00000000
--- a/src/WebSDL/db_routers.py
+++ /dev/null
@@ -1,25 +0,0 @@
-import sys
-
-
-class WebSDLRouter(object):
- def db_for_read(self, model, **hints):
- is_testing = 'test' in sys.argv
- if model._meta.app_label == 'tsa':
- return 'tsa_catalog'
- if model._meta.app_label == 'dataloader' or is_testing:
- return 'odm2'
- return 'default'
-
- def db_for_write(self, model, **hints):
- is_testing = 'test' in sys.argv
- if model._meta.app_label == 'tsa':
- return 'tsa_catalog'
- if model._meta.app_label in ['dataloader', 'dataloaderservices'] or is_testing:
- return 'odm2'
- return 'default'
-
- def allow_migrate(self, db, app_label, **hints):
- is_testing = 'test' in sys.argv
- unmanaged_apps = ['dataloader', 'tsa']
- allow = app_label not in unmanaged_apps or is_testing
- return allow
diff --git a/src/WebSDL/settings/base.py b/src/WebSDL/settings/base.py
index 7e9acf5d..910410db 100644
--- a/src/WebSDL/settings/base.py
+++ b/src/WebSDL/settings/base.py
@@ -153,8 +153,6 @@
USE_I18N = True
USE_L10N = True
LOGIN_URL = '/login/'
-DATABASE_ROUTERS = ['WebSDL.db_routers.WebSDLRouter']
-
# Security and SSL
#
diff --git a/src/WebSDL/settings/settings_template.json b/src/WebSDL/settings/settings_template.json
index 88c32843..00e4ded8 100644
--- a/src/WebSDL/settings/settings_template.json
+++ b/src/WebSDL/settings/settings_template.json
@@ -21,24 +21,6 @@
"password": "{{database password}}",
"host": "{{database server address}}",
"port": "5432"
- },
- {
- "name": "odm2",
- "schema": "odm2",
- "engine": "django.db.backends.postgresql_psycopg2",
- "user": "{{database user}}",
- "password": "{{database password}}",
- "host": "{{database server address}}",
- "port": "5432"
- },
- {
- "name": "tsa_catalog",
- "schema": "tsa_catalog",
- "engine": "django.db.backends.postgresql_psycopg2",
- "user": "{{database user}}",
- "password": "{{database password}}",
- "host": "{{database server address}}",
- "port": "5432"
}
],
"hydroshare_oauth": {
diff --git a/src/dataloader/models.py b/src/dataloader/models.py
index d14664ff..8cc4c996 100644
--- a/src/dataloader/models.py
+++ b/src/dataloader/models.py
@@ -2105,7 +2105,7 @@ class Meta:
psql_schema_fix = 'odm2"."' # for postgres databases
clsmembers = inspect.getmembers(sys.modules[__name__], inspect.isclass)
classes = [model for name, model in clsmembers if issubclass(model, models.Model)]
-database_manager = settings.DATABASES['odm2']['ENGINE']
+database_manager = settings.DATABASES['default']['ENGINE']
for model in classes:
if database_manager == u'sql_server.pyodbc':
diff --git a/src/dataloaderinterface/ajax.py b/src/dataloaderinterface/ajax.py
index 947c637b..8f04eca7 100644
--- a/src/dataloaderinterface/ajax.py
+++ b/src/dataloaderinterface/ajax.py
@@ -11,7 +11,7 @@
import numpy as np
from django.conf import settings
-_dbsettings = settings.DATABASES['odm2']
+_dbsettings = settings.DATABASES['default']
_connection_str = f"postgresql://{_dbsettings['USER']}:{_dbsettings['PASSWORD']}@{_dbsettings['HOST']}:{_dbsettings['PORT']}/{_dbsettings['NAME']}"
_db_engine = sqlalchemy.create_engine(_connection_str, pool_size=30)
diff --git a/src/dataloaderservices/views.py b/src/dataloaderservices/views.py
index 7ab1ba7d..79202346 100644
--- a/src/dataloaderservices/views.py
+++ b/src/dataloaderservices/views.py
@@ -45,15 +45,10 @@
import psycopg2
from django.conf import settings
-_dbsettings = settings.DATABASES['odm2']
+_dbsettings = settings.DATABASES['default']
_connection_str = f"postgresql://{_dbsettings['USER']}:{_dbsettings['PASSWORD']}@{_dbsettings['HOST']}:{_dbsettings['PORT']}/{_dbsettings['NAME']}"
_db_engine = sqlalchemy.create_engine(_connection_str, pool_size=10)
-_dbsettings_loader = settings.DATABASES['default']
-_connection_str_loader = f"postgresql://{_dbsettings_loader['USER']}:{_dbsettings_loader['PASSWORD']}@{_dbsettings_loader['HOST']}:{_dbsettings_loader['PORT']}/{_dbsettings_loader['NAME']}"
-_db_engine_loader = sqlalchemy.create_engine(_connection_str_loader, pool_size=10)
-
-
# TODO: Check user permissions to edit, add, or remove stuff with a permissions class.
# TODO: Use generic api views for create, edit, delete, and list.
@@ -696,8 +691,8 @@ def process_result_value(result_value:TimeseriesResultValueTechDebt) -> Union[st
#dataloader utility function
def get_site_sensor(resultid:str) -> Union[Dict[str, Any],None]:
- with _db_engine_loader.connect() as connection:
- query = text('SELECT * FROM dataloaderinterface_sitesensor ' \
+ with _db_engine.connect() as connection:
+ query = text('SELECT * FROM public.dataloaderinterface_sitesensor ' \
'WHERE "ResultID"=:resultid;'
)
df = pd.read_sql(query, connection, params={'resultid':resultid})
@@ -705,8 +700,8 @@ def get_site_sensor(resultid:str) -> Union[Dict[str, Any],None]:
#dataloader utility function
def update_sensormeasurement(sensor_id:str, result_value:TimeseriesResultValueTechDebt) -> None:
- with _db_engine_loader.connect() as connection:
- query = text('UPDATE dataloaderinterface_sensormeasurement ' \
+ with _db_engine.connect() as connection:
+ query = text('UPDATE public.dataloaderinterface_sensormeasurement ' \
"SET value_datetime=:datetime, " \
"value_datetime_utc_offset = :utc_offset, " \
'data_value = :data_value ' \
@@ -737,8 +732,8 @@ def sync_dataloader_tables(result_value: TimeseriesResultValueTechDebt) -> None:
#dataloader utility function
def set_deployment_date(sample_feature_id:int, date_time:datetime) -> None:
- with _db_engine_loader.connect() as connection:
- query = text('UPDATE dataloaderinterface_siteregistration '\
+ with _db_engine.connect() as connection:
+ query = text('UPDATE public.dataloaderinterface_siteregistration '\
'SET "DeploymentDate"=:date_time '\
'WHERE "DeploymentDate" IS NULL AND ' \
'"SamplingFeatureID"=:sample_feature_id' )
From fb4b8bb8f2131c1e56a0f05bd8e83f38399a1897 Mon Sep 17 00:00:00 2001
From: Paul Tomasula <31142705+ptomasula@users.noreply.github.com>
Date: Mon, 17 Jan 2022 11:15:53 -0500
Subject: [PATCH 02/39] Remove deprecated TSA code.
This TSA helper code was deprecated with the addition of TSV. No need to retain this.
---
src/tsa/__init__.py | 0
src/tsa/admin.py | 6 -
src/tsa/apps.py | 8 --
src/tsa/helpers_deprecated.py | 223 ----------------------------------
src/tsa/models.py | 59 ---------
src/tsa/tests.py | 6 -
src/tsa/views.py | 6 -
7 files changed, 308 deletions(-)
delete mode 100644 src/tsa/__init__.py
delete mode 100644 src/tsa/admin.py
delete mode 100644 src/tsa/apps.py
delete mode 100644 src/tsa/helpers_deprecated.py
delete mode 100644 src/tsa/models.py
delete mode 100644 src/tsa/tests.py
delete mode 100644 src/tsa/views.py
diff --git a/src/tsa/__init__.py b/src/tsa/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/src/tsa/admin.py b/src/tsa/admin.py
deleted file mode 100644
index 13be29d9..00000000
--- a/src/tsa/admin.py
+++ /dev/null
@@ -1,6 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.contrib import admin
-
-# Register your models here.
diff --git a/src/tsa/apps.py b/src/tsa/apps.py
deleted file mode 100644
index 203f06c0..00000000
--- a/src/tsa/apps.py
+++ /dev/null
@@ -1,8 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.apps import AppConfig
-
-
-class TsaConfig(AppConfig):
- name = 'tsa'
diff --git a/src/tsa/helpers_deprecated.py b/src/tsa/helpers_deprecated.py
deleted file mode 100644
index 4ce101db..00000000
--- a/src/tsa/helpers_deprecated.py
+++ /dev/null
@@ -1,223 +0,0 @@
-import json
-import os
-
-from django.conf import settings
-
-from dataloader.models import Organization, Variable, Unit, Result
-from dataloaderinterface.models import SiteSensor, SensorMeasurement
-from tsa.models import DataSeries
-
-
-class TimeSeriesAnalystHelper(object):
- wofpy_get_url = 'http://{server}/wofpy/rest/1_1/GetValues' \
- '?location={sampling_feature_code}&variable={variable_code}&methodCode=2' \
- '&sourceCode={organization_id}&qualityControlLevelCode=Raw&startDate=&endDate='
- influx_get_url = 'https://{database_server}/query?u=web_client&p=password&db=envirodiy' \
- '&q=SELECT%20time,%20DataValue::field,%20UTCOffset::field%20FROM%20{influx_identifier}'
-
- def create_series_from_sensor(self, sensor):
- sensor_queryset = SiteSensor.objects.filter(pk=sensor.pk)
- self.create_series_from_sensors(sensor_queryset)
-
- def create_series_from_sensors(self, sensors):
- server_data = retrieve_server_data()
- odm2_metadata = self.get_sensors_odm2_metadata(sensors)
- series_objects = [self.generate_data_series(sensor, odm2_metadata, server_data) for sensor in sensors]
- DataSeries.objects.bulk_create(series_objects)
-
- def update_series_from_sensor(self, sensor):
- # welp, i could've made the update and create in one method
- # if i hadn't started it with the `rebuild_tsa_catalog` command functionality in mind.
-
- data_series_queryset = DataSeries.objects.filter(result_uuid=sensor.result_uuid)
- if not data_series_queryset.count():
- self.create_series_from_sensor(sensor)
- return
-
- odm2_metadata = self.get_sensors_odm2_metadata(SiteSensor.objects.filter(pk=sensor.pk))
- last_measurement = SensorMeasurement.objects.filter(sensor=sensor).first()
- data_series_queryset.update(
- variable_code=sensor.sensor_output.variable_code,
- variable_name=sensor.sensor_output.variable_name,
- variable_units_name=sensor.sensor_output.unit_name,
- variable_units_type=odm2_metadata['unit_types'][sensor.sensor_output.unit_id],
- variable_units_abbreviation=sensor.sensor_output.unit_abbreviation,
- sample_medium=sensor.sensor_output.sampled_medium,
- general_category=odm2_metadata['variable_types'][sensor.sensor_output.variable_id],
- utc_offset=get_utc_offset(sensor),
- number_observations=odm2_metadata['values_count'][sensor.result_id],
- date_last_updated=last_measurement and last_measurement.value_datetime,
- is_active=get_is_active(sensor)
- )
-
- def update_series_from_site(self, registration):
- result_uuids = [uuid[0] for uuid in registration.sensors.values_list('result_uuid')]
- data_series_queryset = DataSeries.objects.filter(result_uuid__in=result_uuids)
- if not data_series_queryset.count():
- self.create_series_from_sensors(registration.sensors.all())
- return
-
- odm2_metadata = self.get_registration_odm2_metadata(registration)
- data_series_queryset.update(
- site_code=registration.sampling_feature_code,
- site_name=registration.sampling_feature_name,
- latitude=registration.latitude,
- longitude=registration.longitude,
- site_type=registration.site_type,
- source_organization=registration.organization_name or '',
- source_description=odm2_metadata['organization_description'],
- )
-
- def delete_series_for_sensor(self, sensor):
- DataSeries.objects.filter(result_uuid=sensor.result_uuid).delete()
-
- def generate_data_series(self, sensor, odm2_metadata, server_data):
- registration = sensor.registration
- last_measurement = SensorMeasurement.objects.filter(sensor=sensor).first()
-
- return DataSeries(
- result_uuid=str(sensor.result_uuid),
- influx_identifier=get_influx_identifier(sensor),
- source_data_service_id=get_source_id(sensor),
- network=get_network(sensor),
- site_code=registration.sampling_feature_code,
- site_name=registration.sampling_feature_name,
- latitude=registration.latitude,
- longitude=registration.longitude,
- site_type=registration.site_type,
- variable_code=sensor.sensor_output.variable_code,
- variable_name=sensor.sensor_output.variable_name,
- variable_level=get_variable_level(sensor),
- variable_units_name=sensor.sensor_output.unit_name,
- variable_units_type=odm2_metadata['unit_types'][sensor.sensor_output.unit_id],
- variable_units_abbreviation=sensor.sensor_output.unit_abbreviation,
- sample_medium=sensor.sensor_output.sampled_medium,
- value_type=get_value_type(sensor),
- data_type=get_data_type(sensor),
- general_category=odm2_metadata['variable_types'][sensor.sensor_output.variable_id],
- quality_control_level_code=get_processing_level_code(sensor),
- quality_control_level_definition=get_processing_level_definition(sensor),
- quality_control_level_explanation=get_processing_level_explanation(sensor),
- source_organization=registration.organization_name or '',
- source_description=odm2_metadata['organization_descriptions'][registration.organization_id] if registration.organization_id else '',
- begin_datetime=registration.registration_date,
- utc_offset=get_utc_offset(sensor),
- number_observations=odm2_metadata['values_count'][sensor.result_id],
- date_last_updated=last_measurement and last_measurement.value_datetime,
- is_active=get_is_active(sensor),
- get_data_url=self.wofpy_get_url.format(
- server=server_data['server'],
- sampling_feature_code=registration.sampling_feature_code,
- variable_code=sensor.sensor_output.variable_code,
- organization_id=registration.organization_id
- ),
- get_data_influx=self.influx_get_url.format(
- database_server=server_data['influx_server'],
- influx_identifier=get_influx_identifier(sensor)
- ),
- no_data_value=get_no_data_value(sensor)
- )
-
- def get_sensors_odm2_metadata(self, sensors):
- organization_ids = [id[0] for id in sensors.values_list('registration__organization_id')]
- variable_ids = [id[0] for id in sensors.values_list('sensor_output__variable_id')]
- unit_ids = [id[0] for id in sensors.values_list('sensor_output__unit_id')]
- result_ids = [id[0] for id in sensors.values_list('result_id')]
-
- return {
- 'organization_descriptions': {
- id: definition
- for (id, definition) in Organization.objects.filter(organization_id__in=organization_ids).values_list('organization_id', 'organization_description')
- },
- 'variable_types': {
- id: variable_type
- for (id, variable_type) in Variable.objects.filter(variable_id__in=variable_ids).values_list('variable_id', 'variable_type_id')
- },
- 'unit_types': {
- id: unit_type
- for (id, unit_type) in Unit.objects.filter(unit_id__in=unit_ids).values_list('unit_id', 'unit_type_id')
- },
- 'values_count': {
- id: value_count
- for (id, value_count) in Result.objects.filter(result_id__in=result_ids).values_list('result_id', 'value_count')
- }
- }
-
- def get_registration_odm2_metadata(self, registration):
- organization = Organization.objects.filter(organization_id=registration.organization_id).first()
- return {
- 'organization_description': organization and organization.organization_description or '',
- }
-
-def retrieve_server_data():
- server_data = {'server': None, 'database_server': None}
-
- try:
- with open(os.path.join(settings.BASE_DIR, 'settings', 'settings.json')) as data_file:
- data = json.load(data_file)
- server_data['server'] = data['host']
- server_data['database_server'] = next(db_connection['host'] for db_connection in data['databases'] if db_connection['name'] == 'tsa_catalog')
- server_data['influx_server'] = data['influx_server'] if 'influx_server' in data else server_data['database_server']
- except IOError:
- print('Error reading settings.json file')
-
- return server_data
-
-
-def get_influx_identifier(sensor):
- return 'uuid_{}'.format(str(sensor.result_uuid).replace('-', '_'))
-
-
-def get_processing_level_explanation(sensor):
- """ Hook for someone to somehow figure out the right QualityControlLevelExplanation"""
- return ''
-
-
-def get_processing_level_definition(sensor):
- """ Hook for someone to somehow figure out the right QualityControlLevelDefinition"""
- return 'Raw Data'
-
-
-def get_processing_level_code(sensor):
- """ Hook for someone to somehow figure out the right QualityControlLevelCode"""
- return 'Raw'
-
-
-def get_data_type(sensor):
- """ Hook for someone to somehow figure out the right DataType"""
- return 'Average'
-
-
-def get_no_data_value(sensor):
- """ Hook for someone to somehow figure out the right NoDataValue"""
- return -9999
-
-
-def get_is_active(sensor):
- """ Hook for someone to somehow figure out the right IsActive"""
- return 1
-
-
-def get_utc_offset(sensor):
- """ Hook for someone to somehow figure out the right UTCOffset"""
- return 0
-
-
-def get_source_id(sensor):
- """ Hook for someone to somehow figure out the right SourceDataServiceID"""
- return 1
-
-
-def get_network(sensor):
- """ Hook for someone to somehow figure out the right Network"""
- return 'EnviroDIY'
-
-
-def get_variable_level(sensor):
- """ Hook for someone to somehow figure out the right VariableLevel"""
- return 'Common'
-
-
-def get_value_type(sensor):
- """ Hook for someone to somehow figure out the right ValueType"""
- return 'Field Observation'
diff --git a/src/tsa/models.py b/src/tsa/models.py
deleted file mode 100644
index 58255b14..00000000
--- a/src/tsa/models.py
+++ /dev/null
@@ -1,59 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-import uuid
-
-from django.db import models
-from django.utils.six import python_2_unicode_compatible
-
-
-@python_2_unicode_compatible
-class DataSeries(models.Model):
- series_id = models.AutoField(db_column='SeriesID', primary_key=True)
- result_uuid = models.UUIDField(default=uuid.uuid4, editable=False, db_column='ResultUUID', unique=True)
- influx_identifier = models.TextField(db_column='InfluxIdentifier')
- source_data_service_id = models.IntegerField(db_column='SourceDataServiceID')
- network = models.CharField(db_column='Network', max_length=50)
- site_code = models.CharField(db_column='SiteCode', max_length=50)
- site_name = models.CharField(db_column='SiteName', max_length=500)
- latitude = models.FloatField(db_column='Latitude', null=True, blank=True)
- longitude = models.FloatField(db_column='Longitude', null=True, blank=True)
- state = models.CharField(db_column='State', max_length=50, null=True, blank=True)
- county = models.CharField(db_column='County', max_length=50, null=True, blank=True)
- site_type = models.CharField(db_column='SiteType', max_length=50)
- variable_code = models.CharField(db_column='VariableCode', max_length=50)
- variable_name = models.CharField(db_column='VariableName', max_length=255)
- variable_level = models.CharField(db_column='VariableLevel', max_length=50)
- method_description = models.CharField(db_column='MethodDescription', max_length=500)
- variable_units_name = models.CharField(db_column='VariableUnitsName', max_length=255, null=True, blank=True)
- variable_units_type = models.CharField(db_column='VariableUnitsType', max_length=50, null=True, blank=True)
- variable_units_abbreviation = models.CharField(db_column='VariableUnitsAbbreviation', max_length=50)
- sample_medium = models.CharField(db_column='SampleMedium', max_length=50)
- value_type = models.CharField(db_column='ValueType', max_length=50, null=True, blank=True)
- data_type = models.CharField(db_column='DataType', max_length=50, null=True, blank=True)
- general_category = models.CharField(db_column='GeneralCategory', max_length=50, null=True, blank=True)
- time_support = models.FloatField(db_column='TimeSupport', null=True, blank=True)
- time_support_units_name = models.CharField(db_column='TimeSupportUnitsName', max_length=500, null=True, blank=True)
- time_support_units_type = models.CharField(db_column='TimeSupportUnitsType', max_length=50, null=True, blank=True)
- time_support_units_abbreviation = models.CharField(db_column='TimeSupportUnitsAbbreviation', max_length=50, null=True, blank=True)
- quality_control_level_code = models.CharField(db_column='QualityControlLevelCode', max_length=50, null=True, blank=True)
- quality_control_level_definition = models.CharField(db_column='QualityControlLevelDefinition', max_length=500)
- quality_control_level_explanation = models.CharField(db_column='QualityControlLevelExplanation', max_length=500, null=True, blank=True)
- source_organization = models.CharField(db_column='SourceOrganization', max_length=255)
- source_description = models.CharField(db_column='SourceDescription', max_length=500, blank=True, null=True)
- begin_datetime = models.DateTimeField(db_column='BeginDateTime')
- end_datetime = models.DateTimeField(db_column='EndDateTime', blank=True, null=True)
- utc_offset = models.IntegerField(db_column='UTCOffset', null=True, blank=True)
- number_observations = models.IntegerField(db_column='NumberObservations')
- date_last_updated = models.DateTimeField(db_column='DateLastUpdated', blank=True, null=True)
- is_active = models.BigIntegerField(db_column='IsActive')
- get_data_url = models.CharField(db_column='GetDataURL', max_length=500)
- get_data_influx = models.TextField(db_column='GetDataInflux')
- no_data_value = models.BigIntegerField(db_column='NoDataValue', default=-9999)
-
- def __str__(self):
- return "{} {} - {}".format(self.site_code, self.variable_code, self.result_uuid)
-
- class Meta:
- managed = False
- db_table = 'DataSeries'
diff --git a/src/tsa/tests.py b/src/tsa/tests.py
deleted file mode 100644
index 5982e6bc..00000000
--- a/src/tsa/tests.py
+++ /dev/null
@@ -1,6 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.test import TestCase
-
-# Create your tests here.
diff --git a/src/tsa/views.py b/src/tsa/views.py
deleted file mode 100644
index e784a0bd..00000000
--- a/src/tsa/views.py
+++ /dev/null
@@ -1,6 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.shortcuts import render
-
-# Create your views here.
From 315c9a4ec2fc66559157ead10a2da26fda6a70e8 Mon Sep 17 00:00:00 2001
From: Paul Tomasula <31142705+ptomasula@users.noreply.github.com>
Date: Mon, 17 Jan 2022 11:59:13 -0500
Subject: [PATCH 03/39] Remove TSA from installed apps in settings
---
src/WebSDL/settings/base.py | 1 -
src/WebSDL/wsgi.py | 3 ---
2 files changed, 4 deletions(-)
diff --git a/src/WebSDL/settings/base.py b/src/WebSDL/settings/base.py
index 910410db..2b550463 100644
--- a/src/WebSDL/settings/base.py
+++ b/src/WebSDL/settings/base.py
@@ -45,7 +45,6 @@
INSTALLED_APPS = [
# 'debug_toolbar',
'rest_framework',
- 'tsa.apps.TsaConfig',
'accounts.apps.AccountsConfig',
'dataloader.apps.DataloaderConfig',
'dataloaderservices.apps.DataloaderservicesConfig',
diff --git a/src/WebSDL/wsgi.py b/src/WebSDL/wsgi.py
index 88b99d1b..4c8cd40f 100644
--- a/src/WebSDL/wsgi.py
+++ b/src/WebSDL/wsgi.py
@@ -8,12 +8,9 @@
"""
import os
-import crontab_jobs
from django.core.wsgi import get_wsgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "WebSDL.settings.linux_server")
-crontab_jobs.start_jobs()
-
application = get_wsgi_application()
From 7fb62dd0dd9aca70c99188a612491b5d1795163d Mon Sep 17 00:00:00 2001
From: Paul Tomasula <31142705+ptomasula@users.noreply.github.com>
Date: Mon, 17 Jan 2022 12:46:59 -0500
Subject: [PATCH 04/39] Update Django to LTS 3.2
---
environment.yml | 5 ++---
src/accounts/templates/registration/login.html | 2 +-
.../templates/registration/password_reset_complete.html | 2 +-
.../templates/registration/password_reset_confirm.html | 2 +-
src/accounts/templates/registration/password_reset_done.html | 2 +-
src/accounts/templates/registration/password_reset_form.html | 2 +-
src/dataloader/models.py | 2 +-
.../templates/dataloaderinterface/base.html | 2 +-
.../templates/dataloaderinterface/browse-sites.html | 2 +-
.../templates/dataloaderinterface/cookie_policy.html | 2 +-
.../templates/dataloaderinterface/dmca.html | 2 +-
.../templates/dataloaderinterface/footer.html | 2 +-
.../templates/dataloaderinterface/home.html | 2 +-
.../templates/dataloaderinterface/manage_leafpack.html | 2 +-
.../templates/dataloaderinterface/manage_sensors.html | 2 +-
.../templates/dataloaderinterface/my-sites.html | 2 +-
.../templates/dataloaderinterface/privacy.html | 2 +-
.../templates/dataloaderinterface/site_details.html | 2 +-
.../templates/dataloaderinterface/site_registration.html | 2 +-
.../dataloaderinterface/site_registration_update.html | 2 +-
.../templates/dataloaderinterface/status.html | 2 +-
.../templates/dataloaderinterface/terms_of_use.html | 2 +-
.../templates/leafpack/leafpack_detail.html | 2 +-
.../templates/leafpack/leafpack_registration.html | 2 +-
.../templates/timeseries_visualization/tool.html | 2 +-
25 files changed, 26 insertions(+), 27 deletions(-)
diff --git a/environment.yml b/environment.yml
index b89a41d0..6803748a 100644
--- a/environment.yml
+++ b/environment.yml
@@ -7,8 +7,6 @@ channels:
dependencies:
# For ODM2DataSharingPortal migration to AWS
- python =3.8.10 # May 3, 2021: final regular Py 3.8 release. https://www.python.org/downloads/release/python-3810/
- # - django =2.2.* # Installs 2.2.14 but lastest is 2.2.24. Use pip to install.
- # https://docs.djangoproject.com/en/3.2/releases
# # Other Requirements
# - beautifulsoup4 =4.9.3 # with python 3.8
@@ -35,7 +33,7 @@ dependencies:
# Dependency versions not available on conda-forge
- pip:
# - codegen >=1.0 # necessary? Last updated in 2012. https://pypi.org/project/codegen/
- - django ==2.2.24
+ - django ==3.2.*
- django-admin-select2 # >=1.0.1
- django-debug-toolbar # >=1.11.1
- django-discover-runner # >=1.0
@@ -43,6 +41,7 @@ dependencies:
- django-webtest # >=1.8.0
- django-widget-tweaks # >=1.4.1
- djangorestframework
+ - django-utils-six #used for unicode_compatiblity
# - patterns >=0.3 # necessary? Last updated in 2014. https://pypi.org/project/patterns/
# - sqlparse # with django=2.2
# - waitress # with django extensions
diff --git a/src/accounts/templates/registration/login.html b/src/accounts/templates/registration/login.html
index 4d166634..94dfae3e 100644
--- a/src/accounts/templates/registration/login.html
+++ b/src/accounts/templates/registration/login.html
@@ -1,5 +1,5 @@
{% extends "dataloaderinterface/base.html" %}
-{% load widget_tweaks staticfiles %}
+{% load widget_tweaks static %}
{% block content %}
diff --git a/src/accounts/templates/registration/password_reset_complete.html b/src/accounts/templates/registration/password_reset_complete.html
index 7ddcf70d..f385c9ba 100644
--- a/src/accounts/templates/registration/password_reset_complete.html
+++ b/src/accounts/templates/registration/password_reset_complete.html
@@ -1,5 +1,5 @@
{% extends "dataloaderinterface/base.html" %}
-{% load widget_tweaks staticfiles %}
+{% load widget_tweaks static %}
{% block title %}Password reset complete{% endblock %}
diff --git a/src/accounts/templates/registration/password_reset_confirm.html b/src/accounts/templates/registration/password_reset_confirm.html
index 86ea1345..27535d74 100644
--- a/src/accounts/templates/registration/password_reset_confirm.html
+++ b/src/accounts/templates/registration/password_reset_confirm.html
@@ -1,5 +1,5 @@
{% extends "dataloaderinterface/base.html" %}
-{% load widget_tweaks staticfiles %}
+{% load widget_tweaks static %}
{% block title %}Setting New password{% endblock %}
diff --git a/src/accounts/templates/registration/password_reset_done.html b/src/accounts/templates/registration/password_reset_done.html
index 6bafdfb7..7f6cda60 100644
--- a/src/accounts/templates/registration/password_reset_done.html
+++ b/src/accounts/templates/registration/password_reset_done.html
@@ -1,5 +1,5 @@
{% extends "dataloaderinterface/base.html" %}
-{% load widget_tweaks staticfiles %}
+{% load widget_tweaks static %}
{% block title %}Password reset successful{% endblock %}
diff --git a/src/accounts/templates/registration/password_reset_form.html b/src/accounts/templates/registration/password_reset_form.html
index c3b6c2dc..96239684 100644
--- a/src/accounts/templates/registration/password_reset_form.html
+++ b/src/accounts/templates/registration/password_reset_form.html
@@ -1,5 +1,5 @@
{% extends "dataloaderinterface/base.html" %}
-{% load widget_tweaks staticfiles %}
+{% load widget_tweaks static %}
{% block title %}Reset Password{% endblock %}
diff --git a/src/dataloader/models.py b/src/dataloader/models.py
index 8cc4c996..f6e0ff2a 100644
--- a/src/dataloader/models.py
+++ b/src/dataloader/models.py
@@ -15,7 +15,7 @@
FeatureActionQuerySet, TimeSeriesValuesQuerySet, EquipmentModelQuerySet, OrganizationQuerySet
from django.conf import settings
from django.db import models
-from django.utils.encoding import python_2_unicode_compatible
+from six import python_2_unicode_compatible
# TODO: function to handle the file upload folder for file fields.
diff --git a/src/dataloaderinterface/templates/dataloaderinterface/base.html b/src/dataloaderinterface/templates/dataloaderinterface/base.html
index 76597688..400e5573 100644
--- a/src/dataloaderinterface/templates/dataloaderinterface/base.html
+++ b/src/dataloaderinterface/templates/dataloaderinterface/base.html
@@ -1,4 +1,4 @@
-{% load staticfiles %}
+{% load static %}
{% load helpers %}
diff --git a/src/dataloaderinterface/templates/dataloaderinterface/browse-sites.html b/src/dataloaderinterface/templates/dataloaderinterface/browse-sites.html
index 2ad45534..452323db 100644
--- a/src/dataloaderinterface/templates/dataloaderinterface/browse-sites.html
+++ b/src/dataloaderinterface/templates/dataloaderinterface/browse-sites.html
@@ -1,5 +1,5 @@
{% extends 'dataloaderinterface/base.html' %}
-{% load staticfiles %}
+{% load static %}
{% load helpers %}
{% load site %}
diff --git a/src/dataloaderinterface/templates/dataloaderinterface/cookie_policy.html b/src/dataloaderinterface/templates/dataloaderinterface/cookie_policy.html
index 4eb53a4a..7c5e3d71 100644
--- a/src/dataloaderinterface/templates/dataloaderinterface/cookie_policy.html
+++ b/src/dataloaderinterface/templates/dataloaderinterface/cookie_policy.html
@@ -1,5 +1,5 @@
{% extends 'dataloaderinterface/base.html' %}
-{% load staticfiles %}
+{% load static %}
{% block styles %}
{{ block.super }}
diff --git a/src/dataloaderinterface/templates/dataloaderinterface/dmca.html b/src/dataloaderinterface/templates/dataloaderinterface/dmca.html
index d959ef63..da4cdf5a 100644
--- a/src/dataloaderinterface/templates/dataloaderinterface/dmca.html
+++ b/src/dataloaderinterface/templates/dataloaderinterface/dmca.html
@@ -1,5 +1,5 @@
{% extends 'dataloaderinterface/base.html' %}
-{% load staticfiles %}
+{% load static %}
{% block styles %}
{{ block.super }}
diff --git a/src/dataloaderinterface/templates/dataloaderinterface/footer.html b/src/dataloaderinterface/templates/dataloaderinterface/footer.html
index 457cc9e1..7ca196fc 100644
--- a/src/dataloaderinterface/templates/dataloaderinterface/footer.html
+++ b/src/dataloaderinterface/templates/dataloaderinterface/footer.html
@@ -1,4 +1,4 @@
-{% load staticfiles %}
+{% load static %}
diff --git a/src/dataloaderinterface/templates/dataloaderinterface/home.html b/src/dataloaderinterface/templates/dataloaderinterface/home.html
index bb18fb59..10222783 100644
--- a/src/dataloaderinterface/templates/dataloaderinterface/home.html
+++ b/src/dataloaderinterface/templates/dataloaderinterface/home.html
@@ -1,5 +1,5 @@
{% extends 'dataloaderinterface/base.html' %}
-{% load staticfiles %}
+{% load static %}
{% block styles %}
{{ block.super }}
diff --git a/src/dataloaderinterface/templates/dataloaderinterface/manage_leafpack.html b/src/dataloaderinterface/templates/dataloaderinterface/manage_leafpack.html
index 5d954cb0..9e2becf1 100644
--- a/src/dataloaderinterface/templates/dataloaderinterface/manage_leafpack.html
+++ b/src/dataloaderinterface/templates/dataloaderinterface/manage_leafpack.html
@@ -1,5 +1,5 @@
{% extends 'dataloaderinterface/base.html' %}
-{% load staticfiles %}
+{% load static %}
{#{% load helpers %}#}
{% load widget_tweaks %}
diff --git a/src/dataloaderinterface/templates/dataloaderinterface/manage_sensors.html b/src/dataloaderinterface/templates/dataloaderinterface/manage_sensors.html
index f0951f69..90a271a0 100644
--- a/src/dataloaderinterface/templates/dataloaderinterface/manage_sensors.html
+++ b/src/dataloaderinterface/templates/dataloaderinterface/manage_sensors.html
@@ -1,5 +1,5 @@
{% extends 'dataloaderinterface/base.html' %}
-{% load staticfiles %}
+{% load static %}
{#{% load helpers %}#}
{% load widget_tweaks %}
diff --git a/src/dataloaderinterface/templates/dataloaderinterface/my-sites.html b/src/dataloaderinterface/templates/dataloaderinterface/my-sites.html
index 3daa70a2..e13dfa4f 100644
--- a/src/dataloaderinterface/templates/dataloaderinterface/my-sites.html
+++ b/src/dataloaderinterface/templates/dataloaderinterface/my-sites.html
@@ -1,5 +1,5 @@
{% extends 'dataloaderinterface/base.html' %}
-{% load staticfiles %}
+{% load static %}
{% load helpers %}
{% block page_title %}
diff --git a/src/dataloaderinterface/templates/dataloaderinterface/privacy.html b/src/dataloaderinterface/templates/dataloaderinterface/privacy.html
index 22420d8f..bedad4b5 100644
--- a/src/dataloaderinterface/templates/dataloaderinterface/privacy.html
+++ b/src/dataloaderinterface/templates/dataloaderinterface/privacy.html
@@ -1,5 +1,5 @@
{% extends 'dataloaderinterface/base.html' %}
-{% load staticfiles %}
+{% load static %}
{% block styles %}
{{ block.super }}
diff --git a/src/dataloaderinterface/templates/dataloaderinterface/site_details.html b/src/dataloaderinterface/templates/dataloaderinterface/site_details.html
index 35ead80c..ee9f40e9 100644
--- a/src/dataloaderinterface/templates/dataloaderinterface/site_details.html
+++ b/src/dataloaderinterface/templates/dataloaderinterface/site_details.html
@@ -1,5 +1,5 @@
{% extends 'dataloaderinterface/base.html' %}
-{% load staticfiles %}
+{% load static %}
{% load site %}
{% load helpers %}
diff --git a/src/dataloaderinterface/templates/dataloaderinterface/site_registration.html b/src/dataloaderinterface/templates/dataloaderinterface/site_registration.html
index ccfa1df6..3c9024fc 100644
--- a/src/dataloaderinterface/templates/dataloaderinterface/site_registration.html
+++ b/src/dataloaderinterface/templates/dataloaderinterface/site_registration.html
@@ -1,5 +1,5 @@
{% extends 'dataloaderinterface/base.html' %}
-{% load staticfiles %}
+{% load static %}
{% load widget_tweaks %}
{% load site_form_util %}
diff --git a/src/dataloaderinterface/templates/dataloaderinterface/site_registration_update.html b/src/dataloaderinterface/templates/dataloaderinterface/site_registration_update.html
index d3e64334..ce416d08 100644
--- a/src/dataloaderinterface/templates/dataloaderinterface/site_registration_update.html
+++ b/src/dataloaderinterface/templates/dataloaderinterface/site_registration_update.html
@@ -1,5 +1,5 @@
{% extends 'dataloaderinterface/base.html' %}
-{% load staticfiles %}
+{% load static %}
{% load widget_tweaks %}
{% load site_form_util %}
diff --git a/src/dataloaderinterface/templates/dataloaderinterface/status.html b/src/dataloaderinterface/templates/dataloaderinterface/status.html
index c37695d9..25ce8a35 100644
--- a/src/dataloaderinterface/templates/dataloaderinterface/status.html
+++ b/src/dataloaderinterface/templates/dataloaderinterface/status.html
@@ -1,4 +1,4 @@
-{% load site staticfiles %}
+{% load site static %}
{% load helpers %}
diff --git a/src/dataloaderinterface/templates/dataloaderinterface/terms_of_use.html b/src/dataloaderinterface/templates/dataloaderinterface/terms_of_use.html
index d3aadcef..3dc5bd60 100644
--- a/src/dataloaderinterface/templates/dataloaderinterface/terms_of_use.html
+++ b/src/dataloaderinterface/templates/dataloaderinterface/terms_of_use.html
@@ -1,5 +1,5 @@
{% extends 'dataloaderinterface/base.html' %}
-{% load staticfiles %}
+{% load static %}
{% block styles %}
{{ block.super }}
diff --git a/src/dataloaderinterface/templates/leafpack/leafpack_detail.html b/src/dataloaderinterface/templates/leafpack/leafpack_detail.html
index 70b789b5..9db8c035 100644
--- a/src/dataloaderinterface/templates/leafpack/leafpack_detail.html
+++ b/src/dataloaderinterface/templates/leafpack/leafpack_detail.html
@@ -1,5 +1,5 @@
{% extends 'dataloaderinterface/base.html' %}
-{% load staticfiles %}
+{% load static %}
{% load site %}
{% load widget_tweaks %}
diff --git a/src/dataloaderinterface/templates/leafpack/leafpack_registration.html b/src/dataloaderinterface/templates/leafpack/leafpack_registration.html
index 2b77623f..d1be9634 100644
--- a/src/dataloaderinterface/templates/leafpack/leafpack_registration.html
+++ b/src/dataloaderinterface/templates/leafpack/leafpack_registration.html
@@ -1,5 +1,5 @@
{% extends 'dataloaderinterface/base.html' %}
-{% load staticfiles %}
+{% load static %}
{% load site %}
{% load widget_tweaks %}
diff --git a/src/dataloaderinterface/templates/timeseries_visualization/tool.html b/src/dataloaderinterface/templates/timeseries_visualization/tool.html
index 73873193..cb8fbafb 100644
--- a/src/dataloaderinterface/templates/timeseries_visualization/tool.html
+++ b/src/dataloaderinterface/templates/timeseries_visualization/tool.html
@@ -1,5 +1,5 @@
{% extends 'dataloaderinterface/base.html' %}
-{% load staticfiles %}
+{% load static %}
{% block content %}
From 165b6721da52c0455d45e7387db66ec132cb11d2 Mon Sep 17 00:00:00 2001
From: Paul Tomasula <31142705+ptomasula@users.noreply.github.com>
Date: Wed, 19 Jan 2022 18:01:46 -0500
Subject: [PATCH 05/39] SQLAlchemy ODM2 data models
This implements an SQLAlchemy base approach to ODM2 data models. This approach use the automap_base feature map the database schema to a set of classes in the application. The results of this mapping are also cached in a pickle file so that subsequent builds of the application will read the mappings metadata and generate the data models using a declarative_base approach.
---
src/WebSDL/settings/base.py | 5 +-
src/odm2/__init__.py | 11 ++++
src/odm2/base.py | 32 +++++++++
src/odm2/modelcache.pkl | Bin 0 -> 300188 bytes
src/odm2/models/__init__.py | 12 ++++
src/odm2/models/annotations.py | 48 ++++++++++++++
src/odm2/models/core.py | 53 +++++++++++++++
src/odm2/models/cv.py | 86 +++++++++++++++++++++++++
src/odm2/models/dataquality.py | 20 ++++++
src/odm2/models/equipment.py | 41 ++++++++++++
src/odm2/models/extensionproperties.py | 26 ++++++++
src/odm2/models/externalidentifiers.py | 32 +++++++++
src/odm2/models/labanalyses.py | 14 ++++
src/odm2/models/provenance.py | 32 +++++++++
src/odm2/models/results.py | 59 +++++++++++++++++
src/odm2/models/samplingfeatures.py | 23 +++++++
src/odm2/models/simulation.py | 17 +++++
17 files changed, 509 insertions(+), 2 deletions(-)
create mode 100644 src/odm2/__init__.py
create mode 100644 src/odm2/base.py
create mode 100644 src/odm2/modelcache.pkl
create mode 100644 src/odm2/models/__init__.py
create mode 100644 src/odm2/models/annotations.py
create mode 100644 src/odm2/models/core.py
create mode 100644 src/odm2/models/cv.py
create mode 100644 src/odm2/models/dataquality.py
create mode 100644 src/odm2/models/equipment.py
create mode 100644 src/odm2/models/extensionproperties.py
create mode 100644 src/odm2/models/externalidentifiers.py
create mode 100644 src/odm2/models/labanalyses.py
create mode 100644 src/odm2/models/provenance.py
create mode 100644 src/odm2/models/results.py
create mode 100644 src/odm2/models/samplingfeatures.py
create mode 100644 src/odm2/models/simulation.py
diff --git a/src/WebSDL/settings/base.py b/src/WebSDL/settings/base.py
index 2b550463..746c0ec6 100644
--- a/src/WebSDL/settings/base.py
+++ b/src/WebSDL/settings/base.py
@@ -19,12 +19,12 @@
logging.basicConfig(level=logging.INFO)
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
-BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
# Loads settings configuration data from settings.json file
data = {}
try:
- with open(os.path.join(BASE_DIR, 'settings', 'settings.json')) as data_file:
+ with open(os.path.join(BASE_DIR, 'WebSDL','settings', 'settings.json')) as data_file:
data = json.load(data_file)
except IOError:
print("You need to setup the settings data file (see instructions in base.py file.)")
@@ -125,6 +125,7 @@
'CONN_MAX_AGE': 0,
'TEST': database['test'] if 'test' in database else {},
}
+DATAMODELCACHE = os.path.join(BASE_DIR, 'odm2', 'modelcache.pkl')
# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators
diff --git a/src/odm2/__init__.py b/src/odm2/__init__.py
new file mode 100644
index 00000000..c5690567
--- /dev/null
+++ b/src/odm2/__init__.py
@@ -0,0 +1,11 @@
+from odm2 import base as _base
+import pickle as _pickle
+import odm2.models as models
+
+engine = _base.engine
+sessionmaker = _base.sessionmaker
+
+if not _base.cached:
+ _base._model_base.prepare(engine)
+ with open(_base.cache_path, 'wb') as file:
+ _pickle.dump(_base._model_base.metadata, file)
diff --git a/src/odm2/base.py b/src/odm2/base.py
new file mode 100644
index 00000000..37e439e6
--- /dev/null
+++ b/src/odm2/base.py
@@ -0,0 +1,32 @@
+import sqlalchemy
+from sqlalchemy.ext.automap import automap_base
+from sqlalchemy.ext.declarative import declared_attr, declarative_base
+
+import pickle
+
+from django.conf import settings
+
+_dbsettings = settings.DATABASES['default']
+_connection_str = f"postgresql://{_dbsettings['USER']}:{_dbsettings['PASSWORD']}@{_dbsettings['HOST']}:{_dbsettings['PORT']}/{_dbsettings['NAME']}"
+engine = sqlalchemy.create_engine(_connection_str, pool_size=10)
+sessionmaker = sqlalchemy.orm.sessionmaker(engine)
+
+cache_path = settings.DATAMODELCACHE
+
+class Base():
+
+ @declared_attr
+ def __tablename__(self) -> str:
+ cls_name = str(self.__name__)
+ return cls_name.lower()
+
+_model_base = None
+cached = None
+try:
+ with open(cache_path, 'rb') as file:
+ metadata = pickle.load(file=file)
+ _model_base = declarative_base(cls=Base, bind=engine, metadata=metadata)
+ cached = True
+except:
+ metadata = sqlalchemy.MetaData(schema='odm2')
+ _model_base = automap_base(cls=Base, metadata=metadata)
\ No newline at end of file
diff --git a/src/odm2/modelcache.pkl b/src/odm2/modelcache.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..275de06dc9582eb3ffdd911fa70c3bcf976cc7fc
GIT binary patch
literal 300188
zcmd3P2b>(ml|BN2w91K
RbA8lYPM%~Mp(a}wmMXIef9di
z@4b3mUGuEDcka8_oW1a`zJKlMBc+j{N_qRP4YIkRhRxD+eg5Y1MCshpL}~ht>7#F)
zzG1qrzTZS?%SgF4U0)wuGdWQm*)TXXHZoEkny8MA%IO8?mxfBi<#Ve;6LK-G+<$C%
z`-vM$tT0?1N6npbqsm_OeJ{g3(>CT0S8F>;6GN5h24v-(<I8Ais`<}+WP!0
zafp!iv)89r?Q>Om*L3CRN^j-p`o4TB)0M^bxf8o~C_NWoI3XD4T`+Lj
zd5<~o5_hBel0lpqEN$Id9j#8tEl-d-qZ8$A<#DM~+frXptqqP;C(7fck?E=W!X4w)
z?WOTugJ}Nddhg(5tvo(BT$ZaLPZ1X%=2U%tusu)CmnKIhrU&Xvr20;K1&>JA7N2X+
zZX7Ae%^a}4`hu-viUAq%GXR}tL{6vr8+9YsxRcmj?#E(
zdrj0%