From 23579748049996caefa69de3252c86db429cb562 Mon Sep 17 00:00:00 2001 From: darksinge Date: Tue, 5 Dec 2017 17:43:11 -0700 Subject: [PATCH 001/115] added hydroshare model definitions; updated .gitignore --- src/dataloaderinterface/models.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/dataloaderinterface/models.py b/src/dataloaderinterface/models.py index 197757a1..43d142ef 100644 --- a/src/dataloaderinterface/models.py +++ b/src/dataloaderinterface/models.py @@ -163,3 +163,24 @@ def affiliation(self): def can_administer_site(self, registration): return self.user.is_staff or registration.user == self + + +class HydroshareUser(models.Model): + user = models.OneToOneField(ODM2User) + oauth_token = models.CharField(max_length=255) + update_freq = models.IntegerField(default=None) + is_enabled = models.BooleanField(default=False) + + def __str__(self): + return self.user.username + + +class HydroshareSync(models.Model): + # hydroshare_user references a HydroshareUser object (not an ODM2User object). + hydroshare_user = models.OneToOneField(HydroshareUser) + # timestamp for when data was synced with Hydroshare. + sync_date = models.DateTimeField(default=datetime.now()) + # sync type, either 'manual' or 'scheduled'. + sync_type = models.CharField(max_length=255, default='manual') + # resultid corresponds to the PK in the `odm2` database on the `results.resultid` column. + resultid = models.IntegerField(db_column='odm2_results_resultid') From a26db24dedb0d39f9de0b2bdcdb3a8e83bde3360 Mon Sep 17 00:00:00 2001 From: darksinge Date: Wed, 13 Dec 2017 11:26:09 -0700 Subject: [PATCH 002/115] added oauth2 support for HydroShare --- src/WebSDL/settings/base.py | 9 ++- src/WebSDL/urls.py | 1 + src/dataloaderinterface/models.py | 63 ++++++++++++--- .../dataloaderinterface/site_details.html | 18 ++++- .../hydroshare/hydroshare_account.html | 9 +++ .../templates/registration/account.html | 14 +++- src/dataloaderinterface/urls.py | 4 +- src/dataloaderinterface/views.py | 16 +++- src/dataloaderservices/auth.py | 2 +- src/hydroshare_oauth/__init__.py | 0 src/hydroshare_oauth/admin.py | 3 + src/hydroshare_oauth/apps.py | 7 ++ src/hydroshare_oauth/models.py | 5 ++ src/hydroshare_oauth/tests.py | 3 + src/hydroshare_oauth/urls.py | 6 ++ src/hydroshare_oauth/views.py | 80 +++++++++++++++++++ 16 files changed, 220 insertions(+), 20 deletions(-) create mode 100644 src/dataloaderinterface/templates/hydroshare/hydroshare_account.html create mode 100644 src/hydroshare_oauth/__init__.py create mode 100644 src/hydroshare_oauth/admin.py create mode 100644 src/hydroshare_oauth/apps.py create mode 100644 src/hydroshare_oauth/models.py create mode 100644 src/hydroshare_oauth/tests.py create mode 100644 src/hydroshare_oauth/urls.py create mode 100644 src/hydroshare_oauth/views.py diff --git a/src/WebSDL/settings/base.py b/src/WebSDL/settings/base.py index bf035e5a..f80445b2 100644 --- a/src/WebSDL/settings/base.py +++ b/src/WebSDL/settings/base.py @@ -42,6 +42,7 @@ 'dataloader', 'dataloaderservices', 'dataloaderinterface', + 'hydroshare_oauth', 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', @@ -74,7 +75,7 @@ TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [os.path.join(BASE_DIR, 'templates')] + 'DIRS': [os.path.join(BASE_DIR, 'hydroshare')] , 'APP_DIRS': True, 'OPTIONS': { @@ -164,3 +165,9 @@ EMAIL_HOST = EMAIL_SERVER[0] if isinstance(EMAIL_SERVER, tuple) else EMAIL_SERVER DATETIME_FORMAT = "N j, Y, H:m" + +if "hydroshare_oauth" in data: + os.environ.setdefault("HS_CLIENT_ID", data["hydroshare_oauth"]["client_id"]) + os.environ.setdefault("HS_CLIENT_SECRET", data["hydroshare_oauth"]["client_secret"]) + os.environ.setdefault("HS_REDIRECT_URI", data["hydroshare_oauth"]["redirect_uri"]) + os.environ.setdefault("HS_RESPONSE_TYPE", data["hydroshare_oauth"]["response_type"]) diff --git a/src/WebSDL/urls.py b/src/WebSDL/urls.py index 280895d5..e7c9034e 100644 --- a/src/WebSDL/urls.py +++ b/src/WebSDL/urls.py @@ -53,4 +53,5 @@ url(r'^' + BASE_URL + 'api-auth/', include('rest_framework.urls', namespace='rest_framework')), url(BASE_URL, include('dataloaderinterface.urls')), url(BASE_URL, include('dataloaderservices.urls')), + url(r'^' + BASE_URL + 'oauth/hydroshare/', include('hydroshare_oauth.urls', namespace='hydroshare_oauth')) ] diff --git a/src/dataloaderinterface/models.py b/src/dataloaderinterface/models.py index 43d142ef..23475c0b 100644 --- a/src/dataloaderinterface/models.py +++ b/src/dataloaderinterface/models.py @@ -10,6 +10,7 @@ from django.contrib.auth.models import User from django.db import models +HYDROSHARE_SYNC_TYPES = (('M', 'manual'), ('S', 'scheduled')) class SiteRegistration(models.Model): registration_id = models.AutoField(primary_key=True, db_column='RegistrationID') @@ -165,22 +166,66 @@ def can_administer_site(self, registration): return self.user.is_staff or registration.user == self +# HydroshareUser - holds information for user's Hydroshare account class HydroshareUser(models.Model): - user = models.OneToOneField(ODM2User) - oauth_token = models.CharField(max_length=255) - update_freq = models.IntegerField(default=None) + user = models.ForeignKey('ODM2User', db_column='user_id') + account_nickname = models.CharField(max_length=255, unique=True) + access_token = models.CharField(max_length=255, blank=True, default='') + refresh_token = models.CharField(max_length=255, blank=True, default='') is_enabled = models.BooleanField(default=False) + token_expires_in = models.IntegerField(default=0) + oauth_scope = models.CharField(max_length=255, default='read write') + + @property + def scope(self): + return self.hs_oauth_scope.split(",") + + @property + def scope(self, scope): + self.hs_oauth_scope = ",".join(scope) + + def set_token(self, token): + self.access_token = token['access_token'] + self.refresh_token = token['refresh_token'] + self.token_expires_in = token['expires_in'] + self.oauth_scope = token['scope'] + self.save(update_fields=['refresh_token', 'access_token', 'token_expires_in', 'oauth_scope']) def __str__(self): return self.user.username + class Meta: + db_table = 'hydroshare_user' + + +# "Settings" for a Hydroshare account connection +class HydroshareSiteSharing(models.Model): + + site_registration = models.ForeignKey(SiteRegistration, on_delete=models.CASCADE) + sync_type = models.CharField(max_length=255, default='manual', choices=HYDROSHARE_SYNC_TYPES) + resource_id = models.CharField(max_length=255, blank=True) + update_freq = models.IntegerField(null=True) + is_enabled = models.BooleanField(default=False) + last_sync_date = models.DateTimeField() + + class Meta: + db_table = 'hydroshare_site_sharing' + + +# HydroshareSync - tracks scheduled or manual syncs with hydroshares API +class HydroshareSyncLog(models.Model): + + # HydroshareSiteSharing object this log belongs to + site_sharing = models.ForeignKey(HydroshareSiteSharing, on_delete=models.CASCADE) -class HydroshareSync(models.Model): - # hydroshare_user references a HydroshareUser object (not an ODM2User object). - hydroshare_user = models.OneToOneField(HydroshareUser) # timestamp for when data was synced with Hydroshare. - sync_date = models.DateTimeField(default=datetime.now()) + sync_date = models.DateTimeField(auto_now_add=True) + # sync type, either 'manual' or 'scheduled'. - sync_type = models.CharField(max_length=255, default='manual') + sync_type = models.CharField(max_length=255, default='manual', choices=HYDROSHARE_SYNC_TYPES) + # resultid corresponds to the PK in the `odm2` database on the `results.resultid` column. - resultid = models.IntegerField(db_column='odm2_results_resultid') + # resultid = models.IntegerField(db_column='odm2_results_resultid') + + class Meta: + db_table = 'hydroshare_sync_log' diff --git a/src/dataloaderinterface/templates/dataloaderinterface/site_details.html b/src/dataloaderinterface/templates/dataloaderinterface/site_details.html index d59d6e57..e05c3409 100644 --- a/src/dataloaderinterface/templates/dataloaderinterface/site_details.html +++ b/src/dataloaderinterface/templates/dataloaderinterface/site_details.html @@ -28,8 +28,8 @@
+ {% if site.django_user_id != request.user.id %}
- {% if site.django_user_id != request.user.id %}
- {% endif %}
+ {% endif %} {# TODO: make and use template tag #} {% if site.django_user_id == request.user.id or request.user.is_staff %}
-
+
@@ -74,6 +74,16 @@ View code snippet
+
+
+
+ + share Connect To Hydroshare + +
+
+
mode_edit Edit Site
-
+
delete_forever Delete Site diff --git a/src/dataloaderinterface/templates/hydroshare/hydroshare_account.html b/src/dataloaderinterface/templates/hydroshare/hydroshare_account.html new file mode 100644 index 00000000..f57e1e53 --- /dev/null +++ b/src/dataloaderinterface/templates/hydroshare/hydroshare_account.html @@ -0,0 +1,9 @@ +{% extends "dataloaderinterface/base.html" %} + +{% block content %} + + + share Connect To Hydroshare + + +{% endblock %} \ No newline at end of file diff --git a/src/dataloaderinterface/templates/registration/account.html b/src/dataloaderinterface/templates/registration/account.html index 53c4fb67..3159ee9c 100644 --- a/src/dataloaderinterface/templates/registration/account.html +++ b/src/dataloaderinterface/templates/registration/account.html @@ -54,7 +54,7 @@
-{% include 'hydroshare/hydroshare_settings_modal.html' with resource=resource form=form site=site %} +
+ {% include 'hydroshare/hydroshare_settings_modal.html' with resource=resource form=form site=site %} +
+ +{#{% block styles %}#} +{# #} +{#{% endblock %}#} {% block scripts %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/src/dataloaderinterface/views.py b/src/dataloaderinterface/views.py index 7446ceed..89e685a3 100644 --- a/src/dataloaderinterface/views.py +++ b/src/dataloaderinterface/views.py @@ -362,8 +362,10 @@ def post(self, request, *args, **kwargs): hs_resource = self.get_hs_resource(resource_data) # Upload the most recent resource files - # upload_hydroshare_resource_files(site, hs_resource) - call_command('update_hydroshare_resource_files') + try: + upload_hydroshare_resource_files(site, hs_resource) + except Exception as e: + return JsonResponse({'error': e.message}, status=500) # update last sync date on resource_data resource_data.last_sync_date = timezone.now() From b8a3a51240ad8d588879b5a90c7f4273b1023c61 Mon Sep 17 00:00:00 2001 From: Craig Blackburn Date: Wed, 7 Feb 2018 09:48:26 -0700 Subject: [PATCH 056/115] fixed depracation warning for XMLHttpRequest in hydroshare-settings-modal.js --- src/WebSDL/settings/base.py | 2 +- src/dataloaderinterface/apps.py | 4 - .../css/hydroshare-styles.css | 2 +- .../dataloaderinterface/js/device-detail.js | 5 +- .../hydroshare/hydroshare-resource-create.js | 4 +- .../hydroshare/hydroshare-resource-update.js | 2 + .../hydroshare/hydroshare-settings-modal.js | 170 +++++++++--------- .../dataloaderinterface/site_details.html | 6 +- .../hydroshare/hydroshare_site_settings.html | 75 ++++---- src/dataloaderinterface/views.py | 26 +-- 10 files changed, 139 insertions(+), 157 deletions(-) diff --git a/src/WebSDL/settings/base.py b/src/WebSDL/settings/base.py index 0f6e9660..faec072a 100644 --- a/src/WebSDL/settings/base.py +++ b/src/WebSDL/settings/base.py @@ -191,7 +191,7 @@ # crontab job settings CRONTAB_USER = data.get('user', getpass.getuser()) -CRONTAB_LOGFILE_PATH = data.get('crontabLogfile', '/var/log/odm2websdl-cron.log') +CRONTAB_LOGFILE_PATH = data.get('crontabLogFile', '/var/log/odm2websdl-cron.log') CRONTAB_EXECUTE_DAILY_AT_HOUR = data.get('crontabExecuteDailyAtHour', 5) diff --git a/src/dataloaderinterface/apps.py b/src/dataloaderinterface/apps.py index 3c7670eb..bf9c5e3e 100644 --- a/src/dataloaderinterface/apps.py +++ b/src/dataloaderinterface/apps.py @@ -6,7 +6,3 @@ class DataloaderinterfaceConfig(AppConfig): name = 'dataloaderinterface' - - # def ready(self): - # user = True - # scheduledJobs.start_jobs(user=user) diff --git a/src/dataloaderinterface/static/dataloaderinterface/css/hydroshare-styles.css b/src/dataloaderinterface/static/dataloaderinterface/css/hydroshare-styles.css index 36df9c15..8b8d851b 100644 --- a/src/dataloaderinterface/static/dataloaderinterface/css/hydroshare-styles.css +++ b/src/dataloaderinterface/static/dataloaderinterface/css/hydroshare-styles.css @@ -10,4 +10,4 @@ .hs-img-container > img { width: 100%; height: auto; -} \ No newline at end of file +} diff --git a/src/dataloaderinterface/static/dataloaderinterface/js/device-detail.js b/src/dataloaderinterface/static/dataloaderinterface/js/device-detail.js index b8905df8..27e36ab7 100644 --- a/src/dataloaderinterface/static/dataloaderinterface/js/device-detail.js +++ b/src/dataloaderinterface/static/dataloaderinterface/js/device-detail.js @@ -211,6 +211,7 @@ function drawSparklinePlot(seriesInfo, seriesData) { } function getTimeSeriesData(sensorInfo) { + if (sensorInfo['influxUrl'] === 'None' ) { return; } $.ajax({ url: sensorInfo['influxUrl'] }).done(function(influx_data) { @@ -235,10 +236,10 @@ function getTimeSeriesData(sensorInfo) { console.error('No data values were found for this site'); console.info(series.getdatainflux); } - }).fail(function(failedData) { + }).fail(function() { console.log('data failed to load.'); - }) + }); // Papa.parse(sensorInfo['csvPath'], { // download: true, // header: true, diff --git a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hydroshare-resource-create.js b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hydroshare-resource-create.js index 54bc7c40..40ea5574 100644 --- a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hydroshare-resource-create.js +++ b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hydroshare-resource-create.js @@ -9,8 +9,10 @@ $(() => { $.get(template_url) .done(data => { - // console.log(data); $(hydroshareContainer).html(data); + + // called from `hydroshare-resource-modal.js` + initializeHydroShareSettingsDialog(); }) .fail(xhr => { if (xhr.responseJSON) diff --git a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hydroshare-resource-update.js b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hydroshare-resource-update.js index 40451bf0..664e874d 100644 --- a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hydroshare-resource-update.js +++ b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hydroshare-resource-update.js @@ -9,6 +9,8 @@ $(hydroshareContainer).load(resource_template_url, () => { const hydroshareSettingsForm = $('#hydroshare-settings-form'); const hydroshareError = $('p#hydroshare-error'); + initializeHydroShareSettingsDialog(); // called from `hydroshare-resource-modal.js` + $(updateNowButton).click((e) => { $('p#hydroshare-error').text(''); console.log(hydroshareError); diff --git a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hydroshare-settings-modal.js b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hydroshare-settings-modal.js index 3eff0dbd..89e899e3 100644 --- a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hydroshare-settings-modal.js +++ b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hydroshare-settings-modal.js @@ -1,104 +1,98 @@ /** - * hydroshare-settings-modal.js - * @description Determines how the hydroshare settings modal is presented in 'site_details.html'. - */ +* hydroshare-settings-modal.js +* @description Determines how the hydroshare settings modal is presented in 'site_details.html'. +*/ -$(() => { - setTimeout(() => { - console.log("Loading hydroshare-settings-modal.js"); +function initializeHydroShareSettingsDialog() { - const dialog = $('#hydroshare-settings-dialog')[0]; - const showDialogButton = $('#show-hydroshare-settings-dialog')[0]; - const hydroshareSettingsForm = $('#hydroshare-settings-form')[0]; - const scheduledCB = $('input#id_schedule_type_0')[0]; - const manualCB = $('input#id_schedule_type_1')[0]; - const updateFreqSelect = $('select#id_update_freq')[0]; + const dialog = $('#hydroshare-settings-dialog')[0]; + const showDialogButton = $('#show-hydroshare-settings-dialog')[0]; + const hydroshareSettingsForm = $('#hydroshare-settings-form')[0]; + const scheduledCB = $('input#id_schedule_type_0')[0]; + const manualCB = $('input#id_schedule_type_1')[0]; + const updateFreqSelect = $('select#id_update_freq')[0]; - if (dialog && dialog.showModal) { - console.log("registering dialog"); - dialogPolyfill.registerDialog(dialog); - } + if (dialog && dialog.showModal) { + console.log("registering dialog"); + dialogPolyfill.registerDialog(dialog); + } + + showDialogButton.addEventListener('click', () => { + dialog.showModal(); + }); + + dialog.querySelector('.close').addEventListener('click', () => { + dialog.close(); + }); - showDialogButton.addEventListener('click', () => { - dialog.showModal(); - }); + hydroshareSettingsForm.addEventListener('submit', (e) => { + e.preventDefault(); + submitForm(); + }); - dialog.querySelector('.close').addEventListener('click', () => { - dialog.close(); - }); + // dialog.showModal(); - hydroshareSettingsForm.addEventListener('submit', (e) => { - e.preventDefault(); - submitForm(); - }); + $(manualCB).change(onCBChange); + $(scheduledCB).change(onCBChange); + function onCBChange(e) { + const shouldHide = $(scheduledCB).closest(`label[for=id_schedule_type_0]`).hasClass('is-checked'); + $(updateFreqSelect).attr('hidden', shouldHide); + } - // dialog.showModal(); - $(manualCB).change(onCBChange); - $(scheduledCB).change(onCBChange); - function onCBChange(e) { - const shouldHide = $(scheduledCB).closest(`label[for=id_schedule_type_0]`).hasClass('is-checked'); - $(updateFreqSelect).attr('hidden', shouldHide); + function submitForm() { + const inputField = $(hydroshareSettingsForm).find('input[type=submit]')[0]; + + let method = ''; + if (inputField.id === 'create-resource') { + method = 'create'; + } else if (inputField.id === 'update-resource') { + method = 'update'; } + let url = `${hydroshareSettingsForm.baseURI}hsr/${method}/`; + let serializedForm = $(hydroshareSettingsForm).serialize(); + let progressSpinner = $(hydroshareSettingsForm).find('#hydroshare-progress-spinner'); + + progressSpinner.addClass('is-active'); + + $.post(url, serializedForm) + .done(data => { + console.log(data); + if (data.redirect) + window.location.href = data.redirect; + else + dialog.close(); + }) + .fail((xhr) => { + + let errors = xhr.responseJSON; + if (errors) { + console.error(xhr.responseJSON); + for (let [errorName, errorList] of Object.entries(errors)) { + if (Array.isArray(errorList)) { + let fieldContainer = $(`#id_${errorName}`); + let errorContainer = $(fieldContainer).find('ul.errorlist'); + + if (errorContainer.length) { + $(errorContainer).html(''); + } else { + $(fieldContainer).prepend(`
    `); + errorContainer = $(fieldContainer).find('ul.errorlist'); + } - function submitForm() { - const inputField = $(hydroshareSettingsForm).find('input[type=submit]')[0]; - - let method = ''; - if (inputField.id === 'create-resource') { - method = 'create'; - } else if (inputField.id === 'update-resource') { - method = 'update'; - } - - let url = `${hydroshareSettingsForm.baseURI}hsr/${method}/`; - let serializedForm = $(hydroshareSettingsForm).serialize(); - let progressSpinner = $(hydroshareSettingsForm).find('#hydroshare-progress-spinner'); - - progressSpinner.addClass('is-active'); - - $.post(url, serializedForm) - .done(data => { - console.log(data); - if (data.redirect) - window.location.href = data.redirect; - else - dialog.close(); - }) - .fail((xhr) => { - - let errors = xhr.responseJSON; - if (errors) { - console.error(xhr.responseJSON); - for (let [errorName, errorList] of Object.entries(errors)) { - if (Array.isArray(errorList)) { - let fieldContainer = $(`#id_${errorName}`); - let errorContainer = $(fieldContainer).find('ul.errorlist'); - - if (errorContainer.length) { - $(errorContainer).html(''); - } else { - $(fieldContainer).prepend(`
      `); - errorContainer = $(fieldContainer).find('ul.errorlist'); - } - - for (let err of errorList) { - $(errorContainer).append(`
    • ${err}
      • `); - } + for (let err of errorList) { + $(errorContainer).append(`
    • ${err}
      • `); } } - } else { - console.error(xhr.responseText); } - }) - .always(() => { - progressSpinner.removeClass('is-active'); - }) - } - }, 0); - - - -}); \ No newline at end of file + } else { + console.error(xhr.responseText); + } + }) + .always(() => { + progressSpinner.removeClass('is-active'); + }) + } +} diff --git a/src/dataloaderinterface/templates/dataloaderinterface/site_details.html b/src/dataloaderinterface/templates/dataloaderinterface/site_details.html index 81dc4e7f..50a4c64a 100644 --- a/src/dataloaderinterface/templates/dataloaderinterface/site_details.html +++ b/src/dataloaderinterface/templates/dataloaderinterface/site_details.html @@ -415,11 +415,11 @@

      Code Output

      - {% if hs_account and site.django_user_id == request.user.id and resource_is_connected %} + + {% if site.django_user_id == request.user.id and resource_is_connected and hydroshare_account %} - {% elif hs_account and site.django_user_id == request.user.id %} + {% elif site.django_user_id == request.user.id and hydroshare_account %} {% endif %} -{# #} {% endblock %} diff --git a/src/dataloaderinterface/templates/hydroshare/hydroshare_site_settings.html b/src/dataloaderinterface/templates/hydroshare/hydroshare_site_settings.html index 0215ecec..23ab9cec 100644 --- a/src/dataloaderinterface/templates/hydroshare/hydroshare_site_settings.html +++ b/src/dataloaderinterface/templates/hydroshare/hydroshare_site_settings.html @@ -8,11 +8,11 @@ .hydroshare-actions > * { margin: 8px 16px 8px 0; } - p { - margin-bottom: 4px; + .hs-img-container { + margin-bottom: 8px; } - +
      @@ -22,41 +22,50 @@ {% if resource %} -

      Sharing with HydroShare is currently turned {% if resource.is_enabled %}ON{% else %}OFF{% endif %}

      +

      Sharing with HydroShare is currently turned + + {% if resource.is_enabled %} + ON + {% else %} + OFF + {% endif %} + +

      -

      Last updated on {{ resource.last_sync_date|date:'N j, Y' }}

      +

      Last updated on {{ resource.last_sync_date|date:'N j, Y' }}

      - {% if resource.sync_type == 'scheduled' %} -

      Next update scheduled for {{ resource.m_next_sync_date|date:'N j, Y' }}

      - {% endif %} + {% if resource.sync_type == 'scheduled' %} +

      Next update scheduled for {{ resource.m_next_sync_date|date:'N j, Y' }}

      + {% endif %} -{#

      Click Update Now to share latest data with HydroShare.

      #} -

      +{#

      Click Update Now to share latest data with HydroShare.

      #} +

      -
      - +
      + - + + + +
      - -
      {% else %} -
      Share Data On HydroShare
      +
      Share Data On HydroShare
      -

      - Share your data on hydroshare.org. - {# TODO: Implement a 'click here to learn more' link... #} - Click here to learn more. -

      - +

      + Share your data on hydroshare.org. + {# TODO: Implement a 'click here to learn more' link... #} + Click here to learn more. +

      + {% endif %}
      @@ -65,11 +74,3 @@
      Share Data On HydroShare
      {% include 'hydroshare/hydroshare_settings_modal.html' with resource=resource form=form site=site %}
      - -{#{% block styles %}#} -{# #} -{#{% endblock %}#} - -{% block scripts %} - -{% endblock %} diff --git a/src/dataloaderinterface/views.py b/src/dataloaderinterface/views.py index 89e685a3..3ea8db33 100644 --- a/src/dataloaderinterface/views.py +++ b/src/dataloaderinterface/views.py @@ -209,30 +209,16 @@ def get_context_data(self, **kwargs): context['tsa_url'] = settings.TSA_URL context['is_followed'] = self.request.user.is_authenticated and self.request.user.followed_sites.filter(sampling_feature_code=self.object.sampling_feature_code).exists() - hs_account = None try: - hs_account = self.request.user.odm2user.hydroshare_account - context['hs_account'] = hs_account + context["hydroshare_account"] = self.request.user.odm2user.hydroshare_account except AttributeError: pass - if hs_account: - try: - hs_resource = HydroShareResource.objects.get(site_registration=context['site'].pk) - context['resource_is_connected'] = True - # settings_form = HydroShareSettingsForm(initial={ - # 'site_registration': context['site'].pk, - # 'update_freq': hs_resource.update_freq, - # 'schedule_type': hs_resource.sync_type, - # 'enabled': hs_resource.is_enabled, - # 'data_types': hs_resource.data_types.split(",") - # }) - except ObjectDoesNotExist: - # settings_form = HydroShareSettingsForm(initial={'site_registration': context['site'].pk, - # 'data_types': [HydroShareSettingsForm.data_type_choices[0][0]]}) - pass - - # context['hs_settings_form'] = settings_form + try: + resources = HydroShareResource.objects.filter(site_registration=context['site'].pk) + context['resource_is_connected'] = len(resources) > 0 + except ObjectDoesNotExist: + pass return context From 784d348c7091bdb7732cfae411383bf15d6cab70 Mon Sep 17 00:00:00 2001 From: Craig Blackburn Date: Wed, 7 Feb 2018 10:38:49 -0700 Subject: [PATCH 057/115] moved hydroshare oauth route handlers into dataloaderinterface; updated hydroshare model; fixed some small bugs --- src/WebSDL/settings/base.py | 1 - src/WebSDL/urls.py | 3 +- src/dataloaderinterface/models.py | 2 - .../hydroshare/hydroshare-settings-modal.js | 1 - .../hydroshare/hydroshare_account.html | 15 +- src/dataloaderinterface/urls.py | 4 +- src/dataloaderinterface/views.py | 79 ++++ src/hydroshareinterface/__init__.py | 0 src/hydroshareinterface/admin.py | 3 - src/hydroshareinterface/api.py | 373 ------------------ src/hydroshareinterface/apps.py | 7 - src/hydroshareinterface/models.py | 71 ---- src/hydroshareinterface/tests.py | 3 - src/hydroshareinterface/urls.py | 9 - src/hydroshareinterface/views.py | 173 -------- 15 files changed, 85 insertions(+), 659 deletions(-) delete mode 100644 src/hydroshareinterface/__init__.py delete mode 100644 src/hydroshareinterface/admin.py delete mode 100644 src/hydroshareinterface/api.py delete mode 100644 src/hydroshareinterface/apps.py delete mode 100644 src/hydroshareinterface/models.py delete mode 100644 src/hydroshareinterface/tests.py delete mode 100644 src/hydroshareinterface/urls.py delete mode 100644 src/hydroshareinterface/views.py diff --git a/src/WebSDL/settings/base.py b/src/WebSDL/settings/base.py index faec072a..24375602 100644 --- a/src/WebSDL/settings/base.py +++ b/src/WebSDL/settings/base.py @@ -48,7 +48,6 @@ 'dataloader', 'dataloaderservices', 'dataloaderinterface', - 'hydroshareinterface', 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', diff --git a/src/WebSDL/urls.py b/src/WebSDL/urls.py index 6f908b48..c3f47413 100644 --- a/src/WebSDL/urls.py +++ b/src/WebSDL/urls.py @@ -52,6 +52,5 @@ url(r'^' + BASE_URL + 'account/$', UserUpdateView.as_view(), name='user_account'), url(r'^' + BASE_URL + 'api-auth/', include('rest_framework.urls', namespace='rest_framework')), url(BASE_URL, include('dataloaderinterface.urls')), - url(BASE_URL, include('dataloaderservices.urls')), - url(r'^' + BASE_URL + 'hydroshare/', include('hydroshareinterface.urls', namespace='hydroshareinterface')) + url(BASE_URL, include('dataloaderservices.urls')) ] diff --git a/src/dataloaderinterface/models.py b/src/dataloaderinterface/models.py index 58835859..544ccfc1 100644 --- a/src/dataloaderinterface/models.py +++ b/src/dataloaderinterface/models.py @@ -193,8 +193,6 @@ class Meta: # HSUAccount - holds information for user's Hydroshare account class HydroShareAccount(models.Model): - # user = models.ForeignKey('ODM2User', db_column='user_id') - name = models.CharField(max_length=255, default='HydroShare Account') is_enabled = models.BooleanField(default=False) ext_id = models.IntegerField(unique=True) # external hydroshare account id token = models.ForeignKey(OAuthToken, db_column='token_id', null=True, on_delete=models.CASCADE) diff --git a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hydroshare-settings-modal.js b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hydroshare-settings-modal.js index 89e899e3..dee300c8 100644 --- a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hydroshare-settings-modal.js +++ b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hydroshare-settings-modal.js @@ -14,7 +14,6 @@ function initializeHydroShareSettingsDialog() { const updateFreqSelect = $('select#id_update_freq')[0]; if (dialog && dialog.showModal) { - console.log("registering dialog"); dialogPolyfill.registerDialog(dialog); } diff --git a/src/dataloaderinterface/templates/hydroshare/hydroshare_account.html b/src/dataloaderinterface/templates/hydroshare/hydroshare_account.html index 74dbc6a2..327de2cc 100644 --- a/src/dataloaderinterface/templates/hydroshare/hydroshare_account.html +++ b/src/dataloaderinterface/templates/hydroshare/hydroshare_account.html @@ -28,17 +28,6 @@ } -{##} -{# share#} -{# Add HydroShare Connection#} -{##} -{##} -{##} -{# View Shared Resources#} -{##} - {# TODO: Replace with dynamic url #}
      @@ -57,7 +46,7 @@
      -
      +
      {{ form.schedule_type.errors }} {{ form.schedule_type }}
      -
      +
      @@ -89,7 +84,7 @@

      {{ form.abstract.label }}

      {{ form.abstract|add_class:"mdl-textfield__input" }} -{# #} +
      diff --git a/src/dataloaderinterface/templates/hydroshare/hs_site_details.html b/src/dataloaderinterface/templates/hydroshare/hs_site_details.html index 1f723e32..b218cb93 100644 --- a/src/dataloaderinterface/templates/hydroshare/hs_site_details.html +++ b/src/dataloaderinterface/templates/hydroshare/hs_site_details.html @@ -11,9 +11,21 @@ .hs-img-container { margin-bottom: 8px; } - + @media only screen and (min-width: 480px) { + dialog.mdl-dialog { + min-width: 480px; + } + } +{% if resource.ext_id %} + {% include 'hydroshare/hs_delete_dialog.html' with form=delete_form sampling_feature_code=site.sampling_feature_code resource=resource %} +{% endif %} + +
      + {% include 'hydroshare/hs_settings_dialog.html' with resource=resource form=form site=site %} +
      +
      @@ -99,9 +111,3 @@
      Share Data On HydroShare
      {% endif %}
      - -
      - {% include 'hydroshare/hs_settings_dialog.html' with resource=resource form=form site=site %} -
      - -{% include 'hydroshare/hs_delete_dialog.html' with form=delete_form sampling_feature_code=site.sampling_feature_code %} diff --git a/src/dataloaderinterface/views.py b/src/dataloaderinterface/views.py index 24d13b2e..3f098d74 100644 --- a/src/dataloaderinterface/views.py +++ b/src/dataloaderinterface/views.py @@ -380,13 +380,7 @@ def get_context_data(self, **kwargs): 'data_types': hs_resource.data_types.split(",") }) - # TODO: Remove special case for jeff after app is deployed - if self.request.user.email == 'jeff.horsburgh@usu.edu': - # For Jeff... - context['delete_form'] = HydroShareResourceDeleteForm(initial={'delete_external_resource': True}) - else: - # For everyone else... - context['delete_form'] = HydroShareResourceDeleteForm() + context['delete_form'] = HydroShareResourceDeleteForm() resource_util = self.get_hs_resource(hs_resource) try: From e0ba0c1d47056e0e5570c79f75f2b1933935b7b5 Mon Sep 17 00:00:00 2001 From: darksinge Date: Fri, 23 Feb 2018 14:57:35 -0700 Subject: [PATCH 081/115] fixed button not disabling when submitting delete resource form --- .../js/hydroshare/hs-delete-resource.js | 9 ++++----- .../templates/hydroshare/hs_delete_dialog.html | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-delete-resource.js b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-delete-resource.js index 4eb7b9f0..26516862 100644 --- a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-delete-resource.js +++ b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-delete-resource.js @@ -20,15 +20,14 @@ function initializeHydroShareDeleteDialog() { }); deleteForm.submit((e) => { - const submitButton = $(deleteFormDialog).find('input[type=submit]'); - submitButton[0].setAttribute('disabled', ''); - $(submitButton[0]).removeClass('mdl-color--red-400'); - $(submitButton[0]).addClass('mdl-color--grey'); + let dialogButtons = $(deleteFormDialog).find('button'); + $(dialogButtons).removeClass(); + $(dialogButtons).addClass('mdl-button mdl-buton--raised'); + $(dialogButtons).prop('disabled', true); const spinner = $(deleteFormDialog).find('.mdl-js-spinner')[0]; $(spinner).addClass('is-active'); componentHandler.upgradeElement(spinner); - }); deleteResourceCB.change(() => { diff --git a/src/dataloaderinterface/templates/hydroshare/hs_delete_dialog.html b/src/dataloaderinterface/templates/hydroshare/hs_delete_dialog.html index 03ca74d4..6ae5894c 100644 --- a/src/dataloaderinterface/templates/hydroshare/hs_delete_dialog.html +++ b/src/dataloaderinterface/templates/hydroshare/hs_delete_dialog.html @@ -29,7 +29,7 @@

      Stop Sharing Site Data?

    • EnviroDIY and HydroShare will not retain copies of any of your content files.
    • It is highly recommended that you download the latest copy of your resource file(s) before deleting. - Your resource file(s) can be downloaded here. + You can download the resource file(s) here.
      • @@ -38,7 +38,7 @@

      Stop Sharing Site Data?

      From e43a6aee3abb5284ac4ebc5b12a370a877c2f9a5 Mon Sep 17 00:00:00 2001 From: darksinge Date: Fri, 23 Feb 2018 16:12:16 -0700 Subject: [PATCH 082/115] updated hydroshare settings form and dialog --- src/crontab_jobs.py | 2 +- src/dataloaderinterface/forms.py | 2 +- .../update_hydroshare_resource_files.py | 2 +- .../js/hydroshare/hs-settings-dialog.js | 20 ++++-------- .../hydroshare/hs_settings_dialog.html | 32 +++++++++---------- .../templates/hydroshare/hs_site_details.html | 14 +++----- src/dataloaderinterface/views.py | 6 ++-- 7 files changed, 33 insertions(+), 45 deletions(-) diff --git a/src/crontab_jobs.py b/src/crontab_jobs.py index 09781c8e..21dc60f3 100644 --- a/src/crontab_jobs.py +++ b/src/crontab_jobs.py @@ -66,7 +66,7 @@ def start_jobs(user=True): logfile=LOGFILE), comment=JOB_COMMENT_PREPENDER + 'upload_hydroshare_files') job.every().day() # run everyday - job.hour.every(AT_HOUR) # at the time specified by AT_HOUR + job.hour.on(AT_HOUR) # at the time specified by AT_HOUR cron.write() # write, i.e. 'save' crontab job diff --git a/src/dataloaderinterface/forms.py b/src/dataloaderinterface/forms.py index 1540cdf3..4e8a1b00 100644 --- a/src/dataloaderinterface/forms.py +++ b/src/dataloaderinterface/forms.py @@ -47,7 +47,7 @@ def __init__(self, *args, **kwargs): ('monthly', 'Monthly') ) - enabled = forms.BooleanField(initial=True, label='Pause Sharing', required=False) + pause_sharing = forms.BooleanField(initial=False, label='Pause Sharing', required=False) site_registration = forms.CharField(max_length=255) diff --git a/src/dataloaderinterface/management/commands/update_hydroshare_resource_files.py b/src/dataloaderinterface/management/commands/update_hydroshare_resource_files.py index 570589e8..8e30048b 100644 --- a/src/dataloaderinterface/management/commands/update_hydroshare_resource_files.py +++ b/src/dataloaderinterface/management/commands/update_hydroshare_resource_files.py @@ -30,7 +30,7 @@ def handle(self, force_update=False, *args, **options): resource.delete() continue - # Skip resource if not enabled + # Skip resource if not pause_sharing if not resource.is_enabled: upload_skipped_count += 1 continue diff --git a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-settings-dialog.js b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-settings-dialog.js index 20a198ed..a8a7d55d 100644 --- a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-settings-dialog.js +++ b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-settings-dialog.js @@ -25,7 +25,7 @@ function initializeHydroShareSettingsDialog() { showDialogButton.addEventListener('click', () => { dialog.showModal(); - $('label[for="id_enabled"]').removeClass('is-focused'); + $('label[for="id_pause_sharing"]').removeClass('is-focused'); toggleUpdateFreqSelect(!!$(manualCB).attr('checked')) }); @@ -52,20 +52,14 @@ function initializeHydroShareSettingsDialog() { } function submitForm() { - let submitButton = $(hydroshareSettingsForm).find('input[type=submit]'); - $(submitButton).prop('disabled', true); + let dialogButtons = $(hydroshareSettingsForm).find('.mdl-dialog__actions').find('button'); + $(dialogButtons).removeClass(); + $(dialogButtons).addClass('mdl-button mdl-button--raised'); + $(dialogButtons).prop('disabled', true); - let cancelButton = $(hydroshareSettingsForm).find('button.close'); - $(cancelButton).prop('disabled', true); + let submitButton = $(hydroshareSettingsForm).find('button[type=submit]')[0]; - let inputField = $(hydroshareSettingsForm).find('input[type=submit]')[0]; - - let method = ''; - if (inputField.id === 'create-resource') { - method = 'create'; - } else if (inputField.id === 'update-resource') { - method = 'update'; - } + let method = submitButton.id === 'create-resource' ? 'create' : 'update'; let url = `${hydroshareSettingsForm.baseURI}hsr/${method}/`; let serializedForm = $(hydroshareSettingsForm).serialize(); diff --git a/src/dataloaderinterface/templates/hydroshare/hs_settings_dialog.html b/src/dataloaderinterface/templates/hydroshare/hs_settings_dialog.html index 8d2982ae..4b471b0c 100644 --- a/src/dataloaderinterface/templates/hydroshare/hs_settings_dialog.html +++ b/src/dataloaderinterface/templates/hydroshare/hs_settings_dialog.html @@ -11,7 +11,7 @@ {% csrf_token %} {% if resource.id %} -

      Update Settings

      +

      HydroShare Sharing Settings

      {% else %}

      Create Resource

      {% endif %} @@ -19,13 +19,13 @@

      Create Resource

      {{ form.site_registration.as_hidden }} -{# If resource.id is None, then this resource hasn't been created yet (i.e., don't show option to pause sharing) #} + {# If resource.id is None, then this resource hasn't been created yet #} {% if resource.id %}
      -
      @@ -51,6 +51,9 @@

      {{ form.schedule_type.label }}

      + {# Data Types - Time Series, Leaf Packet, etc. #} + {# Yes... #}{% if False %} {#... this "if" statement was intentional #} + {# NOTE: Until there are multiple data types to choose from, do not render! #}

      {{ form.data_types.label }}

      @@ -63,11 +66,12 @@

      {{ form.data_types.label }}

      {% endfor %}
      + {% endif %} {% if not resource.id %} -{# Resource Title#} + {# Resource Title#}

      {{ form.title.label }}

      @@ -78,7 +82,7 @@

      {{ form.title.label }}

      -{# Resource Abstract#} + {# Resource Abstract#}

      {{ form.abstract.label }}

      @@ -94,15 +98,11 @@

      {{ form.abstract.label }}

      - + diff --git a/src/dataloaderinterface/templates/hydroshare/hs_site_details.html b/src/dataloaderinterface/templates/hydroshare/hs_site_details.html index b218cb93..fbccd8f9 100644 --- a/src/dataloaderinterface/templates/hydroshare/hs_site_details.html +++ b/src/dataloaderinterface/templates/hydroshare/hs_site_details.html @@ -19,7 +19,7 @@ {% if resource.ext_id %} - {% include 'hydroshare/hs_delete_dialog.html' with form=delete_form sampling_feature_code=site.sampling_feature_code resource=resource %} + {% include 'hydroshare/hs_delete_dialog.html' with form=delete_form sampling_feature_code=site.sampling_feature_code %} {% endif %}
      @@ -54,15 +54,9 @@ {% else %} -

      Sharing with HydroShare is currently turned - - {% if resource.is_enabled %} - ON - {% else %} - OFF - {% endif %} - -

      + {% if not resource.is_enabled %} +

      Sharing with HydroShare is currently paused.

      + {% endif %}

      Last updated on {{ resource.last_sync_date|date:date_format }}

      diff --git a/src/dataloaderinterface/views.py b/src/dataloaderinterface/views.py index 3f098d74..37ab8c62 100644 --- a/src/dataloaderinterface/views.py +++ b/src/dataloaderinterface/views.py @@ -291,7 +291,7 @@ def get_context_data(self, **kwargs): context['site'] = site context['form'] = HydroShareSettingsForm(initial={'site_registration': site.pk, 'data_types': [initial_datatype], - 'enabled': True, + 'pause_sharing': False, 'title': self.generate_title(site), 'abstract': self.generate_abstract(site)}) return context @@ -376,7 +376,7 @@ def get_context_data(self, **kwargs): 'site_registration': site.pk, 'update_freq': hs_resource.update_freq, 'schedule_type': hs_resource.sync_type, - 'enabled': hs_resource.is_enabled, + 'pause_sharing': not hs_resource.is_enabled, 'data_types': hs_resource.data_types.split(",") }) @@ -421,7 +421,7 @@ def post(self, request, *args, **kwargs): resource_data.data_types = ",".join(form.cleaned_data['data_types']) resource_data.update_freq = form.cleaned_data['update_freq'] resource_data.sync_type = form.cleaned_data['schedule_type'] - resource_data.is_enabled = form.cleaned_data['enabled'] + resource_data.is_enabled = not form.cleaned_data["pause_sharing"] # save resource_data resource_data.save() From 04abfc8fbc310613d42f6f4eb84586ee3135e0df Mon Sep 17 00:00:00 2001 From: darksinge Date: Fri, 23 Feb 2018 16:40:45 -0700 Subject: [PATCH 083/115] added a check to see if a resource is published --- .../templates/hydroshare/hs_site_details.html | 28 +++++++++++++++++++ src/dataloaderinterface/views.py | 1 + 2 files changed, 29 insertions(+) diff --git a/src/dataloaderinterface/templates/hydroshare/hs_site_details.html b/src/dataloaderinterface/templates/hydroshare/hs_site_details.html index fbccd8f9..228a5637 100644 --- a/src/dataloaderinterface/templates/hydroshare/hs_site_details.html +++ b/src/dataloaderinterface/templates/hydroshare/hs_site_details.html @@ -34,12 +34,26 @@ {% if resource %} + {# Message if the resource did not load #} {% if resource_did_not_load %} +

      Error: Failed to connect to hydroshare.org. Updating site data information might not work.

      + + {# Message if the resource has been published #} + {% elif resource_is_published %} + +

      The resource this site is connected to is published. Once a resource is published, you cannot change the resource's content.

      +

      If you would like to continue sharing data from this site to HydroShare:

      +
        +
      1. Click DISCONNECT RESOURCE.
        2. +
      2. Click on START SHARING after the page reloads to start sharing with a new, unpublished resource.
        3. +
      + {% endif %} + {# Actions if the resource was not found #} {% if resource_not_found %}

      HydroShare Resource not found.

      @@ -53,7 +67,19 @@ {% csrf_token %} + + {# Actions if the resource has been published #} + {% elif resource_is_published %} + +
      + {% csrf_token %} + +
      + + + {# All other messages and actions #} {% else %} + {% if not resource.is_enabled %}

      Sharing with HydroShare is currently paused.

      {% endif %} @@ -89,6 +115,8 @@
      {% endif %} + + {# The action if the user is not sharing data with HydroShare #} {% else %}
      Share Data On HydroShare
      diff --git a/src/dataloaderinterface/views.py b/src/dataloaderinterface/views.py index 37ab8c62..22edfdc2 100644 --- a/src/dataloaderinterface/views.py +++ b/src/dataloaderinterface/views.py @@ -385,6 +385,7 @@ def get_context_data(self, **kwargs): resource_util = self.get_hs_resource(hs_resource) try: resource_md = resource_util.get_system_metadata() + context['resource_is_published'] = resource_md.get("published", False) context['resource_not_found'] = resource_md is None except HydroShareNotFound: context['resource_not_found'] = True From e910a04e9e4cebafe417d364ea515c049f56b42e Mon Sep 17 00:00:00 2001 From: darksinge Date: Fri, 23 Feb 2018 17:03:10 -0700 Subject: [PATCH 084/115] fixed a bug preventing the creation of a new site --- .../js/hydroshare/hs-settings-dialog.js | 3 ++- .../templates/hydroshare/hs_site_details.html | 2 +- src/dataloaderinterface/views.py | 7 ++++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-settings-dialog.js b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-settings-dialog.js index a8a7d55d..ff56b475 100644 --- a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-settings-dialog.js +++ b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-settings-dialog.js @@ -100,7 +100,8 @@ function initializeHydroShareSettingsDialog() { } }).always(() => { progressSpinner.removeClass('is-active'); - $(submitButton).prop('disabled', false); + $(dialogButtons).prop('disabled', false); + $(submitButton).addClass('mdl-button--accent'); }); componentHandler.upgradeElement(progressSpinner[0]); // upgradeElement to fix issue where spinner doesn't render diff --git a/src/dataloaderinterface/templates/hydroshare/hs_site_details.html b/src/dataloaderinterface/templates/hydroshare/hs_site_details.html index 228a5637..2e6512e5 100644 --- a/src/dataloaderinterface/templates/hydroshare/hs_site_details.html +++ b/src/dataloaderinterface/templates/hydroshare/hs_site_details.html @@ -124,7 +124,7 @@
      Share Data On HydroShare

      Share your data on hydroshare.org. {# TODO: Implement a 'click here to learn more' link... #} - Click here to learn more. + Click here to learn more.

      + {% endif %} @@ -403,6 +412,7 @@

      Code Output

      + {% if site.django_user_id == request.user.id %} {% if site.django_user_id == request.user.id and resource_is_connected and hydroshare_account %} @@ -410,6 +420,7 @@

      Code Output

      {% elif site.django_user_id == request.user.id and hydroshare_account %} {% endif %} + {% endif %} {% endif %} - {% endif %} - + {% endif %} + {% endblock %} diff --git a/src/dataloaderinterface/templatetags/site.py b/src/dataloaderinterface/templatetags/site.py index ce7ab3b2..3867300b 100644 --- a/src/dataloaderinterface/templatetags/site.py +++ b/src/dataloaderinterface/templatetags/site.py @@ -45,3 +45,14 @@ def add_input_class(value, arg): return re.sub(r'(class=")', r'\1{0} '.format(arg), value) else: return re.sub(r'() # type: Resource +resource = util.get_resource_system_metadata() # type: Resource ``` diff --git a/src/hydroshare_util/adapter.py b/src/hydroshare_util/adapter.py index beff9d50..2b9af6fb 100644 --- a/src/hydroshare_util/adapter.py +++ b/src/hydroshare_util/adapter.py @@ -11,25 +11,40 @@ def __init__(self, hostname=DEFAULT_HOSTNAME, port=None, use_https=True, verify= super(HydroShareAdapter, self).__init__(hostname=hostname, port=port, use_https=use_https, verify=verify, auth=auth) - def _build_params(self, params): # type: (dict) -> str + def _build_params(self, params): # type: (dict) -> str param_vals = ['{param}={val}'.format(param=p, val=v) for p, v in params.iteritems()] return "?{params}".format(params="&".join(param_vals)) - def _request(self, method, url, params=None, data=None, files=None, headers=None, stream=False): - _headers = None + def _request(self, method, url, params=None, data=None, files=None, headers=None, stream=False, **kwargs): + + timeout = None + if 'timeout' in kwargs: + timeout = kwargs.get('timeout', None) + if self._default_headers and headers: - _headers = headers - _headers.update(self._default_headers) + headers = headers + headers.update(self._default_headers) elif self._default_headers: - _headers = self._default_headers + headers = self._default_headers - return super(HydroShareAdapter, self)._request(method, url, params=params, data=data, files=files, stream=stream, headers=_headers) + try: + request = self.session.request(method, url, params=params, data=data, files=files, headers=headers, + stream=stream, verify=self.verify, timeout=timeout) + except requests.ConnectionError: + self._initializeSession() + request = self.session.request(method, url, params=params, data=data, files=files, headers=headers, + stream=stream, verify=self.verify, timeout=timeout) + return request def get_resource_list(self, **kwargs): return self.getResourceList(**kwargs) def get_system_metadata(self, pid, **kwargs): - + """ + Returns system metadata for a resource which includes the dublin core elements + NOTE: get_system_metadata does not call getSystemMetadata() from hs_restclient so that + a timeout can be specified for the request. + """ timeout = None if 'timeout' in kwargs: timeout = kwargs.get('timeout', None) @@ -53,8 +68,8 @@ def get_system_metadata(self, pid, **kwargs): return req.json() - def get_science_metadata_RDF(self, pid): - return self.getScienceMetadata(pid) + def get_science_metadataRDF(self, pid): + return self.getScienceMetadataRDF(pid) def get_science_metadata(self, pid): return self.getScienceMetadata(pid) @@ -68,9 +83,6 @@ def get_resource_map(self, pid): def get_resource(self, pid, destination=None, unzip=False, wait_for_bag_creation=True): return self.getResource(pid, destination=destination, unzip=unzip, wait_for_bag_creation=wait_for_bag_creation) - def get_resource_metadata(self, pid): - return self.get_system_metadata(pid) - def get_resource_types(self): return self.getResourceTypes() diff --git a/src/hydroshare_util/resource.py b/src/hydroshare_util/resource.py index 9a66270b..6f2e1acd 100644 --- a/src/hydroshare_util/resource.py +++ b/src/hydroshare_util/resource.py @@ -86,6 +86,9 @@ def update_file_list(self): def get_system_metadata(self, **kwargs): return self.client.get_system_metadata(self.resource_id, **kwargs) + def get_science_metadata(self): + return self.client.get_science_metadata(self.resource_id) + def update_metadata(self, data=None): if data is None: data = self.to_object() @@ -93,25 +96,25 @@ def update_metadata(self, data=None): def upload_files(self): upload_success_count = 0 - for file in self.files: + for file_ in self.files: try: - self.client.delete_resource_file(self.resource_id, os.path.basename(file)) + self.client.delete_resource_file(self.resource_id, os.path.basename(file_)) except HydroShareNotFound: pass - if not isinstance(file, str): - file = str(file) + if not isinstance(file_, str): + file_ = str(file_) try: - self.client.addResourceFile(self.resource_id, file) - self._r_logger("File upload successful: '{filename}'".format(filename=os.path.basename(file)), + self.client.addResourceFile(self.resource_id, file_) + self._r_logger("File upload successful: '{filename}'".format(filename=os.path.basename(file_)), level=logging.INFO) upload_success_count += 1 except KeyError as e: self._log_error('File upload failed; incorrectly formatted arguments given.', e) raise e except Exception as e: - self._log_error("File upload failed: \n\t{fname}\n".format(fname=os.path.basename(file)), e) + self._log_error("File upload failed: \n\t{fname}\n".format(fname=os.path.basename(file_)), e) raise e self._r_logger("successfully uploaded {count} files".format(count=upload_success_count), level=logging.INFO) @@ -174,7 +177,7 @@ def create(self): metadata.append(coverage_dict) metadata_as_string = self._stringify_metadata(metadata) - # metadata_as_string = '[{"coverage":{"type":"point","value":{"units":"Decimal degrees", "east":-112.01214, "north":41.80126, "projection": "Unknown", "name":"Demo Site (LOCALHOST)"}}}]' + self.resource_id = self.client.create_resource(resource_type=self.resource_type, title=self.title, metadata=metadata_as_string, @@ -184,7 +187,7 @@ def create(self): def _stringify_metadata(self, metadata): string = str(metadata) - string = string.replace("'", '"') # .replace(" ", "") + string = string.replace("'", '"') return string def update(self, metadata=None): @@ -279,4 +282,4 @@ def __repr__(self): return "<{classname}: {title}>".format(classname=self.classname, title=self.title) -__all__ = ["Resource"] \ No newline at end of file +__all__ = ["Resource"] diff --git a/src/hydroshare_util/utility.py b/src/hydroshare_util/utility.py index 42a2336d..a2e6b7b5 100644 --- a/src/hydroshare_util/utility.py +++ b/src/hydroshare_util/utility.py @@ -19,7 +19,7 @@ class HydroShareUtility(HydroShareUtilityBaseClass): XML_COVERAGE_PROTO = "start={start}; end={end}; scheme=W3C-DTF" TIME_FORMAT = '%Y-%m-%dT%H:%M:%S' - def __init__(self, auth=None): # type: (AuthUtil) -> None + def __init__(self, auth=None): # type: (AuthUtil) -> None self.auth = auth @property @@ -56,8 +56,8 @@ def get_resources(self, limit=None, owner=None): return resources - def get_resource_metadata(self, pid): - data = self.client.get_resource_metadata(pid) + def get_resource_system_metadata(self, pid): + data = self.client.get_system_metadata(pid) logging.info("Fetched resource metadata: {json_data}".format(json_data=json.dumps(data))) return Resource(self.client, **data) From f28bca388acaccc9a91f0b09f63dda62543f053b Mon Sep 17 00:00:00 2001 From: darksinge Date: Tue, 27 Feb 2018 09:19:20 -0700 Subject: [PATCH 087/115] fixed bug preventing hydroshare settings from being updated --- src/dataloaderinterface/forms.py | 4 ++-- src/dataloaderinterface/views.py | 4 +++- src/hydroshare_util/auth.py | 36 +++++++++++++++++++++++++------- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/src/dataloaderinterface/forms.py b/src/dataloaderinterface/forms.py index 4e8a1b00..ac7e1bda 100644 --- a/src/dataloaderinterface/forms.py +++ b/src/dataloaderinterface/forms.py @@ -52,7 +52,6 @@ def __init__(self, *args, **kwargs): site_registration = forms.CharField(max_length=255) schedule_type = forms.ChoiceField( - required=True, widget=forms.RadioSelect(renderer=MDLRadioButtonRenderer), choices=schedule_choices, initial='scheduled' @@ -66,7 +65,8 @@ def __init__(self, *args, **kwargs): ) data_types = forms.MultipleChoiceField( - required=True, + # TODO: When EnviroDIY supports multiple data types, the 'required' attribute should be set to True + required=False, widget=forms.CheckboxSelectMultiple, choices=data_type_choices, initial='TS', diff --git a/src/dataloaderinterface/views.py b/src/dataloaderinterface/views.py index 0903c5d9..8d72d83f 100644 --- a/src/dataloaderinterface/views.py +++ b/src/dataloaderinterface/views.py @@ -229,6 +229,9 @@ def get_context_data(self, **kwargs): class HydroShareResourceViewMixin: + def __init__(self): + self.request = None + def get_hs_resource(self, resource): # type: (HydroShareResource) -> Resource """ Creates a 'hydroshare_util.Resource' object """ account = self.request.user.odm2user.hydroshare_account @@ -428,7 +431,6 @@ def post(self, request, *args, **kwargs): resource_data.sync_type = form.cleaned_data['schedule_type'] resource_data.is_enabled = not form.cleaned_data["pause_sharing"] - # save resource_data resource_data.save() success_url = reverse('site_detail', kwargs={'sampling_feature_code': site.sampling_feature_code}) diff --git a/src/hydroshare_util/auth.py b/src/hydroshare_util/auth.py index c371be80..13cf450f 100644 --- a/src/hydroshare_util/auth.py +++ b/src/hydroshare_util/auth.py @@ -7,8 +7,6 @@ from adapter import HydroShareAdapter from . import HydroShareUtilityBaseClass, ImproperlyConfiguredError from django.shortcuts import redirect -# from django.core.exceptions import PermissionDenied -# from django.http import HttpResponseServerError from django.conf import settings import logging as logger @@ -116,6 +114,10 @@ def __init__(self, use_https=True, hostname='www.hydroshare.org', self.__authorization_grant_type = OAUTH_AC def get_token(self): + """ + Get the authorization token dict + :return: a dictionary representing the oauth token + """ token = { 'access_token': self.access_token, 'token_type': self.token_type, @@ -131,8 +133,10 @@ def get_token(self): raise AttributeError("missing attributes(s) for token: {attrs}".format(attrs=missing_attrs)) return token - def get_client(self): - """Provides authentication details to 'hs_restclient.HydroShare' and returns the object""" + def get_client(self): # type: () -> HydroShareAdapter + """ + Passes authentication details to underlying HydroShare object for authorization via OAuth 2.0. + """ if self.auth_type == OAUTH_AC: token = self.get_token() auth = HydroShareAuthOAuth2(self.__client_id, self.__client_secret, token=token) @@ -146,7 +150,15 @@ def get_client(self): return HydroShareAdapter(auth=auth, default_headers=authorization_header) @staticmethod - def authorize_client(response_type=None): # type: (str) -> None + def authorize_client(response_type=None): # type: (str) -> None + """ + Redirects user from the client (data.envirodiy.org) to www.hydroshare.org/o/authorize/. After the user provides + their hydroshare account credentials and authorizes the requesting client, the user is redirected back to the + client website. + + :param response_type: a string representing the auth response type (defaults is 'code'). + :return: None + """ if response_type: auth = OAuthUtil(response_type=response_type) else: @@ -157,6 +169,12 @@ def authorize_client(response_type=None): # type: (str) -> None @staticmethod def authorize_client_callback(code, response_type=None): + """ + Callback handler after a user authorizes the client (data.envirodiy.org). + :param code: a string representing the authorization code recieved by hydroshare.org + :param response_type: a string representing the oauth response_type + :return: a dictionary representing the token + """ # type: (str, str) -> dict if response_type: auth = OAuthUtil(response_type=response_type) @@ -176,7 +194,8 @@ def _set_token(self, **token): if key in self.__dict__: setattr(self, key, value) else: - logger.warning("skipped setting attribute '{attr}' on '{clsname}".format(attr=key, clsname=self.classname)) + logger.warning("skipped setting attribute '{attr}' on '{clsname}".format(attr=key, + clsname=self.classname)) def _refresh_authentication(self): """Does the same thing as 'get_client()', but attempts to refresh 'self.access_token' first""" @@ -206,7 +225,7 @@ def _refresh_authentication(self): return self.get_client() - def _build_oauth_url(self, path, params=None): # type: (str, list) -> str + def _build_oauth_url(self, path, params=None): # type: (str, dict) -> str if params is None: params = {} @@ -271,7 +290,7 @@ def get_token(self): class SelfSignSecurityCertAuth(AuthUtilImplementor): """Used to connect to a development HydroShare server that uses a self-sign security certificate""" - def __init__(self, hostname, port=None): # type: (str, int) -> None + def __init__(self, hostname, port=None): # type: (str, int) -> None self.hostname = hostname self.port = port self.use_https = False @@ -291,6 +310,7 @@ def get_client(self): def get_token(self): return None + class AuthUtilFactory(object): """ Factory class for creating instances of 'AuthUtil'. From 6ded7e8d48879c9856f8e145ac23cf18038c72cd Mon Sep 17 00:00:00 2001 From: darksinge Date: Tue, 27 Feb 2018 10:16:13 -0700 Subject: [PATCH 088/115] implemented access_token refresh ability when a token is about to expire --- src/dataloaderinterface/models.py | 14 +++++++------- src/dataloaderinterface/views.py | 13 +++++++++++-- src/hydroshare_util/auth.py | 17 ++++++++++++----- src/hydroshare_util/utility.py | 2 +- 4 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/dataloaderinterface/models.py b/src/dataloaderinterface/models.py index 482bb66b..fff94e46 100644 --- a/src/dataloaderinterface/models.py +++ b/src/dataloaderinterface/models.py @@ -203,19 +203,22 @@ def save(self, force_insert=False, force_update=False, using=None, super(HydroShareAccount, self).save(force_insert=force_insert, force_update=force_update, using=using, update_fields=update_fields) - # def delete(self, using=None, keep_parents=False): - # super(HydroShareAccount, self).delete(using, keep_parents) - def get_token(self): try: return self.token.to_dict() except ObjectDoesNotExist: return None + def update_token(self, token_dict): # type: (dict) -> None + if isinstance(self.token, OAuthToken): + self.token.delete(keep_parents=True) + self.token = OAuthToken(**token_dict) + self.save() + + def to_dict(self, include_token=True): account = { 'id': self.pk, - 'name': self.name, 'ext_id': self.ext_id, 'is_enabled': self.is_enabled } @@ -225,9 +228,6 @@ def to_dict(self, include_token=True): account['token'] = token return account - def __str__(self): - return self.name - class Meta: db_table = 'hydroshare_account' diff --git a/src/dataloaderinterface/views.py b/src/dataloaderinterface/views.py index 8d72d83f..0ad702ef 100644 --- a/src/dataloaderinterface/views.py +++ b/src/dataloaderinterface/views.py @@ -236,8 +236,17 @@ def get_hs_resource(self, resource): # type: (HydroShareResource) -> Resource """ Creates a 'hydroshare_util.Resource' object """ account = self.request.user.odm2user.hydroshare_account token_json = account.get_token() - client = AuthUtil.authorize(token=token_json).get_client() - hs_resource = Resource(client) + auth_util = AuthUtil.authorize(token=token_json) + + # if the oauth access_token expires in less than a week, refresh the token + if token_json.get('expires_in', None) < 60*60*24*7: + try: + auth_util.refresh_token() + account.update_token(auth_util.get_token()) + except Exception as e: + print(e) + + hs_resource = Resource(auth_util.get_client()) hs_resource.resource_id = resource.ext_id return hs_resource diff --git a/src/hydroshare_util/auth.py b/src/hydroshare_util/auth.py index 13cf450f..49455247 100644 --- a/src/hydroshare_util/auth.py +++ b/src/hydroshare_util/auth.py @@ -32,6 +32,11 @@ def get_client(self): def get_token(self): return self.__implementation.get_token() + def refresh_token(self): + auth_type = self.auth_type + if auth_type == OAUTH_ROPC or auth_type == OAUTH_AC: + return self.__implementation.refresh_access_token() + @staticmethod def authorize_client(response_type=None): return OAuthUtil.authorize_client(response_type=response_type) @@ -197,13 +202,15 @@ def _set_token(self, **token): logger.warning("skipped setting attribute '{attr}' on '{clsname}".format(attr=key, clsname=self.classname)) - def _refresh_authentication(self): - """Does the same thing as 'get_client()', but attempts to refresh 'self.access_token' first""" + def refresh_access_token(self): + """ + Refresh oauth token using the refresh_token + :return: a dictionary representing the refreshed token + """ params = { 'grant_type': 'refresh_token', 'client_id': self.__client_id, 'client_secret': self.__client_secret, - 'redirect_uri': self.__redirect_uri, 'refresh_token': self.refresh_token } @@ -221,9 +228,9 @@ def _refresh_authentication(self): else: # TODO: better exception handling - raise Exception("failed to refresh access token") + raise Exception("failed to refresh access token", response.json()) - return self.get_client() + return self.get_token() def _build_oauth_url(self, path, params=None): # type: (str, dict) -> str if params is None: diff --git a/src/hydroshare_util/utility.py b/src/hydroshare_util/utility.py index a2e6b7b5..349cb18e 100644 --- a/src/hydroshare_util/utility.py +++ b/src/hydroshare_util/utility.py @@ -69,7 +69,7 @@ def get_user_info(self): def get_resource_types(self): try: - Resource.RESOURCE_TYPES = [type for type in self.client.get_resource_types()] + Resource.RESOURCE_TYPES = [type_ for type_ in self.client.get_resource_types()] except Exception as e: logging.error("Failed to get resource types!\n{error}".format(error=e)) From 8e407c57be4afda2ff35bf0ab6fa45ec3ed66163 Mon Sep 17 00:00:00 2001 From: Mauriel Date: Tue, 27 Feb 2018 11:03:36 -0700 Subject: [PATCH 089/115] Tile header change to code --- .../static/dataloaderinterface/css/style.css | 2 ++ .../templates/dataloaderinterface/my-sites.html | 14 +++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/dataloaderinterface/static/dataloaderinterface/css/style.css b/src/dataloaderinterface/static/dataloaderinterface/css/style.css index aee2fecd..a0d796e3 100644 --- a/src/dataloaderinterface/static/dataloaderinterface/css/style.css +++ b/src/dataloaderinterface/static/dataloaderinterface/css/style.css @@ -758,6 +758,8 @@ button:focus { .site-card .mdl-card__title-text a { color: #FFF; font-weight: 300; + text-overflow: ellipsis; + overflow: hidden; } .site-card .mdl-card__title { diff --git a/src/dataloaderinterface/templates/dataloaderinterface/my-sites.html b/src/dataloaderinterface/templates/dataloaderinterface/my-sites.html index 4ca2f867..fb9e6d31 100644 --- a/src/dataloaderinterface/templates/dataloaderinterface/my-sites.html +++ b/src/dataloaderinterface/templates/dataloaderinterface/my-sites.html @@ -56,9 +56,12 @@
      -

      {{ site.sampling_feature_name }} +

      + {{ site.sampling_feature_code }}

      +
      + {{ site.sampling_feature_code }} +
      @@ -145,7 +148,12 @@

      -

      {{ site.sampling_feature_name }}

      +

      + {{ site.sampling_feature_code }} +

      +
      + {{ site.sampling_feature_code }} +

      From f10e1ba76d371f1941813a3add47bc5af24fc2a4 Mon Sep 17 00:00:00 2001 From: Craig Blackburn Date: Thu, 1 Mar 2018 10:58:07 -0700 Subject: [PATCH 090/115] fixed a bug preventing successful oauth token renewal --- src/dataloaderinterface/models.py | 13 ++++++++----- .../templates/dataloaderinterface/site_details.html | 4 ++-- src/dataloaderinterface/views.py | 12 +++++++----- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/dataloaderinterface/models.py b/src/dataloaderinterface/models.py index fff94e46..e9730a32 100644 --- a/src/dataloaderinterface/models.py +++ b/src/dataloaderinterface/models.py @@ -195,8 +195,8 @@ class Meta: # HSUAccount - holds information for user's Hydroshare account class HydroShareAccount(models.Model): is_enabled = models.BooleanField(default=False) - ext_id = models.IntegerField(unique=True) # external hydroshare account id - token = models.ForeignKey(OAuthToken, db_column='token_id', null=True, on_delete=models.CASCADE) + ext_id = models.IntegerField(unique=True) # external hydroshare account id + token = models.ForeignKey(OAuthToken, db_column='token_id', null=True) def save(self, force_insert=False, force_update=False, using=None, update_fields=None): @@ -211,11 +211,14 @@ def get_token(self): def update_token(self, token_dict): # type: (dict) -> None if isinstance(self.token, OAuthToken): - self.token.delete(keep_parents=True) - self.token = OAuthToken(**token_dict) + self.token.access_token = token_dict.get('access_token') + self.token.refresh_token = token_dict.get('refresh_token') + self.token.token_type = token_dict.get('token_type') + self.token.expires_in = token_dict.get('expires_in') + self.token.scope = token_dict.get('scope') + self.token.save() self.save() - def to_dict(self, include_token=True): account = { 'id': self.pk, diff --git a/src/dataloaderinterface/templates/dataloaderinterface/site_details.html b/src/dataloaderinterface/templates/dataloaderinterface/site_details.html index 7384b94c..45243ab3 100644 --- a/src/dataloaderinterface/templates/dataloaderinterface/site_details.html +++ b/src/dataloaderinterface/templates/dataloaderinterface/site_details.html @@ -180,7 +180,7 @@ {###### HydroShare Settings #######} - {% if site.django_user_id == request.user.id %} + {% if site.django_user_id == request.user.id and hydroshare_account %}
      @@ -412,7 +412,7 @@

      Code Output

      - {% if site.django_user_id == request.user.id %} + {% if site.django_user_id == request.user.id and hydroshare_account%} {% if site.django_user_id == request.user.id and resource_is_connected and hydroshare_account %} diff --git a/src/dataloaderinterface/views.py b/src/dataloaderinterface/views.py index 0ad702ef..b8beb655 100644 --- a/src/dataloaderinterface/views.py +++ b/src/dataloaderinterface/views.py @@ -76,12 +76,13 @@ def get_form(self, form_class=None): return form def get_hydroshare_account(self): - hs_account = self.request.user.odm2user.hydroshare_account - if hs_account: - return hs_account - return None + hs_account = None + if self.request.user.odm2user is not None: + hs_account = self.request.user.odm2user.hydroshare_account + return hs_account def get_context_data(self, **kwargs): + user = self.request.user context = super(UserUpdateView, self).get_context_data(**kwargs) context['hs_account'] = self.get_hydroshare_account() @@ -239,7 +240,8 @@ def get_hs_resource(self, resource): # type: (HydroShareResource) -> Resource auth_util = AuthUtil.authorize(token=token_json) # if the oauth access_token expires in less than a week, refresh the token - if token_json.get('expires_in', None) < 60*60*24*7: + seconds_in_week = 60*60*24*7 + if token_json.get('expires_in', seconds_in_week) < seconds_in_week: try: auth_util.refresh_token() account.update_token(auth_util.get_token()) From 741caf4e6437b2005fc559a0ef77085649e0340a Mon Sep 17 00:00:00 2001 From: Craig Blackburn Date: Thu, 1 Mar 2018 11:15:12 -0700 Subject: [PATCH 091/115] updated hydroshare file upload scheduler to check for expiring oauth tokens --- .../update_hydroshare_resource_files.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/dataloaderinterface/management/commands/update_hydroshare_resource_files.py b/src/dataloaderinterface/management/commands/update_hydroshare_resource_files.py index 8e30048b..d338a4fc 100644 --- a/src/dataloaderinterface/management/commands/update_hydroshare_resource_files.py +++ b/src/dataloaderinterface/management/commands/update_hydroshare_resource_files.py @@ -53,9 +53,22 @@ def handle(self, force_update=False, *args, **options): self.stdout.write(colorize('Uploading resource files for: ', fg='blue') + site.sampling_feature_code) try: + token = resource.hs_account.token.to_dict() + # get auth token and upload files to resource - auth = AuthUtil.authorize(token=resource.hs_account.token.to_dict()) - hs_resource = Resource(client=auth.get_client(), resource_id=resource.ext_id) + auth_util = AuthUtil.authorize(token=token) + + # if the oauth access_token expires in less than a week, refresh the token + seconds_in_week = 60 * 60 * 24 * 7 + if token.get('expires_in', seconds_in_week) < seconds_in_week: + try: + auth_util.refresh_token() + account.update_token(auth_util.get_token()) + except Exception as e: + self.stderr.write(colorize('Failed to refresh oauth token. Token expires in {0}: {1}'.format( + token.get('expires_in', 'NA'), e.message), fg='red')) + + hs_resource = Resource(client=auth_util.get_client(), resource_id=resource.ext_id) upload_hydroshare_resource_files(site, hs_resource) upload_success_count += 1 From 7defafecff21d764654328614e274929c9577450 Mon Sep 17 00:00:00 2001 From: darksinge Date: Fri, 2 Mar 2018 14:07:25 -0700 Subject: [PATCH 092/115] fixed issue in update_hydroshare_resource_files.py --- src/WebSDL/settings/base.py | 6 +++--- .../commands/update_hydroshare_resource_files.py | 11 ++++++----- src/dataloaderinterface/views.py | 1 + src/hydroshare_util/resource.py | 4 +++- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/WebSDL/settings/base.py b/src/WebSDL/settings/base.py index 182ded37..45913dc7 100644 --- a/src/WebSDL/settings/base.py +++ b/src/WebSDL/settings/base.py @@ -192,9 +192,9 @@ TSA_URL = data['tsa_url'] if 'tsa_url' in data else '' # crontab job settings -CRONTAB_USER = data.get('user', getpass.getuser()) +CRONTAB_USER = data.get('crontab_user', getpass.getuser()) -CRONTAB_LOGFILE_PATH = data.get('crontabLogFile', '/var/log/odm2websdl-cron.log') +CRONTAB_LOGFILE_PATH = data.get('crontab_log_file', '/var/log/odm2websdl-cron.log') -CRONTAB_EXECUTE_DAILY_AT_HOUR = data.get('crontabExecuteDailyAtHour', 5) +CRONTAB_EXECUTE_DAILY_AT_HOUR = 5 diff --git a/src/dataloaderinterface/management/commands/update_hydroshare_resource_files.py b/src/dataloaderinterface/management/commands/update_hydroshare_resource_files.py index d338a4fc..d89e6e5c 100644 --- a/src/dataloaderinterface/management/commands/update_hydroshare_resource_files.py +++ b/src/dataloaderinterface/management/commands/update_hydroshare_resource_files.py @@ -5,7 +5,7 @@ from hydroshare_util.auth import AuthUtil from django.utils.termcolors import colorize from django.utils import timezone -from datetime import datetime +from datetime import datetime, timedelta class Command(BaseCommand): @@ -63,10 +63,11 @@ def handle(self, force_update=False, *args, **options): if token.get('expires_in', seconds_in_week) < seconds_in_week: try: auth_util.refresh_token() - account.update_token(auth_util.get_token()) + resource.hs_account.update_token(auth_util.get_token()) except Exception as e: - self.stderr.write(colorize('Failed to refresh oauth token. Token expires in {0}: {1}'.format( - token.get('expires_in', 'NA'), e.message), fg='red')) + expire_date = datetime.now() + timedelta(seconds=token['expires_in']) if 'expires_in' in token else 'NA' + self.stderr.write(colorize('Failed to refresh oauth token. Token expires on: {0}\n\t{1}'.format( + expire_date.strftime('%c'), e.message), fg='red')) hs_resource = Resource(client=auth_util.get_client(), resource_id=resource.ext_id) upload_hydroshare_resource_files(site, hs_resource) @@ -80,7 +81,7 @@ def handle(self, force_update=False, *args, **options): except Exception as e: upload_fail_count += 1 - self.stderr.write(colorize('\nERROR: file upload failed, reason given:\n\t{0}'.format(e.message), fg='red')) + self.stderr.write(colorize('\nError: file upload failed, reason given:\n\t{0}'.format(e), fg='red')) self.stdout.write(colorize('\nJob finished uploading resource files to hydroshare.', fg='blue')) self.stdout.write(colorize('\t Uploads successful: ', fg='blue') + str(upload_success_count)) diff --git a/src/dataloaderinterface/views.py b/src/dataloaderinterface/views.py index b8beb655..16655e8d 100644 --- a/src/dataloaderinterface/views.py +++ b/src/dataloaderinterface/views.py @@ -275,6 +275,7 @@ def get_context_data(self, **kwargs): return context def get(self, request, *args, **kwargs): + call_command('update_hydroshare_resource_files', '--force-update') return super(HydroShareResourceUpdateCreateView, self).get(request, args, kwargs) diff --git a/src/hydroshare_util/resource.py b/src/hydroshare_util/resource.py index 6f2e1acd..13defb52 100644 --- a/src/hydroshare_util/resource.py +++ b/src/hydroshare_util/resource.py @@ -1,6 +1,6 @@ import logging import os -import json +from django.http import Http404 from tempfile import mkstemp from re import search as regex_search from hs_restclient import HydroShareNotFound, HydroShareNotAuthorized @@ -141,6 +141,8 @@ def upload_file(self, filename, content): # upload file try: return self.client.add_resource_file(self.resource_id, path, resource_filename=filename) + except HydroShareNotFound: + raise Http404(u"Resource '{0}' was not found".format(self.resource_id)) except Exception as e: raise e finally: From c090e848798d9ef74975d1d9b8735583b00df9f7 Mon Sep 17 00:00:00 2001 From: darksinge Date: Fri, 2 Mar 2018 14:56:47 -0700 Subject: [PATCH 093/115] some hydroshare_util refactoring; added documentation to hydroshare js files --- .../js/hydroshare/hs-create-resource.js | 9 ++- .../js/hydroshare/hs-delete-resource.js | 8 +++ .../js/hydroshare/hs-profile-dialog.js | 7 ++ .../js/hydroshare/hs-settings-dialog.js | 2 +- .../js/hydroshare/hs-update-resource.js | 8 +++ .../templates/hydroshare/hs_profile.html | 12 +--- .../templates/hydroshare/oauth_redirect.html | 2 +- .../templates/registration/account.html | 1 + src/dataloaderinterface/views.py | 7 +- src/hydroshare_util/README.md | 4 +- src/hydroshare_util/adapter.py | 66 +------------------ src/hydroshare_util/resource.py | 38 +++++------ src/hydroshare_util/utility.py | 5 +- 13 files changed, 65 insertions(+), 104 deletions(-) diff --git a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-create-resource.js b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-create-resource.js index b5c0b927..c6e4570a 100644 --- a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-create-resource.js +++ b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-create-resource.js @@ -1,4 +1,11 @@ - +/* +* hs-create-resource.js +* This script is loaded into the browser at /site/:sampling_feature_code/ when a user has a hydroshare account and +* they have not started sharing data with hydroshare.org. +* +* The script makes an ajax request to the server to grab the page content used to start sharing site data with +* HydroShare and loads the page content into the DOM. +* */ $(() => { const template_url = `${window.location.href}hsr/create/`; diff --git a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-delete-resource.js b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-delete-resource.js index 26516862..c42eb9ab 100644 --- a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-delete-resource.js +++ b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-delete-resource.js @@ -1,3 +1,11 @@ +/* +* hs-delete-resource.js +* This script is loaded into the browser at /site/:sampling_feature_code/ when a user has a hydroshare account and +* they have started sharing data with hydroshare.org. +* +* The script performs setup on the hydroshare-delete-modal that allows users to delete the site +* connection to HydroShare and/or delete the associated resource in HydroShare. +* */ 'use strict'; function initializeHydroShareDeleteDialog() { diff --git a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-profile-dialog.js b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-profile-dialog.js index 91c1fc61..bd895c74 100644 --- a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-profile-dialog.js +++ b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-profile-dialog.js @@ -1,3 +1,10 @@ +/* +* hs-profile-dialog.js +* This script is loaded into the browser at /account/. +* +* This script allows a user to authorize/deathorize EnviroDIY to perform actions on the users HydroShare account in +* the users behalf. +* */ (function () { 'use strict'; diff --git a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-settings-dialog.js b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-settings-dialog.js index ff56b475..4ac6c8a9 100644 --- a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-settings-dialog.js +++ b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-settings-dialog.js @@ -1,6 +1,6 @@ /** * hs-settings-dialog.js -* @description Initializes the hydroshare settings modal for presentation on 'site_details.html'. +* This script performs setup for the hydroshare settings modal. */ function initializeHydroShareSettingsDialog() { diff --git a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-update-resource.js b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-update-resource.js index 77609f68..31f67440 100644 --- a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-update-resource.js +++ b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-update-resource.js @@ -1,3 +1,11 @@ +/* +* hs-update-resource.js +* This script is loaded into the browser at /site/:sampling_feature_code/ when a user has a hydroshare account and +* they have started sharing data with hydroshare.org. +* +* The script makes an ajax request to the server to grab the page content used to manage hydroshare-sharing settings +* and loads the page content into the DOM. +* */ 'use strict'; const resource_template_url = `${window.location.href}hsr/update/`; diff --git a/src/dataloaderinterface/templates/hydroshare/hs_profile.html b/src/dataloaderinterface/templates/hydroshare/hs_profile.html index 54746602..6d576da4 100644 --- a/src/dataloaderinterface/templates/hydroshare/hs_profile.html +++ b/src/dataloaderinterface/templates/hydroshare/hs_profile.html @@ -1,12 +1,4 @@ +{% load static %} + {# TODO: Replace with dynamic url #} -
      +
      diff --git a/src/dataloaderinterface/templates/hydroshare/oauth_redirect.html b/src/dataloaderinterface/templates/hydroshare/oauth_redirect.html index 0e130a6d..fad045f2 100644 --- a/src/dataloaderinterface/templates/hydroshare/oauth_redirect.html +++ b/src/dataloaderinterface/templates/hydroshare/oauth_redirect.html @@ -42,9 +42,9 @@

      You are being redirected to hydroshare.org

      {% endblock %} \ No newline at end of file diff --git a/src/dataloaderinterface/templates/registration/account.html b/src/dataloaderinterface/templates/registration/account.html index 462f18ab..3e394120 100644 --- a/src/dataloaderinterface/templates/registration/account.html +++ b/src/dataloaderinterface/templates/registration/account.html @@ -196,6 +196,7 @@

      New Organization

      + {% endblock %} {% block scripts %} diff --git a/src/dataloaderinterface/views.py b/src/dataloaderinterface/views.py index 16655e8d..d8064a0e 100644 --- a/src/dataloaderinterface/views.py +++ b/src/dataloaderinterface/views.py @@ -275,7 +275,7 @@ def get_context_data(self, **kwargs): return context def get(self, request, *args, **kwargs): - call_command('update_hydroshare_resource_files', '--force-update') + # call_command('update_hydroshare_resource_files', '--force-update') return super(HydroShareResourceUpdateCreateView, self).get(request, args, kwargs) @@ -405,11 +405,10 @@ def get_context_data(self, **kwargs): try: resource_md = resource_util.get_system_metadata(timeout=10.0) context['resource_is_published'] = resource_md.get("published", False) - context['resource_not_found'] = resource_md is None except HydroShareNotFound: context['resource_not_found'] = True except requests.exceptions.Timeout: - context['resource_did_not_load'] = True + context['request_timeout'] = True finally: if context.get('resource_not_found', None) is True: context['delete_resource_url'] = reverse('hs_resource_delete', @@ -896,7 +895,7 @@ def get(self, request, *args, **kwargs): return HttpResponse('Error: Authorization failure!') client = auth_utility.get_client() # type: HydroShareAdapter - user_info = client.get_user_info() + user_info = client.getUserInfo() print('\nuser_info: %s', json.dumps(user_info, indent=3)) try: diff --git a/src/hydroshare_util/README.md b/src/hydroshare_util/README.md index 6e45a1c0..95ab4121 100644 --- a/src/hydroshare_util/README.md +++ b/src/hydroshare_util/README.md @@ -56,7 +56,7 @@ def hydroshare_callback(request): #### Authorizing a user using OAuth 2.0 and their username and password: **Note:** From a security standpoint, a user should never be asked for their credentials to *hydroshare.org* from a website existing under a different domain name. - Quoting from [RFC 6742](https://tools.ietf.org/html/rfc6749#section-1.3.3): + From [RFC 6742](https://tools.ietf.org/html/rfc6749#section-1.3.3): > [A user's] credentials should only be used when there is a high degree of trust between the resource owner and the client (e.g., the client is part of the device operating system or a highly privileged @@ -103,7 +103,7 @@ def hydroshare(request): After a user has been authenticated, you can start using `hydroshare_util` to consume HydroShare's REST API. -TO get information about the user: +To get information about the user: ```python # example.py from hydroshare_util.utility import HydroShareUtility diff --git a/src/hydroshare_util/adapter.py b/src/hydroshare_util/adapter.py index 2b9af6fb..d4119d21 100644 --- a/src/hydroshare_util/adapter.py +++ b/src/hydroshare_util/adapter.py @@ -36,14 +36,9 @@ def _request(self, method, url, params=None, data=None, files=None, headers=None stream=stream, verify=self.verify, timeout=timeout) return request - def get_resource_list(self, **kwargs): - return self.getResourceList(**kwargs) - - def get_system_metadata(self, pid, **kwargs): + def getSystemMetadata(self, pid, **kwargs): """ Returns system metadata for a resource which includes the dublin core elements - NOTE: get_system_metadata does not call getSystemMetadata() from hs_restclient so that - a timeout can be specified for the request. """ timeout = None if 'timeout' in kwargs: @@ -67,62 +62,3 @@ def get_system_metadata(self, pid, **kwargs): raise HydroShareHTTPException((url, 'GET', req.status_code)) return req.json() - - def get_science_metadataRDF(self, pid): - return self.getScienceMetadataRDF(pid) - - def get_science_metadata(self, pid): - return self.getScienceMetadata(pid) - - def update_science_metadata(self, pid, metadata): - return self.updateScienceMetadata(pid, metadata) - - def get_resource_map(self, pid): - return self.getResourceMap(pid) - - def get_resource(self, pid, destination=None, unzip=False, wait_for_bag_creation=True): - return self.getResource(pid, destination=destination, unzip=unzip, wait_for_bag_creation=wait_for_bag_creation) - - def get_resource_types(self): - return self.getResourceTypes() - - def create_resource(self, resource_type, title, resource_file=None, resource_filename=None, - abstract=None, keywords=None, - edit_users=None, view_users=None, edit_groups=None, view_groups=None, - metadata=None, extra_metadata=None, progress_callback=None): - return self.createResource(resource_type, title, resource_file=resource_file, - resource_filename=resource_filename, abstract=abstract, keywords=keywords, - edit_users=edit_users, view_users=view_users, edit_groups=edit_groups, - view_groups=view_groups, metadata=metadata, extra_metadata=extra_metadata, - progress_callback=progress_callback) - - def delete_resource(self, pid): - return self.deleteResource(pid) - - def set_access_rules(self, pid, public=False): - return self.setAccessRules(pid, public=public) - - def add_resource_file(self, pid, resource_file, resource_filename=None, progress_callback=None): - return self.addResourceFile(pid, resource_file, resource_filename=resource_filename, - progress_callback=progress_callback) - - def get_resource_file(self, pid, filename, destination=None): - return self.getResourceFile(pid, filename, destination=destination) - - def delete_resource_file(self, pid, filename): - return self.deleteResourceFile(pid, filename) - - def get_resource_file_list(self, pid): - return self.getResourceFileList(pid) - - def get_resource_folder_contents(self, pid, pathname): - return self.getResourceFolderContents(pid, pathname) - - def create_resource_folder(self, pid, pathname): - return self.createResourceFolder(pid, pathname) - - def delete_resource_folder(self, pid, pathname): - return self.deleteResourceFolder(pid, pathname) - - def get_user_info(self): - return self.getUserInfo() diff --git a/src/hydroshare_util/resource.py b/src/hydroshare_util/resource.py index 13defb52..42456504 100644 --- a/src/hydroshare_util/resource.py +++ b/src/hydroshare_util/resource.py @@ -78,27 +78,27 @@ def add_coverage(self, coverage): def update_file_list(self): try: - file_list = [file for file in self.client.get_resource_file_list(self.resource_id)] - self.files = [os.path.basename(file['url']) for file in file_list] + file_list = [file_ for file_ in self.client.getResourceFileList(self.resource_id)] + self.files = [os.path.basename(file_['url']) for file_ in file_list] except Exception as e: self._r_logger("Error updating resource file list", error=e) def get_system_metadata(self, **kwargs): - return self.client.get_system_metadata(self.resource_id, **kwargs) + return self.client.getSystemMetadata(self.resource_id, **kwargs) def get_science_metadata(self): - return self.client.get_science_metadata(self.resource_id) + return self.client.getScienceMetadata(self.resource_id) def update_metadata(self, data=None): if data is None: data = self.to_object() - return self.client.update_science_metadata(self.resource_id, data) + return self.client.updateScienceMetadata(self.resource_id, data) def upload_files(self): upload_success_count = 0 for file_ in self.files: try: - self.client.delete_resource_file(self.resource_id, os.path.basename(file_)) + self.client.deleteResourceFile(self.resource_id, os.path.basename(file_)) except HydroShareNotFound: pass @@ -124,14 +124,14 @@ def upload_file(self, filename, content): raise HydroShareNotFound("resource has no resource_id") try: - self.client.delete_resource_file(self.resource_id, filename) + self.client.deleteResourceFile(self.resource_id, filename) except HydroShareNotFound: # If file doesn't exist on hydroshare, it's okay, just keep breathing. pass except HydroShareNotAuthorized: pass - # Write file to disk... because that's how hs_restclient needs it to be done! Stupid, I know. + # Write file to disk... because that's how hs_restclient needs it to be done! suffix = '.csv' if regex_search('\.csv', filename) else '' fd, path = mkstemp(suffix=suffix) @@ -140,7 +140,7 @@ def upload_file(self, filename, content): # upload file try: - return self.client.add_resource_file(self.resource_id, path, resource_filename=filename) + return self.client.addResourceFile(self.resource_id, path, resource_filename=filename) except HydroShareNotFound: raise Http404(u"Resource '{0}' was not found".format(self.resource_id)) except Exception as e: @@ -161,14 +161,14 @@ def delete_files(self, files=None): url = file_['url'] self._r_logger("Deleting resource file\n\tfile: {file}".format(file=os.path.basename(url)), level=logging.INFO) - self.client.delete_resource_file(self.resource_id, os.path.basename(url)) + self.client.deleteResourceFile(self.resource_id, os.path.basename(url)) def get_coverage_period(self): raise NotImplementedError("method not implemented.") def delete(self): self._r_logger("Deleting resource!", level=logging.INFO) - self.client.delete_resource(self.resource_id) + self.client.deleteResource(self.resource_id) def create(self): @@ -180,11 +180,11 @@ def create(self): metadata_as_string = self._stringify_metadata(metadata) - self.resource_id = self.client.create_resource(resource_type=self.resource_type, - title=self.title, - metadata=metadata_as_string, - keywords=list(self.keywords), - abstract=self.abstract) + self.resource_id = self.client.createResource(resource_type=self.resource_type, + title=self.title, + metadata=metadata_as_string, + keywords=list(self.keywords), + abstract=self.abstract) return self.resource_id def _stringify_metadata(self, metadata): @@ -197,12 +197,12 @@ def update(self, metadata=None): metadata = self.to_object() elif metadata is None: metadata = self.to_object() - return self.client.update_science_metadata(self.resource_id, metadata) + return self.client.updateScienceMetadata(self.resource_id, metadata) def to_object(self, clean=True): metadata = super(Resource, self).to_object(clean=clean) - # replace keyword 'coverages' with keyword 'coverage'... + # replace keyword 'coverages' with 'coverage'... metadata['coverage'] = getattr(metadata, 'coverages', list()) if 'coverages' in metadata: del metadata['coverages'] @@ -220,7 +220,7 @@ def to_object(self, clean=True): return metadata def make_public(self, public=True): - return self.client.set_access_rules(self.resource_id, public=public) + return self.client.setAccessRules(self.resource_id, public=public) def to_dict(self): return { diff --git a/src/hydroshare_util/utility.py b/src/hydroshare_util/utility.py index 349cb18e..39029501 100644 --- a/src/hydroshare_util/utility.py +++ b/src/hydroshare_util/utility.py @@ -7,6 +7,7 @@ import json import time + class HydroShareUtility(HydroShareUtilityBaseClass): """Utility class for accessing and consuming HydroShare's REST API.""" RE_PERIOD = re.compile(r'(?P^start=)(?P[0-9-]{10}T[0-9:]{8}).{2}(?Pend=)' @@ -63,13 +64,13 @@ def get_resource_system_metadata(self, pid): def get_user_info(self): try: - return self.client.get_user_info() + return self.client.getUserInfo() except Exception: return None def get_resource_types(self): try: - Resource.RESOURCE_TYPES = [type_ for type_ in self.client.get_resource_types()] + Resource.RESOURCE_TYPES = [type_ for type_ in self.client.getResourceTypes()] except Exception as e: logging.error("Failed to get resource types!\n{error}".format(error=e)) From 41d9a55116e79648f7662b44cec7a1071ea11822 Mon Sep 17 00:00:00 2001 From: darksinge Date: Fri, 2 Mar 2018 15:06:21 -0700 Subject: [PATCH 094/115] updated settings_template.json --- src/WebSDL/settings/settings_template.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/WebSDL/settings/settings_template.json b/src/WebSDL/settings/settings_template.json index b7dd6675..1379718e 100644 --- a/src/WebSDL/settings/settings_template.json +++ b/src/WebSDL/settings/settings_template.json @@ -31,7 +31,8 @@ "hydroshare_oauth": { "client_id": "client_id_here", "client_secret": "client_secret_here", - "redirect_uri": "http://localhost:8000/oauth/hydroshare/", - "response_type": "code" - } + "redirect_uri": "http://localhost:8000/hydroshare/oauth/" + }, + "crontab_log_file": "/var/log/crontab.log", + "crontab_user": "joesmokum" } From 977c0cb5b047c10d37c32c1512a36b8b47d7aedc Mon Sep 17 00:00:00 2001 From: darksinge Date: Fri, 2 Mar 2018 15:13:37 -0700 Subject: [PATCH 095/115] updated text in hs_site_details.html for published resources --- .../templates/hydroshare/hs_site_details.html | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/dataloaderinterface/templates/hydroshare/hs_site_details.html b/src/dataloaderinterface/templates/hydroshare/hs_site_details.html index 6c3ba068..c35028cf 100644 --- a/src/dataloaderinterface/templates/hydroshare/hs_site_details.html +++ b/src/dataloaderinterface/templates/hydroshare/hs_site_details.html @@ -43,11 +43,12 @@ {# Actions if the resource has been published #} {% elif resource_is_published %} -

      The resource this site is connected to is published. Once a resource is published, you cannot change the resource's content.

      +

      The data from this site are no longer being copied to hydroshare.

      +

      The hydroshare resource this site is connected to has been formally published. Once a resource has been published, you cannot change the resource’s content.

      If you would like to continue sharing data from this site to HydroShare:

        -
      1. Click DISCONNECT RESOURCE.
        2. -
      2. Click on START SHARING after the page reloads to start sharing with a new, unpublished resource.
        3. +
      3. Click Disconnect Resource.
        4. +
      4. Click on Start Sharing after the page reloads to start sharing with a new, unpublished resource.
      From 6f0e9fcce978d3c90662e671c1b60d4af032bcca Mon Sep 17 00:00:00 2001 From: darksinge Date: Tue, 6 Mar 2018 10:55:38 -0700 Subject: [PATCH 096/115] updated and fixed issues for pull request (merge into development) --- src/dataloaderinterface/apps.py | 3 +- .../update_hydroshare_resource_files.py | 2 +- src/dataloaderinterface/models.py | 6 +- .../js/hydroshare/hs-settings-dialog.js | 2 + .../js/mdl-button-resizer.js | 10 -- .../hydroshare/hs_settings_dialog.html | 4 +- .../templates/hydroshare/hs_site_details.html | 2 +- src/dataloaderinterface/views.py | 8 +- src/hydroshare_util/account.py | 35 ------- src/hydroshare_util/adapter.py | 8 +- src/hydroshare_util/coverage.py | 20 ++-- src/hydroshare_util/resource.py | 22 +++-- src/hydroshare_util/utility.py | 16 ++- src/migration.sql | 98 ------------------- 14 files changed, 57 insertions(+), 179 deletions(-) delete mode 100644 src/dataloaderinterface/static/dataloaderinterface/js/mdl-button-resizer.js delete mode 100644 src/hydroshare_util/account.py delete mode 100644 src/migration.sql diff --git a/src/dataloaderinterface/apps.py b/src/dataloaderinterface/apps.py index bf9c5e3e..54b3ebd7 100644 --- a/src/dataloaderinterface/apps.py +++ b/src/dataloaderinterface/apps.py @@ -1,8 +1,7 @@ from __future__ import unicode_literals from django.apps import AppConfig -import crontab_jobs -from django.utils.termcolors import colorize + class DataloaderinterfaceConfig(AppConfig): name = 'dataloaderinterface' diff --git a/src/dataloaderinterface/management/commands/update_hydroshare_resource_files.py b/src/dataloaderinterface/management/commands/update_hydroshare_resource_files.py index d89e6e5c..6147a972 100644 --- a/src/dataloaderinterface/management/commands/update_hydroshare_resource_files.py +++ b/src/dataloaderinterface/management/commands/update_hydroshare_resource_files.py @@ -41,7 +41,7 @@ def handle(self, force_update=False, *args, **options): continue # Skip resources that are not "due" for upload unless 'force_udpate' is True - next_sync = resource.next_sync_date() + next_sync = resource.get_next_sync_date() now = timezone.now() diff = (now - next_sync).total_seconds() if diff < 0 and force_update is False: diff --git a/src/dataloaderinterface/models.py b/src/dataloaderinterface/models.py index e9730a32..641f9190 100644 --- a/src/dataloaderinterface/models.py +++ b/src/dataloaderinterface/models.py @@ -269,8 +269,8 @@ def update_freq_verbose(self): return 'NA' @property - def m_next_sync_date(self): - return self.next_sync_date() + def next_sync_date(self): + return self.get_next_sync_date() @property def sync_at_hour(self): @@ -287,7 +287,7 @@ def get_udpate_freq_index(self): except Exception: return 0 - def next_sync_date(self): + def get_next_sync_date(self): days = 0 minutes = 0 if self.update_freq == 'minute': diff --git a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-settings-dialog.js b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-settings-dialog.js index 4ac6c8a9..394f68e1 100644 --- a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-settings-dialog.js +++ b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-settings-dialog.js @@ -66,6 +66,7 @@ function initializeHydroShareSettingsDialog() { let progressSpinner = $(hydroshareSettingsForm).find('#hydroshare-progress-spinner'); progressSpinner.addClass('is-active'); + $('span#hs-loading-msg').prop('hidden', false); $.post(url, serializedForm) .done(data => { @@ -100,6 +101,7 @@ function initializeHydroShareSettingsDialog() { } }).always(() => { progressSpinner.removeClass('is-active'); + $('span#hs-loading-msg').prop('hidden', true); $(dialogButtons).prop('disabled', false); $(submitButton).addClass('mdl-button--accent'); }); diff --git a/src/dataloaderinterface/static/dataloaderinterface/js/mdl-button-resizer.js b/src/dataloaderinterface/static/dataloaderinterface/js/mdl-button-resizer.js deleted file mode 100644 index fd5f1ca3..00000000 --- a/src/dataloaderinterface/static/dataloaderinterface/js/mdl-button-resizer.js +++ /dev/null @@ -1,10 +0,0 @@ -'use strict'; - -$(function() { - resizeHSConnectionLinkButton(); -}); - -function resizeHSConnectionLinkButton() { - const addHsConnLink = $('a#hs-conn-link'); - addHsConnLink.width(addHsConnLink.innerWidth()); -} \ No newline at end of file diff --git a/src/dataloaderinterface/templates/hydroshare/hs_settings_dialog.html b/src/dataloaderinterface/templates/hydroshare/hs_settings_dialog.html index 4b471b0c..f066f169 100644 --- a/src/dataloaderinterface/templates/hydroshare/hs_settings_dialog.html +++ b/src/dataloaderinterface/templates/hydroshare/hs_settings_dialog.html @@ -53,7 +53,7 @@

      {{ form.schedule_type.label }}

      {# Data Types - Time Series, Leaf Packet, etc. #} {# Yes... #}{% if False %} {#... this "if" statement was intentional #} - {# NOTE: Until there are multiple data types to choose from, do not render! #} + {# Until there are multiple data types to choose from, do not render! #}

      {{ form.data_types.label }}

      @@ -97,6 +97,7 @@

      {{ form.abstract.label }}

      +
      -
      diff --git a/src/dataloaderinterface/templates/hydroshare/hs_site_details.html b/src/dataloaderinterface/templates/hydroshare/hs_site_details.html index c35028cf..cc056d74 100644 --- a/src/dataloaderinterface/templates/hydroshare/hs_site_details.html +++ b/src/dataloaderinterface/templates/hydroshare/hs_site_details.html @@ -67,7 +67,7 @@

      Last updated on {{ resource.last_sync_date|date:date_format }}

      {% if resource.sync_type == 'scheduled' %} -

      Next update scheduled for {{ resource.m_next_sync_date|date:date_format }}

      +

      Next update scheduled for {{ resource.next_sync_date|date:date_format }}

      {% endif %}

      View data from this site in HydroShare

      diff --git a/src/dataloaderinterface/views.py b/src/dataloaderinterface/views.py index d8064a0e..9eab5ed8 100644 --- a/src/dataloaderinterface/views.py +++ b/src/dataloaderinterface/views.py @@ -82,9 +82,7 @@ def get_hydroshare_account(self): return hs_account def get_context_data(self, **kwargs): - user = self.request.user context = super(UserUpdateView, self).get_context_data(**kwargs) - context['hs_account'] = self.get_hydroshare_account() context['organization_form'] = OrganizationForm() return context @@ -275,7 +273,11 @@ def get_context_data(self, **kwargs): return context def get(self, request, *args, **kwargs): - # call_command('update_hydroshare_resource_files', '--force-update') + """ + # uncomment to force a hydroshare resource file update. + # Only do this for debugging purposes! + call_command('update_hydroshare_resource_files', '--force-update') + """ return super(HydroShareResourceUpdateCreateView, self).get(request, args, kwargs) diff --git a/src/hydroshare_util/account.py b/src/hydroshare_util/account.py deleted file mode 100644 index a44dfc14..00000000 --- a/src/hydroshare_util/account.py +++ /dev/null @@ -1,35 +0,0 @@ -from . import HydroShareUtilityBaseClass, HSUClassAttributeError - - -class HSUAccount(HydroShareUtilityBaseClass): - def __init__(self, id=None, email=None, first_name=None, last_name=None, - organization=None, username=None, **kwargs): - self.id = id - self.email = email - self.first_name = first_name - self.last_name = last_name - self.organization = organization - self.username = username - - for key, value in kwargs.iteritems(): - if key in self.__dict__: - setattr(self, key, value) - else: - raise HSUClassAttributeError(self, key) - - def to_dict(self): - account = self.to_object() - return { - 'id': account['id'], - 'email': account['email'], - 'first_name': account['first_name'], - 'last_name': account['last_name'], - 'organization': account['organization'], - 'username': account['username'] - } - - def __repr__(self): - return "<{classname}: {username}>".format(classname=type(self).__name__, username=self.username) - - -__all__ = ["HSUAccount"] diff --git a/src/hydroshare_util/adapter.py b/src/hydroshare_util/adapter.py index d4119d21..817a8661 100644 --- a/src/hydroshare_util/adapter.py +++ b/src/hydroshare_util/adapter.py @@ -39,6 +39,10 @@ def _request(self, method, url, params=None, data=None, files=None, headers=None def getSystemMetadata(self, pid, **kwargs): """ Returns system metadata for a resource which includes the dublin core elements + Note: HydroShareAdapter overrides it's super class method, getSystemMetadata(), so 'timeout' can be + included in **kwargs. By default, the requests library does not have timeout limit for HTTP requests, and + some views wait for getSystemMetadata() to complete an HTTP request to hydroshare.org before the views are + rendered. """ timeout = None if 'timeout' in kwargs: @@ -55,10 +59,10 @@ def getSystemMetadata(self, pid, **kwargs): req = requests.get(url, headers=headers, timeout=timeout) if req.status_code != 200: if req.status_code == 403: - raise HydroShareNotAuthorized(('GET', url)) + raise HydroShareNotAuthorized((req.request.method, url)) elif req.status_code == 404: raise HydroShareNotFound((pid,)) else: - raise HydroShareHTTPException((url, 'GET', req.status_code)) + raise HydroShareHTTPException((url, req.request.method, req.status_code)) return req.json() diff --git a/src/hydroshare_util/coverage.py b/src/hydroshare_util/coverage.py index f09ea707..bb08631a 100644 --- a/src/hydroshare_util/coverage.py +++ b/src/hydroshare_util/coverage.py @@ -27,13 +27,15 @@ def __init__(self, coverage=None, implementation=None): @property def type(self): - return self.__implementation.type + return self.__implementation.type_ def to_dict(self): return self.__implementation.to_dict() class Coverage(CoverageImplementor): + type_ = None + def __init__(self, **kwargs): for attr, value in kwargs.iteritems(): setattr(self, attr, value) @@ -49,6 +51,10 @@ def to_dict(self): class BoxCoverage(Coverage): + """ + BoxCoverage (also known as spatial coverage) is a type of coverage that specifies a geographical area with North, + East, South, and West limits. + """ type_ = 'box' def __init__(self, northlimit=None, eastlimit=None, southlimit=None, westlimit=None, projection=None, units=None, @@ -62,9 +68,6 @@ def __init__(self, northlimit=None, eastlimit=None, southlimit=None, westlimit=N self.units = units def to_dict(self): - # {u'coverages': [{u'type': u'box', u'value': {u'northlimit': 0.0, u'uplimit': None, - # u'projection': u'WGS 84 EPSG:4326', u'downlimit': None, u'zunits': None, u'units': u'Decimal degrees', - # u'southlimit': 0.0, u'westlimit': -180.0, u'eastlimit': 180.0, u'name': None}} return { 'type': self.type_, 'value': { @@ -79,6 +82,7 @@ def to_dict(self): class PointCoverage(Coverage): + """PointCoverage is a type of coverage that specifies an exact geographical location.""" type_ = 'point' DEFAULT_PROJECTION = "WGS 84 EPSG:4326" @@ -93,8 +97,6 @@ def __init__(self, name=None, latitude=None, longitude=None, projection=None, un self.units = units.encode('ascii') if units is not None else None def to_dict(self): - # {'coverages': {u'type': u'point', u'value': { - # u'units': u'Decimal degrees', u'east': -111.946402, u'north': 41.718473, u'projection': u'Unknown'}} return { 'type': self.type_, 'value': { @@ -108,6 +110,10 @@ def to_dict(self): class PeriodCoverage(Coverage): + """ + PeriodCoverage (also known as temporal coverage) is a type of coverage that specifies a period of time over + which data is collected. + """ type_ = 'period' def __init__(self, start=None, end=None, **kwargs): @@ -116,8 +122,6 @@ def __init__(self, start=None, end=None, **kwargs): self.end = end def to_dict(self): - # {u'coverages': [ - # {u'type': u'period', u'value': {u'start': u'1979-01-01 06:00:00', u'end': u'2015-03-31 06:00:00'}} return { 'type': self.type_, 'value': { diff --git a/src/hydroshare_util/resource.py b/src/hydroshare_util/resource.py index 42456504..1b62c55b 100644 --- a/src/hydroshare_util/resource.py +++ b/src/hydroshare_util/resource.py @@ -60,6 +60,8 @@ def __init__(self, client, raw=None, resource_id=None, creator=None, title="", a for coverage in coverages: if isinstance(coverage, dict): self.coverages.append(CoverageFactory(coverage=coverage)) + elif isinstance(coverage, Coverage): + self.coverages.append(coverage) for key, value in kwargs.iteritems(): if key == 'public': @@ -81,7 +83,7 @@ def update_file_list(self): file_list = [file_ for file_ in self.client.getResourceFileList(self.resource_id)] self.files = [os.path.basename(file_['url']) for file_ in file_list] except Exception as e: - self._r_logger("Error updating resource file list", error=e) + self._log("Error updating resource file list", error=e) def get_system_metadata(self, **kwargs): return self.client.getSystemMetadata(self.resource_id, **kwargs) @@ -107,8 +109,8 @@ def upload_files(self): try: self.client.addResourceFile(self.resource_id, file_) - self._r_logger("File upload successful: '{filename}'".format(filename=os.path.basename(file_)), - level=logging.INFO) + self._log("File upload successful: '{filename}'".format(filename=os.path.basename(file_)), + level=logging.INFO) upload_success_count += 1 except KeyError as e: self._log_error('File upload failed; incorrectly formatted arguments given.', e) @@ -116,8 +118,8 @@ def upload_files(self): except Exception as e: self._log_error("File upload failed: \n\t{fname}\n".format(fname=os.path.basename(file_)), e) raise e - self._r_logger("successfully uploaded {count} files".format(count=upload_success_count), - level=logging.INFO) + self._log("successfully uploaded {count} files".format(count=upload_success_count), + level=logging.INFO) def upload_file(self, filename, content): if self.resource_id is None: @@ -159,15 +161,15 @@ def delete_files(self, files=None): for file_ in files: url = file_['url'] - self._r_logger("Deleting resource file\n\tfile: {file}".format(file=os.path.basename(url)), - level=logging.INFO) + self._log("Deleting resource file\n\tfile: {file}".format(file=os.path.basename(url)), + level=logging.INFO) self.client.deleteResourceFile(self.resource_id, os.path.basename(url)) def get_coverage_period(self): raise NotImplementedError("method not implemented.") def delete(self): - self._r_logger("Deleting resource!", level=logging.INFO) + self._log("Deleting resource!", level=logging.INFO) self.client.deleteResource(self.resource_id) def create(self): @@ -253,9 +255,9 @@ def to_dict(self): } def _log_error(self, msg, error): - return self._r_logger(msg, error=error, level=logging.ERROR) + return self._log(msg, error=error, level=logging.ERROR) - def _r_logger(self, msg, error=None, level=None): + def _log(self, msg, error=None, level=None): if error: log = "{msg}\n\tresource_id: {id}\n\t{e}".format(msg=msg, id=self.resource_id, e=error) else: diff --git a/src/hydroshare_util/utility.py b/src/hydroshare_util/utility.py index 39029501..f1d92c49 100644 --- a/src/hydroshare_util/utility.py +++ b/src/hydroshare_util/utility.py @@ -22,6 +22,7 @@ class HydroShareUtility(HydroShareUtilityBaseClass): def __init__(self, auth=None): # type: (AuthUtil) -> None self.auth = auth + self.request_resource_types() @property def client(self): @@ -65,14 +66,21 @@ def get_resource_system_metadata(self, pid): def get_user_info(self): try: return self.client.getUserInfo() - except Exception: - return None + except Exception as e: + logging.error(e) + return None + + def request_resource_types(self): + from multiprocessing import Process + proc = Process(target=self._request_resource_types_async) + proc.start() + proc.join() - def get_resource_types(self): + def _request_resource_types_async(self): try: Resource.RESOURCE_TYPES = [type_ for type_ in self.client.getResourceTypes()] except Exception as e: - logging.error("Failed to get resource types!\n{error}".format(error=e)) + logging.error("Failed to get resource types.\n\t{error}".format(error=e)) __all__ = ["HydroShareUtility"] diff --git a/src/migration.sql b/src/migration.sql deleted file mode 100644 index ebafd0de..00000000 --- a/src/migration.sql +++ /dev/null @@ -1,98 +0,0 @@ -BEGIN; --- --- Create model HydroShareAccount --- -CREATE TABLE "hydroshare_account" ("id" serial NOT NULL PRIMARY KEY, "name" varchar(255) NOT NULL, "is_enabled" boolean NOT NULL, "ext_hydroshare_id" integer NOT NULL UNIQUE); --- --- Create model HydroShareSiteSetting --- -CREATE TABLE "hydroshare_site_setting" ("id" serial NOT NULL PRIMARY KEY, "sync_type" varchar(255) NOT NULL, "resource_id" varchar(255) NOT NULL, "update_freq" interval NOT NULL, "is_enabled" boolean NOT NULL, "last_sync_date" timestamp with time zone NULL, "hs_account_id" integer NULL); --- --- Create model HydroShareSyncLog --- -CREATE TABLE "hydroshare_sync_log" ("id" serial NOT NULL PRIMARY KEY, "sync_date" timestamp with time zone NOT NULL, "sync_type" varchar(255) NOT NULL, "site_sharing_id" integer NOT NULL); --- --- Create model OAuthToken --- -CREATE TABLE "oauth_token" ("id" serial NOT NULL PRIMARY KEY, "access_token" varchar(255) NOT NULL, "refresh_token" varchar(255) NOT NULL, "token_type" varchar(6) NOT NULL, "expires_in" timestamp with time zone NOT NULL, "scope" varchar(255) NOT NULL, "account_id" integer NOT NULL); --- --- Create model ODM2User --- -CREATE TABLE "dataloaderinterface_odm2user" ("id" serial NOT NULL PRIMARY KEY, "affiliation_id" integer NOT NULL, "user_id" integer NOT NULL UNIQUE); --- --- Create model SiteAlert --- -CREATE TABLE "dataloaderinterface_sitealert" ("id" serial NOT NULL PRIMARY KEY, "LastAlerted" timestamp with time zone NULL, "HoursThreshold" integer NOT NULL CHECK ("HoursThreshold" >= 0)); --- --- Create model SiteRegistration --- -CREATE TABLE "dataloaderinterface_siteregistration" ("RegistrationID" serial NOT NULL PRIMARY KEY, "RegistrationToken" varchar(64) NOT NULL UNIQUE, "RegistrationDate" timestamp with time zone NOT NULL, "DeploymentDate" timestamp with time zone NULL, "AffiliationID" integer NOT NULL, "Person" varchar(765) NOT NULL, "Organization" varchar(255) NULL, "SamplingFeatureID" integer NOT NULL, "SamplingFeatureCode" varchar(50) NOT NULL UNIQUE, "SamplingFeatureName" varchar(255) NOT NULL, "Elevation" double precision NULL, "Latitude" double precision NOT NULL, "Longitude" double precision NOT NULL, "SiteType" varchar(765) NOT NULL, "User" integer NOT NULL); -CREATE TABLE "dataloaderinterface_siteregistration_followed_by" ("id" serial NOT NULL PRIMARY KEY, "siteregistration_id" integer NOT NULL, "user_id" integer NOT NULL); --- --- Create model SiteSensor --- -CREATE TABLE "dataloaderinterface_sitesensor" ("id" serial NOT NULL PRIMARY KEY, "ResultID" integer NOT NULL UNIQUE, "ResultUUID" uuid NOT NULL UNIQUE, "ModelName" varchar(255) NOT NULL, "ModelManufacturer" varchar(255) NOT NULL, "VariableName" varchar(255) NOT NULL, "VariableCode" varchar(50) NOT NULL, "UnitsName" varchar(255) NOT NULL, "UnitAbbreviation" varchar(255) NOT NULL, "SampledMedium" varchar(255) NOT NULL, "LastMeasurementID" integer NULL UNIQUE, "LastMeasurementValue" double precision NULL, "LastMeasurementDatetime" timestamp with time zone NULL, "LastMeasurementUtcOffset" integer NULL, "LastMeasurementUtcDatetime" timestamp with time zone NULL, "ActivationDate" timestamp with time zone NULL, "ActivationDateUtcOffset" integer NULL, "RegistrationID" integer NOT NULL); --- --- Add field site_registration to sitealert --- -ALTER TABLE "dataloaderinterface_sitealert" ADD COLUMN "RegistrationID" integer NOT NULL; -ALTER TABLE "dataloaderinterface_sitealert" ALTER COLUMN "RegistrationID" DROP DEFAULT; --- --- Add field user to sitealert --- -ALTER TABLE "dataloaderinterface_sitealert" ADD COLUMN "User" integer NOT NULL; -ALTER TABLE "dataloaderinterface_sitealert" ALTER COLUMN "User" DROP DEFAULT; --- --- Add field site_registration to hydrosharesitesetting --- -ALTER TABLE "hydroshare_site_setting" ADD COLUMN "site_registration_id" integer NOT NULL UNIQUE; -ALTER TABLE "hydroshare_site_setting" ALTER COLUMN "site_registration_id" DROP DEFAULT; --- --- Add field user to hydroshareaccount --- -ALTER TABLE "dataloaderinterface_siteregistration" ADD COLUMN "DeploymentDate" timestamp with time zone NULL; -ALTER TABLE "hydroshare_account" ADD COLUMN "user_id" integer NOT NULL; -ALTER TABLE "hydroshare_account" ALTER COLUMN "user_id" DROP DEFAULT; -ALTER TABLE "hydroshare_site_setting" ADD CONSTRAINT "hydroshare_site_hs_account_id_0aead4e0_fk_hydroshare_account_id" FOREIGN KEY ("hs_account_id") REFERENCES "hydroshare_account" ("id") DEFERRABLE INITIALLY DEFERRED; -CREATE INDEX "hydroshare_site_setting_e1cbf01f" ON "hydroshare_site_setting" ("hs_account_id"); -ALTER TABLE "hydroshare_sync_log" ADD CONSTRAINT "hydrosha_site_sharing_id_af410ff7_fk_hydroshare_site_setting_id" FOREIGN KEY ("site_sharing_id") REFERENCES "hydroshare_site_setting" ("id") DEFERRABLE INITIALLY DEFERRED; -CREATE INDEX "hydroshare_sync_log_96c12680" ON "hydroshare_sync_log" ("site_sharing_id"); -ALTER TABLE "oauth_token" ADD CONSTRAINT "oauth_token_account_id_9ec62be2_fk_hydroshare_account_id" FOREIGN KEY ("account_id") REFERENCES "hydroshare_account" ("id") DEFERRABLE INITIALLY DEFERRED; -CREATE INDEX "oauth_token_8a089c2a" ON "oauth_token" ("account_id"); -ALTER TABLE "dataloaderinterface_odm2user" ADD CONSTRAINT "dataloaderinterface_odm2user_user_id_c7f612f0_fk_auth_user_id" FOREIGN KEY ("user_id") REFERENCES "auth_user" ("id") DEFERRABLE INITIALLY DEFERRED; -ALTER TABLE "dataloaderinterface_siteregistration" ADD CONSTRAINT "dataloaderinterface_siteregistrat_User_ddc425ba_fk_auth_user_id" FOREIGN KEY ("User") REFERENCES "auth_user" ("id") DEFERRABLE INITIALLY DEFERRED; -CREATE INDEX "dataloaderinterface_siteregistration_8f9bfe9d" ON "dataloaderinterface_siteregistration" ("User"); -CREATE INDEX "dataloaderinterface_siteregistr_RegistrationToken_5bd5fe0a_like" ON "dataloaderinterface_siteregistration" ("RegistrationToken" varchar_pattern_ops); -CREATE INDEX "dataloaderinterface_siteregis_SamplingFeatureCode_ff4f3f4d_like" ON "dataloaderinterface_siteregistration" ("SamplingFeatureCode" varchar_pattern_ops); -ALTER TABLE "dataloaderinterface_siteregistration_followed_by" ADD CONSTRAINT "D14b290714bb90087aea85ea12abf527" FOREIGN KEY ("siteregistration_id") REFERENCES "dataloaderinterface_siteregistration" ("RegistrationID") DEFERRABLE INITIALLY DEFERRED; -ALTER TABLE "dataloaderinterface_siteregistration_followed_by" ADD CONSTRAINT "dataloaderinterface_siteregist_user_id_d95fd769_fk_auth_user_id" FOREIGN KEY ("user_id") REFERENCES "auth_user" ("id") DEFERRABLE INITIALLY DEFERRED; -ALTER TABLE "dataloaderinterface_siteregistration_followed_by" ADD CONSTRAINT "dataloaderinterface_siteregis_siteregistration_id_ff0c0787_uniq" UNIQUE ("siteregistration_id", "user_id"); -CREATE INDEX "dataloaderinterface_siteregistration_followed_by_37ede4dc" ON "dataloaderinterface_siteregistration_followed_by" ("siteregistration_id"); -CREATE INDEX "dataloaderinterface_siteregistration_followed_by_e8701ad4" ON "dataloaderinterface_siteregistration_followed_by" ("user_id"); -ALTER TABLE "dataloaderinterface_sitesensor" ADD CONSTRAINT "D0a8daa02bd6a7cd173e6e91f6229a3b" FOREIGN KEY ("RegistrationID") REFERENCES "dataloaderinterface_siteregistration" ("RegistrationID") DEFERRABLE INITIALLY DEFERRED; -CREATE INDEX "dataloaderinterface_sitesensor_58043df5" ON "dataloaderinterface_sitesensor" ("RegistrationID"); -CREATE INDEX "dataloaderinterface_sitealert_58043df5" ON "dataloaderinterface_sitealert" ("RegistrationID"); -ALTER TABLE "dataloaderinterface_sitealert" ADD CONSTRAINT "D83a3e7dcfa6565ec1561ef2bd409ed7" FOREIGN KEY ("RegistrationID") REFERENCES "dataloaderinterface_siteregistration" ("RegistrationID") DEFERRABLE INITIALLY DEFERRED; -CREATE INDEX "dataloaderinterface_sitealert_8f9bfe9d" ON "dataloaderinterface_sitealert" ("User"); -ALTER TABLE "dataloaderinterface_sitealert" ADD CONSTRAINT "dataloaderinterface_sitealert_User_0ee890d1_fk_auth_user_id" FOREIGN KEY ("User") REFERENCES "auth_user" ("id") DEFERRABLE INITIALLY DEFERRED; -ALTER TABLE "hydroshare_site_setting" ADD CONSTRAINT "D6c9323ae832badb58171adc906e94f1" FOREIGN KEY ("site_registration_id") REFERENCES "dataloaderinterface_siteregistration" ("RegistrationID") DEFERRABLE INITIALLY DEFERRED; -CREATE INDEX "hydroshare_account_e8701ad4" ON "hydroshare_account" ("user_id"); -ALTER TABLE "hydroshare_account" ADD CONSTRAINT "hydroshare__user_id_96d6788d_fk_dataloaderinterface_odm2user_id" FOREIGN KEY ("user_id") REFERENCES "dataloaderinterface_odm2user" ("id") DEFERRABLE INITIALLY DEFERRED; -COMMIT; - --- ALTER TABLE dataloaderinterface_sitesensor add column "ResultUUID" uuid NOT NULL; --- ALTER TABLE dataloaderinterface_sitesensor add column "ModelName" character varying(255) NOT NULL; --- ALTER TABLE dataloaderinterface_sitesensor add column "ModelManufacturer" character varying(255) NOT NULL; --- ALTER TABLE dataloaderinterface_sitesensor add column "VariableName" character varying(255) NOT NULL; --- ALTER TABLE dataloaderinterface_sitesensor add column "VariableCode" character varying(50) NOT NULL; --- ALTER TABLE dataloaderinterface_sitesensor add column "UnitsName" character varying(255) NOT NULL; --- ALTER TABLE dataloaderinterface_sitesensor add column "UnitAbbreviation" character varying(255) NOT NULL; --- ALTER TABLE dataloaderinterface_sitesensor add column "SampledMedium" character varying(255) NOT NULL; --- ALTER TABLE dataloaderinterface_sitesensor add column "LastMeasurementID" integer; --- ALTER TABLE dataloaderinterface_sitesensor add column "ActivationDate" timestamp with time zone; --- ALTER TABLE dataloaderinterface_sitesensor add column "ActivationDateUtcOffset" integer; --- ALTER TABLE dataloaderinterface_sitesensor add column "RegistrationID" integer NOT NULL; --- ALTER TABLE dataloaderinterface_sitesensor add column "LastMeasurementDatetime" timestamp with time zone; --- ALTER TABLE dataloaderinterface_sitesensor add column "LastMeasurementUtcOffset" integer; --- ALTER TABLE dataloaderinterface_sitesensor add column "LastMeasurementValue" double precision; --- ALTER TABLE dataloaderinterface_sitesensor add column "LastMeasurementUtcDatetime" time(6) with time zone; From 20f93b5c348060dc5b411a9a3c862205a887b5b7 Mon Sep 17 00:00:00 2001 From: Craig Blackburn Date: Tue, 6 Mar 2018 11:07:28 -0700 Subject: [PATCH 097/115] re-added encoding declaration --- src/dataloaderinterface/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dataloaderinterface/views.py b/src/dataloaderinterface/views.py index 82da0502..7120ebf2 100644 --- a/src/dataloaderinterface/views.py +++ b/src/dataloaderinterface/views.py @@ -1,3 +1,4 @@ +# coding=utf-8 from datetime import datetime from datetime import datetime, timedelta from uuid import uuid4 From 79123e97f451bc8bcc5127c1e30312e4b6f006db Mon Sep 17 00:00:00 2001 From: darksinge Date: Wed, 7 Mar 2018 14:25:01 -0700 Subject: [PATCH 098/115] fixed filename parser regex pattern --- src/dataloaderinterface/views.py | 9 +++++++-- src/hydroshare_util/utility.py | 10 +--------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/dataloaderinterface/views.py b/src/dataloaderinterface/views.py index 82da0502..1b2f59be 100644 --- a/src/dataloaderinterface/views.py +++ b/src/dataloaderinterface/views.py @@ -1,4 +1,4 @@ -from datetime import datetime +# coding=utf-8 from datetime import datetime, timedelta from uuid import uuid4 import json @@ -37,6 +37,7 @@ from dataloaderinterface.models import ODM2User, SiteRegistration, SiteSensor, HydroShareAccount, HydroShareResource, \ SiteAlert, OAuthToken from hydroshare_util import HydroShareNotFound, HydroShareHTTPException +from hydroshare_util.utility import HydroShareUtility from hydroshare_util.adapter import HydroShareAdapter from hydroshare_util.auth import AuthUtil from hydroshare_util.resource import Resource @@ -990,7 +991,11 @@ def get_site_files(site_registration): status = req.status_code if status == 200: - file_name = re.findall('(?<=\")[\S\.]+\.csv(?=\")', req.headers['Content-Disposition']).pop() + # Search inside 'content-disposition' header for a string that looks like a file name and ends in '.csv'. + csv_pattern = re.compile(r'(?<=\")\w*(?=.csv)', re.IGNORECASE) + file_name = csv_pattern.findall(req.headers['Content-Disposition']).pop() + if '.csv' not in file_name: + file_name += '.csv' content = req.content files.append({'file_name': file_name, 'content': content}) elif status == 404: diff --git a/src/hydroshare_util/utility.py b/src/hydroshare_util/utility.py index f1d92c49..21603424 100644 --- a/src/hydroshare_util/utility.py +++ b/src/hydroshare_util/utility.py @@ -10,19 +10,11 @@ class HydroShareUtility(HydroShareUtilityBaseClass): """Utility class for accessing and consuming HydroShare's REST API.""" - RE_PERIOD = re.compile(r'(?P^start=)(?P[0-9-]{10}T[0-9:]{8}).{2}(?Pend=)' - r'(?P[0-9-]{10}T[0-9:]{8}).{2}(?Pscheme=)(?P.+$)', re.I) - XML_NS = {'dc': "http://purl.org/dc/elements/1.1/", - 'dcterms': "http://purl.org/dc/terms/", - 'hsterms': "http://hydroshare.org/terms/", - 'rdf': "http://www.w3.org/1999/02/22-rdf-syntax-ns#", - 'rdfs1': "http://www.w3.org/2001/01/rdf-schema#"} XML_COVERAGE_PROTO = "start={start}; end={end}; scheme=W3C-DTF" - TIME_FORMAT = '%Y-%m-%dT%H:%M:%S' def __init__(self, auth=None): # type: (AuthUtil) -> None self.auth = auth - self.request_resource_types() + # self.request_resource_types() @property def client(self): From ec7c19bac5fe0344e7abeee262919f9aaf965525 Mon Sep 17 00:00:00 2001 From: darksinge Date: Wed, 7 Mar 2018 15:23:45 -0700 Subject: [PATCH 099/115] fixed issue preventing warning message from appearing on hs_delete_dialog.html --- .../js/hydroshare/hs-delete-resource.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-delete-resource.js b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-delete-resource.js index c42eb9ab..4fd86e4f 100644 --- a/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-delete-resource.js +++ b/src/dataloaderinterface/static/dataloaderinterface/js/hydroshare/hs-delete-resource.js @@ -12,7 +12,8 @@ function initializeHydroShareDeleteDialog() { const deleteForm = $('#hs-delete-form'); const deleteFormDialog = $('#hydroshare-delete-dialog')[0]; const showDeleteDialogBtn = $('button#show-hydroshare-delete-dialog'); - const deleteResourceCB = $('label[for="id_delete_external_resource"]'); + const deleteResourceCBLabel = $('label[for="id_delete_external_resource"]'); + const deleteResourceCB = $('input#id_delete_external_resource'); if (!deleteFormDialog.showModal) { dialogPolyfill.registerDialog(deleteFormDialog); @@ -38,9 +39,9 @@ function initializeHydroShareDeleteDialog() { componentHandler.upgradeElement(spinner); }); - deleteResourceCB.change(() => { + deleteResourceCBLabel.change(() => { let warningList = $('div#checkbox-warning-list'); - let isChecked = $(deleteResourceCB[0]).hasClass('is-checked'); + let isChecked = deleteResourceCB[0].checked; $(warningList).prop('hidden', !isChecked); }); From 62854d5b8a71e2d19d20043d6de1a3db4a0b990e Mon Sep 17 00:00:00 2001 From: Craig Blackburn Date: Wed, 14 Mar 2018 09:49:59 -0600 Subject: [PATCH 100/115] Updated django to version 1.11.11 updated requirements.txt updated dataloaderinterface/forms.py: -Renamed MDLRadioButtonRenderer -> MDLRadioButton --- requirements.txt | 2 +- src/dataloaderinterface/forms.py | 11 +++++++---- src/dataloaderinterface/views.py | 8 ++++---- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7e8f2ea2..270c9110 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ beautifulsoup4==4.5.1 codegen==1.0 coverage==4.2 -Django==1.10.3 +Django==1.11.11 django-debug-toolbar==1.6 django-discover-runner==1.0 django-webtest==1.8.0 diff --git a/src/dataloaderinterface/forms.py b/src/dataloaderinterface/forms.py index ac7e1bda..a0e66660 100644 --- a/src/dataloaderinterface/forms.py +++ b/src/dataloaderinterface/forms.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from datetime import datetime, timedelta from dataloader.models import SamplingFeature, People, Organization, Affiliation, Result, Site, EquipmentModel, Medium, \ OrganizationType, ActionBy, SiteType @@ -19,12 +20,14 @@ def create_option(self, name, value, label, selected, index, subindex=None, attr return option -class MDLRadioButtonRenderer(forms.RadioSelect.renderer): - def render(self): +class MDLRadioButton(forms.RadioSelect): + def render(self, name, value, attrs=None, renderer=None): """Adds MDL HTML classes to label and input tags""" - html = mark_safe(u'\n'.join([u'{0}\n'.format(item) for item in self.__iter__()])) + html = super(MDLRadioButton, self).render(name, value, attrs=attrs, renderer=renderer) + html = re.sub(r'', '', html) html = re.sub(r'(