Merge pull request #681 from ODM2/develop

Release 0.15.1

.gitignore

@@ -126,3 +126,4 @@ docker-compose.yml
 settings.json
 settings.*.json
 src/odm2/modelcache.pkl
+local_media/*

environment.yml

@@ -13,16 +13,29 @@ dependencies:
     - hs_restclient >=1.3.7  # https://github.com/hydroshare/hs_restclient
     - markdown >=3.3.4
     - pandas >=1.3
+    - pillow #required for image support
     - psycopg2 >=2.9.1
     - python-crontab >=2.5.1
     - sqlalchemy >=1.4.0,<2.0
 
-    # Dev tools
-    - python-language-server
+    # Dev tools: Language Server Protocol (LSP) (Optional), for...
+      # code navigation + hover suggestions + linters + autocompletion + rename
+    - python-lsp-server >=1.6 # LSP extension for Python (pylsp), including:
+      # Rope for Completions and renaming
+      # Pyflakes linter to detect various errors
+      # McCabe linter for complexity checking
+      # pycodestyle linter for style checking
+      # pydocstyle linter for docstring style checking (disabled by default)
+      # autopep8 for code formatting
+      # YAPF for code formatting (preferred over autopep8)
+      # flake8 for error checking (disabled by default)
+      # pylint for code linting (disabled by default)
+    - pylsp-mypy  # MyPy type checking for Python >=3.7.
 
     # package management
     - conda
     - conda-build
+    - conda-libmamba-solver
     - pip
 
     - pip:

src/WebSDL/settings/base.py

@@ -19,15 +19,19 @@
 logging.basicConfig(level=logging.INFO)
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
-BASE_DIR =  os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 # Loads settings configuration data from settings.json file
 data = {}
 try:
-    with open(os.path.join(BASE_DIR, 'WebSDL','settings', 'settings.json')) as data_file:
+    with open(
+        os.path.join(BASE_DIR, "WebSDL", "settings", "settings.json")
+    ) as data_file:
         data = json.load(data_file)
 except IOError:
-    print("You need to setup the settings data file (see instructions in base.py file.)")
+    print(
+        "You need to setup the settings data file (see instructions in base.py file.)"
+    )
 
 
 # SECURITY WARNING: keep the secret key used in production secret!
@@ -37,112 +41,111 @@
     print("The secret key is required in the settings.json file.")
     exit(1)
 
-ALLOWED_HOSTS = ['*']
+ALLOWED_HOSTS = ["*"]
 
 
 # Application definition
 
 INSTALLED_APPS = [
     # 'debug_toolbar',
-    'rest_framework',
+    "rest_framework",
     #'accounts.apps.AccountsConfig',
-    'dataloader.apps.DataloaderConfig',
-    'dataloaderservices.apps.DataloaderservicesConfig',
-    'dataloaderinterface.apps.DataloaderinterfaceConfig',
-    'hydroshare',
-    'leafpack',
-    'streamwatch',
-    'django.contrib.admin',
-    'django.contrib.auth',
-    'django.contrib.contenttypes',
-    'django.contrib.sessions',
-    'django.contrib.messages',
-    'django.contrib.staticfiles',
-    'widget_tweaks',
-    'requests',
-    'reset_migrations',
-    'timeseries_visualization',
-    'formtools',
-    'accounts.apps.AccountsConfig',
+    "dataloader.apps.DataloaderConfig",
+    "dataloaderservices.apps.DataloaderservicesConfig",
+    "dataloaderinterface.apps.DataloaderinterfaceConfig",
+    "hydroshare",
+    "leafpack",
+    "streamwatch",
+    "django.contrib.admin",
+    "django.contrib.auth",
+    "django.contrib.contenttypes",
+    "django.contrib.sessions",
+    "django.contrib.messages",
+    "django.contrib.staticfiles",
+    "widget_tweaks",
+    "requests",
+    "reset_migrations",
+    "timeseries_visualization",
+    "formtools",
+    "accounts.apps.AccountsConfig",
 ]
 
 MIDDLEWARE = [
-    'django.middleware.security.SecurityMiddleware',
-    'django.contrib.sessions.middleware.SessionMiddleware',
-    'django.middleware.common.CommonMiddleware',
-    'django.middleware.csrf.CsrfViewMiddleware',
-    'django.contrib.auth.middleware.AuthenticationMiddleware',
-    'django.contrib.messages.middleware.MessageMiddleware',
-    'django.middleware.clickjacking.XFrameOptionsMiddleware',
-    'hydroshare_util.middleware.AuthMiddleware',
+    "django.middleware.security.SecurityMiddleware",
+    "django.contrib.sessions.middleware.SessionMiddleware",
+    "django.middleware.common.CommonMiddleware",
+    "django.middleware.csrf.CsrfViewMiddleware",
+    "django.contrib.auth.middleware.AuthenticationMiddleware",
+    "django.contrib.messages.middleware.MessageMiddleware",
+    "django.middleware.clickjacking.XFrameOptionsMiddleware",
+    "hydroshare_util.middleware.AuthMiddleware",
     # 'debug_toolbar.middleware.DebugToolbarMiddleware',
-    'django_cprofile_middleware.middleware.ProfilerMiddleware',
-    'accounts.user_middleware.UserMiddleware',
+    "django_cprofile_middleware.middleware.ProfilerMiddleware",
+    "accounts.user_middleware.UserMiddleware",
 ]
 
 DJANGO_CPROFILE_MIDDLEWARE_REQUIRE_STAFF = False
 
 REST_FRAMEWORK = {
-   'DEFAULT_RENDERER_CLASSES': (
-       'rest_framework.renderers.JSONRenderer',
-       'rest_framework.renderers.BrowsableAPIRenderer',
-#       'rest_framework.renderers.AdminRenderer',
-   )
+    "DEFAULT_RENDERER_CLASSES": (
+        "rest_framework.renderers.JSONRenderer",
+        "rest_framework.renderers.BrowsableAPIRenderer",
+        #       'rest_framework.renderers.AdminRenderer',
+    )
 }
 
-ROOT_URLCONF = 'WebSDL.urls'
+ROOT_URLCONF = "WebSDL.urls"
 
 TEMPLATES = [
     {
-        'BACKEND': 'django.template.backends.django.DjangoTemplates',
-        'DIRS': [os.path.join(BASE_DIR, 'hydroshare')]
-        ,
-        'APP_DIRS': True,
-        'OPTIONS': {
-            'context_processors': [
-                'django.template.context_processors.debug',
-                'django.template.context_processors.request',
-                'django.contrib.auth.context_processors.auth',
-                'django.contrib.messages.context_processors.messages',
+        "BACKEND": "django.template.backends.django.DjangoTemplates",
+        "DIRS": [os.path.join(BASE_DIR, "hydroshare")],
+        "APP_DIRS": True,
+        "OPTIONS": {
+            "context_processors": [
+                "django.template.context_processors.debug",
+                "django.template.context_processors.request",
+                "django.contrib.auth.context_processors.auth",
+                "django.contrib.messages.context_processors.messages",
             ],
         },
     },
 ]
 
-#WSGI_APPLICATION = 'WebSDL.wsgi.application'
+# WSGI_APPLICATION = 'WebSDL.wsgi.application'
 
 
 # Database
 # https://docs.djangoproject.com/en/1.9/ref/settings/#databases
 
 DATABASES = {}
-for database in data['databases']:
-    DATABASES[database['name']] = {
-        'ENGINE': database['engine'],
-        'NAME': database['schema'],
-        'USER': database['user'] if 'user' in database else '',
-        'PASSWORD': database['password'] if 'password' in database else '',
-        'HOST': database['host'] if 'host' in database else '',
-        'PORT': database['port'] if 'port' in database else '',
-        'OPTIONS': database['options'] if 'options' in database else {},
-        'CONN_MAX_AGE': 0,
-        'TEST': database['test'] if 'test' in database else {},
+for database in data["databases"]:
+    DATABASES[database["name"]] = {
+        "ENGINE": database["engine"],
+        "NAME": database["schema"],
+        "USER": database["user"] if "user" in database else "",
+        "PASSWORD": database["password"] if "password" in database else "",
+        "HOST": database["host"] if "host" in database else "",
+        "PORT": database["port"] if "port" in database else "",
+        "OPTIONS": database["options"] if "options" in database else {},
+        "CONN_MAX_AGE": 0,
+        "TEST": database["test"] if "test" in database else {},
     }
-DATAMODELCACHE = os.path.join(BASE_DIR, 'odm2', 'modelcache.pkl')
-DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
+DATAMODELCACHE = os.path.join(BASE_DIR, "odm2", "modelcache.pkl")
+DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
 
 # Password validation
 # https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators
 
 AUTH_PASSWORD_VALIDATORS = [
     {
-        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
     },
     # {
     #     'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
     # },
     {
-        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+        "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
     },
     # {
     #     'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
@@ -153,10 +156,10 @@
 # Internationalization
 # https://docs.djangoproject.com/en/1.9/topics/i18n/
 
-LANGUAGE_CODE = 'en-us'
+LANGUAGE_CODE = "en-us"
 USE_I18N = True
 USE_L10N = True
-LOGIN_URL = '/login/'
+LOGIN_URL = "/login/"
 
 # Security and SSL
 #
@@ -168,55 +171,66 @@
 RECAPTCHA_USER_KEY = data["recaptcha_user_key"] if "recaptcha_user_key" in data else ""
 RECAPTCHA_VERIFY_URL = "https://www.google.com/recaptcha/api/siteverify"
 
-EMAIL_SENDER = data['email_from'] if 'email_from' in data else '',
-NOTIFY_EMAIL = data['notify_email_sender'] if 'notify_email_sender' in data else ''
-DEFAULT_FROM_EMAIL = EMAIL_SENDER[0] if isinstance(EMAIL_SENDER, tuple) else EMAIL_SENDER
-NOTIFY_EMAIL_SENDER = NOTIFY_EMAIL[0] if isinstance(NOTIFY_EMAIL, tuple) else NOTIFY_EMAIL
-EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
-EMAIL_SERVER = data['email_host'] if 'email_host' in data else '',
+EMAIL_SENDER = (data["email_from"] if "email_from" in data else "",)
+NOTIFY_EMAIL = data["notify_email_sender"] if "notify_email_sender" in data else ""
+DEFAULT_FROM_EMAIL = (
+    EMAIL_SENDER[0] if isinstance(EMAIL_SENDER, tuple) else EMAIL_SENDER
+)
+NOTIFY_EMAIL_SENDER = (
+    NOTIFY_EMAIL[0] if isinstance(NOTIFY_EMAIL, tuple) else NOTIFY_EMAIL
+)
+EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend"
+EMAIL_SERVER = (data["email_host"] if "email_host" in data else "",)
 EMAIL_HOST = EMAIL_SERVER[0] if isinstance(EMAIL_SERVER, tuple) else EMAIL_SERVER
-EMAIL_PORT = data['email_port']
-EMAIL_HOST_USER = data['email_username'] if 'email_username' in data else ''
-EMAIL_HOST_PASSWORD = data['email_password'] if 'email_password' in data else ''
+EMAIL_PORT = data["email_port"]
+EMAIL_HOST_USER = data["email_username"] if "email_username" in data else ""
+EMAIL_HOST_PASSWORD = data["email_password"] if "email_password" in data else ""
 EMAIL_USE_TLS = True
 
 DATETIME_FORMAT = "N j, Y g:i a"
 
 HYDROSHARE_UTIL_CONFIG = {
-    'CLIENT_ID': data["hydroshare_oauth"]["client_id"],
-    'CLIENT_SECRET': data["hydroshare_oauth"]["client_secret"],
-    'REDIRECT_URI': data['hydroshare_oauth']['redirect_uri']
+    "CLIENT_ID": data["hydroshare_oauth"]["client_id"],
+    "CLIENT_SECRET": data["hydroshare_oauth"]["client_secret"],
+    "REDIRECT_URI": data["hydroshare_oauth"]["redirect_uri"],
 }
 
 # This data period is measured in days
-SENSOR_DATA_PERIOD = data['sensor_data_period'] if 'sensor_data_period' in data else '2'
+SENSOR_DATA_PERIOD = data["sensor_data_period"] if "sensor_data_period" in data else "2"
 
 # crontab job settings
-CRONTAB_USER = data.get('crontab_user', getpass.getuser())
-CRONTAB_LOGFILE_PATH = data.get('crontab_log_file', '/var/log/odm2websdl-cron.log')
+CRONTAB_USER = data.get("crontab_user", getpass.getuser())
+CRONTAB_LOGFILE_PATH = data.get("crontab_log_file", "/var/log/odm2websdl-cron.log")
 CRONTAB_EXECUTE_DAILY_AT_HOUR = 5
 
-GOOGLE_API_CONF = data.get('google_api_conf', None)
+GOOGLE_API_CONF = data.get("google_api_conf", None)
 
-#AUTH_USER_MODEL = 'cognito.User'
+# AUTH_USER_MODEL = 'cognito.User'
 
-#AWS Congnito
+# AWS Congnito
+COGNITO_SIGNUP_URL = data["cognito_signup_url"]
+COGNITO_SIGNIN_URL = data["cognito_signin_url"]
+COGNITO_REGION = data["cognito_region"]
+COGNITO_ACCESS_KEY = data["cognito_access_key"]
+COGNITO_SECRET_ACCESS_KEY = data["cognito_secret_access_key"]
+COGNITO_USER_POOL_ID = data["cognito_user_pool_id"]
+COGNITO_CLIENT_ID = data["cognito_client_id"]
+COGNITO_CLIENT_SECRET = data["cognito_client_secret"]
+COGNITO_OAUTH_URL = data["cognito_oauth_url"]
+COGNITO_REDIRECT_URL = data["cognito_redirect_url"]
+SESSION_KEY = "_auth_user_id"
+BACKEND_SESSION_KEY = "_auth_user_backend"
+HASH_SESSION_KEY = "_auth_user_hash"
 
-COGNITO_SIGNUP_URL = data['cognito_signup_url']
-COGNITO_SIGNIN_URL = data['cognito_signin_url']
-COGNITO_REGION = data['cognito_region']
-COGNITO_ACCESS_KEY = data['cognito_access_key']
-COGNITO_SECRET_ACCESS_KEY = data['cognito_secret_access_key']
-COGNITO_USER_POOL_ID = data['cognito_user_pool_id']
-COGNITO_CLIENT_ID = data['cognito_client_id']
-COGNITO_CLIENT_SECRET = data['cognito_client_secret']
-COGNITO_OAUTH_URL = data['cognito_oauth_url']
-COGNITO_REDIRECT_URL = data['cognito_redirect_url']
-SESSION_KEY = '_auth_user_id'
-BACKEND_SESSION_KEY = '_auth_user_backend'
-HASH_SESSION_KEY = '_auth_user_hash'
+S3_CLIENT_ID = data["s3_client_id"]
+S3_CLIENT_SECRET = data["s3_client_secret"]
+SITE_PHOTOS_S3_BUCKET = data["s3_site_photos_bucket"]
 
-#Static cache busting
-STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.ManifestStaticFilesStorage'
 
-DEBUG = True if 'debug_mode' in data and data['debug_mode'] == "True" else False
+# Static cache busting
+STATICFILES_STORAGE = "django.contrib.staticfiles.storage.ManifestStaticFilesStorage"
+
+# temp storage for form photos
+MEDIA_ROOT = data["MEDIA_ROOT"] if "MEDIA_ROOT" in data else "./local_media"
+
+DEBUG = True if "debug_mode" in data and data["debug_mode"] == "True" else False

src/WebSDL/settings/settings_template.json

@@ -22,6 +22,10 @@
   "cognito_client_secret": "client secret for cognito user pool",
   "cognito_oauth_url": "",
   "cognito_redirect_url": "The url to redirect users to on successful sign-in",
+
+  "s3_client_id": "Access key for IAM account with S3 access",
+  "s3_client_secret" : "Secret access key for IAM account with S3 access",
+  "site_photos_s3_bucket" : "monitormywatershed-site-photos",
 
   "databases": [
     {