Merge pull request #199 from ODM2/release/0.6.0

Release/0.6.0

.gitignore

@@ -27,6 +27,7 @@ var/
 *.egg-info/
 .installed.cfg
 *.egg
+#resources.csv
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -54,6 +55,7 @@ coverage.xml
 
 # Django stuff:
 *.log
+migrations/
 
 # Sphinx documentation
 docs/_build/
@@ -110,5 +112,13 @@ com_crashlytics_export_strings.xml
 crashlytics.properties
 crashlytics-build.properties
 
+## Visual Studio Code
+.vscode/
 
+## Miscellaneous
+resources.csv
 
+# Docker
+Dockerfile
+docker-start.sh
+docker-compose.yml

requirements.txt

@@ -1,7 +1,7 @@
 beautifulsoup4==4.5.1
 codegen==1.0
 coverage==4.2
-Django==1.10.3
+Django==1.11.11
 django-debug-toolbar==1.6
 django-discover-runner==1.0
 django-webtest==1.8.0
@@ -15,3 +15,8 @@ sqlparse==0.2.2
 waitress==1.0.1
 WebOb==1.6.2
 WebTest==2.0.23
+requests==2.18.4
+hs_restclient==1.2.10
+enum==0.4.6
+unicodecsv==0.14.1
+python-crontab==2.2.8

src/WebSDL/settings/base.py

@@ -12,6 +12,11 @@
 
 import os
 import json
+import getpass
+import logging
+
+# Set application logging level
+logging.basicConfig(level=logging.INFO)
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -76,7 +81,7 @@
 TEMPLATES = [
     {
         'BACKEND': 'django.template.backends.django.DjangoTemplates',
-        'DIRS': [os.path.join(BASE_DIR, 'templates')]
+        'DIRS': [os.path.join(BASE_DIR, 'hydroshare')]
         ,
         'APP_DIRS': True,
         'OPTIONS': {
@@ -137,7 +142,7 @@
 
 USE_L10N = True
 
-LOGIN_URL = 'login'
+LOGIN_URL = '/login/'
 
 DATABASE_ROUTERS = ['WebSDL.db_routers.WebSDLRouter']
 
@@ -168,7 +173,13 @@
 
 EMAIL_HOST = EMAIL_SERVER[0] if isinstance(EMAIL_SERVER, tuple) else EMAIL_SERVER
 
-DATETIME_FORMAT = "N j, Y, H:m"
+DATETIME_FORMAT = "N j, Y g:i a"
+
+HYDROSHARE_UTIL_CONFIG = {
+    'CLIENT_ID': data["hydroshare_oauth"]["client_id"],
+    'CLIENT_SECRET': data["hydroshare_oauth"]["client_secret"],
+    'REDIRECT_URI': data["hydroshare_oauth"]["redirect_uri"],
+}
 
 INFLUX_URL_QUERY = data['influx_query']
 
@@ -180,3 +191,11 @@
 SENSOR_DATA_PERIOD = data['sensor_data_period'] if 'sensor_data_period' in data else '2'
 
 TSA_URL = data['tsa_url'] if 'tsa_url' in data else ''
+
+# crontab job settings
+CRONTAB_USER = data.get('crontab_user', getpass.getuser())
+
+CRONTAB_LOGFILE_PATH = data.get('crontab_log_file', '/var/log/odm2websdl-cron.log')
+
+CRONTAB_EXECUTE_DAILY_AT_HOUR = 5
+

src/WebSDL/settings/linux_sandbox.py

@@ -0,0 +1,16 @@
+from WebSDL.settings.base import *
+
+DEBUG = False
+
+ALLOWED_HOSTS = ['127.0.0.1', 'localhost']
+if "host" in data:
+    ALLOWED_HOSTS.append(data["host"])
+if "host_alt" in data:
+    ALLOWED_HOSTS.append(data["host_alt"])
+if "host_staging" in data:
+    ALLOWED_HOSTS.append(data["host_staging"])
+
+STATIC_ROOT = data["static_root"]
+SITE_ROOT = "/opt/websdlenvironment/"
+STATIC_URL = '/static/'
+SITE_URL = ''

src/WebSDL/settings/macos_sandbox.py

@@ -0,0 +1,48 @@
+from WebSDL.settings.base import *
+
+DEBUG = True
+
+ALLOWED_HOSTS = ['127.0.0.1', 'localhost']
+if "host" in data:
+    ALLOWED_HOSTS.append(data["host"])
+if "host_alt" in data:
+    ALLOWED_HOSTS.append(data["host_alt"])
+if "host_staging" in data:
+    ALLOWED_HOSTS.append(data["host_staging"])
+
+STATIC_ROOT = data["static_root"]
+SITE_ROOT = "/opt/websdlenvironment/"
+STATIC_URL = '/static/'
+SITE_URL = ''
+
+"""
+Installing postgreSQL on MacOS
+------------------------------
+1. Setting up Homebrew
+    1a. If you don't have Homebrew installed, visit https://brew.sh/ for installation instructions
+    1b. If Homebrew is already installed, run:
+        brew doctor
+        brew update
+        
+2. Install postgres by running the following commands:
+    - brew update
+    - brew install postgres
+    - brew tap homebrew/services
+    
+3. To start up postgres, run:
+    brew services start postgresql
+    
+4. To stop postgres, run:
+    brew services stop postgresql
+    
+5. To restart postgres, run:
+    brew services restart postgresql
+    
+6. To create a new user:
+    createuser --interactive
+    
+7. To connect to a database:
+    psql <database_name> [-U <username>]
+
+"""
+

src/WebSDL/settings/settings_template.json

@@ -31,7 +31,8 @@
   "hydroshare_oauth": {
     "client_id": "client_id_here",
     "client_secret": "client_secret_here",
-    "redirect_uri": "http://localhost:8000/oauth/hydroshare/",
-    "response_type": "code"
-  }
+    "redirect_uri": "http://localhost:8000/hydroshare/oauth/"
+  },
+  "crontab_log_file": "/var/log/crontab.log",
+  "crontab_user": "joesmokum"
 }

src/WebSDL/settings/windows_sandbox.py

@@ -0,0 +1,17 @@
+from WebSDL.settings.base import *
+
+DEBUG = True
+
+ALLOWED_HOSTS = ['127.0.0.1', 'localhost']
+
+if "host" in data:
+    ALLOWED_HOSTS.append(data["host"])
+if "host_alt" in data:
+    ALLOWED_HOSTS.append(data["host_alt"])
+if "host_staging" in data:
+    ALLOWED_HOSTS.append(data["host_staging"])
+
+STATIC_ROOT = data["static_root"]
+SITE_ROOT = "C:/Users/fryar/projects/ODM2WebSDL/"
+STATIC_URL = '/static/'
+SITE_URL = ''

src/WebSDL/urls.py

@@ -52,5 +52,5 @@
     url(r'^' + BASE_URL + 'account/$', UserUpdateView.as_view(), name='user_account'),
     url(r'^' + BASE_URL + 'api-auth/', include('rest_framework.urls', namespace='rest_framework')),
     url(BASE_URL, include('dataloaderinterface.urls')),
-    url(BASE_URL, include('dataloaderservices.urls')),
+    url(BASE_URL, include('dataloaderservices.urls'))
 ]

src/WebSDL/wsgi.py

@@ -8,9 +8,12 @@
 """
 
 import os
+import crontab_jobs
 
 from django.core.wsgi import get_wsgi_application
 
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "WebSDL.settings")
 
+crontab_jobs.start_jobs()
+
 application = get_wsgi_application()

src/crontab_jobs.py

@@ -0,0 +1,112 @@
+import os
+import subprocess
+import fnmatch
+from crontab import CronTab
+import re
+from django.utils.termcolors import colorize
+
+from WebSDL.settings.base import CRONTAB_LOGFILE_PATH as LOGFILE
+from WebSDL.settings.base import CRONTAB_EXECUTE_DAILY_AT_HOUR as AT_HOUR
+
+JOB_COMMENT_PREPENDER = "__odm2websdl__"
+
+
+def start_jobs(user=True):
+    """
+    Starts crontab jobs defined in this method.
+
+    Currently the only cronjob in this file is the one to upload files to hydroshare.org on a scheduled basis...
+
+    :param user: A {bool} or a {string} that represents an OS level user (i.e. 'root'). If user is a bool and equal to
+    True, crontab jobs are scheduled under the current OS user.
+    """
+    # check if LOGFILE exists and create if not
+    if not os.path.exists(LOGFILE):
+        try:
+            open(LOGFILE, "a+").close()  # open file for appending
+        except IOError:
+            pass  # most likely will fail because of permission error
+
+    # make sure user has write permissions to LOGFILE
+    if not os.access(LOGFILE, os.W_OK):
+        import getpass
+
+        curr_user = getpass.getuser()
+        msg = "\nWARNING: user '{0}' does not have the write permission to {1}. crontab jobs might not run correctly"\
+            .format(curr_user, LOGFILE)
+        print(colorize(msg, fg='red'))
+
+    else:
+
+        manage_path = locate_file('manage.py')  # get the file path of 'manage.py'
+        if manage_path is None:
+            raise Exception('the file "manage.py" was not found')
+
+        output = subprocess.check_output(['which', 'python'])  # get the python path used by the current process
+        python_path = re.sub(r"(?<=[a-z])\r?\n", "", output)  # remove newlines from output...
+
+        cron = CronTab(user=user)  # get cron object for managing crontab jobs
+
+        # stop jobs to prevent job duplication
+        if len(cron):
+            stop_jobs(cron)
+
+        """
+        Create crontab job for scheduled hydroshare file uploads
+        Example of what the crontab job would look like from this command:
+            0 */5 * * * python manage.py update_hydroshare_resource_files --settings=WebSDL.linux_sandbox >> \ 
+            /crontab.log 2>> /crontab.log # __odm2websdl__upload_hydroshare_files
+        """
+        command_name = 'update_hydroshare_resource_files'
+        job = cron.new(command="""{python} {manage} {command} --settings={settings} >> {logfile} 2>> {logfile}""".format(
+            python=python_path,
+            manage=manage_path,
+            command=command_name,
+            settings=os.environ['DJANGO_SETTINGS_MODULE'],
+            logfile=LOGFILE),
+            comment=JOB_COMMENT_PREPENDER + 'upload_hydroshare_files')
+        job.every().day()  # run everyday
+        job.hour.on(AT_HOUR)  # at the time specified by AT_HOUR
+
+        cron.write()  # write, i.e. 'save' crontab job
+
+        """ 
+        Add additional jobs below here if ever needed... 
+        """
+
+        # print jobs created
+        print(colorize("\nStarted crontab jobs: ", fg='blue'))
+        for job in cron:
+            print(colorize('\t' + str(job), fg='green'))
+
+
+def stop_jobs(cron=None, user=None):
+    """ Stops crontab jobs containing JOB_COMMENT_PREPENDER in the job's comment """
+    print(colorize("\nStopping crontab jobs: ", fg='blue'))
+    if cron is None and user:
+        cron = CronTab(user=user)
+
+    for job in cron:
+        if re.search(re.escape(JOB_COMMENT_PREPENDER), job.comment):
+            job_name = re.sub(re.escape(JOB_COMMENT_PREPENDER), r'', job.comment)
+            print("\t" + colorize(str(job_name), fg='green'))
+            cron.remove(job)
+            cron.write()
+
+
+def locate_file(pattern, root=os.curdir):  # types: (str, str) -> str
+    """
+    locates a file and returns the absolute path
+
+    :param pattern - a string representation of a file to locate.
+    :param root - the root directory to search in for the file.
+    :returns - a string representation of the absolute path of the file if found. Returns None if the file is not found.
+    """
+    for path, dirs, files in os.walk(os.path.abspath(root)):
+        for filename in fnmatch.filter(files, pattern):
+            return os.path.join(path, filename)
+    return None
+
+
+if __name__ == "__main__":
+    start_jobs()