Merge pull request #58 from oPromessa/dev-apr18-issue46

V2.7.7 Compatibility with Windows (use of portalocker, if available), Fix #56

.travis.yml

@@ -120,11 +120,11 @@ before_script:
 # =============================================================================
 # run script for tests
 script:
-# pytest --flakes (Code analysis) ---------------------------------------------
+# pytest --flakes (Code analysis)
   - pytest -v --flakes
-# pytest ----------------------------------------------------------------------
+# pytest
   - pytest -v
-# pytest --doctest-modules ----------------------------------------------------
+# pytest --doctest-modules
   - pytest --doctest-modules
 # flake8
   - flake8
@@ -141,8 +141,6 @@ script:
 # First RUN -------------------------------------------------------------------
   # stdout to file, stderr to file
   - coverage run -a --concurrency multiprocessing uploadr.py $VerboseVar $UploadrOptions > /home/travis/build/oPromessa/flickr-uploader/nohup.log 2> /home/travis/build/oPromessa/flickr-uploader/nohup.lst
-  # stdout to file, stderr to console
-  #- coverage run -a --concurrency multiprocessing uploadr.py $VerboseVar $UploadrOptions > /home/travis/build/oPromessa/flickr-uploader/nohup.log
   - tail -$TAIL_LINES /home/travis/build/oPromessa/flickr-uploader/nohup.log
   - tail -$TAIL_LINES /home/travis/build/oPromessa/flickr-uploader/nohup.lst
 
@@ -159,7 +157,7 @@ script:
 # WAIT TIME BEFORE SECOND RUN -------------------------------------------------
   - sleep $WAIT_TIME
 
-# Second RUN: Remove, Replace, RemoveReplace, ExcludedFolders, addAlbumsMigrate
+# Second RUN ------------------------------------------------------------------
   - if [[ $TestScenario == addAlbumsMigrate ]]; then sqlite3 flickrdb 'PRAGMA user_version="2"'; fi
   - if [[ $TestScenario == addAlbumsMigrate ]]; then export UploadrOptions="$UploadrOptions --add-albums-migrate"; echo $UploadrOptions; fi
 
@@ -191,21 +189,23 @@ script:
 #  - coverage report -m uploadr.py
 #  - sqlite3 flickrdb "SELECT *, datetime( last_modified, 'unixepoch', 'localtime') FROM files;"
 
-# after_failure ---------------------------------------------------------------
-# Attempt to catch "Build times out because no output was received"
+# =============================================================================
+# after_failure
+# No applicable to "Build times out because no output was received"
 after_failure:
 # After failure: Save build related data
   - echo -e "TRAVIS_TEST_RESULT:\t${TRAVIS_TEST_RESULT}\nTRAVIS_PYTHON_VERSION:\t${TRAVIS_PYTHON_VERSION}\nTRAVIS_BUILD_DIR:\t${TRAVIS_BUILD_DIR}\nTRAVIS_BUILD_ID:\t${TRAVIS_BUILD_ID}\nTRAVIS_BUILD_NUMBER:\t${TRAVIS_BUILD_NUMBER}\nTRAVIS_JOB_NUMBER:\t${TRAVIS_JOB_NUMBER}\nTRAVIS_EVENT_TYPE:\t${TRAVIS_EVENT_TYPE}\nTRAVIS_COMMIT:\t\t${TRAVIS_COMMIT}\nTRAVIS_COMMIT_MESSAGE:\t${TRAVIS_COMMIT_MESSAGE}\n" >> build.txt
 #
 # Upload LOG/ERR files to dropbox
-  - python tests/drop.py --token $DROPBOX_TOKEN build.txt $TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER
-  - python tests/drop.py --token $DROPBOX_TOKEN nohup.log $TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER
-  - python tests/drop.py --token $DROPBOX_TOKEN nohup.lst $TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER
+  - python tests/drop.py --token $DROPBOX_TOKEN build.txt $TRAVIS_BUILD_NUMBER/${TRAVIS_JOB_NUMBER}_$TestScenario
+  - python tests/drop.py --token $DROPBOX_TOKEN nohup.log $TRAVIS_BUILD_NUMBER/${TRAVIS_JOB_NUMBER}_$TestScenario
+  - python tests/drop.py --token $DROPBOX_TOKEN nohup.lst $TRAVIS_BUILD_NUMBER/${TRAVIS_JOB_NUMBER}_$TestScenario
 
   - tail -$TAIL_LINES /home/travis/build/oPromessa/flickr-uploader/nohup.log
   - tail -$TAIL_LINES /home/travis/build/oPromessa/flickr-uploader/nohup.lst
 
-# after_script ----------------------------------------------------------------
+# =============================================================================
+# after_script
 after_script:
 # MD5SUM Output for REFERENCE -------------------------------------------------
 #  - if [[ $TestScenario == RemoveReplace ]]; then find "./tests/Test Photo Library" -type f -exec md5sum '{}' \; ; fi
@@ -221,11 +221,12 @@ after_script:
   - echo -e "TRAVIS_TEST_RESULT:\t${TRAVIS_TEST_RESULT}\nTRAVIS_PYTHON_VERSION:\t${TRAVIS_PYTHON_VERSION}\nTRAVIS_BUILD_DIR:\t${TRAVIS_BUILD_DIR}\nTRAVIS_BUILD_ID:\t${TRAVIS_BUILD_ID}\nTRAVIS_BUILD_NUMBER:\t${TRAVIS_BUILD_NUMBER}\nTRAVIS_JOB_NUMBER:\t${TRAVIS_JOB_NUMBER}\nTRAVIS_EVENT_TYPE:\t${TRAVIS_EVENT_TYPE}\nTRAVIS_COMMIT:\t\t${TRAVIS_COMMIT}\nTRAVIS_COMMIT_MESSAGE:\t${TRAVIS_COMMIT_MESSAGE}\n" >> build.txt
 #
 # Upload LOG/ERR files to Dropbox
-  - python tests/drop.py --token $DROPBOX_TOKEN build.txt $TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER
-  - python tests/drop.py --token $DROPBOX_TOKEN nohup.log $TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER
-  - python tests/drop.py --token $DROPBOX_TOKEN nohup.lst $TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER
+  - python tests/drop.py --token $DROPBOX_TOKEN build.txt $TRAVIS_BUILD_NUMBER/${TRAVIS_JOB_NUMBER}_$TestScenario
+  - python tests/drop.py --token $DROPBOX_TOKEN nohup.log $TRAVIS_BUILD_NUMBER/${TRAVIS_JOB_NUMBER}_$TestScenario
+  - python tests/drop.py --token $DROPBOX_TOKEN nohup.lst $TRAVIS_BUILD_NUMBER/${TRAVIS_JOB_NUMBER}_$TestScenario
 
-# Deploy to Heroku ------------------------------------------------------------
+# =============================================================================
+# Deploy to Heroku
 deploy:
   provider: heroku
   api_key: $HEROKU_API_KEY

README.md

@@ -1,6 +1,6 @@
 # flickr-uploader
 -----------------
-by oPromessa, 2017, V2.7.3 [![Master Build Status](https://travis-ci.org/oPromessa/flickr-uploader.svg?branch=master)](https://travis-ci.org/oPromessa/flickr-uploader)
+by oPromessa, 2017, V2.7.7 [![Master Build Status](https://travis-ci.org/oPromessa/flickr-uploader.svg?branch=master)](https://travis-ci.org/oPromessa/flickr-uploader)
 Published on [https://github.com/oPromessa/flickr-uploader/](https://github.com/oPromessa/flickr-uploader/)
 
 
@@ -71,8 +71,10 @@ You should get the following depending on how the setting FULL_SET_NAME is set:
 ## Requirements
 ---------------
 * Python 2.7+ (should work on DSM from Synology (v6.1), Windows and MAC)
+* Also compatile with Python 3.6
 * flicrkapi module. May need to install get-pip.py. (Instructions for
   Synology DSM below.)
+* portalocker module for Windows systems. Not mandatory for Synology.
 * File write access (for the token and local database)
 * Flickr API key (free)
 
@@ -194,8 +196,32 @@ of the upload arguments above correspond to for Flickr's API.
 - Before running uploadr.py make sure you run the command below:
   - To avoid running this command exerytime you log-in into your system, follow the [notes on this link](https://scipher.wordpress.com/2010/05/10/setting-your-pythonpath-environment-variable-linuxunixosx/) to edit file ~/.bashrc and place this command there.
 ```bash
- $  export PYTHONPATH=~/apps/Python/lib/python2.7/site-packages
- $ ./uploadr.py -v
+$  export PYTHONPATH=~/apps/Python/lib/python2.7/site-packages
+```
+
+- On the **first run** you need to authenticate the applicaiton against Flickr.
+   - uploadr.py will provide you a URL/link which you need to run
+```bash
+$ cd dev
+dev$ uploadr.py 
+Importing xml.etree.ElementTree...done. Continuing.
+--------- (V2.7.7) Init:  ---------
+Python version on this system: 3.6.3 (default, Oct  3 2017, 21:45:48) 
+[GCC 7.2.0]
+[2965][2018.04.16 23:55:09]:[12758      ][PRINT   ]:[uploadr] --------- (V2.7.7) Start time: 2018.04.16 23:55:09 ---------(Log:40)
+[2965][2018.04.16 23:55:09]:[12758      ][PRINT   ]:[uploadr] Setting up database:[/home/user/dev/flickrdb]
+[2965][2018.04.16 23:55:09]:[12758      ][PRINT   ]:[uploadr] Database version: [3]
+[2965][2018.04.16 23:55:09]:[12758      ][PRINT   ]:[uploadr] Completed database setup
+[2965][2018.04.16 23:55:09]:[12758      ][PRINT   ]:[uploadr] Checking if token is available... if not will authenticate
+[2965][2018.04.16 23:55:09]:[12758      ][PRINT   ]:[uploadr] Getting new token.
+[2965][2018.04.16 23:55:09]:[12758      ][PRINT   ]:[uploadr] Copy and paste following authorizaiton URL in your browser to obtain Verifier Code.
+https://www.flickr.com/services/oauth/authorize?oauth_token=xxxxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxx&perms=delete
+Verifier code (NNN-NNN-NNN):
+```
+
+- Following **runs** can be launched unattended:
+```
+ dev$ ./uploadr.py -v
 ```
 
 ## Usage/Arguments/Options
@@ -208,21 +234,31 @@ the upload process.
 $ ./uploadr.py
 ```
 To check what files uploadr.py would upload and delete you can run the
-script withe option --dry-run:
+script with option `--dry-run`:
 ```bash
 $ ./uploadr.py --dry-run
 ```
-Run ./uploadrd.py --help for up to the minute information or arguments:
+Run `./uploadrd.py --help` for up to the minute information on arguments:
 ```bash
-usage: uploadr.py [-h] [-v] [-x] [-n] [-i TITLE] [-e DESCRIPTION] [-t TAGS]
-                  [-l N] [-z] [-r] [-p P] [-u] [-d] [-b] [-c] [-s] [-g]
-                  [--add-albums-migrate]
+Importing xml.etree.ElementTree...done. Continuing.
+--------- (V2.7.7) Init:  ---------
+Python version on this system: 3.6.3 (default, Oct  3 2017, 21:45:48) 
+[GCC 7.2.0]
+[2930][2018.04.16 23:47:54]:[12706      ][PRINT   ]:[uploadr] --------- (V2.7.7) Start time: 2018.04.16 23:47:54 ---------(Log:40)
+usage: uploadr.py [-h] [-C filename.ini] [-v] [-x] [-n] [-i TITLE]
+                  [-e DESCRIPTION] [-t TAGS] [-l N] [-z] [-r] [-p P] [-u] [-d]
+                  [-b] [-c] [-s] [-g] [--add-albums-migrate]
 
 Upload files to Flickr. Uses uploadr.ini as config file.
 
 optional arguments:
   -h, --help            show this help message and exit
 
+Configuration related options:
+  -C filename.ini, --config-file filename.ini
+                        Optional configuration file. Default is:
+                        [/home/ruler/uploader/etc/uploadr.ini]
+
 Verbose and dry-run options:
   -v, --verbose         Provides some more verbose output. See also -x option.
                         See also LOGGING_LEVEL value in INI file.
@@ -255,7 +291,9 @@ Processing related options:
   -p P, --processes P   Number of photos to upload simultaneously.
   -u, --not-is-already-uploaded
                         Do not check if file is already uploaded and exists on
-                        flickr prior to uploading.
+                        flickr prior to uploading. Use this option for faster
+                        INITIAL upload. Do not use it in subsequent uploads to
+                        prevent/recover orphan pics without a set.
   -d, --daemon          Run forever as a daemon.Uploading every SLEEP_TIME
                         seconds. Please note it only performs upload/replace.
 
@@ -270,12 +308,11 @@ Handling bad and excluded files:
                         in your Library that flickr does not recognize (Error
                         5) or are too large (Error 8). Check also option -b.
   -s, --list-bad-files  List the badfiles table/list.
-  -g, --remove-excluded, --remove-ignored
+  -g, --remove-excluded
                         Remove previously uploaded files, that are now being
                         excluded due to change of the INI file configuration
-                        EXCLUDED_FOLDERS.NOTE: Please drop use of --remove-
-                        ignored in favor of --remove-excluded or -r. From
-                        version 2.7.0 it will be dropped.
+                        EXCLUDED_FOLDERS.NOTE: Option --remove-ignored was
+                        dropped in favor of --remove-excluded.
 
 Migrate to v2.7.0:
   --add-albums-migrate  From v2.7.0 onwards, uploadr adds to Flickr an album

TODO.md

@@ -11,7 +11,13 @@
 * Would be nice to update ALL tags on replacePhoto and not only the
   mandatory checksum tag as FLICKR maintains the tags from the first load.
 * Change code to insert on database prior to upload and then update result.
-* Test if it Re-upload or not pictures removed from flickr Web interface.
+* Test if it Re-uploads or not pictures removed/deleted from flickr Web
+  interface; while they still exist on local filesystem and local DB.
+  (without the -u option, it should find the file and update database).
+  This should avoid errors on creating sets with invalid primarykey (photo id
+  has changed while the actual checksum/album of the file is actually the same)
+* Consider new option --remove-ignored to address IGNORED_REGEX changes
+  similar to how --remove-excluded handles changes in EXCLUDED_FOLDERS.
 * Align try/except handling within functions like people_get_photos or outside
   like photos_get_not_in_set
 * **[NOT FULLY TESTED YET]** You can try and run (Let me know if it works!)

lib/__version__.py

@@ -5,6 +5,6 @@
     __version__ = Semantic Versioning number Major.Minor.Patch
                   Check https://semver.org
 """
-VERSION = (2, 7, 6)
+VERSION = (2, 7, 7)
 
 __version__ = '.'.join(map(str, VERSION))

requirements.txt

@@ -4,4 +4,7 @@ flickrapi ; python_version >= '2.7'
 pytest-flakes ; python_version >= '2.7'
 coverage ; python_version >= '2.7'
 flake8
-dropbox
+dropbox
+portalocker
+# This is a neccessary dependency for portalocker on Windows
+# pypiwin32==223