Add exponential backoff (#28)

* Remove sleep_to_mimic_human_activity in favor of exp. backoff

* Remove utils.sleep_to_mimic_human_activity()

* Remove sleep_to_mimic_human_activity() from artist.py

* Format with Black

README.md

@@ -32,6 +32,7 @@ A [HiFi Plus](https://tidal.com/pricing) account is **required** in order to ret
    - For Windows, the [FFmpeg download page](http://ffmpeg.org/download.html#build-windows) lists 2 resources; or [`chocolatey`](https://community.chocolatey.org/packages/ffmpeg) is an option
    - The Dockerfile [builds FFmpeg](https://github.com/ebb-earl-co/tidal-wave/blob/trunk/Dockerfile#L12) into the image
  - Only a handful of Python libraries are dependencies:
+   - [`backoff`](https://pypi.org/project/backoff/)
    - [`dataclass-wizard`](https://pypi.org/project/dataclass-wizard/)
    - [`ffmpeg-python`](https://pypi.org/project/ffmpeg-python/)
    - [`mutagen`](https://pypi.org/project/mutagen/)

pyproject.toml

@@ -4,7 +4,7 @@ requires = ["setuptools", "wheel"]
 universal = 0  # Make the generated wheels have "py3" tag
 [project]
 name = "tidal-wave"
-version = "2024.1.1"
+version = "2024.1.2"
 description = "A tool to wave at the TIDAL music service."
 authors = [
     {name = "colinho", email = "pypi@colin.technology"}
@@ -25,6 +25,7 @@ classifiers=[
     "Operating System :: OS Independent"
 ]
 dependencies = [
+    "backoff",
     "dataclass-wizard",
     "ffmpeg-python",
     "mutagen",

requirements.txt

@@ -1,3 +1,4 @@
+backoff
 dataclass-wizard
 ffmpeg-python
 mutagen

tidal_wave/album.py

@@ -10,7 +10,7 @@
 from .models import AlbumsEndpointResponseJSON
 from .requesting import request_albums, request_album_items, request_album_review
 from .track import Track
-from .utils import download_cover_image, sleep_to_mimic_human_activity
+from .utils import download_cover_image
 
 
 @dataclass
@@ -88,7 +88,6 @@ def get_tracks(
                 album=self.metadata,
             )
             track_files[i] = {track.metadata.track_number: track_files_value}
-            sleep_to_mimic_human_activity()
         else:
             self.track_files = track_files
 

tidal_wave/artist.py

@@ -18,7 +18,7 @@
     request_artists_videos,
 )
 from .track import Track
-from .utils import download_cover_image, sleep_to_mimic_human_activity, TIDAL_API_URL
+from .utils import download_cover_image, TIDAL_API_URL
 from .video import Video
 
 logger = logging.getLogger("__name__")
@@ -120,7 +120,6 @@ def get_videos(
                 out_dir=out_dir,
                 metadata=v,
             )
-            sleep_to_mimic_human_activity()
 
     def get(
         self,

tidal_wave/mix.py

@@ -12,7 +12,7 @@
 from .media import AudioFormat
 from .models import TracksEndpointResponseJSON, VideosEndpointResponseJSON
 from .track import Track
-from .utils import sleep_to_mimic_human_activity, TIDAL_API_URL
+from .utils import TIDAL_API_URL
 from .video import Video
 
 logger = logging.getLogger("__name__")
@@ -83,7 +83,6 @@ def get_items(self, session: Session, audio_format: AudioFormat):
         for i, item in enumerate(self.items):
             if item is None:
                 tracks_videos[i] = None
-                sleep_to_mimic_human_activity()
                 continue
             elif isinstance(item, TracksEndpointResponseJSON):
                 track: Track = Track(track_id=item.id)
@@ -94,7 +93,6 @@ def get_items(self, session: Session, audio_format: AudioFormat):
                     metadata=item,
                 )
                 tracks_videos[i] = track
-                sleep_to_mimic_human_activity()
             elif isinstance(item, VideosEndpointResponseJSON):
                 video: Video = Video(video_id=item.id)
                 video.get(
@@ -103,10 +101,8 @@ def get_items(self, session: Session, audio_format: AudioFormat):
                     metadata=item,
                 )
                 tracks_videos[i] = video
-                sleep_to_mimic_human_activity()
             else:
                 tracks_videos[i] = None
-                sleep_to_mimic_human_activity()
                 continue
         else:
             self.tracks_videos: Tuple[

tidal_wave/playlist.py

@@ -13,7 +13,7 @@
 from .models import TracksEndpointResponseJSON, VideosEndpointResponseJSON
 from .requesting import request_playlists
 from .track import Track
-from .utils import download_cover_image, sleep_to_mimic_human_activity, TIDAL_API_URL
+from .utils import download_cover_image
 from .video import Video
 
 logger = logging.getLogger("__name__")
@@ -88,7 +88,6 @@ def get_items(self, session: Session, audio_format: AudioFormat):
         for i, item in enumerate(self.items):
             if item is None:
                 tracks_videos[i] = None
-                sleep_to_mimic_human_activity()
                 continue
             elif isinstance(item, TracksEndpointResponseJSON):
                 track: Track = Track(track_id=item.id)
@@ -99,7 +98,6 @@ def get_items(self, session: Session, audio_format: AudioFormat):
                     metadata=item,
                 )
                 tracks_videos[i] = track
-                sleep_to_mimic_human_activity()
             elif isinstance(item, VideosEndpointResponseJSON):
                 video: Video = Video(video_id=item.id)
                 video.get(
@@ -108,10 +106,8 @@ def get_items(self, session: Session, audio_format: AudioFormat):
                     metadata=item,
                 )
                 tracks_videos[i] = video
-                sleep_to_mimic_human_activity()
             else:
                 tracks_videos[i] = None
-                sleep_to_mimic_human_activity()
                 continue
         else:
             self.tracks_videos: Tuple[

tidal_wave/requesting.py

@@ -23,7 +23,8 @@
 )
 from .utils import TIDAL_API_URL
 
-from requests import HTTPError, PreparedRequest, Request, Session
+import backoff
+from requests import HTTPError, PreparedRequest, Request, Response, Session
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -71,29 +72,43 @@ def function(s, e, i, u, h, p, sc, cf):
         if h is not None:
             kwargs["headers"] = h
 
+        @backoff.on_predicate(
+            backoff.expo,
+            predicate=lambda r: r.status_code == 429,
+            jitter=backoff.random_jitter,
+            max_time=10,
+            logger=logger,
+        )
+        def _get(s: Session, request_kwargs: dict) -> Response:
+            """Return a requests.Response object, from having passed request_kwargs
+            to s.get(), optionally retrying if 429 error occurs."""
+            with s.get(**request_kwargs) as r:
+                return r
+
         data: Optional[sc] = None
         logger.info(f"Requesting from TIDAL API: {e}/{i}{u}")
-        with s.get(**kwargs) as resp:
-            try:
-                resp.raise_for_status()
-            except HTTPError as he:
-                if resp.status_code == 404:
-                    logger.warning(
-                        f"404 Client Error: not found for TIDAL API endpoint {e}/{i}{u}"
-                    )
-                elif resp.status_code == 401:
-                    logger.warning(
-                        f"401 Client Error: Unauthorized for TIDAL API endpoint {e}/{i}{u}"
-                    )
-                else:
-                    logger.exception(he)
+        resp: Response = _get(s=s, request_kwargs=kwargs)
+
+        try:
+            resp.raise_for_status()
+        except HTTPError as he:
+            if resp.status_code == 404:
+                logger.warning(
+                    f"404 Client Error: not found for TIDAL API endpoint {e}/{i}{u}"
+                )
+            elif resp.status_code == 401:
+                logger.warning(
+                    f"401 Client Error: Unauthorized for TIDAL API endpoint {e}/{i}{u}"
+                )
+            else:
+                logger.exception(he)
+        else:
+            if cf:
+                data = sc.from_dict({"credits": resp.json()})
             else:
-                if cf:
-                    data = sc.from_dict({"credits": resp.json()})
-                else:
-                    data = sc.from_dict(resp.json())
-            finally:
-                return data
+                data = sc.from_dict(resp.json())
+        finally:
+            return data
 
     return function(
         s=session,

tidal_wave/utils.py

@@ -3,9 +3,7 @@
 import logging
 import os
 from pathlib import Path
-import random
 import tempfile
-import time
 from typing import Optional, Tuple, Union
 
 from .models import Artist
@@ -139,9 +137,3 @@ def temporary_file(suffix: str = ".mka"):
     finally:
         tf.close()
         os.unlink(tf.name)
-
-
-def sleep_to_mimic_human_activity():
-    _time = random.randint(500, 5000) / 500
-    logger.info(f"Sleeping for {_time} seconds to mimic human activity")
-    time.sleep(_time)