Merge remote-tracking branch 'upstream/main' into mqtt_data

CHANGELOG.md

@@ -6,6 +6,7 @@ This document logs the changes per release of TWCManager.
 * Placeholder - TWC abstraction code
 * Bugfixes
     * (@MikeBishop) - Explicitly request drive_state data to fix apparent issue with older models, and remove endpoints that are not used
+    * (@dtiefnig) - Specify access scope for token refresh
 
 ## v1.3.2 - 2023-03-12
 * (@RichieB2B) - Nicer looking log prefixes for EMS modules

docs/Software_Manual.md

@@ -104,7 +104,7 @@ The following documents provide detail on specific areas of configuration:
 Once the above steps are complete, start the TWCManager script with the following command:
 
 ```
-sudo -u twcmanager python -m TWCManager
+sudo -u twcmanager python3 -m TWCManager
 ```
 
 ### Monitoring the script operation
@@ -158,7 +158,7 @@ From version v1.2.4 of TWCManager and beyond, you can use pip to upgrade TWCMana
 To upgrade TWCManager to the latest version:
 
 ```
-sudo pip install --upgrade twcmanager
+sudo pip3 install --upgrade twcmanager
 ```
 
 ### Development Version

lib/TWCManager/Control/HTTPControl.py

@@ -1265,11 +1265,8 @@ def process_save_settings(self, page="settings"):
                     carapi = master.getModuleByName("TeslaAPI")
                     if key == "carApiBearerToken":
                         carapi.setCarApiBearerToken(self.getFieldValue(key))
-                        # New tokens expire after 8 hours
-                        carapi.setCarApiTokenExpireTime(time.time() + 8 * 60 * 60)
                     elif key == "carApiRefreshToken":
                         carapi.setCarApiRefreshToken(self.getFieldValue(key))
-                        carapi.setCarApiTokenExpireTime(time.time() + 45 * 24 * 60 * 60)
 
                 else:
                     # Write setting to dictionary

lib/TWCManager/EMS/TeslaPowerwall2.py

@@ -238,6 +238,7 @@ def getStormWatch(self):
         carapi = self.master.getModuleByName("TeslaAPI")
         token = carapi.getCarApiBearerToken()
         expiry = carapi.getCarApiTokenExpireTime()
+        baseURL = carapi.getCarApiBaseURL()
         now = time.time()
         key = "CLOUD/live_status"
 
@@ -252,12 +253,14 @@ def getStormWatch(self):
                     "Content-Type": "application/json",
                 }
                 if not self.cloudID:
-                    url = "https://owner-api.teslamotors.com/api/1/products"
+                    url = baseURL.replace("vehicles", "products")
                     bodyjson = None
                     products = list()
 
                     try:
-                        r = self.httpSession.get(url, headers=headers)
+                        r = self.httpSession.get(
+                            url, headers=headers, verify=carapi.verifyCert
+                        )
                         r.raise_for_status()
                         bodyjson = r.json()
                         products = [
@@ -282,16 +285,22 @@ def getStormWatch(self):
                         logger.info("Couldn't find a Powerwall on your Tesla account.")
 
                 if self.cloudID:
-                    url = f"https://owner-api.teslamotors.com/api/1/energy_sites/{self.cloudID}/live_status"
+                    url = baseURL.replace("vehicles", "energy_sites")
+                    url = f"{url}/{self.cloudID}/live_status"
                     bodyjson = None
-                    result = dict()
 
                     try:
-                        r = self.httpSession.get(url, headers=headers)
+                        r = self.httpSession.get(
+                            url, headers=headers, verify=carapi.verifyCert
+                        )
                         r.raise_for_status()
                         bodyjson = r.json()
                         lastData = bodyjson["response"]
                     except:
+                        if r.status_code is 403:
+                            logger.warn(
+                                "Error fetching Powerwall cloud data; does your API token have energy_device_data scope?"
+                            )
                         pass
 
             self.lastFetch[key] = (now, lastData)

lib/TWCManager/Policy/Policy.py

@@ -194,6 +194,11 @@ def enforcePolicy(self, policy, updateLatch=False):
             self.limitOverride = False
             self.fireWebhook("enter")
 
+            # Clear stopAskingToStartCharging so we try charging each car at
+            # least once
+            for vehicle in self.master.getModuleByName("TeslaAPI").getCarApiVehicles():
+                vehicle.stopAskingToStartCharging = False
+
         if updateLatch and "latch_period" in policy:
             policy["__latchTime"] = time.time() + policy["latch_period"] * 60
 
@@ -229,11 +234,6 @@ def enforcePolicy(self, policy, updateLatch=False):
             limit = -1
         self.master.queue_background_task({"cmd": "applyChargeLimit", "limit": limit})
 
-        # Clear stopAskingToStartCharging so we try charging each car at
-        # least once
-        for vehicle in self.master.getModuleByName("TeslaAPI").getCarApiVehicles():
-            vehicle.stopAskingToStartCharging = False
-
         # Report current policy via Status modules
         for module in self.master.getModulesByType("Status"):
             module["ref"].setStatus(

lib/TWCManager/Vehicle/TeslaAPI.py

@@ -124,51 +124,53 @@ def apiRefresh(self):
             "client_id": self.refreshClientID,
             "grant_type": "refresh_token",
             "refresh_token": self.getCarApiRefreshToken(),
+            "scope": "offline_access",
         }
         req = None
         now = time.time()
         try:
             req = requests.post(self.refreshURL, headers=headers, json=data)
             logger.log(logging.INFO2, "Car API request" + str(req))
+            req.raise_for_status()
             apiResponseDict = json.loads(req.text)
         except requests.exceptions.RequestException:
-            logger.log(
-                logging.INFO2, "Request Exception parsing API Token Refresh Response"
-            )
-            pass
-        except ValueError:
-            pass
+            if req.status_code == 401:
+                logger.log(
+                    logging.INFO2,
+                    "TeslaAPI",
+                    "ERROR: Can't access Tesla car via API.  Please supply fresh tokens.",
+                )
+                self.setCarApiBearerToken("")
+                self.setCarApiRefreshToken("")
+            self.updateCarApiLastErrorTime()
+            # Instead of just setting carApiLastErrorTime, erase tokens to
+            # prevent further authorization attempts until user enters password
+            # on web interface. I feel this is safer than trying to log in every
+            # ten minutes with a bad token because Tesla might decide to block
+            # remote access to your car after too many authorization errors.
+            self.master.queue_background_task({"cmd": "saveSettings"})
+            return False
         except json.decoder.JSONDecodeError:
             logger.log(
                 logging.INFO2, "JSON Decode Error parsing API Token Refresh Response"
             )
             pass
+        except ValueError:
+            pass
 
         try:
             logger.log(logging.INFO4, "Car API auth response" + str(apiResponseDict))
             self.setCarApiBearerToken(apiResponseDict["access_token"])
             self.setCarApiRefreshToken(apiResponseDict["refresh_token"])
             self.setCarApiTokenExpireTime(now + apiResponseDict["expires_in"])
             self.master.queue_background_task({"cmd": "saveSettings"})
+            return True
 
-        except KeyError:
-            logger.log(
-                logging.INFO2,
-                "TeslaAPI",
-                "ERROR: Can't access Tesla car via API.  Please log in again via web interface.",
-            )
-            self.updateCarApiLastErrorTime()
-            # Instead of just setting carApiLastErrorTime, erase tokens to
-            # prevent further authorization attempts until user enters password
-            # on web interface. I feel this is safer than trying to log in every
-            # ten minutes with a bad token because Tesla might decide to block
-            # remote access to your car after too many authorization errors.
-            self.setCarApiBearerToken("")
-            self.setCarApiRefreshToken("")
-            self.master.queue_background_task({"cmd": "saveSettings"})
-        except UnboundLocalError:
+        except:
             pass
 
+        return False
+
     def car_api_available(
         self, email=None, password=None, charge=None, applyLimit=None
     ):
@@ -448,6 +450,8 @@ def car_api_available(
                                 # in 15 minutes. We'll show an error about this
                                 # later.
                                 vehicle.delayNextWakeAttempt = 15 * 60
+                            else:
+                                vehicle.delayNextWakeAttempt = 25
 
                         if state == "error":
                             logger.info(
@@ -727,7 +731,7 @@ def car_api_charge(self, charge):
                         if apiResponseDict["response"]["result"] == True:
                             self.resetCarApiLastErrorTime(vehicle)
                         elif charge:
-                            reason = apiResponseDict["response"]["reason"]
+                            reason = self.findReason(apiResponseDict)
                             if reason in [
                                 "complete",
                                 "charging",
@@ -786,7 +790,7 @@ def car_api_charge(self, charge):
                             # Stop charge failed with an error I
                             # haven't seen before, so wait
                             # carApiErrorRetryMins mins before trying again.
-                            reason = apiResponseDict["response"]["reason"]
+                            reason = self.findReason(apiResponseDict)
                             logger.info(
                                 'ERROR "'
                                 + reason
@@ -815,6 +819,13 @@ def car_api_charge(self, charge):
 
         return result
 
+    def findReason(self, apiResponseDict):
+        if "reason" in apiResponseDict["response"]:
+            return apiResponseDict["response"]["reason"]
+        elif "string" in apiResponseDict["response"]:
+            return apiResponseDict["response"]["string"].split(": ")[-1]
+        return ""
+
     def applyChargeLimit(self, limit, checkArrival=False, checkDeparture=False):
         if limit != -1 and (limit < 50 or limit > 100):
             logger.log(logging.INFO8, "applyChargeLimit skipped")
@@ -1120,29 +1131,45 @@ def setCarApiBearerToken(self, token=None):
                 return False
             else:
                 self.carApiBearerToken = token
-                if not self.baseURL:
-                    try:
-                        decoded = jwt.decode(
-                            token,
-                            options={
-                                "verify_signature": False,
-                                "verify_aud": False,
-                                "verify_exp": False,
-                            },
-                        )
+                try:
+                    decoded = jwt.decode(
+                        token,
+                        options={
+                            "verify_signature": False,
+                            "verify_aud": False,
+                            "verify_exp": False,
+                        },
+                    )
+                    if not self.baseURL:
                         if "owner-api" in "".join(decoded.get("aud", "")):
                             self.baseURL = self.regionURL["OwnerAPI"]
                         elif decoded.get("ou_code", "") in self.regionURL:
                             self.baseURL = self.regionURL[decoded["ou_code"]]
-                    except jwt.exceptions.DecodeError:
-                        # Fallback to owner-api if we get an exception decoding jwt token
-                        self.baseURL = self.regionURL["OwnerAPI"]
+
+                    if "exp" in decoded:
+                        self.setCarApiTokenExpireTime(int(decoded["exp"]))
+                    else:
+                        self.setCarApiTokenExpireTime(time.time() + 8 * 60 * 60)
+
+                except jwt.exceptions.DecodeError:
+                    # Fallback to owner-api if we get an exception decoding jwt token
+                    self.baseURL = self.regionURL["OwnerAPI"]
+                    self.setCarApiTokenExpireTime(time.time() + 8 * 60 * 60)
                 return True
         else:
             return False
 
     def setCarApiRefreshToken(self, token):
         self.carApiRefreshToken = token
+        if (
+            token
+            and not self.master.tokenSyncEnabled()
+            and (
+                self.getCarApiBearerToken() == ""
+                or self.getCarApiTokenExpireTime() - time.time() < 60 * 60
+            )
+        ):
+            return self.apiRefresh()
         return True
 
     def setCarApiTokenExpireTime(self, value):
@@ -1233,8 +1260,16 @@ def wakeVehicle(self, vehicle):
         try:
             req = requests.post(url, headers=headers, verify=self.verifyCert)
             logger.log(logging.INFO8, "Car API cmd wake_up" + str(req))
+            req.raise_for_status()
             apiResponseDict = json.loads(req.text)
         except requests.exceptions.RequestException:
+            if req.status_code == 401 and "expired" in req.text:
+                # If the token is expired, refresh it and try again
+                if self.apiRefresh():
+                    return self.wakeVehicle(vehicle)
+            elif req.status_code == 429:
+                # We're explicitly being told to back off
+                self.errorCount = max(30, self.errorCount)
             return False
         except json.decoder.JSONDecodeError:
             return False
@@ -1400,14 +1435,22 @@ def get_car_api(self, url, checkReady=True, provesOnline=True):
         for _ in range(0, 3):
             try:
                 req = requests.get(url, headers=headers, verify=self.verifyCert)
+                req.raise_for_status()
                 logger.log(logging.INFO8, "Car API cmd " + url + " " + str(req))
                 apiResponseDict = json.loads(req.text)
                 # This error can happen here as well:
                 #   {'response': {'reason': 'could_not_wake_buses', 'result': False}}
                 # This one is somewhat common:
                 #   {'response': None, 'error': 'vehicle unavailable: {:error=>"vehicle unavailable:"}', 'error_description': ''}
             except requests.exceptions.RequestException:
-                pass
+                if req.status_code == 401 and "expired" in req.text:
+                    # If the token is expired, refresh it and try again
+                    if self.apiRefresh():
+                        continue
+                elif req.status_code == 429:
+                    # We're explicitly being told to back off
+                    self.errorCount = max(30, self.errorCount)
+                return False, None
             except json.decoder.JSONDecodeError:
                 pass