Fix #1386 add backfill endpoints and cli for backfill_transactions, b…

…ackfill_subscriptions, backfill_persons and backfill_stripe_invoices

subscribie/__init__.py

@@ -14,7 +14,6 @@
 import sys
 import sqlite3
 from .database import database
-import datetime
 from base64 import b64encode
 from flask import (
     Flask,
@@ -43,7 +42,21 @@
 from flask_migrate import Migrate, upgrade
 import click
 from jinja2 import Template
-from .models import PaymentProvider, Person, Company, Module, Plan, PriceList
+from .models import (
+    PaymentProvider,
+    Person,
+    Company,
+    Module,
+    Plan,
+    PriceList,
+)
+from subscribie.utils import (
+    backfill_transactions as call_backfill_transactions,
+    backfill_subscriptions as call_backfill_subscriptions,
+    backfill_persons as call_backfill_persons,
+    backfill_stripe_invoices as call_backfill_stripe_invoices,
+)
+from datetime import datetime
 
 log = logging.getLogger(__name__)
 
@@ -267,6 +280,30 @@ def initdb():
                 log.info("Database already seeded.")
             con.close()
 
+    @app.cli.command()
+    @click.argument("days", type=int)
+    def backfill_transactions(days):
+        click.echo(f"Beginning transaction backfill for {days} days")
+        call_backfill_transactions(days=days)
+
+    @app.cli.command()
+    @click.argument("days", type=int)
+    def backfill_subscriptions(days):
+        click.echo(f"Beginning subscription backfill for {days} days")
+        call_backfill_subscriptions(days=days)
+
+    @app.cli.command()
+    @click.argument("days", type=int)
+    def backfill_persons(days):
+        click.echo(f"Beginning person backfill for {days} days")
+        call_backfill_persons(days=days)
+
+    @app.cli.command()
+    @click.argument("days", type=int)
+    def backfill_stripe_invoices(days):
+        click.echo(f"Beginning stripe invoices backfill for {days} days")
+        call_backfill_stripe_invoices(days=days)
+
     @app.cli.group()
     def translate():
         """Translation and localization commands."""

subscribie/blueprints/admin/__init__.py

@@ -32,6 +32,10 @@
     currencyFormat,
     get_shop_default_country_code,
     dec2pence,
+    backfill_transactions,
+    backfill_subscriptions,
+    backfill_persons,
+    backfill_stripe_invoices
 )
 from subscribie.forms import (
     TawkConnectForm,
@@ -238,7 +242,7 @@ def update_payment_fulfillment(stripe_external_id):
 
 
 @admin.route("/stripe/charge", methods=["POST", "GET"])
-# @login_required
+@login_required
 def stripe_create_charge():
     """Charge an existing subscriber x ammount immediately
 
@@ -2062,3 +2066,38 @@ def check_spam(account_name) -> int:
     from subscribie.anti_spam_subscribie_shop_names.run import detect_spam_shop_name
 
     return str(detect_spam_shop_name(account_name))
+
+
+@admin.route("/backfill/transactions/<int:days>")
+@login_required
+def admin_backfill_transactions(days):
+    backfill_transactions(days)
+    return jsonify({"msg": f"backfill_transactions for {days} days completed"})
+
+
+@admin.route("/backfill/subscriptions/<int:days>")
+@login_required
+def admin_backfill_subscriptions(days):
+    backfill_subscriptions(days)
+    return jsonify({"msg": f"backfill_subscriptions for {days} days completed"})
+
+
+@admin.route("/backfill/persons/<int:days>")
+@login_required
+def admin_backfill_persons(days):
+    backfill_persons(days)
+    return jsonify({"msg": f"backfill_persons for {days} days completed"})
+
+
+@admin.route("/backfill/stripe-invoices/<int:days>")
+@login_required
+def admin_backfill_stripe_invoices(days):
+    backfill_stripe_invoices(days)
+    return jsonify({"msg": f"backfill_stripe_invoices for {days} days completed"})
+
+
+@admin.route("/backfill/stripe-upcoming-invoices/<int:days>")
+@login_required
+def admin_backfill_stripe_upcoming_invoices(days):
+    backfill_stripe_upcoming_invoices(days)
+    return jsonify({"msg": f"backfill_stripe_upcoming_invoices for {days} days completed"})  # noqa: E501

subscribie/utils.py

@@ -615,3 +615,223 @@ def dec2pence(amount: str) -> int:
         raise ValueError(
             "Invalid input: amount should be a string representing a decimal number"
         )
+
+
+def backfill_transactions(days=30):
+    """Backfill transaction data in an idempotent way
+    Useful for fixing webhook delivery misses (such as if all webhook delivery retires
+    exausted), and data corrections from Hotfixes.
+
+    - .e.g created_at See https://github.com/Subscribie/subscribie/issues/1385
+    """
+    from subscribie.models import Transaction
+
+    stripe_connect_account_id = get_stripe_connect_account_id()
+
+    stripe.api_key = get_stripe_secret_key()
+    stripe_connect_account_id = get_stripe_connect_account_id()
+    today = datetime.now()
+    days_before_today = today - timedelta(days=days)
+    days_before_today_timestamp = int(days_before_today.timestamp())
+    paymentIntents = stripe.PaymentIntent.list(
+        stripe_account=stripe_connect_account_id,
+        limit=100,
+        created={"gte": days_before_today_timestamp},
+    )
+    for paymentIntent in paymentIntents.auto_paging_iter():
+        transaction = (
+            database.session.query(Transaction)
+            .filter_by(external_id=paymentIntent.id)
+            .first()
+        )
+
+        if transaction is not None:
+            # Update the transaction in Transaction model
+            msg = f"Current transaction.id {transaction.id} created_at: {transaction.created_at}"  # noqa: E501
+            log.info(msg)
+
+            stripe_transaction_created_at = datetime.fromtimestamp(
+                paymentIntent.created
+            )  # noqa: E501
+            msg = f"Upstream transaction.id {transaction.id} created_at set to {stripe_transaction_created_at}"  # noqa: E501
+            log.info(msg)
+
+            if transaction.created_at != stripe_transaction_created_at:
+                msg = f"Setting transaction.id {transaction.id} created_at to {stripe_transaction_created_at}"  # noqa: E501
+                log.info(msg)
+                transaction.created_at = stripe_transaction_created_at
+                database.session.commit()
+            else:
+                log.info(
+                    "Skipping transaction.created_at update as source data is equal to local"  # noqa: E501
+                )  # noqa: E501
+
+
+def backfill_subscriptions(days=30):
+    """Backfill subscription data in an idempotent way
+    Useful for fixing webhook delivery misses (such as if all webhook delivery retires
+    exausted), and data corrections from Hotfixes.
+
+    - .e.g created_at See https://github.com/Subscribie/subscribie/issues/1385
+    """
+    from subscribie.models import Subscription
+
+    stripe_connect_account_id = get_stripe_connect_account_id()
+
+    stripe.api_key = get_stripe_secret_key()
+    stripe_connect_account_id = get_stripe_connect_account_id()
+    today = datetime.now()
+    days_before_today = today - timedelta(days=days)
+    days_before_today_timestamp = int(days_before_today.timestamp())
+    subscriptions = stripe.Subscription.list(
+        stripe_account=stripe_connect_account_id,
+        limit=100,
+        created={"gte": days_before_today_timestamp},
+    )
+    for stripe_subscription in subscriptions.auto_paging_iter():
+        subscribie_subscription = (
+            database.session.query(Subscription)
+            .filter_by(stripe_subscription_id=stripe_subscription.id)
+            .first()
+        )
+
+        if subscribie_subscription is not None:
+            # Update the subscribie_subscription in Subscription model
+            msg = f"Current subscription.id {subscribie_subscription.id} created_at: {subscribie_subscription.created_at}"  # noqa: E501
+            log.info(msg)
+
+            # TODO Stripe incorporate subscription.start_date into Subscription model
+            # https://docs.stripe.com/api/subscriptions/object#subscription_object-start_date
+            stripe_subscription_created_at = datetime.fromtimestamp(
+                stripe_subscription.created
+            )  # noqa: E501
+            msg = f"Upstream subscription.id {subscribie_subscription.id} created_at set to {stripe_subscription_created_at}"  # noqa: E501
+            log.info(msg)
+
+            if subscribie_subscription.created_at != stripe_subscription_created_at:
+                msg = f"Setting subscription.id {subscribie_subscription.id} created_at to {stripe_subscription_created_at}"  # noqa: E501
+                log.info(msg)
+                subscribie_subscription.created_at = stripe_subscription_created_at
+                database.session.commit()
+            else:
+                log.info(
+                    "Skipping subscription.created_at update for subscription id {subscribie_subscription.id} as source data is equal to local"  # noqa: E501
+                )  # noqa: E501
+
+
+def backfill_persons(days=30):
+    """Backfill person data in an idempotent way
+    Useful for fixing webhook delivery misses (such as if all webhook delivery retires
+    exausted), and data corrections from Hotfixes.
+
+    NOTE: The Stripe session checkout object is used here to
+    signify the earliest known date/time for Person.created_at time
+    since a Person record is created during checkout, this is a reasonable
+    source for created_at time during a backfill recovery run.
+
+    - .e.g created_at See https://github.com/Subscribie/subscribie/issues/1385
+
+    Subscribie stores checkout metadata useful for associating checkouts with
+    a person on the Subscription table(see models.py) these fields include:
+
+    - subscribie_checkout_session_id
+    - stripe_external_id
+    - stripe_subscription_id
+    And a Subscription object is *always* linked to a Person entity.
+
+    """
+    from subscribie.models import Subscription
+
+    stripe_connect_account_id = get_stripe_connect_account_id()
+
+    stripe.api_key = get_stripe_secret_key()
+    stripe_connect_account_id = get_stripe_connect_account_id()
+    today = datetime.now()
+    days_before_today = today - timedelta(days=days)
+    days_before_today_timestamp = int(days_before_today.timestamp())
+    stripe_checkout_sessions = stripe.checkout.Session.list(
+        stripe_account=stripe_connect_account_id,
+        limit=100,
+        created={"gte": days_before_today_timestamp},
+    )
+    for stripe_session in stripe_checkout_sessions.auto_paging_iter():
+        if stripe_session.metadata.get("subscribie_checkout_session_id") is None:
+            log.warning(f"No subscribie_checkout_session_id found on metadata for stripe_session {stripe_session.id}")  # noqa: E501
+            continue
+
+        subscribie_subscription = (
+            database.session.query(Subscription)
+            .filter_by(
+                subscribie_checkout_session_id=stripe_session.metadata[
+                    "subscribie_checkout_session_id"
+                ]
+            )
+            .first()
+        )
+        if subscribie_subscription is not None:
+            if subscribie_subscription.person is None:
+                log.warning(f"Skipping stripe_session {stripe_session.id} as person is None for subscription {subscribie_subscription.id}")  # noqa: E501
+                continue
+            # Update the subscribie_subscription in Subscription model
+            log.debug(f"At stripe_session.id: {stripe_session.id}")
+            msg = f"Current person.created_at: {subscribie_subscription.person.created_at}"  # noqa: E501
+            log.info(msg)
+
+            # TODO Stripe incorporate subscription.start_date into Subscription model
+            # https://docs.stripe.com/api/subscriptions/object#subscription_object-start_date
+            stripe_session_created_at = datetime.fromtimestamp(
+                stripe_session.created
+            )  # noqa: E501
+            msg = f"Infering person create_at from stripe_session_created_at: {stripe_session_created_at}"  # noqa: E501
+            log.info(msg)
+            msg = f"Setting person.created_at to {stripe_session_created_at}"  # noqa: E501
+            log.info(msg)
+            subscribie_subscription.person.created_at = stripe_session_created_at
+            database.session.commit()
+
+
+def backfill_stripe_invoices(days=30):
+    """Backfill stripe_invoice data in an idempotent way
+    Useful for fixing webhook delivery misses (such as if all webhook delivery retires
+    exausted), and data corrections from Hotfixes.
+
+    - .e.g created_at See https://github.com/Subscribie/subscribie/issues/1385
+
+    """
+    from subscribie.models import StripeInvoice
+    stripe_connect_account_id = get_stripe_connect_account_id()
+
+    stripe.api_key = get_stripe_secret_key()
+    stripe_connect_account_id = get_stripe_connect_account_id()
+    today = datetime.now()
+    days_before_today = today - timedelta(days=days)
+    days_before_today_timestamp = int(days_before_today.timestamp())
+    stripe_invoices = stripe.Invoice.list(
+        stripe_account=stripe_connect_account_id,
+        limit=100,
+        created={"gte": days_before_today_timestamp},
+    )
+    for stripe_invoice in stripe_invoices.auto_paging_iter():
+
+        local_stripe_invoice = (
+            database.session.query(StripeInvoice)
+            .filter_by(
+                id=stripe_invoice.id
+            )
+            .first()
+        )
+        if local_stripe_invoice is not None:
+            # Update the local_stripe_invoice in DtripeInvoice model
+            log.debug(f"At local_stripe_invoice.id: {local_stripe_invoice.id}")
+            msg = f"Current local_stripe_invoice.created_at: {local_stripe_invoice.created_at}"  # noqa: E501
+            log.info(msg)
+
+            stripe_invoice_created_at = datetime.fromtimestamp(
+                stripe_invoice.created
+            )  # noqa: E501
+            msg = f"stripe_invoice create_at: {stripe_invoice_created_at}"  # noqa: E501
+            log.info(msg)
+            msg = f"Setting local_stripe_invoice.created_at to {stripe_invoice_created_at}"  # noqa: E501
+            log.info(msg)
+            local_stripe_invoice.created_at = stripe_invoice_created_at
+            database.session.commit()

tests/test_subscribie.py

@@ -1,5 +1,5 @@
 import pytest
-from subscribie.models import User
+from subscribie.models import User, Transaction, Person
 
 from contextlib import contextmanager
 from flask import appcontext_pushed, g
@@ -230,9 +230,35 @@ def test_dec2pence():
         "1.005": 101,  # Tests edge case where rounding up from 1.005
     }
 
-    # Results dictionary
-    results = {}
-
     for test_input, expected_output in test_cases.items():
         result = dec2pence(test_input)
         assert result == expected_output
+
+
+def test_create_at_date_is_not_constant(
+    db_session,
+    app,
+):
+    """Verify that created_at timestamp is changing upon insert and not static
+
+    https://github.com/Subscribie/subscribie/issues/1385
+    """
+
+    transaction1 = Transaction()
+    db_session.add(transaction1)
+    db_session.commit()
+    transaction2 = Transaction()
+    db_session.add(transaction2)
+    db_session.commit()
+
+    assert transaction1.created_at != transaction2.created_at
+
+    person1 = Person()
+    db_session.add(person1)
+    db_session.commit()
+
+    person2 = Person()
+    db_session.add(person2)
+    db_session.commit()
+
+    assert person1.created_at != person2.created_at