Merge pull request #6 from klepp-me/feature/db

Feature/db

.env.test

@@ -5,5 +5,5 @@ AWS_USER_POOL_ID=eu-north-1_...
 
 AWS_S3_ACCESS_KEY_ID=my-aws-id-with-access-to-bucket
 AWS_S3_SECRET_ACCESS_KEY=my-aws-secret-with-access-to-bucket
-
+DATABASE_URL=my-psql-connstr
 ENVIRONMENT=dev

.pre-commit-config.yaml

@@ -1,12 +1,12 @@
-exclude: README.md
+exclude: README.md|migrations
 repos:
   - repo: https://github.com/ambv/black
-    rev: '21.7b0'
+    rev: '22.1.0'
     hooks:
       - id: black
         args: ['--quiet']
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v4.1.0
     hooks:
       - id: check-case-conflict
       - id: end-of-file-fixer
@@ -17,30 +17,30 @@ repos:
       - id: detect-private-key
       - id: double-quote-string-fixer
   - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.3
+    rev: 3.9.2
     hooks:
       - id: flake8
         additional_dependencies: [
-          'flake8-bugbear==20.1.4',  # Looks for likely bugs and design problems
-          'flake8-comprehensions==3.2.3',  # Looks for unnecessary generator functions that can be converted to list comprehensions
+          'flake8-bugbear==22.1.11',  # Looks for likely bugs and design problems
+          'flake8-comprehensions==3.8.0',  # Looks for unnecessary generator functions that can be converted to list comprehensions
           'flake8-deprecated==1.3',  # Looks for method deprecations
-          'flake8-use-fstring==1.1',  # Enforces use of f-strings over .format and %s
-          'flake8-print==3.1.4',  # Checks for print statements
-          'flake8-docstrings==1.5.0',  # Verifies that all functions/methods have docstrings
-          'flake8-annotations==2.4.0',  # Enforces type annotation
+          'flake8-use-fstring==1.3',  # Enforces use of f-strings over .format and %s
+          'flake8-print==4.0.0',  # Checks for print statements
+          'flake8-docstrings==1.6.0',  # Verifies that all functions/methods have docstrings
+          'flake8-annotations==2.7.0',  # Enforces type annotation
         ]
         args: ['--enable-extensions=G']
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.23.3
+    rev: v2.31.1
     hooks:
       - id: pyupgrade
         args: ["--py36-plus"]
   - repo: https://github.com/pycqa/isort
-    rev: 5.9.3
+    rev: 5.10.1
     hooks:
       - id: isort
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v0.930"
+    rev: "v0.941"
     hooks:
       - id: mypy
         exclude: "test_*"
@@ -49,4 +49,7 @@ repos:
             fastapi,
             pydantic,
             starlette,
+            sqlmodel,
+            types-aiofiles,
+            aioboto3
           ]

alembic.ini

@@ -0,0 +1,102 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = migrations
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python-dateutil library that can be
+# installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to migrations/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:migrations/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

app/api/api_v1/api.py

@@ -1,6 +1,6 @@
 from fastapi import APIRouter
 
-from api.api_v1.endpoints import file
+from app.api.api_v1.endpoints import file
 
 api_router = APIRouter()
-api_router.include_router(file.router, tags=['files'])
+api_router.include_router(file.router, tags=['deprecated_api'], deprecated=True)

app/api/api_v1/endpoints/file.py

@@ -4,11 +4,11 @@
 from aiobotocore.client import AioBaseClient
 from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile, status
 
-from api.dependencies import get_boto
-from api.security import cognito_scheme, cognito_scheme_or_anonymous
-from core.config import settings
-from schemas.file import DeletedFileResponse, DeleteFile, FileResponse, HideFile, ListFilesResponse, ShowFile
-from schemas.user import User
+from app.api.dependencies import get_boto
+from app.api.security import cognito_scheme, cognito_scheme_or_anonymous
+from app.core.config import settings
+from app.schemas.file import DeletedFileResponse, DeleteFile, FileResponse, HideFile, ListFilesResponse, ShowFile
+from app.schemas.user import User
 
 router = APIRouter()
 

app/api/api_v2/api.py

@@ -0,0 +1,7 @@
+from fastapi import APIRouter
+
+from app.api.api_v2.endpoints import list_videos, upload
+
+api_router = APIRouter()
+api_router.include_router(list_videos.router, tags=['files'])
+api_router.include_router(upload.router, tags=['files'])

app/api/api_v2/endpoints/list_videos.py

@@ -0,0 +1,67 @@
+import asyncio
+from typing import Optional
+
+from fastapi import APIRouter, Depends, Query
+from sqlalchemy import and_, asc, func, or_
+from sqlalchemy.orm import selectinload
+from sqlmodel import select
+from sqlmodel.ext.asyncio.session import AsyncSession
+
+from app.api.dependencies import yield_db_session
+from app.api.security import cognito_scheme_or_anonymous
+from app.models.klepp import ListResponse, Video, VideoRead
+from app.schemas.user import User as CognitoUser
+
+router = APIRouter()
+
+
+@router.get('/files', response_model=ListResponse[VideoRead])
+async def get_all_files(
+    session: AsyncSession = Depends(yield_db_session),
+    user: CognitoUser | None = Depends(cognito_scheme_or_anonymous),
+    username: Optional[str] = None,
+    hidden: bool = False,
+    tag: Optional[str] = None,
+    offset: int = 0,
+    limit: int = Query(default=100, lte=100),
+) -> dict[str, int | list]:
+    """
+    Get a list of all non-hidden files, unless you're the owner of the file, then you can request
+    hidden files.
+    Works both as anonymous user and as a signed in user.
+    """
+    # Video query
+    video_statement = (
+        select(Video)
+        .options(selectinload(Video.user))
+        .options(selectinload(Video.tags))
+        .options(selectinload(Video.likes))
+        .order_by(asc(Video.uploaded))
+    )
+    if username and username.islower() and username.isalnum():
+        video_statement = video_statement.where(Video.user.has(name=username))  # type: ignore
+    if user and hidden:
+        video_statement = video_statement.where(
+            or_(
+                Video.hidden == False,  # noqa
+                and_(Video.hidden == True, Video.user.has(name=user.username)),  # type:ignore
+            )
+        )
+    else:
+        video_statement = video_statement.where(Video.hidden == False)  # noqa
+    if tag:
+        video_statement = video_statement.where(Video.tags.any(name=tag))  # type: ignore
+
+    # Total count query based on query params, without pagination
+    count_statement = select(func.count('*')).select_from(video_statement)  # type: ignore
+
+    # Add pagination
+    video_statement = video_statement.offset(offset=offset).limit(limit=limit)
+    # Do DB requests async
+    tasks = [
+        asyncio.create_task(session.exec(video_statement)),  # type: ignore
+        asyncio.create_task(session.exec(count_statement)),
+    ]
+    results, count = await asyncio.gather(*tasks)
+    count_number = count.first()
+    return {'total_count': count_number, 'response': results.all()}

app/api/api_v2/endpoints/upload.py

@@ -0,0 +1,96 @@
+import asyncio
+from datetime import datetime, timezone
+from typing import Any, Optional
+from uuid import uuid4
+
+import aiofiles
+from aiobotocore.client import AioBaseClient
+from aiofiles import os
+from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile, status
+from sqlmodel.ext.asyncio.session import AsyncSession
+
+from app.api.dependencies import get_boto, yield_db_session
+from app.api.security import cognito_signed_in
+from app.api.services import await_ffmpeg
+from app.core.config import settings
+from app.models.klepp import User, Video
+from app.schemas.file import FileResponse
+
+router = APIRouter()
+
+
+async def upload_video(boto_session: AioBaseClient, path: str, temp_video_name: str) -> None:
+    """
+    Upload a stored file to s3
+    """
+    async with aiofiles.open(temp_video_name, 'rb+') as video_file:
+        await boto_session.put_object(
+            Bucket=settings.S3_BUCKET_URL,
+            Key=path,
+            Body=await video_file.read(),
+            ACL='public-read',
+        )
+
+
+@router.post('/files', response_model=FileResponse, status_code=status.HTTP_201_CREATED)
+async def upload_file(
+    file: UploadFile = File(..., description='File to upload'),
+    file_name: Optional[str] = Form(
+        default=None, alias='fileName', example='my_file.mp4', regex=r'^[\s\w\d_-]*$', min_length=2, max_length=40
+    ),
+    boto_session: AioBaseClient = Depends(get_boto),
+    user: User = Depends(cognito_signed_in),
+    db_session: AsyncSession = Depends(yield_db_session),
+) -> Any:
+    """
+    Upload a file.
+    """
+    if not file:
+        raise HTTPException(status_code=400, detail='You must provide a file.')
+
+    if file.content_type != 'video/mp4':
+        raise HTTPException(status_code=400, detail='Currently only support for video/mp4 files through this API.')
+
+    upload_file_name = f'{file_name}.mp4' if file_name else file.filename
+    s3_path = f'{user.name}/{upload_file_name}'
+
+    exist = await boto_session.list_objects_v2(Bucket=settings.S3_BUCKET_URL, Prefix=s3_path)
+    if exist.get('Contents'):
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Video already exist in s3.')
+
+    # Save video
+    temp_name = uuid4().hex
+    async with aiofiles.open(f'{temp_name}.mp4', 'wb') as video:
+        while content := await file.read(1024):
+            await video.write(content)  # type: ignore
+
+    upload_task = asyncio.create_task(
+        upload_video(boto_session=boto_session, path=s3_path, temp_video_name=f'{temp_name}.mp4')
+    )
+    ffmpeg_task = asyncio.create_task(await_ffmpeg(url=f'{temp_name}.mp4', name=f'{temp_name}.png'))
+    await asyncio.gather(upload_task, ffmpeg_task)
+
+    async with aiofiles.open(f'{temp_name}.png', 'rb+') as thumbnail_img:
+        await boto_session.put_object(
+            Bucket=settings.S3_BUCKET_URL,
+            Key=s3_path.replace('.mp4', '.png'),
+            Body=await thumbnail_img.read(),
+            ACL='public-read',
+        )
+
+    # Cleanup
+    remove_video = asyncio.create_task(os.remove(f'{temp_name}.mp4'))
+    remove_thumbnail = asyncio.create_task(os.remove(f'{temp_name}.png'))
+    await asyncio.gather(remove_video, remove_thumbnail)
+
+    db_video: Video = Video(
+        path=s3_path, display_name=file_name, user=user, user_id=user.id, uri=f'https://gg.klepp.me/{s3_path}'
+    )
+    db_session.add(db_video)
+    await db_session.commit()
+
+    return {
+        'file_name': file_name,
+        'username': user.name,
+        'datetime': datetime.now(timezone.utc).isoformat(' ', 'seconds'),
+    }