diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml deleted file mode 100644 index 2dc73f8..0000000 --- a/.buildkite/pipeline.yml +++ /dev/null @@ -1,13 +0,0 @@ -steps: - - label: ":partyparrot: Build package" - commands: "sh ./scripts/build.sh" - plugins: - - docker#v3.5.0: - image: "quay.io/pypa/manylinux2010_x86_64" - propagate-environment: true - always-pull: true - # artifact_paths: - # - "dist/*.whl" - branches: "master develop" - agents: - queue: "annalise-build" diff --git a/.coveragerc b/.coveragerc index 40cc691..d0ca3c5 100644 --- a/.coveragerc +++ b/.coveragerc @@ -10,12 +10,12 @@ parallel = true [paths] source = - s3pkgup + pips3 [report] precision = 2 include = - s3pkgup/*.py + pips3/*.py exclude_lines = def __str__ pragma: no cover diff --git a/CHANGELOG.md b/CHANGELOG.md index 16c81fa..a6f83b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,5 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## v0.1 +## [0.1.0] - 27-10-2020 +### Added +* First release into pypi +* Removed annalise.ai specific details +* Improved test coverage +* Configuration via environment variables diff --git a/MANIFEST.in b/MANIFEST.in index 2f29eb6..bf9cb24 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,9 +1,8 @@ include README.rst include versioneer.py -include s3pkgup/_version.py -include s3pkgup/data/index.html.j2 +include pips3/_version.py -recursive-include tests * +recursive-exclude tests * recursive-exclude * __pycache__ recursive-exclude * *.py[co] diff --git a/Makefile b/Makefile index a9f65f6..a2513ae 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ clean-test: ## remove test and coverage artifacts rm -fr .pytest_cache lint: ## check style with flake8 - flake8 s3pkgup tests + flake8 pips3 tests test: ## run tests quickly with the default Python pytest @@ -60,15 +60,15 @@ test-all: ## run tests on every Python version with tox tox coverage: ## check code coverage quickly with the default Python - coverage run --source s3pkgup -m pytest + coverage run --source pips3 -m pytest coverage report -m coverage html $(BROWSER) htmlcov/index.html docs: ## generate Sphinx HTML documentation, including API docs - rm -f docs/s3pkgup.rst + rm -f docs/pips3.rst rm -f docs/modules.rst - sphinx-apidoc -o docs/ s3pkgup + sphinx-apidoc -o docs/ pips3 $(MAKE) -C docs clean $(MAKE) -C docs html $(BROWSER) docs/_build/html/index.html diff --git a/README.md b/README.md new file mode 100644 index 0000000..f90bc4b --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +# pips3 + +Python Boilerplate contains all the boilerplate you need to create a Python package. + +## Features + +* TODO + +### Credits + +This package was created with [Cookiecutter(https://github.com/audreyr/cookiecutter) and the [annalise.ai cookiecutter](https://github.com/AnnaliseAI/pythonpackagecookie) project template. diff --git a/README.rst b/README.rst deleted file mode 100644 index 4c9b321..0000000 --- a/README.rst +++ /dev/null @@ -1,18 +0,0 @@ -======= -s3pkgup -======= - -Python Boilerplate contains all the boilerplate you need to create a Python package. - -Features --------- - -* TODO - -Credits -------- - -This package was created with Cookiecutter_ and the `harrison.ai cookiecutter`_ project template. - -.. _Cookiecutter: https://github.com/audreyr/cookiecutter -.. _`harrison.ai cookiecutter`: https://bitbucket.org/harrison-ai/pythonpackagecookie/src/master/ diff --git a/docs/conf.py b/docs/conf.py index 4f27134..e53704c 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# s3pkgup documentation build configuration file, created by +# pips3 documentation build configuration file, created by # sphinx-quickstart on Fri Jun 9 13:47:02 2017. # # This file is execfile()d with the current directory set to its @@ -22,7 +22,7 @@ import sys sys.path.insert(0, os.path.abspath('..')) -import s3pkgup +import pips3 # -- General configuration --------------------------------------------- @@ -63,7 +63,7 @@ def setup(app): master_doc = 'index' # General information about the project. -project = u's3pkgup' +project = u'pips3' copyright = u"2020, Ben Johnston" author = u"Ben Johnston" @@ -72,9 +72,9 @@ def setup(app): # the built documents. # # The short X.Y version. -version = s3pkgup.__version__ +version = pips3.__version__ # The full version, including alpha/beta/rc tags. -release = s3pkgup.__version__ +release = pips3.__version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -117,7 +117,7 @@ def setup(app): # -- Options for HTMLHelp output --------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = 's3pkgupdoc' +htmlhelp_basename = 'pips3doc' # -- Options for LaTeX output ------------------------------------------ @@ -144,8 +144,8 @@ def setup(app): # (source start file, target name, title, author, documentclass # [howto, manual, or own class]). latex_documents = [ - (master_doc, 's3pkgup.tex', - u's3pkgup Documentation', + (master_doc, 'pips3.tex', + u'pips3 Documentation', u'Ben Johnston', 'manual'), ] @@ -155,8 +155,8 @@ def setup(app): # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 's3pkgup', - u's3pkgup Documentation', + (master_doc, 'pips3', + u'pips3 Documentation', [author], 1) ] @@ -167,10 +167,10 @@ def setup(app): # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 's3pkgup', - u's3pkgup Documentation', + (master_doc, 'pips3', + u'pips3 Documentation', author, - 's3pkgup', + 'pips3', 'One line description of project.', 'Miscellaneous'), ] diff --git a/docs/index.rst b/docs/index.rst index 54d3c32..090b3ad 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,4 +1,4 @@ -Welcome to s3pkgup's documentation! +Welcome to pips3's documentation! ============================================================ .. toctree:: diff --git a/docs/installation.rst b/docs/installation.rst index e7df546..6caa637 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -8,13 +8,13 @@ Installation Stable release -------------- -To install s3pkgup, run this command in your terminal: +To install pips3, run this command in your terminal: .. code-block:: console - $ pip install s3pkgup + $ pip install pips3 -This is the preferred method to install s3pkgup, as it will always install the most recent stable release. +This is the preferred method to install pips3, as it will always install the most recent stable release. If you don't have `pip`_ installed, this `Python installation guide`_ can guide you through the process. @@ -26,19 +26,19 @@ you through the process. From sources ------------ -The sources for s3pkgup can be downloaded from the `Bitbucket repo`_. +The sources for pips3 can be downloaded from the `Bitbucket repo`_. You can either clone the public repository: .. code-block:: console - $ git clone git://bitbucket.org/harrison-ai/s3pkgup + $ git clone git://bitbucket.org/harrison-ai/pips3 Or download the `tarball`_: .. code-block:: console - $ curl -OJL https://bitbucket.org/harrison-ai/s3pkgup/downloads + $ curl -OJL https://bitbucket.org/harrison-ai/pips3/downloads Once you have a copy of the source, you can install it with: @@ -47,5 +47,5 @@ Once you have a copy of the source, you can install it with: $ python setup.py install -.. _Bitbucket repo: https://bitbucket.org/harrison-ai/s3pkgup -.. _tarball: https://bitbucket.org/harrison-ai/s3pkgup/downloads +.. _Bitbucket repo: https://bitbucket.org/harrison-ai/pips3 +.. _tarball: https://bitbucket.org/harrison-ai/pips3/downloads diff --git a/docs/make.bat b/docs/make.bat index 5904496..3eb4e67 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -9,7 +9,7 @@ if "%SPHINXBUILD%" == "" ( ) set SOURCEDIR=. set BUILDDIR=_build -set SPHINXPROJ=s3pkgup +set SPHINXPROJ=pips3 if "%1" == "" goto help diff --git a/docs/usage.rst b/docs/usage.rst index b2d2f08..0c30978 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -2,6 +2,6 @@ Usage ===== -To use s3pkgup in a project:: +To use pips3 in a project:: - import s3pkgup + import pips3 diff --git a/pips3/__init__.py b/pips3/__init__.py new file mode 100644 index 0000000..ddf18b6 --- /dev/null +++ b/pips3/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +"""Top-level package for pips3.""" + +__author__ = """Ben Johnston""" +__email__ = 'ben.johnston@annalise.ai' + +from ._version import get_versions + +__version__ = get_versions()['version'] +del get_versions + +from pips3.base import PipS3, publish_packages diff --git a/s3pkgup/_version.py b/pips3/_version.py similarity index 100% rename from s3pkgup/_version.py rename to pips3/_version.py diff --git a/pips3/base.py b/pips3/base.py new file mode 100644 index 0000000..73e3a8f --- /dev/null +++ b/pips3/base.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Base Class""" + +import logging +import os +import sys +from glob import glob +from typing import Iterable, List, Union + +import boto3 + +from pips3.exceptions import PackageExistsException + +s3 = boto3.client("s3") + +INDEX_TEMPLATE_INTO = "\n\n " +INDEX_TEMPLATE_OUTTRO = "\n \n" + +logging.basicConfig( + format="%(name)s - %(levelname)s - %(message)s", + # stream=sys.stdout, + level=logging.INFO) +logger = logging.getLogger("pips3") + + +class PipS3: + """PipS3 + + Use S3 compliant object storage as a simple pypi repository + + Args: + endpoint (str): The storage endpoint e.g. https://some-bucket.s3-website-ap-southeast-2.amazonaws.com + bucket (str): The name of the bucket storing the build artifacts + prefix (str, optional): The prefix to apply to all s3 keys. Defaults to 'simple' + s3_client (boto3.Session.client, optional): A boto3 S3 session client. Defaults to None, whereby a new + sesion client will be created using the standard AWS + [credentials configuration](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html) + """ + def __init__( + self, + endpoint: str, + bucket: str, + prefix: str = 'simple', # The pypi default https://pypi.org/simple + s3_client: Union[boto3.Session.client, None] = None, + ): + self.endpoint = endpoint + self.bucket = bucket + self.prefix = prefix + + if s3_client is None: + s3_client = boto3.client('s3') + self.s3_client = s3_client + + @staticmethod + def find_package_files( + path: str = 'dist', + pkg_ext: Union[List[str], None] = None, + ) -> Iterable[str]: + """Find Python Packages for Upload + + Args: + path (str, optional): Path to search for packages. Defaults to 'dist'. + pkg_ext (List[str], optional): Valid file extensions of Python packages to upload. Defaults to + None to use standard Python package extensions, .gz and .whl. + + Yields: + Iterable[str]: The paths to the built packages + """ + + if pkg_ext is None: + pkg_ext = ['.gz', '.whl'] + + search_str = f'{path}/*[' + for ext in pkg_ext: + search_str += f'({ext})||' + search_str = search_str[:-2] + ']' + + for pref in glob(search_str): + yield pref + + def list_keys( + self, + max_keys: int = 1000, + project_name: Union[str, None] = None, + continuation_token: Union[str, None] = None) -> Iterable[str]: + """List keys in S3 + + Args: + max_keys (int, optional): The number of keys to retrieve per attempt. Defaults to 1000. + project_name (str, optional): List the keys for the specified project only. Defaults to None, + continuation_token (str, optional): The boto3 continuation token for the next series of responses. Defaults to 1000. + + Yields: + Iterable[str]: The paths to the keys in the bucket + """ + + kwargs = { + "Bucket": self.bucket, + "Prefix": self.prefix + if project_name is None else f"{self.prefix}/{project_name}", + "MaxKeys": max_keys, + } + if continuation_token is not None: + kwargs['ContinuationToken'] = continuation_token + + logger.info("Listing objects in s3://%s/%s", self.bucket, self.prefix) + response = self.s3_client.list_objects_v2(**kwargs) + + for key in response['Contents']: + yield key["Key"] + + if 'NextContinuationToken' in response: + for key in self.list_keys( + max_keys=max_keys, + continuation_token=response['NextContinuationToken']): + yield key + + def generate_index(self, keys: Union[Iterable[str], None] = None) -> str: + """Generate a pypi index file + + Args: + keys (Union[Iterable[str], None], optional): The keys of the s3 bucket. + Defaults to None. If set to None, a list of S3 keys will be generated. + + Returns: + str: The rendered template + """ + + raw_keys = self.list_keys() if keys is None else keys + + template = INDEX_TEMPLATE_INTO + + for key in raw_keys: + basename = os.path.basename(key) + template += f"\n {basename}" + + template += INDEX_TEMPLATE_OUTTRO + return template + + def upload_package(self, pkg_path: str, package_name: str): + """Upload the package to S3 + + Args: + pkg_path (str): The path to the package file to upload + package_name (str): The name of the package + + Raises: + PackageExistsException: If a package file with the same name + already exists for this project + """ + + key = os.path.join(self.prefix, package_name, + os.path.basename(pkg_path)) + logger.info("Uploading %s to s3://%s/%s", pkg_path, self.bucket, key) + + # If the file already exists, do not override + try: + self.s3_client.head_object(Bucket=self.bucket, Key=key) + + # The files does not exist we can upload + except self.s3_client.exceptions.ClientError: + self.s3_client.upload_file(pkg_path, + self.bucket, + key, + ExtraArgs={"ACL": "public-read"}) + return + + raise PackageExistsException( + "Package %s already exists in the S3 Bucket for the project %s", + os.path.basename(pkg_path), package_name) + + def upload_index(self, package_name: str, index: Union[str, None] = None): + """Upload the index file + + Args: + package_name (str): The name of the package + index (Union[str, None], optional): The contents of the index file. Defaults + to None, where an index file will be automatically generated. + """ + generated_index = self.generate_index() if index is None else index + key = f'{self.prefix}/{package_name}/index.html' + logger.info("Uploading index to s3://%s/%s", self.bucket, key) + self.s3_client.put_object(Bucket=self.bucket, + Key=key, + Body=generated_index.encode('utf-8'), + ACL="public-read", + ContentType="text/html") + + +def publish_packages(endpoint: str, bucket: str): + """Publish current package files + + Args: + endpoint (str): The endpoint for the S3-like service + bucket (str): The name of the bucket to use + """ + + uploader = PipS3(endpoint, bucket) + + package_name = None + + for upload_file in PipS3.find_package_files(): + + # Get the package name + if package_name is None: + + package_name = os.path.basename(upload_file).split('-')[0] + package_name = package_name.replace('_', '-') + + uploader.upload_package(upload_file, package_name) + + # Update the index + uploader.upload_index(package_name) diff --git a/pips3/cli.py b/pips3/cli.py new file mode 100644 index 0000000..511eb68 --- /dev/null +++ b/pips3/cli.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +"""Console script for pips3.""" +import os +import sys + +import click + +from pips3 import publish_packages +from pips3.exceptions import InvalidConfig + + +@click.command() +@click.option('--endpoint', default=None, help='S3 Endpoint') +@click.option('--bucket', default=None, help='S3 Bucket') +def main(endpoint, bucket): + """Console script for pips3.""" + + # Try a number of options for determining configuration values + endpoint = os.getenv('PIPS3_ENDPOINT') if endpoint is None else endpoint + bucket = os.getenv('PIPS3_BUCKET') if bucket is None else bucket + + # TODO: #2 Allow retrieving of values from pip.conf + + # If the values are still not specified raise errors + if endpoint is None: + raise InvalidConfig("Error!!! S3 endpoint not specified") + + if bucket is None: + raise InvalidConfig("Error!!! S3 bucket not specified") + + publish_packages(endpoint, bucket) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) # pragma: no cover diff --git a/pips3/exceptions.py b/pips3/exceptions.py new file mode 100644 index 0000000..4667643 --- /dev/null +++ b/pips3/exceptions.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Exceptions""" + + +class PackageExistsException(Exception): + """Package already exists""" + + +class InvalidConfig(Exception): + """Invalid configuration""" \ No newline at end of file diff --git a/pytest.ini b/pytest.ini index e0a3ef3..6b18f81 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,4 @@ [pytest] python_files = tests.py test_*.py *_tests.py pytest_plugins = "pytest_cov", "pep8" -addopts = --doctest-modules --cov-config=.coveragerc --cov=s3pkgup --cov-report=term-missing +addopts = --doctest-modules --cov-config=.coveragerc --cov=pips3 --cov-report=term-missing \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e69de29..8580c3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1,63 @@ +attrs==20.2.0 +aws-sam-translator==1.27.0 +aws-xray-sdk==2.6.0 +boto==2.49.0 +boto3==1.16.4 +botocore==1.19.4 +certifi==2020.6.20 +cffi==1.14.3 +cfn-lint==0.39.0 +chardet==3.0.4 +click==7.1.2 +coverage==5.3 +cryptography==3.2 +decorator==4.4.2 +docker==4.3.1 +ecdsa==0.14.1 +fire==0.3.1 +future==0.18.2 +idna==2.10 +importlib-metadata==2.0.0 +iniconfig==1.1.1 +Jinja2==2.11.2 +jmespath==0.10.0 +jsondiff==1.2.0 +jsonpatch==1.26 +jsonpickle==1.4.1 +jsonpointer==2.0 +jsonschema==3.2.0 +junit-xml==1.9 +MarkupSafe==1.1.1 +mock==4.0.2 +more-itertools==8.5.0 +moto==1.3.16 +networkx==2.5 +packaging==20.4 +pluggy==0.13.1 +py==1.9.0 +pyasn1==0.4.8 +pycparser==2.20 +pyparsing==2.4.7 +pyrsistent==0.17.3 +pytest==6.1.1 +pytest-cov==2.10.1 +python-dateutil==2.8.1 +python-jose==3.2.0 +pytz==2020.1 +PyYAML==5.3.1 +requests==2.24.0 +responses==0.12.0 +rsa==4.6 +s3transfer==0.3.3 +six==1.15.0 +sshpubkeys==3.1.0 +termcolor==1.1.0 +toml==0.10.1 +urllib3==1.25.11 +versioneer==0.18 +websocket-client==0.57.0 +Werkzeug==1.0.1 +wrapt==1.12.1 +xmltodict==0.12.0 +yapf==0.30.0 +zipp==3.4.0 diff --git a/s3pkgup/__init__.py b/s3pkgup/__init__.py deleted file mode 100644 index 5a30a3c..0000000 --- a/s3pkgup/__init__.py +++ /dev/null @@ -1,98 +0,0 @@ -import glob -import logging -import os -import sys - -import boto3 -import pkg_resources -from jinja2 import Template - -s3 = boto3.client("s3") - -# hardcoded values for now -https_endpoint = "http://pypi-prod-annalise-ai.s3-website-ap-southeast-2.amazonaws.com" -bucket = "pypi-prod-annalise-ai" -root_prefix = "simple" - - -index_template = pkg_resources.resource_filename(__name__, "data/index.html.j2") - -artifacts_path = "./artifacts" -os.makedirs(artifacts_path, exist_ok=True) - -logging.basicConfig(stream=sys.stdout, level=logging.INFO) -logger = logging.getLogger("s3pkg") - - -def get_packages(): - for filename in glob.glob('dist/*.*'): - ext = os.path.splitext(filename)[1] - if ext in ['.gz', '.whl']: - yield filename - - -def get_key_name(wheel, project_name): - object = os.path.basename(wheel) - key = f"{root_prefix}/{project_name}/{object}" - return key - - -def upload_to_s3(wheel, key): - try: - logger.info("Uploading %s to S3", key) - s3.upload_file(wheel, bucket, key, ExtraArgs={"ACL": "public-read"}) - except Exception as e: - print(str(e)) - - -def list_keys(prefix): - kwargs = {"Bucket": bucket, "Prefix": prefix, "Delimiter": "/"} - resp = s3.list_objects_v2(**kwargs) - return resp - - -def generate_template(bucket_listing): - listing = [ - x["Key"] for x in bucket_listing["Contents"] if "index.html" not in x["Key"] - ] - keys = [] - for i in listing: - bn = os.path.basename(i) - uri = f"{https_endpoint}/{i}" - keys.append({"uri": uri, "bn": bn}) - with open(index_template) as f: - template = Template(f.read()) - output = template.render(keys=keys) - with open(os.path.join(artifacts_path, "index.html"), "w") as f: - f.write(output) - - -def upload_index(project_name): - uri = f"{root_prefix}/{project_name}/index.html" - try: - s3.upload_file( - os.path.join(artifacts_path, "index.html"), - bucket, - uri, - ExtraArgs={"ACL": "public-read", "ContentType": "text/html"}, - ) - except Exception as e: - print(str(e)) - - -def publish_packages(project_name=None): - - if project_name is None: - project_name = os.environ["BUILDKITE_PIPELINE_SLUG"] - - at_least_one = False - for pkg in get_packages(): - key = get_key_name(pkg, project_name) - upload_to_s3(pkg, key) - prefix = f"{root_prefix}/{project_name}/" - bucket_listing = list_keys(prefix) - index = generate_template(bucket_listing) - upload_index(project_name) - at_least_one = True - if not at_least_one: - raise EnvironmentError("0 packages published") diff --git a/s3pkgup/cli.py b/s3pkgup/cli.py deleted file mode 100644 index 2680672..0000000 --- a/s3pkgup/cli.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- - -"""Console script for s3pkgup.""" -import sys - -import click - -from s3pkgup import publish_packages - - -@click.command() -@click.option('--project', default=None, help='Project Name') -def main(project): - """Console script for s3pkgup.""" - publish_packages(project) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) # pragma: no cover diff --git a/s3pkgup/data/index.html.j2 b/s3pkgup/data/index.html.j2 deleted file mode 100644 index db4a222..0000000 --- a/s3pkgup/data/index.html.j2 +++ /dev/null @@ -1,8 +0,0 @@ - - - - {% for key in keys -%} - {{ key.bn }} - {% endfor %} - - diff --git a/scripts/build.sh b/scripts/build.sh deleted file mode 100755 index a348338..0000000 --- a/scripts/build.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# Build wheel - -set -euo pipefail - -export LANG=en_AU.utf8 - -/opt/python/cp37-cp37m/bin/python setup.py bdist_wheel - -/opt/python/cp37-cp37m/bin/pip install dist/*.whl - -/opt/python/cp37-cp37m/bin/s3pkgup diff --git a/setup.cfg b/setup.cfg index 3381fd1..8fd319c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,6 +14,6 @@ collect_ignore = ['setup.py'] [versioneer] VCS = git style = pep440 -versionfile_source = s3pkgup/_version.py -versionfile_build = s3pkgup/_version.py +versionfile_source = pips3/_version.py +versionfile_build = pips3/_version.py tag_prefix = diff --git a/setup.py b/setup.py index 0d2bdf0..8fe0e4a 100644 --- a/setup.py +++ b/setup.py @@ -1,18 +1,16 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- - """The setup script.""" import versioneer from setuptools import find_packages, setup -with open('README.rst') as readme_file: - readme = readme_file.read() +with open('README.md') as readme_file: + long_description = readme_file.read() requirements = [ - 'boto3>=1.12.20', - 'Jinja2>=2.11.1', - 'Click>=7.0', + 'boto3>=1.16.4', + 'Click>=7.1.2', 'versioneer>=0.18', ] @@ -21,8 +19,8 @@ ] test_requirements = [ - 'pytest>=5.1.2', - 'pytest-cov>=2.7.1', + 'pytest>=6.1.1', + 'pytest-cov>=20.10.1', ] setup( @@ -30,28 +28,30 @@ author_email='ben.johnston@annalise.ai', python_requires='!=2.*, >=3.6', classifiers=[ - 'Development Status :: 2 - Pre-Alpha', 'Intended Audience :: Developers', 'Natural Language :: English', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', ], - description="Python Boilerplate contains all the boilerplate you need to create a Python package.", + description= + "A handy package for creating a simple pypi repo in S3 compliant object storage.", entry_points={ 'console_scripts': [ - 's3pkgup=s3pkgup.cli:main', + 'pips3=pips3.cli:main', ], }, install_requires=requirements, + long_description=long_description, + long_description_content_type="text/markdown", include_package_data=True, - keywords='s3pkgup', - name='s3pkgup', - package_data={'s3pkgup': ['data/index.html.j2']}, - packages=find_packages(include=['s3pkgup']), + keywords=['pips3', 's3', 'pip', 'pypi'], + name='pips3', + packages=find_packages(include=['pips3']), setup_requires=setup_requirements, test_suite='tests', tests_require=test_requirements, - url='https://bitbucket.org/harrison-ai/s3pkgup', + url='https://github.com/AnnaliseAI/pipS3', version=versioneer.get_version(), cmdclass=versioneer.get_cmdclass(), zip_safe=False, diff --git a/tests/__init__.py b/tests/__init__.py index b8a216e..2a21639 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- -"""Unit test package for s3pkgup.""" +"""Unit test package for pips3.""" diff --git a/tests/assets/pips3-0.1.0.dev0.whl b/tests/assets/pips3-0.1.0.dev0.whl new file mode 100644 index 0000000..e69de29 diff --git a/tests/assets/pips3-0.1.0.whl b/tests/assets/pips3-0.1.0.whl new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_base.py b/tests/test_base.py new file mode 100644 index 0000000..95037fe --- /dev/null +++ b/tests/test_base.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Tests for `pips3` package.""" + +import boto3 +import pytest +from unittest.mock import patch +from moto import mock_s3 + +from pips3 import PipS3, publish_packages +from pips3.exceptions import PackageExistsException + +ENDPOINT_URL = "http://localhost:9000" +BUCKET = 'pips3' +PREFIX = 'listing' + + +@mock_s3 +def test_list_bucket(): + """Test listing bucket""" + + s3_client = boto3.client('s3', region_name='us-east-1') + + ## Populate the bucket with some files + s3_client.create_bucket(Bucket=BUCKET) + + expected_keys = [] + + for i in range(11): + + key = f'{PREFIX}/{i}.bin' + s3_client.put_object( + Bucket=BUCKET, + Body=f'{i}'.encode(), + Key=key, + ) + expected_keys.append(key) + + # Create PipS3 object + obj = PipS3(ENDPOINT_URL, BUCKET, PREFIX, s3_client) + + keys = [key for key in obj.list_keys(max_keys=2)] + + expected_keys.sort() + keys.sort() + + assert keys == expected_keys + + +def test_find_packages(): + """Test finding packages""" + + packages = [ + pkg for pkg in PipS3.find_package_files(path='docs', + pkg_ext=['.rst', '.bat']) + ] + + expected = [ + 'docs/index.rst', 'docs/installation.rst', 'docs/readme.rst', + 'docs/usage.rst', 'docs/make.bat' + ] + + packages.sort() + expected.sort() + + assert packages == expected + + +def test_generate_index_template(): + """Test generating index template""" + + obj = PipS3(ENDPOINT_URL, BUCKET, PREFIX) + some_keys = [f'{PREFIX}/foo.whl', f'{PREFIX}/bar.whl'] + + expected_template = f""" + + + foo.whl + bar.whl + +""" + + assert obj.generate_index(keys=some_keys) == expected_template + + +@mock_s3 +def test_upload_package(): + """Test uploading a package to the correct project""" + + s3_client = boto3.client('s3', region_name='us-east-1') + s3_client.create_bucket(Bucket=BUCKET) + + fake_pkg = 'README.md' + + obj = PipS3(ENDPOINT_URL, BUCKET, PREFIX) + + project_name = "pips3" + + obj.upload_package(fake_pkg, project_name) + + # Check the file exists at the expected path + s3_client.head_object(Bucket=BUCKET, + Key=f'{PREFIX}/{project_name}/{fake_pkg}') + + # Test error when trying to upload twice + with pytest.raises(PackageExistsException): + obj.upload_package(fake_pkg, project_name) + + +@mock_s3 +def test_upload_index(): + """Test uploading an index""" + + s3_client = boto3.client('s3', region_name='us-east-1') + s3_client.create_bucket(Bucket=BUCKET) + + obj = PipS3(ENDPOINT_URL, BUCKET, PREFIX) + + project_name = "pips3" + index = "index.html" + + obj.upload_index(project_name, index) + + # Check the file exists at the expected path + s3_client.head_object(Bucket=BUCKET, + Key=f'{PREFIX}/{project_name}/index.html') + + +@mock_s3 +@patch('pips3.base.PipS3.find_package_files', + return_value=[ + 'tests/assets/pips3-0.1.0.dev0.whl', + 'tests/assets/pips3-0.1.0.whl', + ]) +def test_publish_packages(files_mock): + """Publish packages""" + + s3_client = boto3.client('s3', region_name='us-east-1') + s3_client.create_bucket(Bucket=BUCKET) + + prefix = "simple" + + publish_packages(ENDPOINT_URL, BUCKET) + + # Get the index + index = s3_client.get_object(Bucket=BUCKET, + Key=f'{prefix}/pips3/index.html') + index = index['Body'].read().decode('utf-8') + + expected_index = f""" + + + pips3-0.1.0.dev0.whl + pips3-0.1.0.whl + +""" + + assert index == expected_index diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..6298702 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Tests for `pips3` cli.""" + +from unittest.mock import patch + +from click.testing import CliRunner +import pytest + +from pips3 import cli +from pips3.exceptions import InvalidConfig + +URL = 'http://localhost:9000' +BUCKET = 'somebucket' +PACKAGE = 'pips3' + + +@patch('pips3.cli.publish_packages') +def test_command_line_interface(publish_mock): + """Test the CLI.""" + runner = CliRunner() + + result = runner.invoke(cli.main, ['--endpoint', URL, '--bucket', BUCKET]) + + assert result.exit_code == 0 + publish_mock.assert_called_with(URL, BUCKET) + + +@patch('pips3.cli.publish_packages') +def test_command_line_interface_envars(publish_mock, monkeypatch): + """Test the CLI using envars""" + + monkeypatch.setenv('PIPS3_ENDPOINT', URL) + monkeypatch.setenv('PIPS3_BUCKET', BUCKET) + + runner = CliRunner() + + result = runner.invoke(cli.main) + + assert result.exit_code == 0 + publish_mock.assert_called_with(URL, BUCKET) + + +def test_cli_errors(monkeypatch): + """Test the cli responds to errors""" + + runner = CliRunner() + + result = runner.invoke(cli.main) + + assert result.exit_code != 0 + assert isinstance(result.exception, InvalidConfig) + assert 'endpoint not specified' in str(result.exception) + + monkeypatch.setenv('PIPS3_ENDPOINT', URL) + result = runner.invoke(cli.main) + + assert result.exit_code != 0 + assert isinstance(result.exception, InvalidConfig) + assert 'bucket not specified' in str(result.exception) + + monkeypatch.setenv('PIPS3_BUCKET', URL) + result = runner.invoke(cli.main) \ No newline at end of file diff --git a/tests/test_s3pkgup.py b/tests/test_s3pkgup.py deleted file mode 100644 index b586771..0000000 --- a/tests/test_s3pkgup.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -"""Tests for `s3pkgup` package.""" - -import pytest -from click.testing import CliRunner - -from s3pkgup import s3pkgup -from s3pkgup import cli - -@pytest.fixture -def response(): - """Sample pytest fixture. - - See more at: http://doc.pytest.org/en/latest/fixture.html - """ - # import requests - # return requests.get('https://github.com/audreyr/cookiecutter-pypackage') - - -def test_content(response): - """Sample pytest test function with the pytest fixture as an argument.""" - # from bs4 import BeautifulSoup - # assert 'GitHub' in BeautifulSoup(response.content).title.string - - -def test_command_line_interface(): - """Test the CLI.""" - runner = CliRunner() - result = runner.invoke(cli.main) - assert result.exit_code == 0 - assert 's3pkgup.cli.main' in result.output - help_result = runner.invoke(cli.main, ['--help']) - assert help_result.exit_code == 0 - assert '--help Show this message and exit.' in help_result.output