diff --git a/requirements.txt b/requirements.txt index 8bb979d..425e37e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,6 @@ gtrending>=0.3.0,<1.0.0 requests>=2.22.0 rich>=4.0.0,<11.0.0 xdg>=5.1.1,<6.0.0 +matplotlib=3.2.2 +pandas=1.1.5 +PyGithub=1.55.0 \ No newline at end of file diff --git a/starcli/__main__.py b/starcli/__main__.py index 5d8141b..52937d7 100644 --- a/starcli/__main__.py +++ b/starcli/__main__.py @@ -15,6 +15,7 @@ search_github_trending, search_error, status_actions, + draw_stargraph, ) @@ -24,7 +25,8 @@ @click.command() -@click.option("--lang", "-l", type=str, default="", help="Language filter eg: python") +@click.option("--lang", "-l", type=str, default="", + help="Language filter eg: python") @click.option( "--spoken-language", "-S", @@ -32,12 +34,11 @@ default="", help="Spoken Language filter eg: en for English, zh for Chinese", ) -@click.option( - "--created", - "-c", - default="", - help="Specify repo creation date in YYYY-MM-DD, use >date, <=date etc to be more specific.", -) +@click.option("--created", + "-c", + default="", + help="Specify repo creation date in YYYY-MM-DD, use >date, <=date etc to be more specific.", + ) @click.option( "--topic", "-t", @@ -83,12 +84,14 @@ is_flag=True, help="Print the actual stats number (1300 instead of 1.3k)", ) -@click.option( - "--date-range", - "-d", - type=click.Choice(["today", "this-week", "this-month"], case_sensitive=False), - help="View stars received within time, choose from: today, this-week, this-month. Uses GitHub trending for fetching results, hence some other filter options may not work.", -) +@click.option("--date-range", + "-d", + type=click.Choice(["today", + "this-week", + "this-month"], + case_sensitive=False), + help="View stars received within time, choose from: today, this-week, this-month. Uses GitHub trending for fetching results, hence some other filter options may not work.", + ) @click.option( "--user", "-u", @@ -102,6 +105,33 @@ default="", help="Optionally use GitHub personal access token in the format 'username:password'.", ) +@click.option("--stargraph", + "-G", + is_flag=True, + default=False, + help="Display Star graph over time, Sample Format : starcli --stargraph --auth 'your-username:token' --repo 'hedyhli/starcli' --savepath '/content/' --duration 'yearly'/'monthly'", + ) +@click.option( + "--duration", + "-D", + type=str, + default="", + help="Save Star graph in the path in the format 'monthly' or 'yearly", +) +@click.option( + "--savepath", + "-H", + type=str, + default="", + help="Show monthly or yearly data", +) +@click.option( + "--repo", + "-R", + type=str, + default="", + help="Full repo name", +) @click.option( "--pager", "-P", @@ -109,7 +139,8 @@ default=False, help="Use $PAGER to page output. (put -r in $LESS to enable ANSI styles)", ) -@click.option("--debug", is_flag=True, default=False, help="Turn on debugging mode") +@click.option("--debug", is_flag=True, default=False, + help="Turn on debugging mode") def cli( lang, spoken_language, @@ -125,6 +156,10 @@ def cli( user, debug=False, auth="", + stargraph=False, + savepath="", + repo="", + duration="", pager=False, ): """Find trending repos on GitHub""" @@ -188,18 +223,38 @@ def cli( return else: # Cache results tmp_repos.append({"time": str(datetime.now())}) - with open(CACHED_RESULT_PATH, "a+") as f: - if os.path.getsize(CACHED_RESULT_PATH) == 0: # file is empty - result_dict = {options_key: tmp_repos} - f.write(json.dumps(result_dict, indent=4)) - else: # file is not empty - f.seek(0) - result_dict = json.load(f) - result_dict[options_key] = tmp_repos - f.truncate(0) - f.write(json.dumps(result_dict, indent=4)) + try: + with open(CACHED_RESULT_PATH, "a+") as f: + if os.path.getsize( + CACHED_RESULT_PATH) == 0: # file is empty + result_dict = {options_key: tmp_repos} + f.write(json.dumps(result_dict, indent=4)) + else: # file is not empty + f.seek(0) + result_dict = json.load(f) + result_dict[options_key] = tmp_repos + f.truncate(0) + f.write(json.dumps(result_dict, indent=4)) + except BaseException: + pass repos = tmp_repos[0:limit_results] + if stargraph: + if auth and not re.search(".:.", auth): # Check authentication format + click.secho( + f"Invalid authentication format: {auth} must be 'username:token'", + fg="bright_red", + ) + auth = None + + if repo and not re.search("./.", repo): # Check authentication format + click.secho( + f"Invalid authentication format: {repo} must be 'username/reponame'", + fg="bright_red", + ) + repo = None + + draw_stargraph(repo, auth, savepath, duration) if not long_stats: # shorten the stat counts when not --long-stats for repo in repos: diff --git a/starcli/search.py b/starcli/search.py index d07a82c..6fc424e 100644 --- a/starcli/search.py +++ b/starcli/search.py @@ -6,6 +6,12 @@ import logging from random import randint import re +import os +import math +import github +from github import Github +import pandas as pd +import matplotlib.pyplot as plt # Third party imports import requests @@ -17,7 +23,10 @@ API_URL = "https://api.github.com/search/repositories" -date_range_map = {"today": "daily", "this-week": "weekly", "this-month": "monthly"} +date_range_map = { + "today": "daily", + "this-week": "weekly", + "this-month": "monthly"} status_actions = { "retry": "Failed to retrieve data. Retrying in ", @@ -107,7 +116,7 @@ def get_valid_request(url, auth=""): secho("Internet connection error...", fg="bright_red") return None - if not request.status_code in (200, 202): + if request.status_code not in (200, 202): handling_code = search_error(request.status_code) if handling_code == "retry": for i in range(15, 0, -1): @@ -121,7 +130,9 @@ def get_valid_request(url, auth=""): secho(status_actions[handling_code], fg="bright_yellow") return None else: - secho("An invalid handling code was returned.", fg="bright_red") + secho( + "An invalid handling code was returned.", + fg="bright_red") return None else: break @@ -181,9 +192,8 @@ def search( if not created: # if created not provided # creation date: the time now minus a random number of days # 100 to 400 days - which was stored in day_range - created_str = ">=" + (datetime.utcnow() + timedelta(days=day_range)).strftime( - date_format - ) + created_str = ">=" + (datetime.utcnow() + + timedelta(days=day_range)).strftime(date_format) else: # if created is provided created_str = get_date(created) if not created_str: @@ -192,9 +202,8 @@ def search( if not pushed: # if pushed not provided # pushed date: start, is the time now minus a random number of days # 100 to 400 days - which was stored in day_range - pushed_str = ">=" + (datetime.utcnow() + timedelta(days=day_range)).strftime( - date_format - ) + pushed_str = ">=" + (datetime.utcnow() + + timedelta(days=day_range)).strftime(date_format) else: # if pushed is provided pushed_str = get_date(pushed) if not pushed_str: @@ -207,10 +216,12 @@ def search( query += f"stars:{stars}+created:{created_str}" # construct query query += f"+pushed:{pushed_str}" # add pushed info to query - query += f"+language:{language}" if language else "" # add language to query + # add language to query + query += f"+language:{language}" if language else "" query += f"".join(["+topic:" + i for i in topics]) # add topics to query - url = f"{API_URL}?q={query}&sort=stars&order={order}" # use query to construct url + # use query to construct url + url = f"{API_URL}?q={query}&sort=stars&order={order}" if debug: logger.debug("Search: url:" + url) # print the url when debugging if debug and auth: @@ -226,8 +237,11 @@ def search( def search_github_trending( - language=None, spoken_language=None, order="desc", stars=">=10", date_range=None -): + language=None, + spoken_language=None, + order="desc", + stars=">=10", + date_range=None): """Returns trending repositories from github trending page""" if date_range: gtrending_repo_list = fetch_repos( @@ -256,7 +270,93 @@ def search_github_trending( if order == "asc": return sorted(repositories, key=lambda repo: repo["stargazers_count"]) - return sorted(repositories, key=lambda repo: repo["stargazers_count"], reverse=True) + return sorted( + repositories, + key=lambda repo: repo["stargazers_count"], + reverse=True) + + +def draw_stargraph(reponame, auth, path, duration=""): + g = Github() + try: + repo = g.get_repo(reponame) + dictnry = dict() + request_header = {'Accept': 'application/vnd.github.v3.star+json'} + + for pages in range(1, math.ceil(repo.stargazers_count / 30) + 1): + # requesting the repo information via github REST API + request = 'https://api.github.com/repos/{}/stargazers?page={}'.format( + reponame, pages) + + r = requests.get( + request, + auth=( + auth.split(":")[0], + auth.split(":")[1]), + headers=request_header) + + for items in r.json(): + items['starred_at'] = items['starred_at'].replace( + 'T', " ") # cleaning the collected dates + items['starred_at'] = items['starred_at'].replace('Z', " ") + + # Converting to Year-Month format + month = '{:%Y-%m}'.format( + datetime.strptime( + items['starred_at'].split(" ")[0], + '%Y-%m-%d')) + # Adding the star counts monthly for each year + if month not in dictnry.keys(): + dictnry[month] = 1 + else: + dictnry[month] += 1 + + for i in range(len(dictnry.keys()) + ): # arranging the star counts cummulatively over the time period + if i >= 1: + dictnry[list(dictnry.keys())[i]] = dictnry[list(dictnry.keys())[ + i - 1]] + dictnry[list(dictnry.keys())[i]] + + data = pd.DataFrame(dictnry.items(), columns=['Year_Month', 'stars']) + if duration == "yearly": + data['Year_Month'] = pd.to_datetime(data['Year_Month']) + data = data.set_index('Year_Month') + + # Plotting the star counts over the time period collected from the API + fig, ax = plt.subplots( + figsize=( + len(dictnry) // 2, len(dictnry) // 3), constrained_layout=True) + ax.plot( + data.index, + data.stars, + color='tab:orange', + label='Stars Count') + ax.set_xlabel('Year-Month', size=30) + ax.set_ylabel('Stars', size=30) + ax.set_title('Stargazers', size=30) + plt.xticks(rotation=90) + ax.grid(True) + plt.yticks(fontsize=30) + plt.xticks(fontsize=25) + if max(dictnry.values()) < 1000: + fontsize = 10 + else: + fontsize = 20 + for a, b in zip(data.index, data.stars): + plt.text( + a, + b, + str(b), + verticalalignment='bottom', + horizontalalignment='right', + color='black', + fontsize=fontsize) + ax.legend(loc='upper left') + plt.savefig(os.path.join(path, "output.png")) + plt.close() + except github.UnknownObjectException as e: + if e.status == 404: + print("Repository Not found") def convert_repo_dict(gtrending_repo): @@ -266,7 +366,8 @@ def convert_repo_dict(gtrending_repo): repo_dict["html_url"] = gtrending_repo.get("url") repo_dict["stargazers_count"] = gtrending_repo.get("stars", -1) repo_dict["language"] = gtrending_repo.get("language") - # gtrending_repo has key `description` and value is empty string if it's empty + # gtrending_repo has key `description` and value is empty string if it's + # empty repo_dict["description"] = ( gtrending_repo.get("description") if gtrending_repo.get("description") != ""