Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Added new feature Stargraph of a repository in monthly or yearly pattern #105

Closed
wants to merge 1 commit into from
Closed

Added new feature Stargraph of a repository in monthly or yearly pattern #105

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,6 @@ gtrending>=0.3.0,<1.0.0
requests>=2.22.0
rich>=4.0.0,<11.0.0
xdg>=5.1.1,<6.0.0
matplotlib=3.2.2
pandas=1.1.5
PyGithub=1.55.0
103 changes: 79 additions & 24 deletions starcli/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
search_github_trending,
search_error,
status_actions,
draw_stargraph,
)


Expand All @@ -24,20 +25,20 @@


@click.command()
@click.option("--lang", "-l", type=str, default="", help="Language filter eg: python")
@click.option("--lang", "-l", type=str, default="",
help="Language filter eg: python")
@click.option(
"--spoken-language",
"-S",
type=str,
default="",
help="Spoken Language filter eg: en for English, zh for Chinese",
)
@click.option(
"--created",
"-c",
default="",
help="Specify repo creation date in YYYY-MM-DD, use >date, <=date etc to be more specific.",
)
@click.option("--created",
"-c",
default="",
help="Specify repo creation date in YYYY-MM-DD, use >date, <=date etc to be more specific.",
)
@click.option(
"--topic",
"-t",
Expand Down Expand Up @@ -83,12 +84,14 @@
is_flag=True,
help="Print the actual stats number (1300 instead of 1.3k)",
)
@click.option(
"--date-range",
"-d",
type=click.Choice(["today", "this-week", "this-month"], case_sensitive=False),
help="View stars received within time, choose from: today, this-week, this-month. Uses GitHub trending for fetching results, hence some other filter options may not work.",
)
@click.option("--date-range",
"-d",
type=click.Choice(["today",
"this-week",
"this-month"],
case_sensitive=False),
help="View stars received within time, choose from: today, this-week, this-month. Uses GitHub trending for fetching results, hence some other filter options may not work.",
)
@click.option(
"--user",
"-u",
Expand All @@ -102,14 +105,42 @@
default="",
help="Optionally use GitHub personal access token in the format 'username:password'.",
)
@click.option("--stargraph",
"-G",
is_flag=True,
default=False,
help="Display Star graph over time, Sample Format : starcli --stargraph --auth 'your-username:token' --repo 'hedyhli/starcli' --savepath '/content/' --duration 'yearly'/'monthly'",
)
@click.option(
"--duration",
"-D",
type=str,
default="",
help="Save Star graph in the path in the format 'monthly' or 'yearly",
)
@click.option(
"--savepath",
"-H",
type=str,
default="",
help="Show monthly or yearly data",
)
@click.option(
"--repo",
"-R",
type=str,
default="",
help="Full repo name",
)
@click.option(
"--pager",
"-P",
is_flag=True,
default=False,
help="Use $PAGER to page output. (put -r in $LESS to enable ANSI styles)",
)
@click.option("--debug", is_flag=True, default=False, help="Turn on debugging mode")
@click.option("--debug", is_flag=True, default=False,
help="Turn on debugging mode")
def cli(
lang,
spoken_language,
Expand All @@ -125,6 +156,10 @@ def cli(
user,
debug=False,
auth="",
stargraph=False,
savepath="",
repo="",
duration="",
pager=False,
):
"""Find trending repos on GitHub"""
Expand Down Expand Up @@ -188,18 +223,38 @@ def cli(
return
else: # Cache results
tmp_repos.append({"time": str(datetime.now())})
with open(CACHED_RESULT_PATH, "a+") as f:
if os.path.getsize(CACHED_RESULT_PATH) == 0: # file is empty
result_dict = {options_key: tmp_repos}
f.write(json.dumps(result_dict, indent=4))
else: # file is not empty
f.seek(0)
result_dict = json.load(f)
result_dict[options_key] = tmp_repos
f.truncate(0)
f.write(json.dumps(result_dict, indent=4))
try:
with open(CACHED_RESULT_PATH, "a+") as f:
if os.path.getsize(
CACHED_RESULT_PATH) == 0: # file is empty
result_dict = {options_key: tmp_repos}
f.write(json.dumps(result_dict, indent=4))
else: # file is not empty
f.seek(0)
result_dict = json.load(f)
result_dict[options_key] = tmp_repos
f.truncate(0)
f.write(json.dumps(result_dict, indent=4))
except BaseException:
pass

repos = tmp_repos[0:limit_results]
if stargraph:
if auth and not re.search(".:.", auth): # Check authentication format
click.secho(
f"Invalid authentication format: {auth} must be 'username:token'",
fg="bright_red",
)
auth = None

if repo and not re.search("./.", repo): # Check authentication format
click.secho(
f"Invalid authentication format: {repo} must be 'username/reponame'",
fg="bright_red",
)
repo = None

draw_stargraph(repo, auth, savepath, duration)

if not long_stats: # shorten the stat counts when not --long-stats
for repo in repos:
Expand Down
131 changes: 116 additions & 15 deletions starcli/search.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@
import logging
from random import randint
import re
import os
import math
import github
from github import Github
import pandas as pd
import matplotlib.pyplot as plt

# Third party imports
import requests
Expand All @@ -17,7 +23,10 @@

API_URL = "https://api.github.com/search/repositories"

date_range_map = {"today": "daily", "this-week": "weekly", "this-month": "monthly"}
date_range_map = {
"today": "daily",
"this-week": "weekly",
"this-month": "monthly"}

status_actions = {
"retry": "Failed to retrieve data. Retrying in ",
Expand Down Expand Up @@ -107,7 +116,7 @@ def get_valid_request(url, auth=""):
secho("Internet connection error...", fg="bright_red")
return None

if not request.status_code in (200, 202):
if request.status_code not in (200, 202):
handling_code = search_error(request.status_code)
if handling_code == "retry":
for i in range(15, 0, -1):
Expand All @@ -121,7 +130,9 @@ def get_valid_request(url, auth=""):
secho(status_actions[handling_code], fg="bright_yellow")
return None
else:
secho("An invalid handling code was returned.", fg="bright_red")
secho(
"An invalid handling code was returned.",
fg="bright_red")
return None
else:
break
Expand Down Expand Up @@ -181,9 +192,8 @@ def search(
if not created: # if created not provided
# creation date: the time now minus a random number of days
# 100 to 400 days - which was stored in day_range
created_str = ">=" + (datetime.utcnow() + timedelta(days=day_range)).strftime(
date_format
)
created_str = ">=" + (datetime.utcnow() +
timedelta(days=day_range)).strftime(date_format)
else: # if created is provided
created_str = get_date(created)
if not created_str:
Expand All @@ -192,9 +202,8 @@ def search(
if not pushed: # if pushed not provided
# pushed date: start, is the time now minus a random number of days
# 100 to 400 days - which was stored in day_range
pushed_str = ">=" + (datetime.utcnow() + timedelta(days=day_range)).strftime(
date_format
)
pushed_str = ">=" + (datetime.utcnow() +
timedelta(days=day_range)).strftime(date_format)
else: # if pushed is provided
pushed_str = get_date(pushed)
if not pushed_str:
Expand All @@ -207,10 +216,12 @@ def search(

query += f"stars:{stars}+created:{created_str}" # construct query
query += f"+pushed:{pushed_str}" # add pushed info to query
query += f"+language:{language}" if language else "" # add language to query
# add language to query
query += f"+language:{language}" if language else ""
query += f"".join(["+topic:" + i for i in topics]) # add topics to query

url = f"{API_URL}?q={query}&sort=stars&order={order}" # use query to construct url
# use query to construct url
url = f"{API_URL}?q={query}&sort=stars&order={order}"
if debug:
logger.debug("Search: url:" + url) # print the url when debugging
if debug and auth:
Expand All @@ -226,8 +237,11 @@ def search(


def search_github_trending(
language=None, spoken_language=None, order="desc", stars=">=10", date_range=None
):
language=None,
spoken_language=None,
order="desc",
stars=">=10",
date_range=None):
"""Returns trending repositories from github trending page"""
if date_range:
gtrending_repo_list = fetch_repos(
Expand Down Expand Up @@ -256,7 +270,93 @@ def search_github_trending(

if order == "asc":
return sorted(repositories, key=lambda repo: repo["stargazers_count"])
return sorted(repositories, key=lambda repo: repo["stargazers_count"], reverse=True)
return sorted(
repositories,
key=lambda repo: repo["stargazers_count"],
reverse=True)


def draw_stargraph(reponame, auth, path, duration=""):
g = Github()
try:
repo = g.get_repo(reponame)
dictnry = dict()
request_header = {'Accept': 'application/vnd.github.v3.star+json'}

for pages in range(1, math.ceil(repo.stargazers_count / 30) + 1):
# requesting the repo information via github REST API
request = 'https://api.github.com/repos/{}/stargazers?page={}'.format(
reponame, pages)

r = requests.get(
request,
auth=(
auth.split(":")[0],
auth.split(":")[1]),
headers=request_header)

for items in r.json():
items['starred_at'] = items['starred_at'].replace(
'T', " ") # cleaning the collected dates
items['starred_at'] = items['starred_at'].replace('Z', " ")

# Converting to Year-Month format
month = '{:%Y-%m}'.format(
datetime.strptime(
items['starred_at'].split(" ")[0],
'%Y-%m-%d'))
# Adding the star counts monthly for each year
if month not in dictnry.keys():
dictnry[month] = 1
else:
dictnry[month] += 1

for i in range(len(dictnry.keys())
): # arranging the star counts cummulatively over the time period
if i >= 1:
dictnry[list(dictnry.keys())[i]] = dictnry[list(dictnry.keys())[
i - 1]] + dictnry[list(dictnry.keys())[i]]

data = pd.DataFrame(dictnry.items(), columns=['Year_Month', 'stars'])
if duration == "yearly":
data['Year_Month'] = pd.to_datetime(data['Year_Month'])
data = data.set_index('Year_Month')

# Plotting the star counts over the time period collected from the API
fig, ax = plt.subplots(
figsize=(
len(dictnry) // 2, len(dictnry) // 3), constrained_layout=True)
ax.plot(
data.index,
data.stars,
color='tab:orange',
label='Stars Count')
ax.set_xlabel('Year-Month', size=30)
ax.set_ylabel('Stars', size=30)
ax.set_title('Stargazers', size=30)
plt.xticks(rotation=90)
ax.grid(True)
plt.yticks(fontsize=30)
plt.xticks(fontsize=25)
if max(dictnry.values()) < 1000:
fontsize = 10
else:
fontsize = 20
for a, b in zip(data.index, data.stars):
plt.text(
a,
b,
str(b),
verticalalignment='bottom',
horizontalalignment='right',
color='black',
fontsize=fontsize)
ax.legend(loc='upper left')
plt.savefig(os.path.join(path, "output.png"))
plt.close()
except github.UnknownObjectException as e:
if e.status == 404:
print("Repository Not found")


def convert_repo_dict(gtrending_repo):
Expand All @@ -266,7 +366,8 @@ def convert_repo_dict(gtrending_repo):
repo_dict["html_url"] = gtrending_repo.get("url")
repo_dict["stargazers_count"] = gtrending_repo.get("stars", -1)
repo_dict["language"] = gtrending_repo.get("language")
# gtrending_repo has key `description` and value is empty string if it's empty
# gtrending_repo has key `description` and value is empty string if it's
# empty
repo_dict["description"] = (
gtrending_repo.get("description")
if gtrending_repo.get("description") != ""
Expand Down