diff --git a/main.ipynb b/main.ipynb index 29ae20e..61f66c9 100644 --- a/main.ipynb +++ b/main.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Data management imports" + ] + }, { "cell_type": "code", "execution_count": null, @@ -9,6 +16,26 @@ "import pandas as kungfupanda # pandas for data manipulation and markdown\n", "from pandas import DataFrame # export\n", "\n", + "import argparse # For command line arguments when calling py script with flags\n", + "import pickle # for saving/loading json records and file \n", + " # modification date history\n", + "\n", + "from questionDataclass import questionDataclass as Question" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "OS and directory management imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "from os import listdir # for file retrieval and path calculations\n", "from os.path import isfile, join\n", "from os import stat\n", @@ -23,33 +50,60 @@ "from os.path import abspath, dirname # relative paths used are from this script's\n", "import sys # location rather than the calling location\n", " # e.g. if you call `python someFolder/main.py`\n", - " # then it will still work.\n", - "\n", - "import subprocess\n", - "\n", - "from os.path import getmtime, getctime # retreiving file creation/modification times\n", - "from datetime import datetime\n", - "import time\n", - "\n", + " # then it will still work." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Environment variable imports + file log and creation time imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "from os import getenv, environ # for environment variables\n", "from dotenv import load_dotenv, find_dotenv # for config purposes (.env file)\n", "\n", + "import subprocess # tracing git log history for ctimes and mtimes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from os.path import getmtime, getctime # retreiving file creation/modification times\n", + "from datetime import datetime\n", + "import time" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "QOL and anti-reundancy imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "from typing import Set, Dict, List, Tuple # misc. QOL imports\n", "from collections import defaultdict\n", "from icecream import ic # for debugging / outputs\n", "\n", - "from questionDataclass import questionDataclass as Question\n", - "\n", - "# TQDM import done below to check if this is \n", - "# a .py or .ipynb file\n", - "\n", "import re # for regex file name matching / question number matching\n", + "from functools import cache # for redundancy protection\n", "\n", - "import argparse # For command line arguments when calling py script with flags\n", - "import pickle # for saving/loading json records and file \n", - " # modification date history\n", - "\n", - "from functools import cache # for redundancy protection" + "# TQDM import done separately below after checking if this is a .py or .ipynb file" ] }, { @@ -73,10 +127,18 @@ "# loading env variables\n", "print('Default .env activated from script directory (.readme_updater/)')\n", "load_dotenv(find_dotenv(), override=True)\n", + "\n", "if '.env' in listdir('../') :\n", " print('.env found in ../ directory. Overriding default...')\n", - " load_dotenv(find_dotenv('../.env'), override=True)\n", - "\n", + " load_dotenv(find_dotenv('../.env'), override=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "# NOTE: if the script is being run from a jupyter notebook, then it should\n", "# already be in the correct directory.\n", "IS_NOTEBOOK = True\n", @@ -89,19 +151,36 @@ " print('Working directory already set to script location. No need for adjustment')\n", "except NameError:\n", " print('NameError')\n", - " pass\n", + " pass\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# TQDM import based off if current running script is a jupyter notebook\n", + "# or a python script\n", "\n", "if IS_NOTEBOOK :\n", - " import tqdm.notebook as tqdm\n", + " from tqdm.notebook import tqdm\n", "else :\n", - " from tqdm import tqdm\n", - "\n", - "\n", + " from tqdm import tqdm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "# README_ABS_DIR will get confirmed in if name==main prior to running\n", - "README_ABS_DIR = getcwd().replace('\\\\', '/')\n", - "NOTEBOOK_ABS_DIR = README_ABS_DIR\n", - "print(f'{NOTEBOOK_ABS_DIR = }')\n", - "MAIN_DIRECTORY = NOTEBOOK_ABS_DIR[NOTEBOOK_ABS_DIR.rfind('/')+1:]" + "README_ABS_DIR = getcwd().replace('\\\\', '/')\n", + "NOTEBOOK_ABS_DIR = README_ABS_DIR\n", + "MAIN_DIRECTORY = NOTEBOOK_ABS_DIR[NOTEBOOK_ABS_DIR.rfind('/')+1:]\n", + "\n", + "print(f'{NOTEBOOK_ABS_DIR = }')" ] }, { @@ -583,6 +662,7 @@ " print(f'Level not found. Defaulting to \"Unknown\"')\n", " level = 'Unknown'\n", "\n", + "\n", " creationtime, modificationtime = getCtimeMtimes(join(README_PATH, path))\n", "\n", " if path not in fileLatestTimes or max(creationtime, modificationtime) > fileLatestTimes[path] :\n", @@ -597,9 +677,11 @@ " categories = set()\n", " language = leetcodeFile[leetcodeFile.rfind('.') + 1:]\n", "\n", + "\n", " if len(altTitle) > 0 :\n", " title = altTitle + ' - ' + title\n", "\n", + "\n", " # Question is from a contest folder\n", " if contest :\n", " temp = re.findall('q\\d{1}', leetcodeFile) # Checking if file name has a question number (e.g. q1 of the contest)\n", @@ -622,24 +704,17 @@ " contestTitle=contest,\n", " contestQNo=contestQNo,\n", " fileLatestTimes=fileLatestTimes)\n", - " return True\n", - " \n", - " questionData[number] = addCase(level=level, \n", - " number=number, \n", - " title=title,\n", - " categories=categories, \n", - " language=language, \n", - " notebook_path=join(README_PATH, path), \n", - " readme_path=path,\n", - " contestTitle=contest,\n", - " contestQNo=contestQNo,\n", - " fileLatestTimes=fileLatestTimes)\n", - " \n", - " if number >= 3220 :\n", - " print(f'{questionData[number] = }')\n", - " print(f'{path = }')\n", - " print(questionDetailsDict[number])\n", - " \n", + " else :\n", + " questionData[number] = addCase(level=level, \n", + " number=number, \n", + " title=title,\n", + " categories=categories, \n", + " language=language, \n", + " notebook_path=join(README_PATH, path), \n", + " readme_path=path,\n", + " contestTitle=contest,\n", + " contestQNo=contestQNo,\n", + " fileLatestTimes=fileLatestTimes)\n", " return True" ] }, @@ -784,7 +859,7 @@ "'''\n", "\n", "@cache\n", - "def getList(fileName, filePath) -> set[int] :\n", + "def getList(fileName, filePath) -> Set[int] :\n", " output = set() # can change to dict later if we want to output category info\n", "\n", " count = 0\n", @@ -892,6 +967,7 @@ " *,\n", " questionDetailsDict: dict = retrieveQuestionDetails(),\n", " export: bool = False) -> str :\n", + " \n", " if questionNo in questionData :\n", " questionData = questionData[questionNo]\n", "\n", @@ -1103,6 +1179,7 @@ " includeQuestions: set[int] = set(),\n", " relativeFolderAdjustment: int = 0,\n", " includeMarkdownFolder: bool = False) -> DataFrame :\n", + " \n", " questionData = convertDataToMatrix(questionData, \n", " sortBy=sortBy, \n", " includeDate=includeDate, \n", @@ -1113,7 +1190,7 @@ " # Protects against empty cases (e.g. if you have no daily files)\n", " dfQuestions = kungfupanda.DataFrame()\n", " if questionData :\n", - " dfQuestions = kungfupanda.DataFrame(data=questionData, columns=COLUMNS[:len(questionData[0])])\n", + " dfQuestions = kungfupanda.DataFrame(data=questionData, columns=COLUMNS[:len(questionData[0])])\n", " \n", " # dfQuestions = dfQuestions.astype(TYPE_CLARIFICATION[:len(questionData[0])])\n", "\n", @@ -1220,7 +1297,7 @@ "\n", " # For the overal hosting markdown\n", " OVERALL_FILE_NOTEBOOK_PATH = join(README_PATH, MARKDOWN_PATH, 'Topics.md')\n", - " OVERALL_FILE_README_PATH = join(MARKDOWN_PATH, 'Topics.md')\n", + " OVERALL_FILE_README_PATH = join(MARKDOWN_PATH, 'Topics.md')\n", "\n", " if not isdir(NOTEBOOK_PATH) :\n", " mkdir(NOTEBOOK_PATH)\n", @@ -1288,13 +1365,13 @@ " \n", " easyMarkdown = convertQuestionDataToDataframe(easyQuestions, \n", " includeDate=True, \n", - " includeMarkdownFolder=True)\n", + " includeMarkdownFolder=False)\n", " mediumMarkdown = convertQuestionDataToDataframe(mediumQuestions, \n", " includeDate=True, \n", - " includeMarkdownFolder=True)\n", + " includeMarkdownFolder=False)\n", " hardMarkdown = convertQuestionDataToDataframe(hardQuestions, \n", " includeDate=True, \n", - " includeMarkdownFolder=True)\n", + " includeMarkdownFolder=False)\n", " \n", " \n", " easy_path = join(DIFFICULTY_MARKDOWNS_PATH, 'Easy.md')\n", @@ -1351,6 +1428,7 @@ " code_length: bool = False,\n", " recent: bool = False,\n", " daily: bool = False) -> str : # output path\n", + " \n", " df = None\n", " fileName = None\n", " header_data = None\n", diff --git a/main.py b/main.py index e780ac4..2788e55 100644 --- a/main.py +++ b/main.py @@ -1,12 +1,26 @@ #!/usr/bin/env python # coding: utf-8 +# Data management imports + # In[ ]: import pandas as kungfupanda # pandas for data manipulation and markdown from pandas import DataFrame # export +import argparse # For command line arguments when calling py script with flags +import pickle # for saving/loading json records and file + # modification date history + +from questionDataclass import questionDataclass as Question + + +# OS and directory management imports + +# In[ ]: + + from os import listdir # for file retrieval and path calculations from os.path import isfile, join from os import stat @@ -23,32 +37,40 @@ # e.g. if you call `python someFolder/main.py` # then it will still work. -import subprocess + +# Environment variable imports + file log and creation time imports + +# In[ ]: + + +from os import getenv, environ # for environment variables +from dotenv import load_dotenv, find_dotenv # for config purposes (.env file) + +import subprocess # tracing git log history for ctimes and mtimes + + +# In[ ]: + from os.path import getmtime, getctime # retreiving file creation/modification times from datetime import datetime import time -from os import getenv, environ # for environment variables -from dotenv import load_dotenv, find_dotenv # for config purposes (.env file) + +# QOL and anti-reundancy imports + +# In[ ]: + from typing import Set, Dict, List, Tuple # misc. QOL imports from collections import defaultdict from icecream import ic # for debugging / outputs -from questionDataclass import questionDataclass as Question - -# TQDM import done below to check if this is -# a .py or .ipynb file - import re # for regex file name matching / question number matching - -import argparse # For command line arguments when calling py script with flags -import pickle # for saving/loading json records and file - # modification date history - from functools import cache # for redundancy protection +# TQDM import done separately below after checking if this is a .py or .ipynb file + # # Script Configuration # #### `.env` variables and `working directories` @@ -63,10 +85,15 @@ # loading env variables print('Default .env activated from script directory (.readme_updater/)') load_dotenv(find_dotenv(), override=True) + if '.env' in listdir('../') : print('.env found in ../ directory. Overriding default...') load_dotenv(find_dotenv('../.env'), override=True) + +# In[ ]: + + # NOTE: if the script is being run from a jupyter notebook, then it should # already be in the correct directory. IS_NOTEBOOK = True @@ -81,17 +108,28 @@ print('NameError') pass + +# In[ ]: + + +# TQDM import based off if current running script is a jupyter notebook +# or a python script + if IS_NOTEBOOK : - import tqdm.notebook as tqdm + from tqdm.notebook import tqdm else : from tqdm import tqdm +# In[ ]: + + # README_ABS_DIR will get confirmed in if name==main prior to running -README_ABS_DIR = getcwd().replace('\\', '/') -NOTEBOOK_ABS_DIR = README_ABS_DIR +README_ABS_DIR = getcwd().replace('\\', '/') +NOTEBOOK_ABS_DIR = README_ABS_DIR +MAIN_DIRECTORY = NOTEBOOK_ABS_DIR[NOTEBOOK_ABS_DIR.rfind('/')+1:] + print(f'{NOTEBOOK_ABS_DIR = }') -MAIN_DIRECTORY = NOTEBOOK_ABS_DIR[NOTEBOOK_ABS_DIR.rfind('/')+1:] # In[ ]: @@ -520,6 +558,7 @@ def parseCase(leetcodeFile: str, # file name print(f'Level not found. Defaulting to "Unknown"') level = 'Unknown' + creationtime, modificationtime = getCtimeMtimes(join(README_PATH, path)) if path not in fileLatestTimes or max(creationtime, modificationtime) > fileLatestTimes[path] : @@ -534,9 +573,11 @@ def parseCase(leetcodeFile: str, # file name categories = set() language = leetcodeFile[leetcodeFile.rfind('.') + 1:] + if len(altTitle) > 0 : title = altTitle + ' - ' + title + # Question is from a contest folder if contest : temp = re.findall('q\d{1}', leetcodeFile) # Checking if file name has a question number (e.g. q1 of the contest) @@ -559,24 +600,17 @@ def parseCase(leetcodeFile: str, # file name contestTitle=contest, contestQNo=contestQNo, fileLatestTimes=fileLatestTimes) - return True - - questionData[number] = addCase(level=level, - number=number, - title=title, - categories=categories, - language=language, - notebook_path=join(README_PATH, path), - readme_path=path, - contestTitle=contest, - contestQNo=contestQNo, - fileLatestTimes=fileLatestTimes) - - if number >= 3220 : - print(f'{questionData[number] = }') - print(f'{path = }') - print(questionDetailsDict[number]) - + else : + questionData[number] = addCase(level=level, + number=number, + title=title, + categories=categories, + language=language, + notebook_path=join(README_PATH, path), + readme_path=path, + contestTitle=contest, + contestQNo=contestQNo, + fileLatestTimes=fileLatestTimes) return True @@ -699,7 +733,7 @@ def getLists() -> List[str] : ''' @cache -def getList(fileName, filePath) -> set[int] : +def getList(fileName, filePath) -> Set[int] : output = set() # can change to dict later if we want to output category info count = 0 @@ -786,6 +820,7 @@ def generate_markdown(questionNo: int, *, questionDetailsDict: dict = retrieveQuestionDetails(), export: bool = False) -> str : + if questionNo in questionData : questionData = questionData[questionNo] @@ -980,6 +1015,7 @@ def convertQuestionDataToDataframe(questionData: dict, includeQuestions: set[int] = set(), relativeFolderAdjustment: int = 0, includeMarkdownFolder: bool = False) -> DataFrame : + questionData = convertDataToMatrix(questionData, sortBy=sortBy, includeDate=includeDate, @@ -990,7 +1026,7 @@ def convertQuestionDataToDataframe(questionData: dict, # Protects against empty cases (e.g. if you have no daily files) dfQuestions = kungfupanda.DataFrame() if questionData : - dfQuestions = kungfupanda.DataFrame(data=questionData, columns=COLUMNS[:len(questionData[0])]) + dfQuestions = kungfupanda.DataFrame(data=questionData, columns=COLUMNS[:len(questionData[0])]) # dfQuestions = dfQuestions.astype(TYPE_CLARIFICATION[:len(questionData[0])]) @@ -1066,7 +1102,7 @@ def topicBasedMarkdowns(questionData: dict, # For the overal hosting markdown OVERALL_FILE_NOTEBOOK_PATH = join(README_PATH, MARKDOWN_PATH, 'Topics.md') - OVERALL_FILE_README_PATH = join(MARKDOWN_PATH, 'Topics.md') + OVERALL_FILE_README_PATH = join(MARKDOWN_PATH, 'Topics.md') if not isdir(NOTEBOOK_PATH) : mkdir(NOTEBOOK_PATH) @@ -1126,13 +1162,13 @@ def generateDifficultyLevelMarkdowns(questionData: dict) -> Tuple[Tuple[int, str easyMarkdown = convertQuestionDataToDataframe(easyQuestions, includeDate=True, - includeMarkdownFolder=True) + includeMarkdownFolder=False) mediumMarkdown = convertQuestionDataToDataframe(mediumQuestions, includeDate=True, - includeMarkdownFolder=True) + includeMarkdownFolder=False) hardMarkdown = convertQuestionDataToDataframe(hardQuestions, includeDate=True, - includeMarkdownFolder=True) + includeMarkdownFolder=False) easy_path = join(DIFFICULTY_MARKDOWNS_PATH, 'Easy.md') @@ -1176,6 +1212,7 @@ def miscMarkdownGenerations(questionData: dict, code_length: bool = False, recent: bool = False, daily: bool = False) -> str : # output path + df = None fileName = None header_data = None