Added logic to fetch README files, documentation, commit messages, an…

website/management/commands/update_projects.py

@@ -1,9 +1,12 @@
+import base64
+
 import requests
 from django.conf import settings
 from django.core.management.base import BaseCommand
 from django.utils.dateparse import parse_datetime
 
-from website.models import Project
+from website.models import Project, Tag
+from website.utils import ai_summary, markdown_to_text
 
 
 class Command(BaseCommand):
@@ -51,6 +54,55 @@ def handle(self, *args, **kwargs):
                 project.size = repo_data.get("size", 0)
                 project.last_commit_date = parse_datetime(repo_data.get("pushed_at"))
 
+                tags = []
+                for topic in repo_data.get("topics", []):
+                    tag, created = Tag.objects.get_or_create(name=topic)
+                    tags.append(tag)
+
+                project.tags.set(tags)  # This assigns the tags to the project
+
+                # Fetch README
+                url = f"https://api.github.com/repos/{repo_name}/readme"
+                response = requests.get(url, headers=headers)
+                if response.status_code == 200:
+                    readme_data = response.json()
+                    readme_content_encoded = readme_data.get("content", "")
+
+                    # Decode the Base64 content
+                    try:
+                        readme_content = base64.b64decode(readme_content_encoded).decode("utf-8")
+                        project.readme_content = readme_content
+                        readme_text = markdown_to_text(readme_content)
+                        project_tags = [tag.name for tag in project.tags.all()]
+                        project.ai_summary = ai_summary(readme_text, project_tags)
+
+                    except (base64.binascii.Error, UnicodeDecodeError) as e:
+                        self.stdout.write(
+                            self.style.WARNING(f"Failed to decode README for {repo_name}: {e}")
+                        )
+                        project.readme_content = ""
+                        project.ai_summary = ""
+                else:
+                    self.stdout.write(
+                        self.style.WARNING(
+                            f"Failed to fetch README for {repo_name}: {response.status_code}"
+                        )
+                    )
+
+                # Fetch Recent Commit Messages
+                url = f"https://api.github.com/repos/{repo_name}/commits"
+                response = requests.get(url, headers=headers)
+                if response.status_code == 200:
+                    commits_data = response.json()
+                    commit_messages = [commit["commit"]["message"] for commit in commits_data[:5]]
+                    project.recent_commit_messages = "\n".join(commit_messages)
+                else:
+                    self.stdout.write(
+                        self.style.WARNING(
+                            f"Failed to fetch recent commits for {repo_name}: {response.status_code}"
+                        )
+                    )
+
                 # Fetch counts of issues and pull requests using the Search API
                 def get_issue_count(repo_name, query, headers):
                     url = f"https://api.github.com/search/issues?q=repo:{repo_name}+{query}"

website/migrations/0170_project_ai_summary_project_readme_content_and_more.py

@@ -0,0 +1,27 @@
+# Generated by Django 5.1.3 on 2024-12-14 12:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("website", "0169_dailystatusreport_current_mood_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="project",
+            name="ai_summary",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="readme_content",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="recent_commit_messages",
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

website/migrations/0173_merge_20241219_1258.py

@@ -0,0 +1,12 @@
+# Generated by Django 5.1.3 on 2024-12-19 12:58
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("website", "0170_project_ai_summary_project_readme_content_and_more"),
+        ("website", "0172_merge_20241218_0505"),
+    ]
+
+    operations = []

website/migrations/0174_merge_20241219_1316.py

@@ -0,0 +1,12 @@
+# Generated by Django 5.1.3 on 2024-12-19 13:16
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("website", "0173_merge_20241219_1258"),
+        ("website", "0173_remove_company_admin_remove_company_integrations_and_more"),
+    ]
+
+    operations = []

website/migrations/0175_merge_20241219_1501.py

@@ -0,0 +1,12 @@
+# Generated by Django 5.1.3 on 2024-12-19 15:01
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("website", "0174_merge_20241219_1305"),
+        ("website", "0174_merge_20241219_1316"),
+    ]
+
+    operations = []

website/models.py

@@ -921,6 +921,9 @@ class Project(models.Model):
     closed_issues = models.IntegerField(default=0)
     size = models.IntegerField(default=0)
     commit_count = models.IntegerField(default=0)
+    readme_content = models.TextField(null=True, blank=True)
+    recent_commit_messages = models.TextField(null=True, blank=True)
+    ai_summary = models.TextField(null=True, blank=True)
 
     def __str__(self):
         return self.name

website/templates/submit_roadmap_pr.html

@@ -14,15 +14,6 @@ <h1 class="text-3xl font-bold mb-8">Submit Pull Request For Analysis</h1>
               action="{% url 'submit-roadmap-pr' %}"
               class="bg-white shadow-md rounded px-8 pt-8 pb-8 mb-6">
             {% csrf_token %}
-            <div class="mb-6">
-                <label for="pr-link" class="block text-gray-700 text-2xl font-semibold mb-3">GitHub PR Link</label>
-                <input type="url"
-                       id="pr-link"
-                       name="pr_link"
-                       placeholder="Enter GitHub PR link"
-                       class="shadow appearance-none border rounded w-full py-3 px-4 text-gray-700 leading-tight focus:outline-none focus:shadow-outline"
-                       required>
-            </div>
             <div class="mb-6">
                 <label for="issue-link"
                        class="block text-gray-700 text-2xl font-semibold mb-3">GitHub Issue Link (Roadmap)</label>
@@ -33,6 +24,15 @@ <h1 class="text-3xl font-bold mb-8">Submit Pull Request For Analysis</h1>
                        class="shadow appearance-none border rounded w-full py-3 px-4 text-gray-700 leading-tight focus:outline-none focus:shadow-outline"
                        required>
             </div>
+            <div class="mb-6">
+                <label for="pr-link" class="block text-gray-700 text-2xl font-semibold mb-3">GitHub PR Link</label>
+                <input type="url"
+                       id="pr-link"
+                       name="pr_link"
+                       placeholder="Enter GitHub PR link"
+                       class="shadow appearance-none border rounded w-full py-3 px-4 text-gray-700 leading-tight focus:outline-none focus:shadow-outline"
+                       required>
+            </div>
             <div class="flex items-center justify-center">
                 <button type="submit"
                         class="bg-red-500 hover:bg-red-700 text-white font-bold py-3 px-6 rounded focus:outline-none focus:shadow-outline">

website/templates/website/project_list.html

@@ -135,7 +135,13 @@ <h3>{{ project.name }}</h3>
                                 </li>
                             </ul>
                         </div>
-                        <p>{{ project.description }}</p>
+                        <div class="project-info">
+                            <p>{{ project.description }}</p>
+                            <div class="ai-summary">
+                                <h4>Summary</h4>
+                                <p>{{ project.ai_summary }}</p>
+                            </div>
+                        </div>
                         <!-- Contributors Section -->
                         <div class="project-stats flex items-center gap-4">
                             <p>
@@ -154,7 +160,7 @@ <h3>{{ project.name }}</h3>
                                     </a>
                                 {% empty %}
                                     <div class="contributor-avatar p-10 placeholder text-center">
-                                        <span className="text-xs">No Avatar</span>
+                                        <span class="text-xs">No Avatar</span>
                                     </div>
                                 {% endfor %}
                             </div>
@@ -176,10 +182,10 @@ <h4>
                         <!-- Tags Section -->
                         <div class="project-tags">
                             <h4>
-                                <i class="fas fa-tags"></i>Tags
+                                <i class="fas fa-tags"></i> Tags
                             </h4>
-                            <ul>
-                                {% for tag in project.tags.all %}<li>{{ tag.name }}</li>{% endfor %}
+                            <ul class="tags-list">
+                                {% for tag in project.tags.all %}<li class="tag-item badge badge-primary">{{ tag.name }}</li>{% endfor %}
                             </ul>
                         </div>
                     </div>
@@ -449,26 +455,60 @@ <h4>
         .external-links a:hover {
             text-decoration: underline;
         }
-
         .project-tags {
             margin-top: 1rem;
         }
 
-        .project-tags ul {
+        .project-tags ul, .tags-list {
             list-style-type: none;
             padding: 0;
+            margin: 0;
         }
 
-        .project-tags li {
+        .project-tags li, .tag-item {
             display: inline-block;
-            background-color: #f1f1f1;
-            padding: 5px 10px;
-            margin-right: 5px;
-            border-radius: 5px;
+            margin: 0.2em 5px 0.2em 0;
+            padding: 0.4em 0.8em;
+            background-color: #007bff;
+            color: #ffffff;
+            border-radius: 0.2em;
+            font-size: 0.875em;
         }
 
-        .project-tags h4 i {
+        .project-tags li {
+            background-color: #070707;
+        }
+
+        .project-tags h4 i, .project-tags li i {
             margin-right: 0.5rem;
         }
+
+        .tag-item:hover {
+            background-color: #0056b3;
+            cursor: pointer;
+        }
+
+        .project-info {
+            margin-top: 1rem;
+        }
+
+        .project-description, .ai-summary {
+            margin-bottom: 1rem;
+        }
+
+        .ai-summary h4 {
+            margin-bottom: 0.5rem;
+            color: #333;
+            font-weight: bold; 
+        }
+
+        .ai-summary p {
+            margin: 0;
+            color: #181616; 
+        }
+
+        .ai-summary{
+            margin-top: 15px;
+        }
     </style>
 {% endblock content %}

website/utils.py

@@ -4,6 +4,10 @@
 from collections import deque
 from urllib.parse import urlparse, urlsplit, urlunparse
 
+import markdown
+from openai import OpenAI
+
+client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
 import requests
 from bs4 import BeautifulSoup
 from django.core.exceptions import ValidationError
@@ -174,12 +178,38 @@ def format_timedelta(td):
     return f"{hours}h {minutes}m {seconds}s"
 
 
-import openai
+def markdown_to_text(markdown_content):
+    """Convert Markdown to plain text."""
+    html_content = markdown.markdown(markdown_content)
+    text_content = BeautifulSoup(html_content, "html.parser").get_text()
+    return text_content
+
 
-openai.api_key = os.getenv("OPENAI_API_KEY")
 GITHUB_API_TOKEN = os.getenv("GITHUB_ACCESS_TOKEN")
 
 
+def ai_summary(text, tags=None):
+    """Generate an AI-driven summary using OpenAI's GPT, including GitHub topics."""
+    try:
+        tags_str = ", ".join(tags) if tags else "No topics provided."
+        prompt = f"Generate a brief summary of the following text, focusing on key aspects such as purpose, features, technologies used, and current status. Consider the following GitHub topics to enhance the context: {tags_str}\n\n{text}"
+
+        response = client.chat.completions.create(
+            model="gpt-3.5-turbo",
+            messages=[
+                {"role": "system", "content": "You are a helpful assistant."},
+                {"role": "user", "content": prompt},
+            ],
+            max_tokens=150,
+            temperature=0.5,
+        )
+
+        summary = response.choices[0].message.content.strip()
+        return summary
+    except Exception as e:
+        return f"Error generating summary: {str(e)}"
+
+
 def fetch_github_data(owner, repo, endpoint, number):
     """
     Fetch data from GitHub API for a given repository endpoint.
@@ -199,22 +229,36 @@ def analyze_pr_content(pr_data, roadmap_data):
     """
     Use OpenAI API to analyze PR content against roadmap priorities.
     """
-    prompt = f"""
-    Compare the following pull request details with the roadmap priorities and provide:
-    1. A priority alignment score (1-10) with reasoning.
-    2. Key recommendations for improvement.
-    3. Assess the quality of the pull request based on its description, structure, and potential impact.
-
-    ### PR Data:
-    {pr_data}
+    try:
+        prompt = f"""
+        Compare the following pull request details with the roadmap priorities and provide:
+        1. A priority alignment score (1-10) with reasoning.
+        2. Key recommendations for improvement.
+        3. Assess the quality of the pull request based on its description, structure, and potential impact.
+
+        ### PR Data:
+        {pr_data}
+
+        ### Roadmap Data:
+        {roadmap_data}
+        """
+
+        response = client.chat.completions.create(
+            model="gpt-4",
+            messages=[
+                {
+                    "role": "system",
+                    "content": "You are an experienced code reviewer and project manager.",
+                },
+                {"role": "user", "content": prompt},
+            ],
+            temperature=0.7,
+            max_tokens=1000,  # Added to ensure sufficient response length
+        )
 
-    ### Roadmap Data:
-    {roadmap_data}
-    """
-    response = openai.ChatCompletion.create(
-        model="gpt-4", messages=[{"role": "user", "content": prompt}], temperature=0.7
-    )
-    return response["choices"][0]["message"]["content"]
+        return response.choices[0].message.content
+    except Exception as e:
+        return f"Error analyzing PR content: {str(e)}"
 
 
 def save_analysis_report(pr_link, issue_link, analysis):