Signals

fivem/FXServer.conf.json

@@ -1,4 +1,4 @@
 {
-  "ArtifactVersion": 7290,
+  "ArtifactVersion": 10367,
   "Location": "fxserver"
 }

src/IntelliPed.ConsoleApp/IntelliPed.ConsoleApp.csproj

@@ -9,10 +9,10 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="8.0.5" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="6.0.1" />
-    <PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="8.0.10" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="8.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="8.0.1" />
   </ItemGroup>
 
   <ItemGroup>

src/IntelliPed.ConsoleApp/Program.cs

@@ -13,7 +13,29 @@
     Model = "gpt-3.5-turbo-0125"
 };
 
-Agent agent = new(openAiOptions);
+PersonalInfo personalInfo = new()
+{
+    Name = "Brian Limond",
+    Age = 28,
+    DateOfBirth = new DateTime(1995, 3, 15),
+    Address = "456 Oak Lane, Smalltown, USA",
+    PersonalityTraits = ["Hardworking", "Difficult", "Dry Humour"],
+    Interests = ["Fishing", "Cooking", "Watching movies"],
+    Occupation = "Electrician",
+    Education = "High School Diploma",
+    Goals = ["HIGH PRIORITY: Find my lost dog", "Buy a house", "Start a family"],
+    Skills = ["Electrical wiring", "Problem-solving", "Time management"],
+    Preferences = new() { { "FavoriteFood", "Steak" }, { "FavoriteColor", "Green" } },
+    Biography = "Timmy grew up in Smalltown and has always been known for his reliability and kindness. He found his passion in working with his hands and became an electrician...",
+    Family = ["Father: Mark Johnson", "Mother: Susan Johnson", "Sister: Emma Johnson"],
+    SignificantLifeEvents = ["Graduated high school in 2013", "Completed electrician apprenticeship in 2016"],
+    SocialCircle = ["Best Friend: Jack", "Neighbor: Mrs. Smith"],
+    CommunicationStyle = "Friendly and straightforward",
+    EmotionalState = "Content and optimistic",
+    Values = ["Honesty", "Loyalty", "Hard work"]
+};
+
+Agent agent = new(personalInfo, openAiOptions);
 
 await agent.Start();
 

src/IntelliPed.Core/Agents/Agent.cs

@@ -1,5 +1,6 @@
 using IntelliPed.Core.Plugins;
 using IntelliPed.Core.Signals;
+using IntelliPed.Messages.Heartbeats;
 using IntelliPed.Messages.Signals;
 using Microsoft.AspNetCore.SignalR.Client;
 using Microsoft.Extensions.DependencyInjection;
@@ -12,12 +13,20 @@ public class Agent
 {
     public Kernel Kernel { get; }
     public HubConnection HubConnection { get; }
+    public PersonalInfo PersonalInfo { get; }
+    public Heartbeat? LatestHeartbeat { get; private set; }
+
     private readonly SignalProcessor _signalProcessor;
 
-    public Agent(OpenAiOptions openAiOptions)
+    public Agent(PersonalInfo personalInfo, OpenAiOptions openAiOptions)
     {
+        PersonalInfo = personalInfo;
         _signalProcessor = new(this);
 
+        HubConnection = new HubConnectionBuilder()
+            .WithUrl("http://localhost:5000/agent-hub")
+            .Build();
+
         HubConnection = new HubConnectionBuilder()
             .WithUrl("http://localhost:5000/agent-hub")
             .Build();
@@ -41,11 +50,15 @@ public async Task Start()
     {
         await HubConnection.StartAsync();
 
-        await HubConnection.InvokeAsync("CreatePuppet");
-
         HubConnection.On<DamageSignal>("DamageReceived", _signalProcessor.Handle);
         HubConnection.On<SpeechSignal>("SpeechHeard", _signalProcessor.Handle);
+        HubConnection.On<Heartbeat>("Heartbeat", Heartbeat);
 
         _signalProcessor.Start();
     }
+
+    private void Heartbeat(Heartbeat heartbeat)
+    {
+        LatestHeartbeat = heartbeat;
+    }
 }

src/IntelliPed.Core/Agents/PersonalInfo.cs

@@ -0,0 +1,51 @@
+using System.Text;
+
+namespace IntelliPed.Core.Agents;
+
+public record PersonalInfo
+{
+    public required string Name { get; init; }
+    public required int Age { get; init; }
+    public DateTime? DateOfBirth { get; init; }
+    public string? Address { get; init; }
+    public List<string>? PersonalityTraits { get; init; }
+    public List<string>? Interests { get; init; }
+    public string? Occupation { get; init; }
+    public string? Education { get; init; }
+    public List<string>? Goals { get; init; }
+    public List<string>? Skills { get; init; }
+    public Dictionary<string, string>? Preferences { get; init; } // e.g., {"FavoriteFood": "Pizza"}
+    public string? Biography { get; init; }
+    public List<string>? Family { get; init; } // e.g., {"Father: John Doe", "Mother: Jane Doe"}
+    public List<string>? SignificantLifeEvents { get; init; } // e.g., {"Graduated college in 2010"}
+    public List<string>? SocialCircle { get; init; } // e.g., {"Friend: Alice", "Mentor: Bob"}
+    public string? CommunicationStyle { get; init; }
+    public string? EmotionalState { get; init; }
+    public List<string>? Values { get; init; } // e.g., {"Honesty", "Loyalty"}
+
+    public override string ToString()
+    {
+        StringBuilder builder = new();
+
+        builder.AppendLine($"Name: {Name}");
+        builder.AppendLine($"Age: {Age}");
+        if (DateOfBirth.HasValue) builder.AppendLine($"Date of Birth: {DateOfBirth:MMMM dd, yyyy}");
+        if (!string.IsNullOrWhiteSpace(Address)) builder.AppendLine($"Address: {Address}");
+        if (PersonalityTraits != null && PersonalityTraits.Any()) builder.AppendLine($"Personality Traits: {string.Join(", ", PersonalityTraits)}");
+        if (Interests != null && Interests.Any()) builder.AppendLine($"Interests: {string.Join(", ", Interests)}");
+        if (!string.IsNullOrWhiteSpace(Occupation)) builder.AppendLine($"Occupation: {Occupation}");
+        if (!string.IsNullOrWhiteSpace(Education)) builder.AppendLine($"Education: {Education}");
+        if (Goals != null && Goals.Any()) builder.AppendLine($"Goals: {string.Join(", ", Goals)}");
+        if (Skills != null && Skills.Any()) builder.AppendLine($"Skills: {string.Join(", ", Skills)}");
+        if (Preferences != null && Preferences.Any()) builder.AppendLine($"Preferences: {string.Join(", ", Preferences.Select(kv => $"{kv.Key}: {kv.Value}"))}");
+        if (!string.IsNullOrWhiteSpace(Biography)) builder.AppendLine($"Biography: {Biography}");
+        if (Family != null && Family.Any()) builder.AppendLine($"Family: {string.Join(", ", Family)}");
+        if (SignificantLifeEvents != null && SignificantLifeEvents.Any()) builder.AppendLine($"Significant Life Events: {string.Join(", ", SignificantLifeEvents)}");
+        if (SocialCircle != null && SocialCircle.Any()) builder.AppendLine($"Social Circle: {string.Join(", ", SocialCircle)}");
+        if (!string.IsNullOrWhiteSpace(CommunicationStyle)) builder.AppendLine($"Communication Style: {CommunicationStyle}");
+        if (!string.IsNullOrWhiteSpace(EmotionalState)) builder.AppendLine($"Emotional State: {EmotionalState}");
+        if (Values != null && Values.Any()) builder.AppendLine($"Values: {string.Join(", ", Values)}");
+
+        return builder.ToString();
+    }
+}

src/IntelliPed.Core/IntelliPed.Core.csproj

@@ -7,12 +7,13 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="8.0.5" />
-    <PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
-    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="8.0.0" />
-    <PackageReference Include="Microsoft.SemanticKernel" Version="1.13.0" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="8.0.10" />
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.2" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="8.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="8.0.1" />
+    <PackageReference Include="Microsoft.KernelMemory.SemanticKernelPlugin" Version="0.79.241014.2" />
+    <PackageReference Include="Microsoft.SemanticKernel" Version="1.22.0" />
   </ItemGroup>
 
   <ItemGroup>

src/IntelliPed.Core/Plugins/GtaNativeFunctionsPlugin.cs

@@ -0,0 +1,27 @@
+using System.ComponentModel;
+using System.Diagnostics;
+using Microsoft.SemanticKernel;
+
+namespace IntelliPed.Core.Plugins;
+
+/*
+ * The idea is to allow the agent to execute native GTA V functions.
+ * Still need to explore further.
+ *
+ * I think GPT has decent knowledge of GTA V natives, but it's not perfect.
+ * It would be good to embed documentation.
+ */
+
+public class GtaNativeFunctionsPlugin
+{
+    [KernelFunction]
+    [Description("Executes a native GTA V function. Make use of your knowledge of native GTA V / FiveM functions.")]
+    public async Task<string> ExecuteNativeFunction(
+        Kernel kernel,
+        [Description("The native hash e.g. GET_ENTITY_HEALTH, TASK_GO_TO_ENTITY, TASK_SMART_FLEE_PED")] string nativeHash,
+        [Description("The arguments required for the native function.")] params object[] args)
+    {
+        Debug.WriteLine($"Used native hash {nativeHash} with args {string.Join(", ", args)}");
+        return "Executed native.";
+    }
+}

src/IntelliPed.Core/Plugins/NavigationPlugin.cs

@@ -1,5 +1,6 @@
 using System.ComponentModel;
 using IntelliPed.Core.Agents;
+using IntelliPed.Messages.Common;
 using IntelliPed.Messages.Navigation;
 using Microsoft.AspNetCore.SignalR.Client;
 using Microsoft.SemanticKernel;
@@ -31,25 +32,27 @@ public async Task<string> NavigateTo(
     [Description("Returns the co-ordinates of the specified location.")]
     [return: Description("The co-ordinates of the location in the format of: (x, y, z).")]
     public async Task<Coordinates> GetLocation(
-        Kernel kernel,
         [Description("The location.")] string location)
     {
         await Task.Delay(250);
         Console.WriteLine($"The co-ordinates of {location} are (-831, 172, 70)");
         return new Coordinates(-831, 172, 70);
     }
-}
-
-public class Coordinates
-{
-    public float X { get; set; }
-    public float Y { get; set; }
-    public float Z { get; set; }
 
-    public Coordinates(float x, float y, float z)
+    [KernelFunction]
+    [Description("Flees from the specified ped.")]
+    public async Task<string> FleeFrom(
+        Kernel kernel,
+        [Description("The network ID of the ped to flee from.")] int pedNetworkIdToFleeFrom)
     {
-        X = x;
-        Y = y;
-        Z = z;
+        Agent agent = kernel.GetRequiredService<Agent>();
+
+        await agent.HubConnection.InvokeAsync("FleeFrom", new FleeFromRequest
+        {
+            PedNetworkId = pedNetworkIdToFleeFrom,
+        });
+
+        Console.WriteLine($"Fleeing from {pedNetworkIdToFleeFrom}");
+        return $"You have started fleeing from the ped with network id {pedNetworkIdToFleeFrom}";
     }
 }

src/IntelliPed.Core/Plugins/SpeechPlugin.cs

@@ -9,7 +9,7 @@ namespace IntelliPed.Core.Plugins;
 public class SpeechPlugin
 {
     [KernelFunction]
-    [Description("Speaks out loud in the world so that other peds can hear you.")]
+    [Description("Speaks out loud in the world so that other people can hear you.")]
     public async Task<string> Speak(
         Kernel kernel,
         [Description("What to speak.")] string message)
@@ -22,6 +22,6 @@ public async Task<string> Speak(
         });
 
         Console.WriteLine("Successfully spoke");
-        return "Successfully spoke.";
+        return "You successfully spoke.";
     }
 }

src/IntelliPed.Core/Signals/SignalProcessor.cs

@@ -1,6 +1,8 @@
 using System.Collections.Concurrent;
 using IntelliPed.Core.Agents;
+using IntelliPed.Messages.AgentStatus;
 using IntelliPed.Messages.Signals;
+using Microsoft.AspNetCore.SignalR.Client;
 using Microsoft.SemanticKernel;
 using Microsoft.SemanticKernel.ChatCompletion;
 using Microsoft.SemanticKernel.Connectors.OpenAI;
@@ -22,8 +24,21 @@ public SignalProcessor(Agent agent)
     public void Start()
     {
         if (IsProcessing) throw new InvalidOperationException("Signal processor is already running.");
+
         IsProcessing = true;
-        Task.Run(() => ProcessSignals(_cancellationTokenSource.Token));
+
+        CancellationToken cancellationToken = _cancellationTokenSource.Token;
+        cancellationToken.Register(() =>
+        {
+            Task task = _agent.HubConnection.InvokeAsync("SetAgentStatus", new SetAgentStatusRequest
+            {
+                IsThinking = false
+            }, CancellationToken.None);
+
+            task.Wait(CancellationToken.None);
+        });
+
+        Task.Run(() => ProcessSignals(cancellationToken), cancellationToken);
     }
 
     public void Stop()
@@ -51,26 +66,44 @@ private async Task ProcessSignals(CancellationToken cancellationToken)
             // Process signal
             IChatCompletionService chatService = _agent.Kernel.GetRequiredService<IChatCompletionService>();
 
-            ChatHistory chat = new(
-                """
-                 You are a ped in Grand Theft Auto V who is fully autonomous. Your goals are to freeroam. 
-
-                 Your decisions must always be made independently without seeking user assistance. 
-                 Play to your strengths as an LLM and pursue simple strategies with no legal complications.
-
-                 You must make use of your reasoning and decision-making capabilities to respond to the signal.
-                 Be realistic and think about what a ped would do in this situation.
-
-                 You may invoke kernel functions.
-                 """);
+            ChatHistory chat = new();
 
             chat.AddUserMessage(signal.ToString());
 
+            await _agent.HubConnection.InvokeAsync("SetAgentStatus", new SetAgentStatusRequest
+            {
+                IsThinking = true
+            }, cancellationToken);
+
             ChatMessageContent result = await chatService.GetChatMessageContentAsync(chat, new OpenAIPromptExecutionSettings
             {
-                ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions
+                ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions,
+                Temperature = 0.5f,
+                ChatSystemPrompt =
+                    $"""
+                     You are a person living in Grand Theft Auto V who is fully autonomous. Your goals are to freeroam.
+
+                     Your decisions must always be made independently without seeking user assistance.
+                     Play to your strengths as an LLM and pursue simple strategies with no legal complications.
+
+                     You must make use of your reasoning and decision-making capabilities to respond to the signal.
+                     Be realistic and think about what your character would do in this situation.
+
+                     You should invoke kernel functions to achieve your goals.
+
+                     **Personal Information**
+                     {_agent.PersonalInfo}
+
+                     **Current Status**
+                     {_agent.LatestHeartbeat?.ToString() ?? "All good."}
+                     """,
             }, kernel: _agent.Kernel, cancellationToken: cancellationToken);
 
+            await _agent.HubConnection.InvokeAsync("SetAgentStatus", new SetAgentStatusRequest
+            {
+                IsThinking = false
+            }, cancellationToken);
+
             Console.WriteLine($"Result: {result}");
         }
     }

src/IntelliPed.FiveM.Client/Extensions/PedExtensions.cs

@@ -0,0 +1,16 @@
+using CitizenFX.Core.Native;
+using CitizenFX.Core;
+
+namespace IntelliPed.FiveM.Client.Extensions;
+
+public static class PedExtensions
+{
+    public static string GetCurrentStreet(this Entity entity)
+    {
+        uint streetHashKey = 0;
+        uint crossingRoad = 0;
+        API.GetStreetNameAtCoord(entity.Position.X, entity.Position.Y, entity.Position.Z, ref streetHashKey, ref crossingRoad);
+        string streetName = API.GetStreetNameFromHashKey(streetHashKey);
+        return streetName;
+    }
+}