Arabic resposne issue from Openai LLM

[rupeshKumarAmt]

Checked other resources

• I added a very descriptive title to this question.
• I searched the LangChain documentation with the integrated search.
• I used the GitHub search to find a similar question and didn't find it.

Commit to Help

• I commit to help with one of those options 👆

Example Code

import os
import json
from dotenv import load_dotenv
from chromadb import HttpClient
from chromadb.config import Settings
from langchain.prompts import PromptTemplate
from langchain.chat_models import ChatOpenAI
from langchain.vectorstores.chroma import Chroma
from langchain.embeddings import OpenAIEmbeddings
import config
from openai._exceptions import RateLimitError
from langchain.schema.messages import HumanMessage, SystemMessage
import re
import unicodedata

def retrieve_answer(question: str, uuid, open_api_key):
    question = clean_user_query(question=question)

    try:

        llm = get_llm_object(open_api_key)

        chroma_client = get_chroma_client()

        # Check collection exist or not
        try:
            uuid = str(uuid)
            chroma_client.get_collection(name=uuid)
        except Exception as e:
            raise ValueError(json.loads(e.args[0]).get('error'))

        chroma_initializer = Chroma(
            embedding_function=OpenAIEmbeddings(api_key=open_api_key),
            client=chroma_client,
            collection_name=uuid,
        )

        vector_similarity_search = chroma_initializer.similarity_search(k=6, query=question)
        print('Vector Similarity search results ...... ', vector_similarity_search)

        template = """
            You are a helpful assistant who provides data based on the provided context.
            Use strictly the context to search on question provided below.
            If you don't know the answer, Just say that "It's out current context".
            Avoid the cached data.
            Avoid any data from other sources apart from the context.
            Strictly detect context language, but strictly respond in the language of the question.
            Keep the answer concise and short and simple 250 words.
            The confidence level is value of how much the question is related to the context.
            The default confidence level is 0 
            Higher the question matching to context greater the confidence level which is in between 0 and 1.
            The output should be in json format with message, confidence_level and source in any condition
            Context: {context} 
            Question: {question}
            Helpful Answer: """

        prompt = PromptTemplate(template=template, input_variables=["question", "context"]).format(question=question,
                                                                                                   context=vector_similarity_search)

        message = [
            SystemMessage(content=prompt),
            HumanMessage(content=question),
        ]

        res = llm(message)

        embeddings_sources = chroma_initializer.get().get("metadatas")

        result_to_pass = res.content.strip('"')

        try:
            if result_to_pass is not None:
                result_new = json.loads(result_to_pass)
                source_dict = {"source": result_new.get("source")}

                if source_dict not in embeddings_sources:
                    result_new = config.api["gpt-out-of-context-message"]
            else:
                raise ValueError("No answer found.")
        except json.JSONDecodeError:
            result_new = config.api["gpt-out-of-context-message"]
        return result_new
    except RateLimitError as e:
        raise ValueError(json.loads(e.response.text)["error"]["message"])
    except ValueError as e:
        raise e
    except Exception as e:
        raise Exception(str(e))

Description

We are trying to do RAG on a arabic document.
We are able to add the arabic context to chroma db and via similarity search its retrieveing the data for arabic question.

The issue here we are facing is that its giving the response as not its out of context contain data.

if same question is being asked in english its responding back.

System Info

langchain==0.0.346
langchain-core==0.0.10
langchainhub==0.1.14
langsmith==0.0.69

[rupeshKumarAmt]

Checked other resources

• I added a very descriptive title to this question.
• I searched the LangChain documentation with the integrated search.
• I used the GitHub search to find a similar question and didn't find it.

Commit to Help

• I commit to help with one of those options 👆

Example Code

import os
import json
from dotenv import load_dotenv
from chromadb import HttpClient
from chromadb.config import Settings
from langchain.prompts import PromptTemplate
from langchain.chat_models import ChatOpenAI
from langchain.vectorstores.chroma import Chroma
from langchain.embeddings import OpenAIEmbeddings
import config
from openai._exceptions import RateLimitError
from langchain.schema.messages import HumanMessage, SystemMessage
import re
import unicodedata

def retrieve_answer(question: str, uuid, open_api_key):
    question = clean_user_query(question=question)

    try:

        llm = get_llm_object(open_api_key)

        chroma_client = get_chroma_client()

        # Check collection exist or not
        try:
            uuid = str(uuid)
            chroma_client.get_collection(name=uuid)
        except Exception as e:
            raise ValueError(json.loads(e.args[0]).get('error'))

        chroma_initializer = Chroma(
            embedding_function=OpenAIEmbeddings(api_key=open_api_key),
            client=chroma_client,
            collection_name=uuid,
        )

        vector_similarity_search = chroma_initializer.similarity_search(k=6, query=question)
        print('Vector Similarity search results ...... ', vector_similarity_search)

        template = """
            You are a helpful assistant who provides data based on the provided context.
            Use strictly the context to search on question provided below.
            If you don't know the answer, Just say that "It's out current context".
            Avoid the cached data.
            Avoid any data from other sources apart from the context.
            Strictly detect context language, but strictly respond in the language of the question.
            Keep the answer concise and short and simple 250 words.
            The confidence level is value of how much the question is related to the context.
            The default confidence level is 0 
            Higher the question matching to context greater the confidence level which is in between 0 and 1.
            The output should be in json format with message, confidence_level and source in any condition
            Context: {context} 
            Question: {question}
            Helpful Answer: """

        prompt = PromptTemplate(template=template, input_variables=["question", "context"]).format(question=question,
                                                                                                   context=vector_similarity_search)

        message = [
            SystemMessage(content=prompt),
            HumanMessage(content=question),
        ]

        res = llm(message)

        embeddings_sources = chroma_initializer.get().get("metadatas")

        result_to_pass = res.content.strip('"')

        try:
            if result_to_pass is not None:
                result_new = json.loads(result_to_pass)
                source_dict = {"source": result_new.get("source")}

                if source_dict not in embeddings_sources:
                    result_new = config.api["gpt-out-of-context-message"]
            else:
                raise ValueError("No answer found.")
        except json.JSONDecodeError:
            result_new = config.api["gpt-out-of-context-message"]
        return result_new
    except RateLimitError as e:
        raise ValueError(json.loads(e.response.text)["error"]["message"])
    except ValueError as e:
        raise e
    except Exception as e:
        raise Exception(str(e))

Description

We are trying to do RAG on a arabic document. We are able to add the arabic context to chroma db and via similarity search its retrieveing the data for arabic question.

The issue here we are facing is that its giving the response as not its out of context contain data.

if same question is being asked in english its responding back.

System Info

langchain==0.0.346 langchain-core==0.0.10 langchainhub==0.1.14 langsmith==0.0.69

Please Help its kind of urgent !