!pip install -q openai==1.61.1

   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 0.0/463.1 kB ? eta -:--:--
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╸ 460.8/463.1 kB 16.3 MB/s eta 0:00:01
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 463.1/463.1 kB 10.5 MB/s eta 0:00:00

from google.colab import drive
drive.mount('/content/drive')

Mounted at /content/drive

import os
project_path = "./Mid_Term_Project"
os.listdir(project_path)

['JHU_AGAI_W9_Mid_Term_Project_Learners_Notebook.ipynb',
 'Alex_emails_march_04.csv',
 'config.json']

os.chdir('./Mid_Term_Project')
os.listdir()

['JHU_AGAI_W9_Mid_Term_Project_Learners_Notebook.ipynb',
 'Alex_emails_march_04.csv',
 'config.json']

# @title Loading the `config.json` file
import json, os

# Load the JSON file and extract values
file_name = 'config.json'
with open(file_name, 'r') as file:
    config = json.load(file)
    os.environ['OPENAI_API_KEY']  = config.get("API_KEY") # Loading the API Key
    os.environ["OPENAI_BASE_URL"] = config.get("OPENAI_API_BASE") # Loading the API Base Url

model_name = "gpt-4o-mini"

from openai import OpenAI

# Initialize OpenAI client
client = OpenAI()

# @title LLM function
# @markdown Once the API details are filled, the notebook will automatically load the configuration, and learners can generate model outputs using the llm() function.


def llm(system_prompt, user_prompt):
  try:
      # Craft the messages to pass to chat.completions.create
      prompt = [
          {'role':'system', 'content': system_prompt},
          {'role': 'user', 'content': user_prompt}
      ]

      response = client.chat.completions.create(
          model=model_name,
          messages=prompt,
          temperature=0
      )

      return response.choices[0].message.content.strip()

  except Exception as e:
      prediction = f'Sorry, I encountered the following error: \n {e}'
      print(prediction)

# @title Step 1: Load the Dataset
# Data Loading

import pandas as pd
df = pd.read_csv("./emails.csv", index_col="email_id", encoding='latin-1')      #Add the data file location
df

df.head()

# @title Step 2: Apply Yesterbox Filtering
# @markdown The Yesterbox approach involves processing emails from the previous day first before tackling today's emails.

# @markdown For this dataset, consider today's date as 4th March 2025.

# @markdown We filter the dataset to only include emails received on 3rd March 2025 (yesterday)
# (Yesterbox Approach)(Today: 4 march)



from datetime import datetime, timedelta

yesterday_date = pd.to_datetime("3/3/2025").strftime('%m/%d/%Y')

df['date_received'] = pd.to_datetime(df['date_received']).dt.strftime('%m/%d/%Y')

yesterday_emails = df[df['date_received'] == yesterday_date].reset_index(drop=True)
print(f"Filtered Emails Count: {len(yesterday_emails)}")

Filtered Emails Count: 51

df.shape

(60, 5)

# Here we see only 51 emails, as in 10 email had the date of 4th March 2025
yesterday_emails.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 51 entries, 0 to 50
Data columns (total 5 columns):
 #   Column          Non-Null Count  Dtype 
---  ------          --------------  ----- 
 0   date_received   51 non-null     object
 1   sender          51 non-null     object
 2   subject         51 non-null     object
 3   body            51 non-null     object
 4   main_recipient  51 non-null     object
dtypes: object(5)
memory usage: 2.1+ KB

# @title Code to add categories to the dataset
categories = ['Urgent & High-Priority Emails',
              'Deadline-Driven Emails',
              'Routine Updates & Check-ins',
              'Non-Urgent Informational Emails',
              'Personal & Social Emails',
              'Spam/Unimportant Emails']

system_prompt = """

You are an AI Email CLassifier. Your task is to read a given email and classify them according to the below fixed categories:
1. Urgent & High-Priority Emails
2. Deadline-Driven Emails
3. Routine Updates & Check-ins
4. Non-Urgent Informational Emails
5. Personal & Social Emails
6. Spam/Unimportant Emails
Rules - The response should include only the fixed category names as written above and no addtional explanation or punctuations. Do not invent any new categories or change the category names in any way. If the email is promotional or irrelevant classify it as "Spam/Unimportant Emails".
Example input and output -
  Example Email : 'You Won a Free iPhone 15 Pro! ?? Click to Claim'
  Expected Response : 'Spam/Unimportant Emails'

"""

user_prompt = f"""

Classify the given email into exactly ONE of the predefined categories.


Respond with ONLY the category name from the list below:
- Urgent & High-Priority Emails
- Deadline-Driven Emails
- Routine Updates & Check-ins
- Non-Urgent Informational Emails
- Personal & Social Emails
- Spam/Unimportant Emails

"""

# @title Categorizing the emails
from tqdm import tqdm  # Import the tqdm library for the progress bar

# Ensure the 'category' column exists in the DataFrame
if 'category' not in yesterday_emails.columns:
  yesterday_emails['category'] = None  # Create the column if it does not exist


for index, row in tqdm(yesterday_emails.iterrows(), total=df.shape[0], desc='Processing emails'):
  prompt = f"""{user_prompt}
    Please find the attached email below from yesterday that need to be analyzed:
    ```
    {row.to_string()}
    ```
  """

  category_by_llm = llm(system_prompt, prompt)

  # Append the category generated by the LLM to the 'category' column in the same row
  if category_by_llm in categories:
    yesterday_emails.at[index, 'category'] = category_by_llm
  else:
    yesterday_emails.at[index, 'category'] = ""

Processing emails:  85%|████████▌ | 51/60 [00:37<00:06,  1.35it/s]

# Display the DataFrame which has categories
yesterday_emails.head()

yesterday_emails['category'].value_counts()

total_yesterday = len(yesterday_emails)
category_counts = yesterday_emails['category'].value_counts().to_dict()

urgent_count = category_counts.get('Urgent & High-Priority Emails', 0)
deadline_count = category_counts.get('Deadline-Driven Emails', 0)

critical_count = urgent_count + deadline_count
non_critical_count = total_yesterday - critical_count

# @title System Prompt
# Write your system prompt here
system_prompt = """

You are an Executive Email Assistant. Your task is to create an executive dashboard and give a top level summary of the emails in your inbox. The emails you are summarizing are only yesterday's emails and you need to provide a summary of those emails
where we know the count of the number of emails per category, highlight the number of critical emails that require immediate action and mention the emails that can be handled later.
Rules - Focus on being clear and concise, Highlight what needs attention today, Use a professional tone, do NOT invent data outside of the emails provided.

"""

# @title User Prompt
# Write your user prompt here
user_prompt = f"""

Generate an Executive Summary of Emails using the Yesterbox approach.

Task:
1. Analyze the dataset of classified emails below.
2. Count the number of emails in each category.
3. Use these counts to generate the executive dashboard.

Rules:
1. Derive all numbers only from the dataset below.
2. Do noy assume or invent any values.
3. All totals and subtotals must be internally consistent.

See Emails from Yesterday:

```
{yesterday_emails.to_string()}
```
An example of how the output should look:
Executive Summary of Emails

1. Total Number of Emails Received: <number>
2. Total Number of Emails from Yesterday: <number>
3. Email Breakdown by Categories (Count Only):
   - Urgent & High-Priority Emails: <count>
   - Deadline-Driven Emails: <count>
   - Routine Updates (Review & Acknowledge): <count>
   - Non-Urgent Informational Emails: <count>
   - Personal & Social Emails: <count>
   - Spam/Unimportant Emails: <count>

4. AI Conclusion:
   - Critical Emails Requiring Immediate Attention:
      - Urgent & High-Priority Emails: <count>
      - Deadline-Driven Emails: <count>
      - Total Critical Emails: <sum>
   - Emails That Can Be Reviewed Later:
      - Routine Updates: <count>
      - Non-Urgent Informational Emails: <count>
      - Personal & Social Emails: <count>
      - Spam/Unimportant Emails: <count>
      - Total Non-Critical Emails: <sum>

Summary Insights:
- Provide a short executive insight (2–3 sentences) about workload, urgency, and prioritization.
- Use a professional leadership tone.

"""

# @title  Calling the model and display the summary


response_1 = llm(system_prompt, user_prompt)        # llm is the model using gpt-4o-mini
response_1

from IPython.display import display, Markdown
display(Markdown(response_1))

# @title System Prompt

# Write your system prompt here
system_prompt = """
You are an AI-powered Executive Email Assistant.Your role is to analyze the urgent and high-priority emails received yesterday and summarize them as part of today’s to-do list.

Rules to follow:
1. Assume all provided emails were received yesterday.
2. Treat all emails as requiring immediate action today.
3. For each email, generate a concise summary and a clear, specific next step.
4. Do not merge emails and do not invent details beyond the email content.
5. Use a professional, executive tone.
6. Follow the exact output format requested by the user.

"""

# Filtering out the emails that are urgent and high-priority
urgent_emails = yesterday_emails[yesterday_emails['category'] == 'Urgent & High-Priority Emails']

# @title User Prompt

# Write your user prompt here
user_prompt = f"""

Below is a list of Urgent & High-Priority emails that were received yesterday and require immediate action today.

Your task is to:
1. Generate a concise summary for each email.
2. Clearly state the next step that should be taken today.
3. Ensure the next step is specific to that email.
4. Follow the exact format below for every email.

REQUIRED FORMAT (repeat for each email):

Subject:
Received:
Sender Name:
Summary:
Next Step:

IMPORTANT:
1. Add numbering for each new email..
2. Do not add headings or explanations.
3. Do not include any text outside the required format.
4. Ensure all emails are treated as part of today’s to-do list.

Below is the attached DataFrame, which contains all the emails that needs to be summarized:
```
{urgent_emails.to_string()}
```
"""

# @title  Calling the model and display the summary

response_2 = llm(system_prompt, user_prompt)

from IPython.display import display, Markdown
display(Markdown(response_2))

# @title System Prompt

# Write your system prompt here
system_prompt = """

You are an AI-powered Executive Email Assistant. Your role is to analyze the time-sensitive and deadline-driven emails
that were received yesterday and must be addressed today.

Guidelines to follow:
1. Each email must clearly involve a deadline occurring today or an impending delivery timeline.
2. For each email, generate a concise summary and a specific next step focused on meeting the deadline.
3. Do not include emails categorized as Urgent & High-Priority.
4. Do not invent deadlines or details not present in the email content.
5. Use a professional, executive tone.
6. Follow the exact output format requested by the user.

"""

# Filtering out the emails that are Time Sensitive & Deadline-Driven
deadline_emails = yesterday_emails[yesterday_emails['category'] == 'Deadline-Driven Emails']

# @title User Prompt


# Write your user prompt here
user_prompt = f"""

You are given a set of time sensitive and deadline-driven emails that were received yesterday
and must be addressed today.

Your task is to summarize only the deadline-driven emails and clearly outline
the required actions to meet deadlines.

Instructions to folloe:
1. The response should have the exact structure and headings shown below.
2. Do not include Urgent & High-Priority emails.
3. Each email must clearly involve a deadline that is today, overdue, or tied to an imminent meeting or delivery timeline.
4. Do not invent deadlines.
5. Do not add extra explanations outside the requested sections.

Required Output Format:
Subject: (The email's subject line)
Received: (The time and date the email was received)
Sender Name: (Name of the person who sent the email)
Summary: (A concise summary of the email's content)
Next Step: (A specific action that needs to be taken based on the email, with a clear focus on meeting deadlines)

Example Output:
Deadline-Driven Emails Summary

1. Email Exceeding Deadline (if applicable)
   - Subject:
   - Received:
   - Sender Name:
   - Summary:
   - Next Step:

2. Email Requiring Action Today
   - Subject:
   - Received:
   - Sender Name:
   - Summary:
   - Next Step:

(Repeat section 2 for each remaining deadline-driven email)

Final Count of Deadline-Driven Emails
 - Total Deadline-Driven Emails: <number> (mention if any have exceeded their deadline)

Summary of Actionable Items
 - List concise action items required today to meet deadlines.


Below is the attached DataFrame, which contains all the emails that needs to be summarized:
```
{deadline_emails.to_string()}
```
"""

# @title  Calling the model and display the summary

response_3 = llm(system_prompt, user_prompt)

from IPython.display import display, Markdown
display(Markdown(response_3))

# @title System Prompt
# Write your system prompt here
system_prompt = """

You are an AI-powered Executive Email Assistant. Your role is to draft a professional first response for critical business emails
(Urgent & High-Priority + Deadline-Driven) received yesterday.

Guidelines:
1. Acknowledge the sender’s request clearly and politely.
2. Address the key points, queries, or actions raised in the original email.
3. State a clear next step or decision.
4. Maintain a formal, professional corporate tone.
5. Do not introduce new commitments beyond what is implied in the email.
6. Follow the exact output format requested by the user.

"""

# Filtering out the emails that are Critical Emails, i.e. ('Urgent & High-Priority Emails' + 'Deadline-Driven Emails')
critical_emails = yesterday_emails[yesterday_emails['category'].isin(['Urgent & High-Priority Emails', 'Deadline-Driven Emails'])]

# @title User Prompt

# You may format the classifying method according to you


# Write your user prompt here
user_prompt = f"""

You are given a list of critical emails (cateogories - Urgent & High-Priority and Deadline-Driven)
that were received yesterday and require an initial response today.

Your task is to:
1. Draft a professional first-response email for each provided email.
2. The response should acknowledge the sender, address their request, and state the next step.
3. Keep responses concise, polite, and professional in tone.

Output format required (repeat for each email):

Subject: (Subject of the email)
Sender Name: (Name of the Sender)
AI Drafted Reply: (Your AI-generated response)

Important:
1. Do not add numbering, bullet points, or section headers.
2. Do not include explanations or commentary outside the draft.
3. Each draft should read like a real email reply.
4. Address each sender in the body of the email
5. Sign off each email with the sender name: Alex.

Below is the attached DataFrame, which contains all the critical emails that needs to be replied:
```
{critical_emails.to_string()}
```
"""

# @title  Calling the model and display the summary

response_4 = llm(system_prompt, user_prompt)
from IPython.display import display, Markdown
display(Markdown(response_4))

# @title Evaluation System Prompt

eval_system_prompt = """

You are an impartial LLM-as-a-Judge. You need to assess the quality,correctness,and effectiveness of responses generated by another LLM.
Your role is to evaluate AI-generated email summaries and responses based on predefined qualitative criteria.

The objective is to:+
1. Assess the provided email summary.
2. Assign numeric scores from 1 to 5 on criteria such as relevance, clarity and actionability.
3. Provide concise justifications for each score.
4. Highlight the key strengths of the summaries and suggest areas for improvement.
5. Output the evaluation in a valid JSON format as mentioned by the user.
6. Provide a overall justification for the evaluation and mention key observations.

Guidelines:
1. Do not include or generate any text outside the JSON object.
2. Do not include markdown, explanations, or headings.

"""

# @title Evaluation User Prompt

eval_user_prompt = """

  Evaluate the following AI-generated summary using the criteria below.

  Evaluation Criteria:
  1. Relevance:
    -How well does the summary address the original task or content of the email?
    -Identify the key information that is captured or missing.
    -Provide a score from 1 to 5 with justification.

  2. Clarity:
    -How clear and easy to understand is the summary?
    -Highlight any ambiguity or confusing phrasing.
    -Provide a score from 1 to 5 with justification.

  3. Actionability:
    -Does the summary clearly specify next steps or actions?
    -Are the actions practical and next steps well-defined?
    -Provide a score from 1 to 5 with justification.

  Additionally, provide:
  -Strengths: Highlight the key strengths of the summary.
  -Improvements: Suggest 1-2 areas for improvement.
  -Overall Justification: Provide a 2-3 line summary evaluation, including key observations.

  Output Requirements:
  - Respond only in valid JSON.
  - Separate each drafted response using the delimiter: ---------
  - Use the following structure and keys:

  {
    "Relevance": {"score": "", "justification": ""},
    "Clarity": {"score": "", "justification": ""},
    "Actionability": {"score": "", "justification": ""},
    "Strengths": "",
    "Improvements": "",
    "Overall_Justification": ""
  }

"""

# @title Evaluation Function & User Prompt

def evaluate_summary(eval_system_prompt, eval_user_prompt, summary, eval_model="gpt-4o-mini"):
    try:

        modified_prompt = f""" {eval_user_prompt}
                          Here is the summary:
                          ```
                          {summary}
                          ```
                          """

        eval_response = client.chat.completions.create(
            model=eval_model,
            messages=[
                {"role": "system", "content": eval_system_prompt},
                {"role": "user", "content": modified_prompt}
            ],
            temperature=0
        )
        return eval_response.choices[0].message.content.strip()

    except Exception as e:
        print(f"Error evaluating prompt: {e}")
        return "{}"  # Return empty JSON structure on error

responses = response_4.split("Subject:")            # splitting response on the basis of delimiter, so we get each response for the email indivisually
for _ in responses[1:-1]:                      # Excluding First and Last Element as they are nothing but blanks .i.e. (' ')
  display(Markdown(_))
  print("+"*100)

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

print(response_4)

# @title Evaluation for Each Generated Response
evaluation_results = [evaluate_summary(eval_system_prompt, eval_user_prompt, summary) for summary in responses[1:-1]]        # Excluding first and last elements as they are not the actual emails and some LLM generated support texts

display(Markdown(evaluation_results[0]))

# @title Converting JSON Scores into DataFrame
import pandas as pd
import json

scores = []
justifications = []
strengths = []
improvements = []


for result in evaluation_results:
    try:
        result_dict = json.loads(result)
        score_dict = {k: v.get("score", "NA") for k, v in result_dict.items() if isinstance(v, dict) and "score" in v}
        justification = result_dict.get("Overall_Justification", "NA")
        strength = result_dict.get("Strengths", "NA")
        improvement = result_dict.get("Improvements", "NA")

        # Append Results
        scores.append(score_dict)
        justifications.append(justification)
        strengths.append(strength)
        improvements.append(improvement)

    except (json.JSONDecodeError, KeyError, TypeError):
        scores.append({"Relevance": None, "Clarity": None, "Conciseness": None, "Coherence": None, "Actionability": None})
        justifications.append("")
        strengths.append("")
        improvements.append("")

# @title Creating Final Evaluation DataFrame
pd.set_option('max_colwidth', 0)
df_scores = pd.DataFrame(scores)
df_scores["Strengths"] = strengths
df_scores["Improvements"] = improvements
df_scores["Justification"] = justifications

# @title Display Final Evaluation Table
df_scores

	date_received	sender	subject	body	main_recipient	category
0	03/03/2025	Julia Martin	Approval Request: Budget Approval Needed by EOD	Hi Alex,\n\nI hope you're doing well. As we ap...	Alex	Deadline-Driven Emails
1	03/03/2025	Fiona White	Are Your APIs Secure? Reddit & Discord Sound t...	Hi Alex,\n\nA heated Discord discussion in the...	Alex	Non-Urgent Informational Emails
2	03/03/2025	Samantha Lee	Approval Needed: Project Scope Adjustment for ...	Hi Alex,\n\nWeve encountered an unexpected AP...	Alex	Urgent & High-Priority Emails
3	03/03/2025	James Patel	Subject: Daily Update Project Titan (March 3)	Hey Alex,\n\nQuick update on Project Titan for...	Alex	Routine Updates & Check-ins
4	03/03/2025	David Whitmore	[URGENT] Dashboard Syncing Issues Production...	Hey Alex,\n\nWeve got a big issue right nowl...	Alex	Urgent & High-Priority Emails

	Relevance	Clarity	Actionability	Strengths	Improvements	Justification
0	5	4	5	The summary is concise and directly addresses the key points of the original email, including the urgency and the commitment to action.	Clarify the reference to 'next quarter's projects' to avoid ambiguity and ensure all parties understand which projects are being discussed.	The summary is effective in conveying the essential information and next steps regarding the budget approval request. It is relevant and actionable, with minor clarity issues that could be improved.
1	4	5	4	The summary effectively conveys the approval needed and the specific action to be taken, maintaining a professional tone throughout.	It could include more context regarding the reasons for the project scope adjustment and provide a clearer timeline for the implementation steps.	Overall, the summary is relevant, clear, and actionable, though it could enhance its effectiveness by providing additional context and detailed next steps.
2	4	5	4	The summary is clear and concise, effectively acknowledging the issue and committing to a timely response.	Include more specific details about the discrepancies and suggest any immediate actions for the sender, if relevant.	The summary effectively addresses the issue and is clear in its communication. However, it could enhance relevance and actionability by providing more details and immediate next steps.
3	4	5	4	The summary effectively conveys the urgency of the request and the timeline for approval, maintaining a professional tone throughout.	It could include more specific details about the resources being requested and clarify any actions Nathan needs to take following the approval.	The summary is generally effective, with strong clarity and relevance. However, it could benefit from additional details to enhance actionability and provide a more comprehensive overview.
4	4	5	4	The summary effectively captures the urgency of the issue and proposes a clear next step, maintaining a professional tone.	1. Include more context about the implications of the data sync failure. 2. Suggest a timeframe for the proposed call to enhance actionability.	The summary is relevant and clear, effectively addressing the issue at hand. However, it could benefit from additional context and more defined next steps.
5	4	5	3	The summary effectively conveys the urgency of the request and acknowledges the need for additional time, demonstrating understanding and professionalism.	The summary could include specific reasons for the extension request and a clearer timeline for when the decision will be communicated.	The summary is relevant and clear, but it falls short in providing actionable next steps. Including more details about the request and a timeline would enhance its effectiveness.
6	4	5	4	The summary effectively acknowledges the urgency of the issue and maintains a professional tone. It also sets a clear expectation for a follow-up discussion.	The summary could benefit from including more specific details about the investigation process and any immediate actions being taken. Additionally, mentioning the expected timeline for updates would enhance clarity.	Overall, the summary is relevant and clear, with a good acknowledgment of the issue. However, it could improve in actionability by providing more specific next steps and details about the investigation.
7	4	5	3	The summary effectively acknowledges the issue and expresses a commitment to follow up, demonstrating responsiveness.	It could benefit from including specific implications of the delay and a clearer timeline for the next steps.	The summary is relevant and clear, but it falls short in providing actionable next steps. Overall, it effectively communicates the situation but could enhance its effectiveness with more detail.
8	5	4	4	The summary effectively conveys the urgency of the situation and the commitment to address the issue promptly.	Clarify the expected timeframe for the update and specify any immediate actions being taken by the team.	The summary is relevant and conveys urgency well, but it could improve in clarity and actionability by providing more specific details about next steps.
9	4	5	4	The summary effectively conveys gratitude for the meeting invite and confirms attendance, demonstrating professionalism and engagement.	Including specific agenda items or topics for discussion would enhance relevance. Additionally, suggesting any preparatory actions could improve actionability.	The summary is generally effective, with strong clarity and relevance. However, it could benefit from more detailed content regarding the meeting agenda and actionable steps.
10	4	5	4	The summary is clear and acknowledges the issue promptly, demonstrating a proactive approach to resolving the problem.	Include more specific details about the authentication issue and outline a timeline or specific next steps for resolution.	The summary effectively addresses the issue and is clear in its communication. However, it could benefit from additional details and a more defined action plan.
11	4	5	4	The summary effectively conveys urgency and clarity in the response, ensuring that the recipient understands the commitment to act promptly.	1. Include more context about the implications of the approval. 2. Mention any additional actions required from the sender or other stakeholders.	The summary is strong in clarity and relevance but could enhance its actionability by providing more context and detailing further steps. Overall, it effectively communicates the urgency of the situation.
12	5	4	4	The summary effectively conveys the urgency of the situation and the commitment to address the issue promptly.	1. Clarify the timeline for the investigation and status update. 2. Specify any immediate actions that the recipient should take, if applicable.	The summary is relevant and conveys urgency well, but it could improve in clarity and actionability by providing more specific details about next steps.
13	4	5	3	The summary is clear and acknowledges the sender's concerns, maintaining a professional tone.	Include more specific details about the server issue and outline clear next steps or a timeline for resolution.	The summary effectively addresses the follow-up on server downtime with clarity but lacks some details and actionable next steps that would enhance its effectiveness.
14	4	5	4	The summary effectively captures the main issue and expresses a willingness to collaborate, demonstrating a positive tone.	It could benefit from including more specific details about the meeting agenda and outlining any preparatory actions that Alex might take before the meeting.	The summary is generally effective, with strong clarity and relevance. However, enhancing the actionability by specifying next steps would improve its overall effectiveness.
15	4	5	4	The summary effectively communicates the main points of the email and maintains a professional tone.	1. Include more details about the expected benefits of the upgrade. 2. Specify any actions required from the sender or other parties involved in the approval process.	The summary is relevant and clear, effectively conveying the request and next steps. However, it could benefit from additional details and clearer action items to enhance its effectiveness.
16	4	5	4	The summary effectively captures the urgency of the situation and provides a clear response from Alex, demonstrating a commitment to resolving the issue.	The summary could include specific details about the invoice and outline a timeline for the approval process to enhance clarity and actionability.	The summary is relevant and clear, effectively conveying the urgency of the overdue invoice. However, it could benefit from additional details and a clearer outline of next steps.
17	4	5	4	The summary effectively acknowledges the issue and expresses a commitment to investigate, demonstrating responsiveness and professionalism.	1. Include more context about the importance of the firewall logs. 2. Specify a timeline for when the sender can expect further updates.	The summary is relevant, clear, and actionable, but it could benefit from additional context and a clearer timeline for follow-up. Overall, it effectively addresses the sender's concern.
18	4	5	4	The summary effectively conveys urgency and clarity in the response, ensuring the recipient understands the importance of the issue.	The summary could benefit from including more specific details about the medication alerts and outlining the expected outcomes of the emergency call.	The summary is relevant and clear, effectively addressing the urgency of the situation. However, it could be improved by providing more specific information and clearer next steps.
19	4	5	4	The summary effectively conveys the main points of the email and maintains a professional tone.	1. Include specific details about the budget or context of the AWS Summit 2025. 2. Specify a timeline for when the approval will be provided.	The summary is relevant and clear, effectively communicating the request and response. However, it could be improved by adding more context and a timeline for action.
20	4	5	3	The summary is clear and concise, effectively acknowledging the meeting invite and expressing a willingness to resolve the issue.	Include more specific details about the invoice dispute to enhance relevance and suggest concrete next steps or actions to improve actionability.	The summary is generally effective, with strong clarity and relevance, but it could benefit from additional details and actionable steps to enhance its overall utility.

Business Problem - Automating Email Prioritization & Responses¶

The Case Study¶

Objective¶

LLM Setup¶

Data Setup¶

Background¶

About You¶

TASK - Categorization of emails¶

Task 1: Awareness of your email: Get to know the detailed summary of your email received¶

1A. Executive Dashboard (Top-Level Summary of Yesterday’s Inbox)¶

TASK¶

1B. Urgent Emails from Yesterday (🛑 Must-Do First Today)¶

TASK¶

1C: Deadline-Driven Emails from Yesterday (⚡ Needs Attention Today)¶

TASK¶

Task 2: AI-Generated "First Response" Drafts for Critical Email¶

Critical Emails are combination of the Urgent & High-Priority Emails + Deadline-Driven Emails¶

TASK¶

Task 3: Evaluation¶

Evaluation of Prompt Outputs with LLM-as-a-Judge¶

2. Evaluation Criteria¶

TASK¶

Task 4: Summary and Recommendation¶

	date_received	sender	subject	body	main_recipient
email_id
1	3/3/2025	Julia Martin	Approval Request: Budget Approval Needed by EOD	Hi Alex,\n\nI hope you're doing well. As we ap...	Alex
2	3/3/2025	Fiona White	Are Your APIs Secure? Reddit & Discord Sound t...	Hi Alex,\n\nA heated Discord discussion in the...	Alex
3	3/3/2025	Samantha Lee	Approval Needed: Project Scope Adjustment for ...	Hi Alex,\n\nWeve encountered an unexpected AP...	Alex
4	3/3/2025	James Patel	Subject: Daily Update Project Titan (March 3)	Hey Alex,\n\nQuick update on Project Titan for...	Alex
5	3/3/2025	David Whitmore	[URGENT] Dashboard Syncing Issues Production...	Hey Alex,\n\nWeve got a big issue right nowl...	Alex