# TF$-$IDF and PPMI Code Template

$"""$

This template will help you implement TF$-$IDF and PPMI calculations using the NLTK library and the Brown corpus.

You will preprocess the corpus, compute term frequencies, document frequencies, TF$-$IDF scores, and create

a word co$-$occurrence matrix to compute Positive Pointwise Mutual Information $($PPMI$)$ scores.

$"""$

import nltk

from nltk$.$corpus import brown

from collections import defaultdict, Counter

import math

import numpy as np

# Download the Brown corpus if not already downloaded

nltk$.$download$(\text{'}$brown$\text{'})$

# Preprocess the corpus: Tokenize, lowercase, and add start$/$end tokens

def preprocess$($corpus$)$:

$"""$

Preprocess the corpus by tokenizing, converting to lowercase, and adding and tokens.

Args:

corpus $($list$)$: List of sentences from the corpus.

Returns:

list: Preprocessed and tokenized corpus.

$"""$

tokenized$\_$corpus $=[]$

for sentence in corpus:

# TODO: Implement tokenization and lowercasing

# HINT: Use list comprehension and str$.$lower$()$

# TODO: Add $\text{'}\text{'}$ at the start and $\text{'}\text{'}$ at the end of the sentence

pass # Remove this line after implementing

return tokenized$\_$corpus

# Calculate Term Frequency $($TF$)$

def compute$\_$tf$($corpus$)$:

$"""$

Calculate the term frequency for each word in each document.

Args:

corpus $($list$)$: Preprocessed corpus where each document is a list of words.

Returns:

dict: Term frequencies for each document.

$"""$

tf $=$ defaultdict$($Counter$)$

# TODO: For each document, count the occurrences of each word

# HINT: Use enumerate to get document index and Counter to count words

pass # Remove this line after implementing

return tf

# Calculate Document Frequency $($DF$)$

def compute$\_$df$($tf$)$:

$"""$

Calculate the document frequency for each word across all documents.

Args:

tf $($dict$)$: Term frequencies for each document.

Returns:

Counter: Document frequencies for each word.

$"""$

df $=$ Counter$()$

# TODO: For each word, count the number of documents it appears in

# HINT: Use a set of words for each document to avoid counting duplicates

pass # Remove this line after implementing

return df

# Calculate TF$-$IDF for each word

def compute$\_$tfidf$($tf$,$ df$,$ num$\_$docs$)$:

$"""$

Calculate the TF$-$IDF score for each word in each document.

Args:

tf $($dict$)$: Term frequencies for each document.

df $($Counter$)$: Document frequencies for each word.

num$\_$docs $($int$)$: Total number of documents.

Returns:

dict: TF$-$IDF scores for each word in each document.

$"""$

tfidf $=$ defaultdict$($dict$)$

# TODO: For each document and word, calculate TF$-$IDF score

# TF$-$IDF formula: TF$($word$)*$ log$($N $/(1+$ DF$($word$)))$

# HINT: Use math.log$()$ for logarithm

pass # Remove this line after implementing

return tfidf

# Create a word co$-$occurrence matrix

def create$\_$cooccurrence$\_$matrix$($corpus$,$ window$\_$size$=5)$:

$"""$

Create a word co$-$occurrence matrix from the corpus.

Args:

corpus $($list$)$: Preprocessed corpus where each document is a list of words.

window$\_$size $($int$)$: The size of the context window.

Returns:

tuple: Co$-$occurrence matrix, word to index mapping, and index to word mapping.

$"""$

# TODO: Build the vocabulary of unique words

# HINT: Use a set to store unique words

pass # Remove this line after implementing

# TODO: Initialize co$-$occurrence matrix

# HINT: Use numpy to create a zero matrix of size vocab$\_$size x vocab$\_$size

pass # Remove this line after implementing

# TODO: Fill in the co$-$occurrence matrix

# HINT: For each word, consider a window of words around it

pass # Remove this line after implementing

return cooccurrence$\_$matrix, word$\_$to$\_$id$,$ id$\_$to$\_$word

# Calculate PPMI from co$-$occurrence matrix

def compute$\_$ppmi$($cooccurrence$\_$matrix$)$:

$"""$

Compute the Positive Pointwise Mutual Information $($PPMI$)$ matrix from the co$-$occurrence matrix.

Args:

cooccurrence$\_$matrix $($numpy$.$ndarray$)$: Co$-$occurrence matrix of word counts.

Returns:

numpy.ndarray: PPMI matrix.

$"""$

# TODO: Calculate total sum of all co$-$occurrences

# HINT: Use numpy.sum$()$

pass # Remove this line after implementing

# TODO: Calculate sum over rows $($word occurrence counts$)$

# HINT: Use numpy.sum$()$ with axis$=1$

pass # Remove this line after implementing

# Initialize PPMI matrix with zeros

ppmi$\_$matrix $=$ np$.$zeros$($cooccurrence$\_$matrix.shape$)$

# TODO: Compute PPMI for each cell in the matrix

# HINT: Use nested loops to iterate over the matrix indices

# Remember to check if pij $>0$ before computing PMI

pass # Remove this line after implementing

return ppmi$\_$matrix

# Main execution

if $\_\_$name$\_\_=="\_\_$main$\_$