import string

from collections import Counter

# Computed from "A Tale of Two Cities". Compare Table $1\mathrm{.}3$ in Hoffstein, Pipher, Silverman.

english$\_$freq $=\{$

$\text{'}$a$\text{'}$: $0\mathrm{.}0803,$

$\text{'}$b$\text{'}$: $0\mathrm{.}014,$

$\text{'}$c$\text{'}$: $0\mathrm{.}0232,$

$\text{'}$d$\text{'}$: $0\mathrm{.}0467,$

$\text{'}$e$\text{'}$: $0\mathrm{.}1247,$

$\text{'}$f$\text{'}$: $0\mathrm{.}0226,$

$\text{'}$g$\text{'}$: $0\mathrm{.}0209,$

$\text{'}$h$\text{'}$: $0\mathrm{.}065,$

$\text{'}$i$\text{'}$: $0\mathrm{.}0683,$

$\text{'}$j$\text{'}$: $0\mathrm{.}0012,$

$\text{'}$k$\text{'}$: $0\mathrm{.}008,$

$\text{'}$l$\text{'}$: $0\mathrm{.}0367,$

$\text{'}$m$\text{'}$: $0\mathrm{.}0255,$

$\text{'}$n$\text{'}$: $0\mathrm{.}0706,$

$\text{'}$o$\text{'}$: $0\mathrm{.}0776,$

$\text{'}$p$\text{'}$: $0\mathrm{.}0166,$

$\text{'}$q$\text{'}$: $0\mathrm{.}0011,$

$\text{'}$r$\text{'}$: $0\mathrm{.}0621,$

$\text{'}$s$\text{'}$: $0\mathrm{.}0626,$

$\text{'}$t$\text{'}$: $0\mathrm{.}0902,$

$\text{'}$u$\text{'}$: $0\mathrm{.}0279,$

$\text{'}$v$\text{'}$: $0\mathrm{.}0087,$

$\text{'}$w$\text{'}$: $0\mathrm{.}0236,$

$\text{'}$x$\text{'}$: $0\mathrm{.}0012,$

$\text{'}$y$\text{'}$: $0\mathrm{.}0203,$

$\text{'}$z$\text{'}$: $0\mathrm{.}0004$

$\}$

def only$\_$letters$($X$,$ case$=$None$)$:

$\text{'}\text{'}\text{'}$Returns the string obtained from X by removing everything but the letters.

If case$=$"upper" or case$=$"lower", then the letters are all

converted to the same case.$\text{'}\text{'}\text{'}$

X $=\text{'}\text{'}.$join$($c for c in X if c in string.ascii$\_$letters$)$

if len$($X$)==0$:

return None

if case is None:

return X

elif case $==$ "lower":

return X$.$lower$()$

elif case $==$ "upper":

return X$.$upper$()$

def shift$\_$char$($ch$,$ shift$\_$amt$)$:

$\text{'}\text{'}\text{'}$Shifts a specific character by shift$\_$amt.

Example:

shift$\_$char$("$Y$",3)$ returns $"$B$"$

$\text{'}\text{'}\text{'}$

if ch in string.ascii$\_$lowercase:

base $=\text{'}$a$\text{'}$

elif ch in string.ascii$\_$uppercase:

base $=\text{'}$A$\text{'}$

# It's not clear what shifting should mean in other cases

# so if the character is not upper or lower$-$case, we leave it unchanged

else:

return ch

return chr$(($ord$($ch$)-$ord$($base$)+$shift$\_$amt$)\%26+$ord$($base$))$

def shift$\_$string$($X$,$ shift$\_$amt$)$:

$\text{'}\text{'}\text{'}$Shifts all characters in X by the same amount.$\text{'}\text{'}\text{'}$

return $\text{'}\text{'}.$join$($shift$\_$char$($ch$,$ shift$\_$amt$)$ for ch in X$)$

def count$\_$substrings$($X$,$n$)$:

$\text{'}\text{'}\text{'}$Returns a Python Counter object of all n$-$grams in X$.\text{'}\text{'}\text{'}$

if not X:

return $\{\}$

X $=$ only$\_$letters$($X$)$

shifts $=[$X$[$i:$]$ for i in range$($n$)]$

grams $=[\text{'}\text{'}.$join$($chrs$)$ for chrs in zip$(*$shifts$)]$

return Counter$($grams$)$

def get$\_$freq$($X$)$:

$\text{'}\text{'}\text{'}$Returns the proportion that each letter occurs in $"$X$".$

I might change this later, but for now, it converts everything to lowercase.

The reason is to match what appears in the english$\_$freq dictionary.$\text{'}\text{'}\text{'}$

X $=$ only$\_$letters$($X$,$ case$=$"lower"$)$

n $=$ len$($X$)$

ctr $=$ count$\_$substrings$($X$,1)$

output $=\{\}$

for char in string.ascii$\_$lowercase:

output$[$char$]=$ ctr$[$char$]/$n

return output

def mut$\_$ind$\_$co$($d$1,$ d$2)$:

$\text{'}\text{'}\text{'}$For letter frequency dictionaries d$1$ and d$2,$ return the Mutual Index of Coincidence.

See Equation $(5\mathrm{.}9)$ on page $222$ in Hoffstein, Pipher, Silverman.$\text{'}\text{'}\text{'}$

s $=0$

for k in d$1.$keys$()$:

s $+=$ d$1.$get$($k$,0)*$d$2.$get$($k$,0)$

return s

$*$ Paste in a block of English text, surrounded by triple quotation marks $($this helps prevent errors due to apostrophes$),$ and store the resulting string using the variable $`$s$`$:

$```$

s $=\text{'}\text{'}\text{'}$Your text goes here. It can be long and include linebreaks.$\text{'}\text{'}\text{'}$

$```$

$*$ How do these values compare to the percentages shown in Hoffstein$-$Pipher$-$Silverman $($HPS$),$ Table $1\mathrm{.}3$ on page $6?($Your answer will depend on how long your block of text was. Notice that HPS reports percentages, and the $`$get$\_$freq$`$ function reports probabilities or proportions.$)$

Briefly answer this question in a markdown cell $($you can click the three lines on the right side of this cell and select "add markdown cell$).$

For your same string $`$s$`$ as above, compute the Mutual Index of Coincidence with $`$english$\_$freq$`.$ For suitably long English text, the value should be around $`0\mathrm{.}065`.($As long as the value is above $`0\mathrm{.}06`$ and below $`0\mathrm{.}075`,$ that is a good range.$)$

Make a new string $`$t$`$ of random text $($not English, just hit random keys on the keyboard, but try not to repeat the same letters to an extreme degree$).$

Again compute the Mutual Index of Coincidence with $`$english$\_$freq$`.$ The new value should be closer to $`0\mathrm{.}04`$ or $`0\mathrm{.}05`.$