# models.py

import numpy as np

import torch

import torch.nn as nn

from transformer import Transformer

from utils import Indexer

class LanguageModel$($object$)$:

$$ def get$\_$next$\_$char$\_$log$\_$probs$($self$,$ context$)->$ np$.$ndarray:

$"""$

$$ Returns a log probability distribution over the next characters given a context.

$$ The log should be base e

$$ NOTE: You should make sure you call model.eval$()$ to determinize inference here $($turns off dropout

$$ layers in TransformerEncoder$).$

$$ :param context: the string context that the LM conditions on

$$ :return: A numpy vector log P$($y $|$ context$)$ where y ranges over the output vocabulary.

$"""$

$$ raise Exception$("$Only implemented in subclasses"$)$

$$ def get$\_$log$\_$prob$\_$sequence$($self$,$ next$\_$chars, context$)->$ float:

$"""$

$$ Scores a bunch of characters following context. That is$,$ returns

$$ log P$($nc$1,$ nc$2,$ nc$3,...|$ context$)=$ log P$($nc$1|$ context$)+$ log P$($nc$2|$ context, nc$1),...$

$$ The log should be base e

$$ NOTE: You should make sure you call model.eval$()$ to determinize inference here $($turns off dropout

$$ layers in TransformerEncoder$).$

$$ :param next$\_$chars:

$$ :param context:

$$ :return: The float probability

$"""$

$$ raise Exception$("$Only implemented in subclasses"$)$

class UniformLanguageModel$($LanguageModel$)$:

$$ def $\_\_$init$\_\_($self$,$ voc$\_$size$)$:

$$ self.voc$\_$size $=$ voc$\_$size

$$ def get$\_$next$\_$char$\_$log$\_$probs$($self$,$ context$)$:

$$ return np$.$ones$([$self$.$voc$\_$size$])*$ np$.$log$(1\mathrm{.}0/$self$.$voc$\_$size$)$

$$ def get$\_$log$\_$prob$\_$sequence$($self$,$ next$\_$chars, context$)$:

$$ return np$.$log$(1\mathrm{.}0/$self$.$voc$\_$size$)*$ len$($next$\_$chars$)$