OCR Performance Metric

trex = rcompile('[^a-zA-Z 0-9]')
tokenize = lambda x: trex.sub('', x.lower().replace(',', ' ').replace('-', ' '))

def read_transcript(file_name):
    file_names = glob(f'{file_name}*')
    ret = ''
    for fname in file_names:
        with open(fname) as fd:
            ret += fd.read()+'\n\n'
    return ret

#Some of the reference transcripts have missing information and those files have
#filename with '(missing page <number>)' substrings - filter these files out for
#now.
rex = rcompile('[0-9]+.txt')
reference_transcripts_file_roots = [x.split('.')[0] for x in reference_transcripts_files if rex.match(x) is not None]

with Path(reference_transcripts_dir):
    reference_transcripts = [read_transcript(f'{x}.txt') for x in reference_transcripts_file_roots]

reference_transcripts = list(map(tokenize, reference_transcripts))

 Instantiate vectorizer object
tfidf = TfidfVectorizer(stop_words='english',
                        ngram_range=(1,2),
                        min_df=1
                       )
# Create a vocabulary and get word counts per document
ref_dtm = tfidf.fit_transform(reference_transcripts)

features = tfidf.get_feature_names()
#display(len(features), features[:50])

# Get feature names to use as dataframe column headers
ref_dtm = pd.DataFrame(ref_dtm.todense(), columns=features)

ishape = ref_dtm.shape[1] # 22296
# Create Model 
input_img = Input(shape=(ishape, ))

x = Dense(512)(input_img)

x = Dense(256)(x)

x = Dense(128)(x)

encoded = Dense(64)(x)

x = Dense(128)(encoded)

x = Dense(256)(x)

x = Dense(512, activation='sigmoid')(x)
decoded = Dense(ishape, activation='sigmoid')(x)


rmodel = Model(input_img, decoded)
rmodel.compile(loss='mse', optimizer=Adam(learning_rate=0.01))

rmodel.fit(ref_dtm, ref_dtm, batch_size=512, epochs=5)

encoder = Model(input_img, encoded)

encoded_ref_dtm = encoder.predict(ref_dtm)

from joblib import dump
MODELS_DIR = '../models/'

TFIDF = MODELS_DIR+'tfidf.pkl'
ENCODED_REF_DTM = MODELS_DIR+'encoded_ref_dtm.pkl'
ENCODER = MODELS_DIR+'encoder.h5'
FILE_ROOTS = MODELS_DIR+'file_roots.pkl'

dump(tfidf, TFIDF)
encoder.save(ENCODER)
dump(encoded_ref_dtm, ENCODED_REF_DTM)
dump(reference_transcripts_file_roots, FILE_ROOTS)

        # Load the model saved in encoder.h5
        self.encoder = load_model(ENCODER)
        # Load the TfIDF vectorizer saved in tfidf.pkl
        self.tfidf = load(TFIDF)
        # Load the encoded reference DTM saved in encoded_ref_dtm.pkl
        self.encoded_ref_dtm = load(ENCODED_REF_DTM)
        # Load transcript file roots list from file_roots.pkl
        self.file_roots = load(FILE_ROOTS)

# Match characters that are not blank space, digits or
# letters in the alphabet.
rex = rcompile('[^a-zA-Z 0-9]')

def tokenize(x):
    return rex.sub('', x.lower().replace(',', ' ').replace('-', ' '))

        def read_transcript(file_name):
            '''
            Read and return the content of all transcript files
            that have file_name as prefix.
            e.g. A file name of .../google_transcripts/3102
            would mean that the conents of
            .../google_transcripts/3102-1
            and
            .../google_transcripts/3102-2
            would be read and returned.
            '''

            # generate the list of all file names that match
            # the wild carded file name
            file_names = glob(f'{file_name}*')
            ret = ''

            for fname in file_names:
                with open(fname) as fd:
                    ret += fd.read()+'\n\n'

            return ret

        with Path(transcripts_dir):
            self.transcripts = [read_transcript(x) for x in self.file_roots]

        self.transcripts = list(map(tokenize, self.transcripts))

        # Use the TfIDF vectorizer to vectorize the transcripts
        dtm = self.tfidf.transform(self.transcripts).todense()
        # Run the vectors through the encoder to reduce
        # dimensionality
        self.encoded_dtm = self.encoder.predict(dtm)

# Convert cosine_similarity value to cos error
# cosine_similarity of 1 implies identical text
# so 1 minus that value is a measure of error.


def COS_ERROR(x): return abs(round((1-x)*1e7, 0))

        cossim = []
        for idx, ref in enumerate(self.encoded_ref_dtm):
            cossim.append(
                cosine_similarity([ref],
                                  [self.encoded_dtm[idx]])[0][0])

        # Convert cosine_similarity values into COS_ERROR values
        # and store in coserr
        coserr = [COS_ERROR(x) for x in cossim]

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

#DATA_DIR is set to location of all the error files
GOOGLE_COS_ERROR = DATA_DIR + 'google_cos_error.csv'
REFERENCE_COS_ERROR = DATA_DIR + 'ref_cos_error.csv'
TESSERACT_SSQ_COS_ERROR = DATA_DIR + 'tess_ssq_cos_error.csv'
TESSERACT_STORYSQUAD_COS_ERROR = DATA_DIR + 'tess_storysquad_cos_error.csv'

reference_error = pd.read_csv(REFERENCE_COS_ERROR)
google_error = pd.read_csv(GOOGLE_COS_ERROR)
ssq_error = pd.read_csv(TESSERACT_SSQ_COS_ERROR)
storysquad_error = pd.read_csv(TESSERACT_STORYSQUAD_COS_ERROR)

xticks = reference_error.index.to_list()
xlabels = reference_error.img_name.to_list()

sns.set(font_scale=4)
fig, ax = plt.subplots(1,1,figsize=(130,50), subplot_kw={'ylim': (0,10000), 'xlim':(0,164)})

ax.set_facecolor('white')
reference_error['ref_cos_error'].plot(ax = ax, linewidth=20,color='green', label='Reference');
google_error['google_cos_error'].plot(ax = ax, linewidth=7,color='blue', label='Google');
ssq_error['tess_ssq_cos_error'].plot(ax = ax, linewidth=7,color='cyan', label='Tesseract ssq');
storysquad_error['tess_storysquad_cos_error'].plot(ax = ax, linewidth=7,color='red', label='Tesseract storysquad');

plt.xticks(xticks, xlabels, rotation='vertical')
plt.legend(prop={'size': 60})
plt.savefig('cos_error_plot.png');