Semantic textual search with vector embeddings

!pip install -qU pinecone-client ipywidgets

import pinecone

# Load Pinecone API key
import os
api_key = os.getenv("PINECONE_API_KEY") or "YOUR-API-KEY"
pinecone.init(api_key=api_key, environment='us-west1-gcp')
# List all indexes currently present for your key
pinecone.list_indexes()

!pip install -qU wordcloud pandas-profiling
!pip install -qU sentence-transformers --no-cache-dir

import pandas as pd
import numpy as np
import time
import re
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
import matplotlib.pyplot as plt
import sqlite3

pd.set_option('display.max_colwidth', 200)

# Pick a name for the new index
index_name = 'semantic-text-search'

# Check whether the index with the same name already exists
if index_name in pinecone.list_indexes():
    pinecone.delete_index(index_name)

pinecone.create_index(name=index_name, dimension=300, metric='cosine', shards=1)

index = pinecone.Index(index_name=index_name)

import requests, os
DATA_DIR = 'tmp'
URL = "https://www.dropbox.com/s/b2cyb85ib17s7zo/all-the-news.db?dl=1"
FILE = f"{DATA_DIR}/all-the-news.db"

def download_data():
    os.makedirs(DATA_DIR, exist_ok=True)

    if not os.path.exists(FILE):
        r = requests.get(URL)  # create HTTP response object
        with open(FILE, "wb") as f:
            f.write(r.content)

download_data()

cnx = sqlite3.connect(FILE)
data = pd.read_sql_query("SELECT * FROM longform", cnx)
data.set_index('id', inplace=True)
data.head()

# Define number of test articles
NUM_OF_TEST_ARTICLES = 2

# Remove test articles from data and keep them in separate dataframe
test_articles = data[['title','content']][97::81][:NUM_OF_TEST_ARTICLES]
data.drop(list(test_articles.index), inplace=True)

from sentence_transformers import SentenceTransformer

model = SentenceTransformer('average_word_embeddings_komninos')

from typing import Iterator

class BatchGenerator:
    """ Models a simple batch generator that make chunks out of an input DataFrame. """
    
    def __init__(self, batch_size: int = 10) -> None:
        self.batch_size = batch_size
    
    def to_batches(self, df: pd.DataFrame) -> Iterator[pd.DataFrame]:
        """ Makes chunks out of an input DataFrame. """
        splits = self.splits_num(df.shape[0])
        if splits <= 1:
            yield df
        else:
            for chunk in np.array_split(df, splits):
                yield chunk
    
    def splits_num(self, elements: int) -> int:
        """ Determines how many chunks DataFrame contians. """
        return round(elements / self.batch_size)
    
    __call__ = to_batches

df_batcher = BatchGenerator(300)

# Fill missing and remove redundant data
data['title'] = data['title'].fillna('')

# Create vector embeddings based on the title column
print('Encoding titles...')
encoded_titles = model.encode(data['title'].tolist(), show_progress_bar=True)
data['title_vector'] = encoded_titles.tolist()

data['vector_id'] = data.index
data['vector_id'] = data['vector_id'].apply(str)

# Upsert title vectors in title namespace
print("Uploading vectors to title namespace..")
for batch_df in df_batcher(data):
    index.upsert(vectors=zip(batch_df.vector_id, batch_df.title_vector), namespace='title')

Uploading vectors to title namespace..

# Fill missing data
data['content'] = data['content'].fillna('')

# Extract only first few sentences of each article for quicker vector calculations
data['content'] = data.content.apply(lambda x: ' '.join(re.split(r'(?<=[.:;])\s', x)[:10]))

# Create vector embeddings based on the content column
print('Encoding content...')
encoded_content = model.encode(data['content'].tolist(), show_progress_bar=True)
data['content_vector'] = encoded_content.tolist()

# Upsert title vectors in content namespace
print("Uploading vectors to content namespace..")
for batch_df in df_batcher(data):
    index.upsert(vectors=zip(batch_df.vector_id, batch_df.content_vector), namespace='content')

# Check index size for each namespace
index.describe_index_stats()

{'dimension': 300,
 'namespaces': {'content': {'vector_count': 4998},
                'title': {'vector_count': 4998}}}

# Print test articles
display(test_articles)

titles_mapped = dict(zip(data.index, data.title))
content_mapped = dict(zip(data.index, data.content))

def get_wordcloud_for_article(recommendations, namespace):
    'Generates word cloud for the recommendations (titles or content).'

    stopwords = set(STOPWORDS).union([np.nan, 'NaN', 'S'])
    wordcloud = WordCloud(
                   max_words=50000, 
                   min_font_size =12, 
                   max_font_size=50, 
                   relative_scaling = 0.9, 
                   stopwords=set(STOPWORDS),
                   normalize_plurals= True
                  )
    
    if namespace == 'title':
        clean_titles = [word for word in recommendations.title.values if word not in stopwords]
        wordcloud = wordcloud.generate(' '.join(clean_titles))
    else:
        clean_content = [word for word in recommendations.content.values if word not in stopwords]
        wordcloud = wordcloud.generate(' '.join(clean_content))

    plt.imshow(wordcloud, interpolation='bilinear')
    plt.axis("off")
    plt.show()


def print_query_results(query_result, query, namespace, show_options={'wordcloud':True, 'tabular':True}):
    'Prints query result with wordcloud.'
      
    print(f'\nMost similar results querying {query} in "{namespace}" namespace:\n')
    if not query_result.results:
        print('no query result')
    
    matches = query_result.results[0].matches
    ids = [res.id for res in matches]
    scores = [res.score for res in matches]
    df = pd.DataFrame({'id':ids, 
                       'score':scores,
                       'title': [titles_mapped[int(_id)] if int(_id) in titles_mapped else ' '  for _id in ids],
                       'content': [content_mapped[int(_id)] if int(_id) in content_mapped else ' '  for _id in ids],
                       })
    if show_options['tabular']:
        display(df.head(5))
    if show_options['wordcloud']:
        get_wordcloud_for_article(df, namespace)
    print('\n')

def query_article_title(test_article, namespace, top_k=5, show_options={'wordcloud':True, 'tabular':True}):
    '''Queries an article using its title in the specified
     namespace and prints results.'''

    # Create vector embeddings based on the title column
    encoded_titles = model.encode(test_article['title'], 
                                  show_progress_bar=False)
    test_article['title_vector'] = encoded_titles.tolist()

    # Query namespace passed as parameter using title vector
    query_result_titles = index.query(queries=[test_article.title_vector], 
                                      namespace=namespace, 
                                      top_k=top_k)

    # Print query results 
    if show_options['wordcloud'] or show_options['tabular']:
        print_query_results(query_result_titles, query='title', 
                            namespace=namespace, 
                            show_options=show_options)

    return query_result_titles

def query_article_content(test_article, namespace, top_k=5, show_options={'wordcloud':True, 'tabular':True}):
    '''Queries an article using its content in the specified 
    namespace and prints results.'''

    # Create vector embeddings based on the content column
    encoded_content = model.encode(test_article['content'], 
                                   show_progress_bar=False)
    test_article['content_vector'] = encoded_content.tolist()

    # Query content namespace using content vector
    query_result_content = index.query(queries=[test_article.content_vector], 
                                       namespace=namespace, 
                                       top_k=top_k)

    # Print query results 
    if show_options['wordcloud'] or show_options['tabular']:
        print_query_results(query_result_content, 
                            query='content', 
                            namespace=namespace, 
                            show_options=show_options)

    return query_result_content

def aggregate_results(article):
    '''Aggregates results after querying both namespaces
       for both the article's title and content.'''

    results = []
    
    results.append(query_article_title(article, namespace='title', top_k=30, show_options={'wordcloud':False, 'tabular':False}))
    results.append(query_article_title(article, namespace='content', top_k=30, show_options={'wordcloud':False, 'tabular':False}))
    results.append(query_article_content(article, namespace='title', top_k=30, show_options={'wordcloud':False, 'tabular':False}))
    results.append(query_article_content(article, namespace='content', top_k=30, show_options={'wordcloud':False, 'tabular':False}))

    articles_scores = {}
    articles_count = {}

    for res in results:
        ids = [r.id for r in res.results[0].matches]
        scores = [r.score for r in res.results[0].matches]
        for id, score in zip(ids, scores):
            if id not in articles_scores:
                articles_scores[id] = score
                articles_count[id] = 1
            else:
                articles_scores[id] += score
                articles_count[id] += 1
    
    return articles_scores , articles_count

def show_aggregated_results(results_dict, counts, show_options={'wordcloud':True, 'tabular':True}):
    '''Shows results after aggregation. Values are sorted based
    on the number of queries they appear (1-4) and based on their
    average score.'''
    
    df = pd.DataFrame({'id':results_dict.keys(), 
                       'count': counts.values(),
                       'average_score':[round(r/c, 3) for r, c in zip(results_dict.values(),counts.values())],
                       'title': [titles_mapped[int(_id)] if int(_id) in titles_mapped else ' '  for _id in results_dict.keys()],
                       'content': [content_mapped[int(_id)] if int(_id) in content_mapped else ' '  for _id in results_dict.keys()],
                       })
    df.sort_values(by=['count', 'average_score'], ascending=False, inplace=True)
    
    if show_options['tabular']:
        print('\nMost similar results after aggregation:\n')
        display(df.head(5))
    if show_options['wordcloud']:
        print('\nWordcloud for titles and content after aggregation:')
        print('-Titles:')
        get_wordcloud_for_article(df[:10], 'title')
        print('-Content:')
        get_wordcloud_for_article(df[:10], 'content')
    print('\n')

# Query index using simple and cross namespace approach
for e, (_, test_article) in enumerate(test_articles.iterrows()):
    print(f'\nArticle {e+1}')
    print(f'\n Title: {test_article.title}')
    print(f' Content: {test_article.content[:200].strip()}' + ('...' if len(test_article.content) > 200 else ''))
    
    # Uncomment to query the titles in title namespace
    #query_article_title(test_article, 'title',  show_options={'wordcloud':True, 'tabular':False})

    # Uncomment to query the content in content namespace
    #query_article_content(test_article, namespace='content', show_options={'wordcloud':True, 'tabular':False})

    # Cross namespace query
    aggregated_results, counts = aggregate_results(test_article)
    show_aggregated_results(aggregated_results, counts, show_options={'wordcloud':False, 'tabular':True})

Article 1

 Title: The Rise and Fall and Rise of Virtual Reality
 Content: In the wake of Facebook's purchase of Oculus VR, can this revolutionary technology triumph anew?

Most similar results after aggregation:

pinecone.delete_index(index_name)