Python | Positional index

# importing libraries

import numpy as np

import os

import nltk

from nltk.stem import PorterStemmer

from nltk.tokenize import TweetTokenizer

from natsort import natsorted

import string

def read_file (filename):

with open (filename, ’r’ , encoding = "ascii" , errors = "surrogateescape" ) as f:

stuff = f.read ()

f.close ()

# Remove header and footer.

stuff = remove_header_footer (stuff)

return stuff

def remove_header_footer (final_string):

new_final_string = ""

tokens = final_string.split ( ’ ’ )

# Remove [0] tokens and [-1] tokens

for token in tokens [ 1 : - 1 ]:

  new_final_string + = token + ""

return new_final_string

def preprocessing (final_string):

# Tokenize.

tokenizer = TweetTokenizer ()

token_list = tokenizer.tokenize (final_string)

  Remove punctuation marks.

table = str . maketrans (’ ’, ’ ’, ’ ’)

  token_list = [word.translate (table) for word in token_list]

punctuations = (string.punctuation) .replace ( "" " ," ")

  trans_table = str . maketrans (’ ’,’ ’, punctuations)

stripped_words = [word.translate (trans_table) for word in token_list]

token_list = [ str for str in stripped_words if str ]

# Change to lowercase.

token_list = [word.lower () for word in token_list]

return token_list

 
# In this example, we create a positional index for folder 1 only.

folder_names = [ "comp.graphics" ]

 
# Initialize stemmer.

stemmer = PorterStemmer ()

 
# Initialize file number

fileno = 0

 
# Initialize the dictionary.

pos_index = {}

 
# Initialize file association (fileno -" filename).

file_map = { }

for folder_name in folder_names:

# Open files.

file_names = natsorted (os.listdir ( "20_newsgroups /" + folder_name))

# For each file.

for file_name in file_names:

# Read file content.

  stuff = read_file ( " 20_newsgroups / " + folder_name + "/" + file_name)

# This is a list of words in text order.

# We need to keep the order because we want positions.

# The function "pre liner processing "performs basic punctuation removal,

  # remove stop words, etc. .

final_token_list = preprocessing (stuff)

# For position and term in tokens.

  for pos, term in enumerate (final_token_list):

# Stop the term first.

  term = stemmer.stem (term)

# If the term already exists in the positional index dictionary.

  if term in pos_index:

# Increase the overall frequency by 1.

pos_index [term] [ 0 ] = pos_index [term] [ 0 ] + 1

# Check if the term previously existed in this DocID.

  if fileno in pos_index [term] [ 1 ]:

  pos_index [term] [ 1 ] [fileno] .append (pos)

else :

  pos_index [term] [ 1 ] [fileno] = [pos]

# If the term does not exist in positional index dictionary

# (first meeting).

else :

# Initialize the list.

pos_index [term] = []

# Common frequency 1.

pos_index [term] .append ( 1 )

# The message list is initially empty.

pos_index [term] .append ({}) 

# Add document ID to message list.

pos_index [term] [ 1 ] [fileno] = [pos]

# No file map. to the file name.

file_map [fileno] = " 20_newsgroups / " + folder_name + "/" + file_name

# Increase the file number. counter for displaying document ID

fileno + = 1

  
# Example positional index for code review.

sample_pos_idx = pos_index [ " andrew " ]

print ( "Positional Index" )

print (sample_pos_idx)

file_list = sample_pos_idx [ 1 ]

print ( "Filename, [Positions]" )

for fileno,positions in file_list.items ():

print (file_map [fileno], positions)