#!/usr/bin/env python

# extract-definitions.py - given a number of configurations, output sentences matching a grammer looking (somewhat) like a definition

# Eric Lease Morgan <eric_morgan@infomotions.com>

# June 26, 2022 - first cut; not perfect, but fun and good enough
# June 27, 2022 - added parallel processing and a few modes


# configure
CARREL     = 'lead-pipe'
LEXICON    = '/Users/eric/Desktop/lexicon.txt'
DEFINITION = [ 'is', 'are', 'were', 'was' ]
MODALITY   = [ 'can', 'could', 'may', 'might', 'must', 'shall', 'should', 'will', 'would' ]
POSSESION  = [ 'have', 'has', 'had' ]
PATTERN    = '*.txt'
GRAMMAR    = '''
	  NOUNPRASE: {<DT>?<JJ.*>*<NN.?>+}
	  PREDICATE: {<VB.*>}
	DECLARATION: {<NOUNPRASE><PREDICATE><NOUNPRASE>}
'''

# sub-configuration; what type of verbs are desired
verbs = DEFINITION

# require
import nltk
import rdr
import re
from multiprocessing import Pool

# given a file, a list of words, and a list of verbs, return a set of sentences
def extractSentences( file, parser, lexicon, verbs ) :

	# slurp up the given file; read the text
	with open( file ) as handle : text = handle.read()

	# get and process all sentences in the text
	sentences = nltk.sent_tokenize( text )
	for sentence in sentences : 
	
		# do rudimentary normalization and tokenization
		sentence = re.sub( '\n', ' ', sentence )
		sentence = re.sub( '\t', ' ', sentence )
		sentence = re.sub( ' +', ' ', sentence )
		sentence = re.sub( '- ', '',  sentence )
		sentence = re.sub( '- ', '',  sentence )
		
		# tokenize and normalize
		tokens = nltk.word_tokenize( sentence )
		tokens = [ token.lower() for token in tokens ]

		# check for given nouns
		if set( lexicon ).intersection( set( tokens ) ) :
	
			# get parts-of-speech and create an NLTK tree
			pos  = nltk.pos_tag( tokens )
			tree = parser.parse( pos )
		
			# process each definition
			for definition in tree.subtrees( lambda t : t.label() == 'DECLARATION' ) :

				# get all subject words; re-check for given nouns
				subjects = []
				leaves   = definition[ 0 ].leaves()
				for leaf in leaves : subjects.append( leaf[ 0 ] )
				if set( lexicon ).intersection( set( subjects ) ) :	
		
					# get the verb; check for given verbs and output, conditionally
					verb = definition[ 1 ].flatten().leaves()[ 0 ][ 0 ]
					if verb in verbs : print( sentence, '\n' )
	
	# done
	return( True )
	

# do the work
if __name__ == '__main__' : 

	# initialize
	pool   = Pool()
	parser = nltk.RegexpParser( GRAMMAR )
	with open( LEXICON ) as handle : lexicon = handle.read().split( '\n' )

	# get and parallel process each text file in the given carrel
	localLibrary = rdr.configuration( 'localLibrary' )
	filenames    = localLibrary/CARREL/rdr.TXT
	pool.starmap( extractSentences, [ [ str( filename ), parser, lexicon, verbs ] for filename in filenames.glob( PATTERN ) ] )

	# clean up and done
	pool.close()
	exit()