import sys
import nltk

lines = open(sys.argv[1]).readlines()

index = 0
curr = ""
num_para = 0
num_segments = 0
num_tokens = 0
vocab = set()

for line in lines:
	line = line.strip()
	if len(line) == 0:
		continue
	if line.startswith("=============="):
		index = 0
		num_para += 1
		curr = ""
		continue
	if line.startswith("$$$$$$$$$$$$$"):
		index += 1
		curr = ""
		continue
	curr = curr + line
	if index == 4:
		inst = curr.lower()
		segments = inst.split(".") 
		segments = list(filter(lambda x: len(x) >= 3, segments))
		num_segments += len(segments)
		for segment in segments:
			tk_seq = nltk.word_tokenize(segment)
			for tk in tk_seq:
				vocab.add(tk)
			num_tokens += len(tk_seq)
		
print("Vocab ", len(vocab))
print("Num paragraphs ", num_para)
print("Average segment per para is ", num_segments/float(num_para))		
print("Average tokens per segment is ", num_tokens/float(num_segments))