-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathTfidf.py
More file actions
43 lines (29 loc) · 1.35 KB
/
Tfidf.py
File metadata and controls
43 lines (29 loc) · 1.35 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
# importing the required libraries
import Functions
from Functions import *
# importing TfidfVectorizer class to convert a collection of raw documents to a matrix of TF-IDF features.
from sklearn.feature_extraction.text import TfidfVectorizer
# importing cosine_similarity function to compute the cosine similarity between two vectors.
from sklearn.metrics.pairwise import cosine_similarity
# importing nlargest to return the n largest elements from an iterable in descending order.
from heapq import nlargest
from nltk.tokenize import (
sent_tokenize,
word_tokenize,
)
def generate_summary(text, n):
# Tokenize the text into individual sentences
sentences = sent_tokenize(text)
# Create the TF-IDF matrix
vectorizer = TfidfVectorizer(stop_words="english")
tfidf_matrix = vectorizer.fit_transform(sentences)
# Compute the cosine similarity between each sentence and the document
sentence_scores = cosine_similarity(tfidf_matrix[-1], tfidf_matrix[:-1])[0]
# Select the top n sentences with the highest scores
summary_sentences = nlargest(
n, range(len(sentence_scores)), key=sentence_scores.__getitem__
)
summary_tfidf = " ".join([sentences[i] for i in sorted(summary_sentences)])
return summary_tfidf
#text = Functions.open_file("Output/CleanText")
#save_file("Output/Tfidf", str(generate_summary(text, 50)))