from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords

tokenizer = RegexpTokenizer('\s+', gaps=True)
english_stopwords = set(stopwords.words('english'))
words = tokenizer.tokenize("Can't is a contraction.")
print(words)
filtered = [word for word in words if word not in english_stopwords]
print(filtered)

# stopwords of all languages
# print(stopwords.words())

# languages
print(stopwords.fileids())

70942452edd56290f3b18a4f0162a67e42f9a30d
<mask token>

<mask token>
print(words)
<mask token>
print(filtered)
print(stopwords.fileids())

<mask token>
tokenizer = RegexpTokenizer('\\s+', gaps=True)
english_stopwords = set(stopwords.words('english'))
words = tokenizer.tokenize("Can't is a contraction.")
print(words)
filtered = [word for word in words if word not in english_stopwords]
print(filtered)
print(stopwords.fileids())

from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords
tokenizer = RegexpTokenizer('\\s+', gaps=True)
english_stopwords = set(stopwords.words('english'))
words = tokenizer.tokenize("Can't is a contraction.")
print(words)
filtered = [word for word in words if word not in english_stopwords]
print(filtered)
print(stopwords.fileids())

from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords

tokenizer = RegexpTokenizer('\s+', gaps=True)
english_stopwords = set(stopwords.words('english'))
words = tokenizer.tokenize("Can't is a contraction.")
print(words)
filtered = [word for word in words if word not in english_stopwords]
print(filtered)

# stopwords of all languages
# print(stopwords.words())

# languages
print(stopwords.fileids())
