app/django/utils/stopwords.py
author Pawel Solyga <Pawel.Solyga@gmail.com>
Tue, 20 Jan 2009 22:01:02 +0000
changeset 852 c94bf642be8d
parent 54 03e267d67478
permissions -rw-r--r--
Replace css class 'notice' that is used in User profile template field help text with new 'fieldhelptext' class. Notice is part of CSS for notification bar and we want width limit just for field help text not for notification bar text. Patch by: Pawel Solyga Review by: to-be-reviewed

# Performance note: I benchmarked this code using a set instead of
# a list for the stopwords and was surprised to find that the list
# performed /better/ than the set - maybe because it's only a small
# list.

stopwords = '''
i
a
an
are
as
at
be
by
for
from
how
in
is
it
of
on
or
that
the
this
to
was
what
when
where
'''.split()

def strip_stopwords(sentence):
    "Removes stopwords - also normalizes whitespace"
    words = sentence.split()
    sentence = []
    for word in words:
        if word.lower() not in stopwords:
            sentence.append(word)
    return u' '.join(sentence)