app/django/utils/stopwords.py
author Lennard de Rijk <ljvderijk@gmail.com>
Sun, 12 Jul 2009 14:03:35 +0200
changeset 2610 95949d4c45d9
parent 54 03e267d67478
permissions -rw-r--r--
Add scope view for GradingSurveyGroup and set access checks. The access checks are now properly set for create, edit and show. To facilitate the scope view the access check have an extra check built in to ensure that a scope is actually present before cheking for an existing Program.

# Performance note: I benchmarked this code using a set instead of
# a list for the stopwords and was surprised to find that the list
# performed /better/ than the set - maybe because it's only a small
# list.

stopwords = '''
i
a
an
are
as
at
be
by
for
from
how
in
is
it
of
on
or
that
the
this
to
was
what
when
where
'''.split()

def strip_stopwords(sentence):
    "Removes stopwords - also normalizes whitespace"
    words = sentence.split()
    sentence = []
    for word in words:
        if word.lower() not in stopwords:
            sentence.append(word)
    return u' '.join(sentence)