scripts/stats.py
author Sverre Rabbelier <srabbelier@gmail.com>
Sat, 11 Apr 2009 13:19:23 +0000
changeset 2151 f58515b0b2e1
parent 2148 0345046ed7a5
child 2218 84b0ce492cf5
permissions -rwxr-xr-x
Style fix in function names in stats.py Patch by: Sverre Rabbelier
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
2047
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     1
#!/usr/bin/python2.5
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     2
#
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     3
# Copyright 2009 the Melange authors.
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     4
#
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     5
# Licensed under the Apache License, Version 2.0 (the "License");
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     6
# you may not use this file except in compliance with the License.
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     7
# You may obtain a copy of the License at
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     8
#
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
     9
#   http://www.apache.org/licenses/LICENSE-2.0
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    10
#
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    11
# Unless required by applicable law or agreed to in writing, software
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    12
# distributed under the License is distributed on an "AS IS" BASIS,
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    13
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    14
# See the License for the specific language governing permissions and
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    15
# limitations under the License.
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    16
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    17
"""Starts an interactive shell with statistic helpers.
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    18
"""
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    19
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    20
__authors__ = [
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    21
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    22
]
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    23
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    24
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    25
import cPickle
2079
086c60aad3d8 Minor tweaks to orgStats
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2063
diff changeset
    26
import datetime
2047
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    27
import operator
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    28
import sys
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    29
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    30
import interactive
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    31
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    32
2058
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    33
def dateFetch(queryGen, last=None, batchSize=100):
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    34
  """Iterator that yields an entity in batches.
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    35
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    36
  Args:
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    37
    queryGen: should return a Query object
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    38
    last: used to .filter() for last_modified_on
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    39
    batchSize: how many entities to retrieve in one datastore call
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    40
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    41
  Retrieved from http://tinyurl.com/d887ll (AppEngine cookbook).
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    42
  """
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    43
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    44
  from google.appengine.ext import db
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    45
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    46
   # AppEngine will not fetch more than 1000 results
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    47
  batchSize = min(batchSize,1000)
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    48
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    49
  query = None
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    50
  done = False
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    51
  count = 0
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    52
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    53
  while not done:
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    54
    print count
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    55
    query = queryGen()
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    56
    query.order('last_modified_on')
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    57
    if last:
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    58
      query.filter("last_modified_on > ", last)
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    59
    results = query.fetch(batchSize)
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    60
    for result in results:
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    61
      count += 1
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    62
      yield result
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    63
    if batchSize > len(results):
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    64
      done = True
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    65
    else:
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    66
      last = results[-1].last_modified_on
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    67
773b13d86309 Add dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2052
diff changeset
    68
2047
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    69
def addKey(target, fieldname):
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    70
  """Adds the key of the specified field.
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    71
  """
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    72
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    73
  result = target.copy()
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    74
  result['%s_key' % fieldname] = target[fieldname].key().name()
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    75
  return result
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    76
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    77
2060
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
    78
def getEntities(model):
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
    79
  """Returns all users as dictionary.
2052
a723a2509e21 Add some additional functions for stats to stats.py script.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2047
diff changeset
    80
  """
a723a2509e21 Add some additional functions for stats to stats.py script.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2047
diff changeset
    81
2060
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
    82
  def wrapped():
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
    83
    gen = lambda: model.all()
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
    84
    it = interactive.deepFetch(gen)
2052
a723a2509e21 Add some additional functions for stats to stats.py script.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2047
diff changeset
    85
2060
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
    86
    entities = [(i.key().name(), i) for i in it]
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
    87
    return dict(entities)
2052
a723a2509e21 Add some additional functions for stats to stats.py script.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2047
diff changeset
    88
2060
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
    89
  return wrapped
2052
a723a2509e21 Add some additional functions for stats to stats.py script.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2047
diff changeset
    90
a723a2509e21 Add some additional functions for stats to stats.py script.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2047
diff changeset
    91
2062
9c739b37c367 Refactor getProps to use dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2061
diff changeset
    92
def getProps(last=None):
2047
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    93
  """Returns all proposals as a list of dictionaries.
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    94
  """
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    95
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    96
  key_order = [
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
    97
      'link_id', 'scope_path', 'title', 'abstract', 'content',
2062
9c739b37c367 Refactor getProps to use dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2061
diff changeset
    98
      'additional_info', '_mentor', 'possible_mentors', 'score',
9c739b37c367 Refactor getProps to use dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2061
diff changeset
    99
      'status', '_org', 'created_on', 'last_modified_on']
2047
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   100
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   101
  from soc.models.student_proposal import StudentProposal
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   102
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   103
  gen = lambda: StudentProposal.all()
2062
9c739b37c367 Refactor getProps to use dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2061
diff changeset
   104
9c739b37c367 Refactor getProps to use dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2061
diff changeset
   105
  it = dateFetch(gen, last)
2047
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   106
2062
9c739b37c367 Refactor getProps to use dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2061
diff changeset
   107
  proposals = [(i.key().name(), i.toDict(key_order)) for i in it]
2079
086c60aad3d8 Minor tweaks to orgStats
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2063
diff changeset
   108
  if proposals:
086c60aad3d8 Minor tweaks to orgStats
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2063
diff changeset
   109
    last = i.last_modified_on # last modified entity
086c60aad3d8 Minor tweaks to orgStats
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2063
diff changeset
   110
  else:
086c60aad3d8 Minor tweaks to orgStats
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2063
diff changeset
   111
    last = datetime.datetime.now()
2047
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   112
2079
086c60aad3d8 Minor tweaks to orgStats
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2063
diff changeset
   113
  return dict(proposals), last
2047
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   114
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   115
2079
086c60aad3d8 Minor tweaks to orgStats
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2063
diff changeset
   116
def orgStats(target, orgs):
2047
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   117
  """Retrieves org stats.
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   118
  """
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   119
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   120
  from soc.logic import dicts
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   121
2079
086c60aad3d8 Minor tweaks to orgStats
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2063
diff changeset
   122
  orgs = [(v.key(), v) for k, v in orgs.iteritems()]
086c60aad3d8 Minor tweaks to orgStats
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2063
diff changeset
   123
  orgs = dict(orgs)
086c60aad3d8 Minor tweaks to orgStats
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2063
diff changeset
   124
2062
9c739b37c367 Refactor getProps to use dateFetch
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2061
diff changeset
   125
  grouped = dicts.groupby(target.values(), '_org')
2047
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   126
2079
086c60aad3d8 Minor tweaks to orgStats
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2063
diff changeset
   127
  grouped = [(orgs[k], v) for k, v in grouped.iteritems()]
086c60aad3d8 Minor tweaks to orgStats
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2063
diff changeset
   128
  popularity = [(k.link_id, len(v)) for k, v in grouped]
086c60aad3d8 Minor tweaks to orgStats
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2063
diff changeset
   129
086c60aad3d8 Minor tweaks to orgStats
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2063
diff changeset
   130
  return dict(grouped), dict(popularity)
2047
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   131
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   132
2052
a723a2509e21 Add some additional functions for stats to stats.py script.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2047
diff changeset
   133
def countStudentsWithProposals():
a723a2509e21 Add some additional functions for stats to stats.py script.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2047
diff changeset
   134
  """Retrieves number of Students who have submitted at least one Student Proposal.
a723a2509e21 Add some additional functions for stats to stats.py script.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2047
diff changeset
   135
  """
2060
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
   136
2052
a723a2509e21 Add some additional functions for stats to stats.py script.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2047
diff changeset
   137
  proposals = getStudentProposals()
a723a2509e21 Add some additional functions for stats to stats.py script.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2047
diff changeset
   138
  students = {}
2060
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
   139
2052
a723a2509e21 Add some additional functions for stats to stats.py script.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2047
diff changeset
   140
  for proposal_key in proposals.keys():
a723a2509e21 Add some additional functions for stats to stats.py script.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2047
diff changeset
   141
    students[proposals[proposal_key].scope_path] = True
2060
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
   142
2052
a723a2509e21 Add some additional functions for stats to stats.py script.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2047
diff changeset
   143
  return len(students)
a723a2509e21 Add some additional functions for stats to stats.py script.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2047
diff changeset
   144
a723a2509e21 Add some additional functions for stats to stats.py script.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2047
diff changeset
   145
2047
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   146
def printPopularity(popularity):
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   147
  """Prints the popularity for the specified proposals.
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   148
  """
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   149
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   150
  g = operator.itemgetter(1)
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   151
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   152
  for item in sorted(popularity.iteritems(), key=g, reverse=True):
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   153
    print "%s: %d" % item
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   154
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   155
2148
0345046ed7a5 Make it possible to save any value with saveValues
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2090
diff changeset
   156
def saveValues(values, saver):
2047
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   157
  """Saves the specified popularities.
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   158
  """
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   159
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   160
  import logging
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   161
  from google.appengine.ext import db
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   162
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   163
  from soc.models.organization import Organization
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   164
2148
0345046ed7a5 Make it possible to save any value with saveValues
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2090
diff changeset
   165
  def txn(key, value):
0345046ed7a5 Make it possible to save any value with saveValues
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2090
diff changeset
   166
    org = Organization.get_by_key_name(key)
0345046ed7a5 Make it possible to save any value with saveValues
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2090
diff changeset
   167
    saver(org, value)
2047
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   168
    org.put()
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   169
2148
0345046ed7a5 Make it possible to save any value with saveValues
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2090
diff changeset
   170
  for key, value in sorted(values.iteritems()):
2047
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   171
    print key
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   172
    db.run_in_transaction_custom_retries(10, txn, key, value)
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   173
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   174
  print "done"
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   175
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   176
2090
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   177
def addFollower(follower, proposals, add_public=True, add_private=True):
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   178
  """Adds a user as follower to the specified proposals.
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   179
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   180
  Args:
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   181
    follower: the User to add as follower
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   182
    proposals: a list with the StudnetProposals that should be subscribed to
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   183
    add_public: whether the user is subscribed to public updates
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   184
    add_private: whether the user should be subscribed to private updates
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   185
  """
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   186
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   187
  from soc.models.review_follower import ReviewFollower
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   188
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   189
  result = []
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   190
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   191
  for i in proposals:
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   192
     properties = {
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   193
       'user': follower,
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   194
       'link_id': follower.link_id,
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   195
       'scope': i,
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   196
       'scope_path': i.key().name(),
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   197
       'key_name': '%s/%s' % (i.key().name(), follower.link_id),
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   198
       'subscribed_public': add_public,
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   199
       'subscribed_private': add_private,
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   200
     }
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   201
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   202
     entity = ReviewFollower(**properties)
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   203
     result.append(entity)
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   204
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   205
  return result
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   206
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   207
2047
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   208
def loadPickle(name):
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   209
  """Loads a pickle.
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   210
  """
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   211
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   212
  f = open(name + '.dat')
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   213
  return cPickle.load(f)
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   214
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   215
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   216
def dumpPickle(target, name):
2061
c7ba473b091e Style fixes
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2060
diff changeset
   217
  """Dumps a pickle.
2047
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   218
  """
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   219
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   220
  f = open("%s.dat" % name, 'w')
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   221
  cPickle.dump(target, f)
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   222
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   223
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   224
def main(args):
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   225
  """Main routine.
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   226
  """
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   227
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   228
  interactive.setup()
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   229
2060
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
   230
  from soc.models.organization import Organization
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
   231
  from soc.models.user import User
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
   232
  from soc.models.student import Student
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
   233
  from soc.models.mentor import Mentor
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
   234
  from soc.models.org_admin import OrgAdmin
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
   235
2151
f58515b0b2e1 Style fix in function names in stats.py
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2148
diff changeset
   236
  def slotSaver(org, value):
2148
0345046ed7a5 Make it possible to save any value with saveValues
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2090
diff changeset
   237
    org.slots = value
2151
f58515b0b2e1 Style fix in function names in stats.py
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2148
diff changeset
   238
  def popSaver(org, value):
2148
0345046ed7a5 Make it possible to save any value with saveValues
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2090
diff changeset
   239
    org.nr_applications = value
2151
f58515b0b2e1 Style fix in function names in stats.py
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2148
diff changeset
   240
  def rawSaver(org, value):
2148
0345046ed7a5 Make it possible to save any value with saveValues
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2090
diff changeset
   241
    org.slots_calculated = value
0345046ed7a5 Make it possible to save any value with saveValues
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2090
diff changeset
   242
2047
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   243
  context = {
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   244
      'load': loadPickle,
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   245
      'dump': dumpPickle,
2052
a723a2509e21 Add some additional functions for stats to stats.py script.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2047
diff changeset
   246
      'orgStats': orgStats,
2047
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   247
      'printPopularity': printPopularity,
2148
0345046ed7a5 Make it possible to save any value with saveValues
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2090
diff changeset
   248
      'saveValues': saveValues,
2060
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
   249
      'getOrgs': getEntities(Organization),
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
   250
      'getUsers': getEntities(User),
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
   251
      'getStudents': getEntities(Student),
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
   252
      'getMentors': getEntities(Mentor),
2063
319d9f05955a Fixed a typo in stats.py
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2062
diff changeset
   253
      'getOrgAdmins': getEntities(OrgAdmin),
2047
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   254
      'getProps': getProps,
2060
45029d87be4a Add and use a getEntities method
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2058
diff changeset
   255
      'countStudentsWithProposals': countStudentsWithProposals,
2090
a07ff1e1bd1f Add a script to subscribe someone to all updates
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2089
diff changeset
   256
      'addFollower': addFollower,
2089
b8a9691da6e7 Add the entities to the context too
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2079
diff changeset
   257
      'Organization': Organization,
b8a9691da6e7 Add the entities to the context too
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2079
diff changeset
   258
      'User': User,
b8a9691da6e7 Add the entities to the context too
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2079
diff changeset
   259
      'Student': Student,
b8a9691da6e7 Add the entities to the context too
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2079
diff changeset
   260
      'Mentor': Mentor,
b8a9691da6e7 Add the entities to the context too
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2079
diff changeset
   261
      'OrgAdmin': OrgAdmin,
2151
f58515b0b2e1 Style fix in function names in stats.py
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2148
diff changeset
   262
      'slotSaver': slotSaver,
f58515b0b2e1 Style fix in function names in stats.py
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2148
diff changeset
   263
      'popSaver': popSaver,
f58515b0b2e1 Style fix in function names in stats.py
Sverre Rabbelier <srabbelier@gmail.com>
parents: 2148
diff changeset
   264
      'rawSaver': rawSaver,
2047
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   265
  }
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   266
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   267
  interactive.remote(args, context)
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   268
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   269
if __name__ == '__main__':
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   270
  if len(sys.argv) < 2:
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   271
    print "Usage: %s app_id [host]" % (sys.argv[0],)
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   272
    sys.exit(1)
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   273
7e9656691c8e Added a stats module to scripts
Sverre Rabbelier <srabbelier@gmail.com>
parents:
diff changeset
   274
  main(sys.argv[1:])