import datetime
import itertools
from django.contrib import messages
from django.utils.functional import curry
from django.contrib.auth.models import User
from django.http import HttpResponseRedirect
from django.utils import timezone
from guardian.decorators import permission_required
from activity.forms import ActivationActivityFormset, ActivationActivityOtherDetailsForm, ActivationActivityForm
from activity.models import ActivityStatus
from administration.decorators import next, sessioned_post
from communication.forms import EmailMessageForm
from communication.models.correspondence import Correspondence
from communication.models.message import Message, InternalRecipient
from engagement.models import Engagement
from fund.models import Fund
from django.shortcuts import render, redirect
from django.template import RequestContext
from administration.helpers import get_abbreviated_name
from django.db.models import Q
from django.core.urlresolvers import reverse
from django.shortcuts import render_to_response, get_object_or_404
from django.contrib.auth.decorators import login_required
from administration.views import to_csv, to_csv2
from django.forms.models import inlineformset_factory, formset_factory, modelformset_factory
from functools import partial, wraps
from manager.models import Manager, ManagerIndex
from meeting.views import __get_manager_and_contact_info
from meeting.models import Meeting
from django.template.defaultfilters import linebreaksbr
from activity.models import Activity
from fund.models import FundIndex
from fund.forms import AddFundIndexForm
from client.models import Client
from manager.forms import AddManagerForm
from meeting.forms import AddContactForm
from engagement.filter_forms import FilterForm, BillingFilterForm, TemplateTaskFilterForm, TaskFilterForm, EngagementReviewFilterForm, \
    EngagementReviewCriteriaFilterForm
from engagement.forms import EngagementForm, AddClientForm, FundActivitiesManagerForm, \
    FundActivitiesInline, FundActivitiesForm, fund_form_wrapper
from engagement_task.forms import EngagementTaskTemplateForm, TemplateAttachmentFormset, TaskAttachmentFormset, \
    EngagementTaskActivationForm, EngagementReviewCriteriaForm, \
    EngagementReviewCriteriaRoleFormset, EngagementReviewForm, \
    EngagementReviewScoreFormset, TaskCompleteForm, \
    EngagementTaskFormFactory
from engagement_task.models import EngagementTask, TemplateAttachment, EngagementTaskTemplate, \
    EngagementTaskAssignee, TaskAttachment, EngagementReview, EngagementReviewCriteria, EngagementReviewScore
from trip.forms import TripForm, TripLocationForm
from trip.models import Trip, TripLocation

from administration.helpers import build_query_from_filter, get_headings_and_fieldnames, get_ordering_data, \
    format_list_view, build_query_from_search_param, exists_with_value, addIfExists
from billing.forms import OopForm, OopFormSet, InvoiceForm, InvoiceItemForm
from billing.models import ClientBilling, Invoice, InvoiceItem
from django.http import Http404

@permission_required("engagement.view_engagement", return_403=True)
@sessioned_post
#Engagement List View
def index(request, csv=False, skipPost=False):
    def __get_results_and_headings(group_by, result, csv):
        id = -1
        headings = ['Status', 'Client(s)', 'Onsite Type', 'Att', 'Manager', 'Location', 'Meeting Date', 'Delivery Date', 'Priority', '', '', '', '']

        if csv:
            headings = ['Fund(s)'] + headings
            position=[8,9,10,11]
            csvhead=['AnalystName','Lead Analyst','Reviewer1','Reviewer2']
            for i in csvhead:
                headings.insert(position[csvhead.index(i)],i)

        if group_by == 'Trip':
            h = ['Trip'] + headings
            #Sort by trip code, placing engagements without a trip at the top of the list
            result = sorted(result, key=lambda x: (addIfExists(x['Engagement'], 'trip.code', datetime.date(1900, 1, 1).strftime("%Y-%m")) if x['Trip'] != 'N/A'
                                                   else datetime.date(datetime.MAXYEAR, 1, 1).strftime("%Y-%m"), x['Engagement'].status))
        elif group_by == 'Manager':
            id = 4
            result = sorted(result, key=lambda x: (addIfExists(x['Manager'],'name','')))
        elif group_by == 'Analyst':
            h = ['Lead'] + headings
            result = sorted(result, key=lambda x: (addIfExists(x['Engagement'], 'lead_analyst.first_name', '')))
        elif group_by == 'Location':
            h = [headings[5]] + headings[0:5] + ['Trip'] + headings[6:]
            result = sorted(result, key=lambda x: (addIfExists(x['Manager'], 'primary_city.name', '')))
        elif group_by == 'Meeting Month':
            id = 6
            result = sorted(result, key=lambda x: (x['Meeting_Month'], x['Engagement'].status))
            result.reverse()
        elif group_by == 'CRM':
            h = ['CRM'] + headings
            result = sorted(result, key=lambda x: (addIfExists(x['CRM'], 'first_name', '')))
        elif group_by == 'Client':
            id = 1
            result = sorted(result, key=lambda x: (addIfExists(x['Clients'][0], 'name', '')))
        else:
            h = headings
            result = sorted(result, key=lambda x: (x['Engagement'].get_status_display()))

        if id > 0:
           h = [headings[id]] + (headings[0:id] if id > 0 else []) + headings[id + 1:]

        headings = h
        return (group_by, headings, result)

    form = FilterForm(request.POST or None)
    query = []

    if request.method == 'POST' and not skipPost:
        [query, group_by] = __get_engagement_params(form, query, request)
    else:
        # By default filter by today's month and year
        if not skipPost:
            today = datetime.date.today()
            query.append(Q(target_meeting_date__month = today.month, target_meeting_date__year = today.year))
            query.append(Q(status__in=[Engagement.BOOKED, Engagement.SCHEDULE, Engagement.COMPLETE, Engagement.REPORT_IN_PROG]))

            #TODO: Strip form data from here
            form.fields["status"].initial = [Engagement.BOOKED, Engagement.SCHEDULE, Engagement.COMPLETE, Engagement.REPORT_IN_PROG]
            form.fields["egmnt_meeting_month"].initial = [str(today.month) + "_" + str(today.year)]

        #TODO: Verify this is still true and not status as above
        # By default group by trip
        group_by = 'Trip'

    #TODO: 'manager__primary_city' should reduce the number of queries; but the manager isn't being used ><
    engagements = Engagement.objects\
                    .select_related("priority", "lead_analyst", "manager", "manager__primary_city",
                                    "manager__primary_city__state", "manager__primary_city__state__country", "manager__primary_city__country", "trip",
                                    'desk_analyst', 'reviewer_1', 'reviewer_2')\
                    .prefetch_related('funds', 'activities', 'activities__activity_type', 'activities__client_concerned', 'activities__fund', 'clients',
                                      'trip__locations','clients__primary_cha_representative', 'activities__activity_type','activities__analyst')\
                    .filter(*query).only('id', 'status', 'trip_required', 'onsite_type', 'manager', 'clients', 'funds', 'lead_analyst', 'trip', 'priority', 'desk_analyst',
                                         'reviewer_1', 'reviewer_2', 'target_meeting_date', 'client_attending', 'target_delivery_date', 'notes',
                                         'priority__name', 'priority__id',).order_by()
    if len(engagements) > 0:
        engagements = list(set(engagements))

        result = []
        headings = []
        for e in engagements:
            #Verify the fund name is included in the search criteria
            if exists_with_value(form.data, 'fundSearchField'):
                fund_list = e.funds.all().filter(id__in = [form.data['fundSearchField']])
            else:
                fund_list = e.funds.all()
                if not fund_list:
                    fund_list = ["TBD"]
            names=''
            for x in e.activities.all():
                if x.analyst:
                   k=(get_abbreviated_name( x.analyst ))
                   names=names +' ' + k

            tmp = {'Engagement': e, 'Manager': e.manager,
                   'Meeting_Month': datetime.date(addIfExists(e, 'target_meeting_date.year', datetime.date.min.year), addIfExists(e, 'target_meeting_date.month', 1), 1),
                   'Location': addIfExists(e, 'manager.primary_city'),
                   'Trip': 'N/A' if not e.trip_required else e.trip, 'Review_Level': e.review_level, 'Analyst': e.lead_analyst, 'Status': e.status ,'analyst':names ,'Lead_Analyst':get_abbreviated_name(e.lead_analyst),'Reviewer_1' :get_abbreviated_name( e.reviewer_1),'Reviewer_2':get_abbreviated_name(e.reviewer_2)}

            if group_by == 'Client':
                for c in e.clients.all():
                    #If grouped by clients, output a row for each engagement for the respective client, collapsing the fund list into a single field
                    tmp.update({'Clients': [c], 'Client': c, 'Funds': list(set([x.fund for x in e.activities.all() if x.fund in fund_list and c in x.client_concerned.all()]))})
                    result.append(tmp)
            else:
                if group_by == 'CRM':
                    tmp.update({'CRM': [addIfExists(c, 'primary_cha_representative', '--') for c in e.clients.all()][0]})

                # tmp.update({'Clients': e.clients.all()})
                #TODO: In 1.7 http://stackoverflow.com/questions/12973929/why-does-djangos-prefetch-related-only-work-with-all-and-not-filter
                #TODO: Can set the QS because the lookup below is expensive for useless fields
                #We're now grouping the clients and funds together, so populate the funds dictionary by client to make things easier
                #as well as to give us control to only display the funds this client has subscriptions for
                if csv:
                    funds = []
                else:
                    funds = {}
                activities = {x.fund: [j.pk for j in x.client_concerned.all()] for x in e.activities.all() if x.fund in fund_list}

                for c in e.clients.all():
                    if not csv:
                        funds[c] = []
                    for f, cs in activities.iteritems():
                        if c.pk in cs:
                            if csv:
                                funds.append(f)
                            else:
                                funds[c].append(f)
                tmp.update({'Funds': funds, 'Clients': e.clients.all()})
                result.append(tmp)

        if len(result) > 0:
            [group_by, headings, result] = __get_results_and_headings(group_by, result, csv)

        cols = len(headings)

    #If there are no results, just output an empty table
    else:
        result = []
        headings = []
        cols = 0

    engmnt_tabs = form.get_engmnt_tabs()

    context = dict(results=result, headings=headings, body_id = "tools", page_title = "Engagements", form = form,
                   group_by = group_by, cols = cols, engmnt_tabs=engmnt_tabs)

    if csv:
        context.update({'child_template': 'engagement/components/row_display.html', 'filename': 'engagement_list'})
        return to_csv2(context)
    else:
        context.update({'parent_template': 'administration/components/table_wrapper.html'})

    return render_to_response("engagement/engagement_list.html", context, context_instance=RequestContext(request))

#from engagement.helpers import __get_engagement_params

@login_required
#Engagement Billing List View
def billing_view(request, csv=False):

    form = BillingFilterForm(request.POST or None)
    query = []
    group_by = None
    by_invoice_date = False

    #Default view is just pull everything atm
    #Otherwise parse the form and build the query string
    if request.method == 'POST':

        query = build_query_from_filter(form, request, [{'form_field': 'status', 'model_field': 'status'},
                                                        ], query)

        query = build_query_from_search_param(form, [{'form_field': 'clientSearch', 'model_field': 'clients__id'},
                                                     {'form_field': 'managerSearch', 'model_field': 'manager__id'},
                                                     {'form_field': 'fundSearch', 'model_field': 'funds__id'},
                                                     {'form_field':'invoiceSearch_name','model_field': 'billings__invoice_items__invoice__number'},], query)

        if form.data.has_key('egmnt_meeting_date_from') and form.data.has_key('egmnt_meeting_date_to'):
            if form.data['egmnt_meeting_date_from'] != '':
                query.append(Q(target_meeting_date__gte = datetime.datetime.strptime(form.data['egmnt_meeting_date_from'],"%Y-%m-%d")))
            if form.data['egmnt_meeting_date_to'] != '':
                query.append(Q(target_meeting_date__lte = datetime.datetime.strptime(form.data['egmnt_meeting_date_to'],"%Y-%m-%d")))

        if form.data.has_key('invoiceSearch') and form.data['invoiceSearch'] != '':
            query.append(Q(billings__invoice_items__invoice__number__contains = form.data['invoiceSearch_name']))

        if form.data.has_key('invoice_date_from') and form.data.has_key('invoice_date_to') and form.data['invoice_date_from'] != '' and form.data['invoice_date_to'] != '':
            by_invoice_date = True
            query.append(Q(billings__invoice_items__invoice__date__gte = datetime.datetime.strptime(form.data['invoice_date_from'],"%Y-%m-%d")))
            query.append(Q(billings__invoice_items__invoice__date__lte = datetime.datetime.strptime(form.data['invoice_date_to'],"%Y-%m-%d")))

        [order_by_field, group_by] = get_ordering_data(form, {'Client': {'o': '-number', 'g': 'client'},                       #Don't order by a m2m field
                                                            'Manager': {'o': '-number', 'g': 'manager'},
                                                            'default': {'o': '-number', 'g': 'client'}})

    else:
        order_by_field = '-status'
        group_by = 'client_id'

        four_months = datetime.timedelta(weeks=16)

        query.append(~Q(status__in = [Engagement.CANCELLED]))
        query.append(Q(target_delivery_date__gte = datetime.date.today() - four_months))

    engagements = Engagement.objects.select_related("priority", "lead_analyst", "manager", "trip").prefetch_related('funds', 'activities', 'activities__fund', 'activities__fund__versions__fund_structure', 'activities__fund__versions__fund_strategy', 'activities__activity_type', 'clients', 'billings').filter(*query).distinct()

    if len(engagements) > 0:
         #Ordering by a FK field sometimes causes duplicates
        engagements = list(set(engagements))

        result = []

        [mappings, field_names, headings] = get_headings_and_fieldnames(form, csv, group_by, user=request.user)

        #if group_by == 'invoice.number':

        #else:
        #    idx = field_names.index('invoice.number')
        #    field_names = field_names[:idx] + field_names[idx:]
        #    headings = headings[:idx] + headings[idx:]

        #Structure the query results for the template
        for e in engagements:
            fund_list = e.funds.all()
            if not fund_list:
                fund_list = ["TBD"]

            clients = e.clients.all()
            billings = ClientBilling.objects.filter(client__in = clients, engagement = e).prefetch_related('client')

            #Verify the fund name is included in the search criteria
            if form.data.has_key('fundSearch') and form.data['fundSearch'] != '' and fund_list != ["TBD"]:
                #TODO: pretty sure this runs another query
                fund_list = fund_list.filter(id__in = [form.data['fundSearch']])

            if not fund_list == ["TBD"]:
                if by_invoice_date:
                    #TODO: Consider moving this outside of the loop
                    invoices = InvoiceItem.objects.filter(billing__client__in = clients, invoice__date__gte = datetime.datetime.strptime(form.data['invoice_date_from'], "%Y-%m-%d"), \
                                                          invoice__date__lte = datetime.datetime.strptime(form.data['invoice_date_to'], "%Y-%m-%d"), fund__in = fund_list, billing__engagement = e).select_related('fund', 'invoice', 'billing', 'billing__client')
                else:
                    invoices = InvoiceItem.objects.filter(billing__client__in = clients, fund__in = fund_list, billing__engagement = e).select_related('fund', 'invoice', 'billing', 'billing__client')
            else:
                invoices = []

            #If grouped by clients, output a row for each engagement for the respective client, collapsing the fund list into a single field
            if group_by == 'client_id':
                for c in clients:
                    result.append(format_list_view(e, {'funds': fund_list, 'clients': [c], 'invoices': [x for x in invoices if x.billing.client == c],
                                                       'billings': [x for x in billings if x.client == c]}, field_names, mappings))

            #Otherwise, group both the clients and funds into single fields
            else:
                result.append(format_list_view(e, {'funds': fund_list, 'clients': clients, 'billings': billings, 'invoices': invoices}, field_names, mappings))

        if csv:
            for r in result:
                r[field_names.index('fund_id')]['value'] = r[field_names.index('fund_id')]['value'].replace('
', "/")
                r[field_names.index('review_level')]['value'] = r[field_names.index('review_level')]['value'].replace('
', "/")

        cols = len(result[0]) if len(result) > 0 else 0

        result = sorted(result, key = lambda x: x[0]['value'])
    #If there are no results, just output an empty table
    else:
        result = []
        headings = []
        cols = 0

    if csv:
        result = [map(lambda x: x['value'], r) for r in result]

        keywords = {'data': result, 'filename': 'engagement_list', 'titles': headings}
        return to_csv(request, **keywords)

    context = dict(results=result, headings=headings, body_id = "tools", page_title = "Engagements", form = form,
                   group_by = group_by, cols = cols, )
    return render_to_response("engagement/billing_list.html", context, context_instance=RequestContext(request))

@login_required
@permission_required("engagement.view_engagement", return_403=True)
def engagement_to_csv(request):
    return index(request, csv=True, skipPost=request.POST.dict() == {})

@login_required
@permission_required("engagement_task.change_engagementtask", return_403=True)
@sessioned_post
def engagementtask_list(request, csv = None):

    form = TaskFilterForm(request.POST or None)
    group_by = None
    query = []
    group_by_non_visible_attr = []

    #Default view is just pull everything atm
    #Otherwise parse the form and build the query string
    if request.method == 'POST':

        query = build_query_from_filter(form, request, [{'form_field': 'services', 'model_field': 'template__related_services__activity__activity_type__id'},
                                                        {'form_field': 'category', 'model_field': 'template__category__id '},
                                                        {'form_field': 'assignee', 'model_field': 'assignees__id'},
                                                        {'form_field': 'estatus', 'model_field': 'engagement__status'}], query)

        query = build_query_from_search_param(form, [{'form_field': 'taskSearch', 'model_field': 'template__pk'},
                                                     {'form_field': 'managerSearch', 'model_field': 'engagement__manager__id'},
                                                     {'form_field': 'clientSearch', 'model_field': 'engagement__clients__id'},], query)

        [order_by_field, group_by] = get_ordering_data(form, {'Category': {'o': '-category', 'g': 'template.category.alias'},
                                                            'Engagement': {'o': 'engagement', 'g':'engagement.name'},
                                                            'Assigned To': {'o': '-assignees', 'g': 'assignee_set'},
                                                            'Client': {'o': '-engagement__clients__id', 'g': 'client'},
                                                            'Fund': {'o': '-funds__fund_id', 'g': 'funds'},
                                                            'Status': {'o': '-status', 'g': 'status'},
                                                            'default': {'o': '-status', 'g': None}},)

        engagements_tasks = EngagementTask.objects.select_related("template", 'template__category', 'template__predecessor_task', "engagement").order_by('-id').prefetch_related('funds', 'assignees', 'engagement__funds', 'engagement__clients').filter(*query)
    else:
        engagements_tasks = []

    display_msg = False
    if len(engagements_tasks) > 0 and len(engagements_tasks) <= 1000:
        result = []
        [mappings, field_names, headings] = get_headings_and_fieldnames(form, csv, group_by, user=request.user)

        # Add the group by non visible fields array which do not show up as a column always by default
        if group_by != 'engagement.name':
            i = headings.index('Engagement')
            headings = headings[:i] + headings[i + 1:]
            field_names = field_names[:i] + field_names[i + 1:]

        # Fetch the status filters and filter records  here since there is no status field in engagement task model
        status_filters = []
        if form.data.has_key('status'):
            status_filters = request.POST.getlist('status')

        tmp = dict(EngagementTask.TASK_STATUS_CHOICES)
        status_dict = dict((v,k) for k, v in tmp.iteritems())

        engagements_tasks = list(set(engagements_tasks))

        tasks_found = []
        for e in engagements_tasks:
            # Calculate the status for the task
            clients = e.engagement.clients.all()
            assignees = e.assignees.all()
            funds = e.funds.all()
            if group_by == 'funds':
                for f in e.funds.all():
                    (tf, r) =__get_etask_list(e, field_names, mappings, assignees, [f], clients, status_filters, status_dict)
            elif group_by == 'client':
                for c in e.engagement.clients.all():
                    (tf, r) = __get_etask_list(e, field_names, mappings, assignees, funds, [c], status_filters, status_dict)
            elif group_by == 'assignee_set':
                for a in e.assignees.all():
                    (tf, r) = __get_etask_list(e, field_names, mappings, [a], funds, clients, status_filters, status_dict)
            else:
                (tf, r) = __get_etask_list(e, field_names, mappings, assignees, funds, clients, status_filters, status_dict)

            if r:
                result.append(r)
            if tf:
                tasks_found.append(tf.pk)

        batchformSet = modelformset_factory(EngagementTask, form=TaskCompleteForm, can_delete=False, can_order=False, extra=0)

        if request.user.has_perm("engagement_task.can_perform_batch_operations"):
            batchform = batchformSet(initial={'complete': False}, queryset=EngagementTask.objects.filter(pk__in = tasks_found), prefix="batch")
        else:
            batchform=batchformSet(queryset=None)

        cols = len(result[0]) if len(result) > 0 else 0

        #Sort in any case, because the default index (status) is not an attribute and cannot be filtered by in the query
        i = 0

        if result and result[0]:
            while not 'value' in result[0][i]:
                i = i + 1

            result = sorted(result, key = lambda x: x[i]['value'])

    else:
        if len(engagements_tasks) > 1000:
            display_msg = True
        result = []
        headings = []
        cols = 0
        batchform = None

    #batchform = BatchSetAttrForm(initial={'update_type': 'S', 'attribute': 'complete', 'set_to': 'true'})
    context = dict(results=result, headings=headings, body_id = "tools", show_msg = display_msg,
                   page_title = "Engagement Tasks", form = form, group_by = group_by, cols = cols, batchform = batchform)

    return render_to_response("engagement/engagementtask_list.html", context, context_instance=RequestContext(request))

@login_required
@permission_required("engagement.change_engagement", return_403=True)
def et_batch_complete(request):
    d = request.POST.dict()
    date = d.get('completed_date', None)

    if date:
        date = datetime.datetime.strptime(date, "%Y-%m-%d")

    batchformSet = modelformset_factory(EngagementTask, form=TaskCompleteForm, can_delete=False, can_order=False, extra=0)
    batchform = batchformSet(request.POST or None, prefix="batch")

    failed = []

    for f in batchform:
        original = f.instance.updated_by
        f.instance.updated_by = request.user
        if f.is_valid(user=request.user, date=date, override=True) and 'complete' in f.cleaned_data:
            f.save(user=request.user, notify=False)
        else:
            failed.append(f.instance.id)
            f.instance.updated_by = original

    return HttpResponseRedirect(reverse('engagementtask_list'))

@login_required
@permission_required("engagement_task.change_engagementtasktemplate", return_403=True)
def engagementtasktemplate_list(request, csv=False):

    form = TemplateTaskFilterForm(request.POST or None)
    group_by = None
    query = []

    if request.method == 'POST':

        query = build_query_from_filter(form, request, [{'form_field': 'services', 'model_field': 'related_services__activity__activity_type_id'},
                                                        {'form_field': 'category', 'model_field': 'category_id'},
                                                        {'form_field': 'assignee', 'model_field': 'assignee__icontains'}], query)

        query = build_query_from_search_param(form, [{'form_field': 'taskSearch', 'model_field': 'pk'}], query)

        [order_by_field, group_by] = get_ordering_data(form, {'Category': {'o': '-category', 'g':'category.alias'},
                                                            'Services': {'o': '-related_services__activity__activity_type__name', 'g': 'services'},
                                                            'default': {'o': '-category', 'g': None}},)

        engagements_template_tasks = EngagementTaskTemplate.objects.select_related("category", "predecessor_task").order_by(order_by_field, '-id').prefetch_related('related_services').filter(*query).distinct()
    else:
        # Null values are ignored in the default sorting
        engagements_template_tasks = list(itertools.chain(
            EngagementTaskTemplate.objects.select_related("category", "predecessor_task")
            .filter(sequence__isnull=False)
            .order_by('sequence')
            .prefetch_related('related_services')
            .distinct(),

            EngagementTaskTemplate.objects.select_related("category", "predecessor_task")
            .filter(sequence__isnull=True)
            .prefetch_related('related_services')
            .distinct()
        ))

    if len(engagements_template_tasks) > 0:
        result = []
        [mappings, field_names, headings] = get_headings_and_fieldnames(form, False, group_by, user=request.user)

        for e in engagements_template_tasks:
            if group_by == 'services':
                for s in e.related_services.all():
                    result.append(format_list_view(e, {'services': [s]}, field_names, mappings))
            else:
                result.append(format_list_view(e, {'services': e.related_services.all()}, field_names, mappings))

        cols = len(result[0]) if len(result) > 0 else 0

        if group_by:
            result = sorted(result, key = lambda x: x[0]['value'])
    else:
        result = []
        headings = []
        cols = 0

    context = dict(results=result, headings=headings, body_id = "tools",
                   page_title = "Engagement Template Tasks", form = form, group_by = group_by, cols = cols)

    return render_to_response("engagement/engagementtasktemplate_list.html", context, context_instance=RequestContext(request))

#TODO: Refactor this form
@login_required
@permission_required("engagement.change_engagement", return_403=True)
@next
def engagement_form(request, engagement_id = None):
    contact_info = None
    manager = None
    manager_office = None
    #location = None
    manager_data = None
    activation_task = None
    initial_data = {}
    e = Engagement()
    data = request.POST.dict()
    funds = None
    clients = None

    trips = Trip.objects.all().prefetch_related('locations', 'triplocation_set')

    invoice_item_formset = inlineformset_factory(Invoice, InvoiceItem, can_delete = False, extra = 0, form=InvoiceItemForm)

    if engagement_id:
        mode = 'EDIT'
        e = get_object_or_404(Engagement.objects.select_related('manager', 'desk_analyst', 'lead_analyst', 'reviewer_1', 'reviewer_2').prefetch_related('funds', 'clients', 'activities', 'tasks'), pk=engagement_id)
        form = EngagementForm(request.POST or None, instance=e, initial={'funds': e.funds.all().order_by('pk'), 'clients': e.clients.all().order_by('pk')})
        #location = e.manager.get_ajax_data()
        manager_data = __get_manager_and_contact_info(e.manager.pk, None)
        invoice_form = InvoiceForm(prefix='invoice', initial={'engagement': e}, clients=e.clients.all())
        invoice_items = invoice_item_formset(prefix="invoice_formset")
        page_title = 'Edit Engagement'
        initial_data = __format_initial_data(e.activities.select_related('fund', 'activity_type').prefetch_related('client_concerned').all())

        # Determine whether the engagement has an activation task, this will toggle the activate button
        try:
            activation_task = e.tasks.filter(template__enable_activation=True)[0]
        except IndexError:
            pass

        funds = e.funds.all()
        clients = e.clients.all().order_by('pk')
    else:
        mode = 'ADD'
        form = EngagementForm(request.POST or None)
        invoice_form = None   #Only show the invoice form on page edits
        invoice_items = None
        page_title = 'Add Engagement'

    ffmanager = FundActivitiesManagerForm(request.POST or None, prefix="ffmanager")
    FundFormSet = formset_factory(wraps(FundActivitiesForm)(partial(FundActivitiesForm, e = e, mprefix=ffmanager.prefix)), formset=FundActivitiesInline, extra = 0, can_delete=False)
    fund_form = FundFormSet(request.POST or None, initial = initial_data, prefix="fundset")

    cityFormSet = inlineformset_factory(Trip, TripLocation, can_delete = True, extra = 1, form = TripLocationForm)

    trip_form = TripForm()

    #Formset for billing
    oop_formset = inlineformset_factory(Engagement, ClientBilling, form=OopForm, formset=OopFormSet, extra=0, can_delete=True)
    oop_form = oop_formset(request.POST or None, instance=e, prefix='billing')

    #Require at least a single location to be included
    trip_locations = cityFormSet()
    #TODO: verify this is working
    trip_locations.forms[0].empty_permitted = False

    flag = True
    #Invoice Forms, Trip Forms are submitted through AJAX, so ignore them here
    if request.method == 'POST':
        if form.is_valid():
            ffmanager = FundActivitiesManagerForm(request.POST or None, prefix="ffmanager", funds = form.cleaned_data['funds'])

            if ffmanager.is_valid():
                old_activities = []
                if engagement_id:
                    old_activities = e.activities.select_related('fund', 'activity_type').prefetch_related('client_concerned').all()

                obj = form.save(update = False)

                #check if funds_same_for_all was selected
                if ffmanager.cleaned_data.get('same_for_all', True):
                    if request.user.has_perm('billing.view_clientbilling'):
                        oop_form = oop_formset(request.POST, instance = obj, prefix = 'billing')
                        if not oop_form.is_valid():
                            flag = False

                    if flag:
                        FFWRAPPER = fund_form_wrapper(engagement = obj, manager_prefix=ffmanager.prefix, manager=ffmanager.cleaned_data)
                        fund_form = FFWRAPPER(request.POST, initial = initial_data, prefix="fundset")

                        if fund_form.is_valid():
                            #Order is important here - we need to save the engagement *after* we've deleted the activities
                            fund_form.save(engagement = obj, old_acts = old_activities)

                            if request.user.has_perm('billing.view_clientbilling'):
                                oop_form.save()

                            obj = form.save(update = True, hasBillingPerm = request.user.has_perm('billing.view_clientbilling'))

                            # Activation redirection
                            if request.POST.get("save-with-activation", None) is not None and activation_task:
                                return HttpResponseRedirect(reverse('engagementtask_activation', args=(activation_task.id, )))

                            return HttpResponseRedirect(request.POST["next"])
                else:
                    if request.user.has_perm('billing.view_clientbilling'):
                        oop_form = oop_formset(request.POST, instance = obj, prefix = 'billing')
                        if not oop_form.is_valid():
                            flag = False
                    if flag:
                        FFWRAPPER = fund_form_wrapper(engagement = obj)
                        fund_form = FFWRAPPER(request.POST, initial = initial_data, prefix="fundset")
                        if fund_form.is_valid():
                            fund_form.save(engagement = obj, old_acts = old_activities)

                            if request.user.has_perm('billing.view_clientbilling'):
                                oop_form.save()

                            obj = form.save(update = True, hasBillingPerm = request.user.has_perm('billing.view_clientbilling'))

                            # Activation redirection
                            if request.POST.get("save-with-activation", None) is not None:
                                return HttpResponseRedirect(reverse('engagementtask_activation', args=(activation_task.id, )))

                            return HttpResponseRedirect(request.POST["next"])

            #Else handle many errors
        else:
            if request.user.has_perm('billing.view_clientbilling'):
                oop_form = oop_formset(request.POST, instance = e, prefix = 'billing')

    if hasattr(form, 'cleaned_data'):
        #Bind the data incase POST failed
        #Super hacky solution
        if exists_with_value(form.cleaned_data, 'funds'):
            funds = FundIndex.objects.filter(pk__in = form.cleaned_data['funds'])

        if exists_with_value(form.cleaned_data, 'clients'):
            clients = Client.objects.filter(pk__in = form.cleaned_data['clients']).order_by('id')

        if len(initial_data):
            initial_data = initial_data[0]
        else:
            initial_data = []

        ffmanager = FundActivitiesManagerForm(request.POST or None, funds=funds, prefix="ffmanager")
        FundFormSet = formset_factory(wraps(FundActivitiesForm)(partial(FundActivitiesForm, e = e, initial=initial_data, funds=funds, clients=clients)), formset=FundActivitiesInline, extra = 0, can_delete=False)
        fund_form = FundFormSet(request.POST or None, prefix="fundset")

    if activation_task is None:
        messages.add_message(request, messages.INFO, "You cannot activate this engagement yet, there is no "
                                                     "activation task.")

    if (ffmanager.prefix + "-same_for_all" in data):
        showMinor = data[ffmanager.prefix + "-same_for_all"] != 'on'
    else:
        showMinor = True

    if engagement_id or exists_with_value(data, 'manager'):
        if engagement_id:
            manager = e.manager
        else:
            manager = ManagerIndex.objects.get(pk = data['manager'])

        manager_office = {}
        manager_office['city'] = manager.primary_city
        manager_office['office'] = manager.primary_office_address

        #TODO: if contact was set and post failed?
        if e.contact:
            contact_info = e.contact.get_ajax_data()

    contact = AddContactForm(prefix="contact")
    meetings_trips = Meeting.objects.all().filter(engagements__id =engagement_id )
    for k in meetings_trips:
        ch_antendees= User.objects.values('first_name','last_name').filter(meeting_attendees=k.id)
        names = ", ".join([i["first_name"]+" "+i["last_name"] for i in ch_antendees if len(i["first_name"])>0])
        setattr(k, "template_first_name", names)

    return render(request, 'engagement/engagement_form.html',
                  {'body_id': 'tools', 'contact_info': contact_info, 'page_title': page_title, 'manager': manager,
                   'manager_office': manager_office, 'mode': mode, 'i': e.pk, 'manager_data': manager_data,
                   'trip_form_as_modal': True,
                   'trips': trips,
                   #Forms
                   'invoice_form': invoice_form,
                   'billing_form': oop_form,
                   'manager_form': AddManagerForm(prefix="manager"),
                   'edit_manager_form': AddManagerForm(prefix="manager"),
                   'manager_contact_form': AddContactForm(prefix="contact"),
                   'client_form': AddClientForm(),
                   'engagement_form': form,
                   'fund_form': fund_form,
                   'contact_form': contact,
                   'ffm': ffmanager,
                   'add_fund_form': AddFundIndexForm(prefix="fundindex"),
                   'trip_form': trip_form,
                   'trip_locations': trip_locations,
                   'ii': invoice_items,
                   'activation_task': activation_task,
                   'funds': funds,
                   'clients': clients,
                   'showfferrors': showMinor,
                    'meetings_trips': meetings_trips

                    })

@login_required
@permission_required("engagement.delete_engagement", return_403=True)
def engagement_delete(request, engagement_id=None):
    engagement = get_object_or_404(Engagement, pk=engagement_id)

    if engagement.status not in [Engagement.COMPLETE]:
        engagement.delete()

    return HttpResponseRedirect(reverse('engagement_index'))

@login_required
@permission_required("engagement_task.change_engagementtasktemplate", return_403=True)
def engagementtasktemplate_add(request):
    context = dict()

    if request.POST:
        templateform = EngagementTaskTemplateForm(request.POST)
        attachmentform = TemplateAttachmentFormset(request.POST, request.FILES)

        if templateform.is_valid():
            template = templateform.save(commit=False)
            attachmentform.instance = template

            if attachmentform.is_valid():
                # Object log update
                templateform.instance.updated_by = request.user

                templateform.save()
                attachmentform.save()

                return HttpResponseRedirect(reverse('engagementtasktemplate_list'))
    else:
        template = EngagementTaskTemplate()
        templateform = EngagementTaskTemplateForm()
        attachmentform = TemplateAttachmentFormset(instance=template)

    context.update(
        templateform=templateform,
        attachmentform=attachmentform,
        page_title='Add Task Template'
    )

    return render(request, "engagement/engagement_task_template_form.html", context)

@login_required
@permission_required("engagement_task.change_engagementtasktemplate", return_403=True)
def engagementtasktemplate_edit(request, tasktemplate_id=None):
    templateform = None
    attachmentform = None
    context = dict()
    template = get_object_or_404(EngagementTaskTemplate, pk=tasktemplate_id)

    if request.POST:
        templateform = EngagementTaskTemplateForm(request.POST, instance=template)

        if templateform.is_valid():
            templateform.save(commit=False)
            attachmentform = TemplateAttachmentFormset(request.POST, request.FILES, instance=template)

            if attachmentform.is_valid():
                # Object log update
                templateform.instance.updated_by = request.user

                templateform.save()
                attachmentform.save()

                return HttpResponseRedirect(reverse('engagementtasktemplate_list'))

    else:
        templateform = EngagementTaskTemplateForm(instance=template)
        attachmentform = TemplateAttachmentFormset(instance=template)

    context.update(
        template=template,
        templateform=templateform,
        attachmentform=attachmentform,
        mode="EDIT",
        body_id = 'tools',
        page_title='Edit Task Template'
    )

    return render(request, "engagement/engagement_task_template_form.html", context)

@login_required
@permission_required("engagement_task.delete_engagementtasktemplate", return_403=True)
def engagementtasktemplate_delete(request, tasktemplate_id=None):
    tasktemplate = get_object_or_404(EngagementTaskTemplate, pk=tasktemplate_id)

    predecessors = EngagementTaskTemplate.objects.filter(predecessor_task=tasktemplate)

    if not predecessors.exists():
        tasktemplate.delete()
    else:
        message = "Template cannot be deleted, it is predecessor of the templates: " + \
                  ",".join([ett.name for ett in predecessors])
        messages.add_message(request, messages.WARNING, message)

    return HttpResponseRedirect(reverse('engagementtasktemplate_list'))

@login_required
@permission_required("engagement_task.change_engagementtask", return_403=True)
def engagementtask_edit(request, task_id=None):
    taskform = None
    attachmentform = None
    context = dict()
    task = get_object_or_404(EngagementTask, pk=task_id)
    activity = None

    is_reviewtask = task.is_reviewtask

    if (is_reviewtask):
        reviewer_role, reviewee_role = task.get_reviewer_reviewee_roles()
        reviewer = getattr(task.engagement, reviewer_role.lower(), None)
        reviewee = getattr(task.engagement, reviewee_role.lower(), None)

        is_reviewtask = reviewee and reviewer

    related_docs = task.template.attachments.all()
    task_details = task.template.notes
    notified_on_completion = None
    # This parameter indicates the insertion of an activate button in the UI
    enable_activation = task.template.enable_activation

    if task.template.notify is not None:
        notified_on_completion = task.get_assignee(task.template.notify)

    if request.POST:
        taskform = EngagementTaskFormFactory(request.user, request.POST, instance=task)

        if task.template.allow_upload:
            attachmentform = TaskAttachmentFormset(request.POST, request.FILES, instance=task)

        # Object log update, it is important that this is called before is_valid as it carries the request.user to
        # clean() function. Validator functions cannot access to the request.
        taskform.instance.updated_by = request.user

        if taskform.is_valid():
            goto_review = True if taskform.data.get("save-with-review", None) is not None else False
            goto_activate = True if taskform.data.get("save-with-activate", None) is not None else False
            t = taskform.save(user=request.user)

            # Do the reassignment
            if taskform.cleaned_data.get("reassignment", None) is not None and not taskform.instance.complete and \
                    request.user.has_perm("engagement.reassign_tasks"):

                t.was_reassigned = 1
                t.save()
                for assignee in taskform.cleaned_data["reassignment"]:
                    if assignee not in taskform.instance.assignees.all():
                        EngagementTaskAssignee.objects.create(
                            assignee=assignee,
                            task=taskform.instance,
                            role=EngagementTaskAssignee.REASSIGNED
                        )

                for assignee in taskform.instance.assignees.all():
                    if assignee not in taskform.cleaned_data["reassignment"]:
                        EngagementTaskAssignee.objects.filter(
                            assignee=assignee,
                            task=taskform.instance
                        ).delete()

            if task.template.allow_upload and attachmentform.is_valid():
                attachmentform.save()

            # Go the edit or add review
            if goto_review:
                _qs = EngagementReview.objects.filter(engagementtask=task)
                if _qs.exists():
                    review = _qs[0]
                    return HttpResponseRedirect(reverse("engagementreview_edit", args=(review.id, )))
                else:
                    return HttpResponseRedirect(reverse("engagementreview_add", args=(task.id, )))

            # Got to activate
            if goto_activate:
                return HttpResponseRedirect(reverse("engagementtask_activation", args=(task.id, )))

            return HttpResponseRedirect(reverse('employee_dashboard'))

    else:
        taskform = EngagementTaskFormFactory(request.user, instance=task)

        if task.template.allow_upload:
            attachmentform = TaskAttachmentFormset(instance=task)

        taskform.fields["complete"].initial = task.complete

    if len(task.funds.all()) > 0:
        waiting_tasks = EngagementTask.objects.filter(
            template__predecessor_task=task.template,
            engagement=task.engagement,
            funds__in=task.funds.all()
        )
    else:
        waiting_tasks = EngagementTask.objects.filter(
            template__predecessor_task=task.template,
            engagement=task.engagement
        )

    # Since tasks are created for each activity type and we need to paint the waiting assignees only once
    if len(waiting_tasks) > 0:
        waiting_tasks = waiting_tasks[0]

    task_deadline_delta = False

    if task.deadline is not None:
        today = datetime.date.today()
        task_deadline_delta = task.deadline - today
        task_deadline_delta = task_deadline_delta.days

    context.update(
        task=task,
        engagement_roles = task.engagement.return_roles(),
        body_id = 'tools',
        taskform=taskform,
        attachmentform=attachmentform,
        related_docs=related_docs,
        task_details=task_details,
        notified_on_completion=notified_on_completion,
        enable_activation=enable_activation,
        task_deadline_delta=task_deadline_delta,
        waiting_tasks=waiting_tasks,
        is_reviewtask=is_reviewtask,
        is_activationtask=task.template.enable_activation,
        page_title='Edit Task'
    )

    return render(request, "engagement/engagement_task_form.html", context)

@login_required
@permission_required("engagement_task.delete_engagementtask", return_403=True)
def engagementtask_delete(request, task_id=None):
    task = get_object_or_404(EngagementTask, pk=task_id)

    task.delete()

    return HttpResponseRedirect(reverse('engagementtask_list'))

@login_required
@permission_required("engagement_task.change_engagementreview", return_403=True)
def engagementreview_list(request):
    form = EngagementReviewFilterForm(request.POST or None)
    group_by = None
    query = []
    ers=EngagementReviewScore.objects.all()
    erc=EngagementReviewCriteria.objects.all()

    if request.method == 'POST':
        [query, group_by] = __get_engagement_review_params(form, query, request)
        engagement_reviews = EngagementReview.objects.prefetch_related('reviewscores','engagementtask__engagement').select_related('reviewer','reviewee').filter(*query).distinct()
    else:
        engagement_reviews = EngagementReview.objects.prefetch_related('reviewscores','engagementtask__engagement').select_related('reviewer','reviewee')

    if not request.user.has_perm("engagement_task.view_all_engagementreview"):
        engagement_reviews = engagement_reviews.filter(Q(reviewer=request.user) | Q(reviewee=request.user))
    if len(engagement_reviews) > 0:
        engagement_reviews=list(set(engagement_reviews))
        result = []
        headings=[]
        criteria_scores={}
        heading_list=[]
        rename={"1":"Written Quality","4":"Timeliness","5":"Technial/Issue Identification","6":"Review process","7":"Manager Interaction",
                "8":"Training","9":"Review","10":"Overall"}
        for c in erc:
            criteria_scores[c.id]='N/A'
            temp=c.text
            id=c.id
            k=str(id).replace("L","")
            heading_list.append(rename[k] if k in rename.keys() else c.text)
        var=heading_list[-1]
        heading_list.insert(0,var)
        heading_list.pop()

        for e in engagement_reviews:
            for c in erc:
                criteria_scores[c.id]='N/A'
            reviewids=ers.filter(review=e.id)
            ([criteria_scores.update({x.criteria.id:x.score})for x in reviewids])
            tmp={'Engagement':e.engagement_reformat,'Complexity':e.engagementtask.engagement.complexity,'Reviewer':e.reviewer_with_role,'Reviewee':e.reviewee_with_role,'Status':e.status,'Completion_Date':e.date,'Engagement_Review':e}
            tmp.update(criteria_scores)
            result.append(tmp)

        if len(result) > 0:
            headings = ['Engagement','Reviewer','Reviewee','Complexity']
            for k in heading_list:
                headings.append(str(k))
            headings.append('Status')
            headings.append('Completion Date')
            for x in range(0, 3):
                headings.append(" ")

            [group_by, headings, result] = __get_results_and_headings_engagementreview(group_by,headings, result)

        cols = len(result[0]) if len(result) > 0 else 0
    else:
        result = []
        headings = []
        cols = 0
    context = dict(
        results=result,
        headings=headings,
        body_id = "tools",
        page_title = "Engagement Review Criterias",
        form = form,
        group_by = group_by,
        cols = cols
    )
    return render_to_response("engagement/engagementreview_list.html", context, context_instance=RequestContext(request))

def __get_results_and_headings_engagementreview(group_by,headings, result):
    id = -1
    if group_by=="Complexity":
        id = 1
        result = sorted(result, key=lambda x: (x['Complexity']))
    elif group_by=="Reviewer":
        id = 2
        result = sorted(result, key=lambda x: (x['Reviewer']))
    elif group_by=="Reviewee":
        id = 3
        result = sorted(result, key=lambda x: (x['Reviewee']))
    else:
        h = headings
        result = sorted(result, key=lambda x: (x['Engagement']))
    if id > 0:
        h = [headings[id]] + (headings[0:id] if id > 0 else []) + headings[id + 1:]
    headings = h

    return (group_by, headings, result)

def __get_engagement_review_params(form, query, request):
    query = build_query_from_filter(form, request, [{'form_field': 'status', 'model_field': 'status'}], query)

    query = build_query_from_search_param(form, [{'form_field': 'reviewersSearch', 'model_field': 'reviewer__id'},
                                                     {'form_field': 'revieweesSearch', 'model_field': 'reviewee__id'},
                                                     {'form_field': 'managersSearch', 'model_field': 'engagementtask__engagement__manager__id'},
                                                     {'form_field': 'fromSearch', 'model_field': 'date__gte'},
                                                     {'form_field': 'toSearch', 'model_field': 'date__lte'}], query)

    if exists_with_value(form.data, 'fromSearch'):
        query.append(Q(date__gte = datetime.datetime.strptime(form.data['fromSearch'], "%Y-%m-%d")))

    if exists_with_value(form.data, 'toSearch'):
        query.append(Q(date__lte = datetime.datetime.strptime(form.data['toSearch'], "%Y-%m-%d")))

    group_by = exists_with_value(form.data, 'group_by')
    return query, group_by

@login_required
@permission_required("engagement_task.add_engagementreview", return_403=True)
def engagementreview_add(request, engagementtask_id=None):
    engagementtask = get_object_or_404(EngagementTask, pk=engagementtask_id)
    reviewform = None
    reviewscoreform = None
    context = dict()
    bundles = list()
    reviewer_role, reviewee_role = engagementtask.get_reviewer_reviewee_roles()
    reviewer = getattr(engagementtask.engagement, reviewer_role.lower(), None)
    reviewee = getattr(engagementtask.engagement, reviewee_role.lower(), None)
    criterias = EngagementReviewCriteria.objects.filter(
        roles__reviewer_role=reviewer_role,
        roles__reviewee_role=reviewee_role
    ).distinct()

    #If there is no reviewee there is no one to review and we shouldn't be accessing this page
    if not reviewee:
        raise Http404("This task has no one to review.")

    try:
        review = EngagementReview.objects.get(engagementtask = engagementtask, reviewer = reviewer, reviewee = reviewee)

        #If a review already asks for this tuple, then just edit that
        if review.pk:
            return engagementreview_edit(request, review_id = review.pk)

    except EngagementReview.DoesNotExist:
        pass

    if request.POST:
        reviewform = EngagementReviewForm(request.POST)
        reviewscoreform = EngagementReviewScoreFormset(request.POST)

        if reviewform.is_valid():
            criteria = reviewform.save(commit=False)

            if request.POST.get("submit", False) is not False:
                reviewform.instance.status = EngagementReview.SUBMITTED

            reviewscoreform.instance = criteria

            if reviewscoreform.is_valid():
                reviewform.save()
                reviewscoreform.save()

                return HttpResponseRedirect(reverse('engagementreview_list'))
    else:
        review = EngagementReview(
            engagementtask=engagementtask,
            reviewer=reviewer,
            reviewee=reviewee
        )
        fake_POST = {
            "reviewscores-TOTAL_FORMS": unicode(len(criterias)),
            "reviewscores-MAX_NUM_FORMS": u"1000",
            "reviewscores-INITIAL_FORMS": u"0",
        }

        count = 0
        for criteria in criterias:
            count_str = str(count)
            fake_POST.update({
                "reviewscores-" + count_str + "-review": u"",
                "reviewscores-" + count_str + "-criteria": unicode(criteria.id),
                "reviewscores-" + count_str + "-score": u"0",
            })
            count += 1

        reviewform = EngagementReviewForm(instance=review)
        reviewscoreform = EngagementReviewScoreFormset(fake_POST, instance=review)

    # Bundles Criteria objects with the corresponding engagement
    for idx in range(len(criterias)):
        bundles.append((criterias[idx], reviewscoreform.forms[idx]))

    context.update(
        reviewform=reviewform,
        reviewscoreform=reviewscoreform,
        bundles=bundles,
        reviewee=reviewee,
        errors=len(reviewform.errors) > 0, # or len(reviewscoreform.errors[0]) > 0,
        funds=" ,".join([fund.fund_name + " (" + services + ")"
                         for fund, services in engagementtask.engagement.get_funds_with_services()]),
        reviewee_role=dict(EngagementTaskTemplate.ASSIGNEE_ROLES)[reviewee_role],
        score_options=EngagementReviewScore.SCORES,
        page_title='Add Review',
        body_class='med-pop'
    )

    return render(request, "engagement/engagement_review_form.html", context)

@login_required
@permission_required("engagement_task.change_engagementreview", return_403=True)
def engagementreview_edit(request, review_id=None):
    engagementreview = get_object_or_404(EngagementReview, pk=review_id)
    context = dict()
    bundles = list()
    reviewform = None
    reviewscoreform = None
    funds = " ,".join([fund.fund_name + " (" + services + ")"
                       for fund, services in engagementreview.engagementtask.engagement.get_funds_with_services()])
    already_submitted = engagementreview.status == EngagementReview.SUBMITTED
    reviewer_role, reviewee_role = engagementreview.engagementtask.get_reviewer_reviewee_roles()
    reviewer = getattr(engagementreview.engagementtask.engagement, reviewer_role.lower(), None)
    reviewee = getattr(engagementreview.engagementtask.engagement, reviewee_role.lower(), None)
    criterias = EngagementReviewCriteria.objects.filter(
        roles__reviewer_role=reviewer_role,
        roles__reviewee_role=reviewee_role
    ).distinct()

    if request.POST:
        reviewform = EngagementReviewForm(request.POST, instance=engagementreview)
        reviewscoreform = EngagementReviewScoreFormset(request.POST, instance=engagementreview)

        if reviewform.is_valid() and reviewscoreform.is_valid():
            reviewform.save(commit=False)

            if request.POST.get("submit", False) is not False:
                reviewform.instance.status = EngagementReview.SUBMITTED

            reviewform.save()
            reviewscoreform.save()

            return HttpResponseRedirect(reverse('engagementreview_list'))
    else:
        reviewform = EngagementReviewForm(instance=engagementreview)
        reviewscoreform = EngagementReviewScoreFormset(instance=engagementreview)

    # Bundles Criteria objects with the corresponding engagement
    for idx in range(len(criterias)):
        bundles.append((criterias[idx], reviewscoreform.forms[idx]))

    context.update(
        reviewform=reviewform,
        reviewscoreform=reviewscoreform,
        bundles=bundles,
        reviewee=reviewee,
        errors=len(reviewform.errors) > 0 or len(reviewscoreform.errors) > 0,
        funds=funds,
        reviewee_role=dict(EngagementTaskTemplate.ASSIGNEE_ROLES)[reviewee_role],
        score_options=EngagementReviewScore.SCORES,
        already_submitted=already_submitted,
        page_title="Edit Review",
        mode="EDIT"
    )

    return render(request, "engagement/engagement_review_form.html", context)

@login_required
@permission_required("engagement_task.delete_engagementreview", return_403=True)
def engagementreview_delete(request, review_id=None):
    get_object_or_404(EngagementReview, pk=review_id).delete()

    return HttpResponseRedirect(reverse('engagementreview_list'))

@login_required
@permission_required("engagement_task.change_engagementreviewcriteria", return_403=True)
def engagementreviewcriteria_list(request):
    form = EngagementReviewCriteriaFilterForm(request.POST or None)
    group_by = None
    query = []
    context = dict()

    if request.method == 'POST':

        query = build_query_from_filter(form, request, [{'form_field': 'services', 'model_field': 'related_services__activity__activity_type_id'},
                                                        {'form_field': 'reviewers', 'model_field': 'roles__reviewer_role'},
                                                        {'form_field': 'reviewees', 'model_field': 'roles__reviewee_role'}], query)

        query = build_query_from_search_param(form, [{'form_field': 'criteriaSearch', 'model_field': 'text__icontains'},
                                                     {'form_field': 'descriptionSearch', 'model_field': 'description__icontains'}], query)

        [order_by_field, group_by] = get_ordering_data(form, {'Services': {'o': '-related_services__activity__activity_type__name', 'g': 'services'},
                                                              'Role Pairs': {'o': '-order', 'g': 'roles'},
                                                              'default': {'o': '-order', 'g': None}},)

        engagement_review_criterias = EngagementReviewCriteria.objects.order_by(order_by_field, '-id').prefetch_related('related_services', 'roles').filter(*query).distinct()
    else:
        # Null values are ignored in the default sorting
        engagement_review_criterias = list(itertools.chain(
            EngagementReviewCriteria.objects.filter(order__isnull=False)
            .order_by('order')
            .prefetch_related('related_services','roles')
            .distinct(),

            EngagementReviewCriteria.objects.filter(order__isnull=True)
            .prefetch_related('related_services', 'roles')
            .distinct()
        ))

    engagement_review_criterias = list(set(engagement_review_criterias))

    if len(engagement_review_criterias) > 0:
        result = []
        [mappings, field_names, headings] = get_headings_and_fieldnames(form, False, group_by, user=request.user)

        for e in engagement_review_criterias:
            if group_by == 'services':
                for s in e.related_services.all():
                    result.append(format_list_view(e, {'services': [s], 'roles': e.roles.all()}, field_names, mappings))
            elif group_by == 'roles':
                for r in e.roles.all():
                    result.append(format_list_view(e, {'services': e.related_services.all(), 'roles': [r]}, field_names, mappings))
            else:
                result.append(format_list_view(e, {'services': e.related_services.all(), 'roles': e.roles.all()}, field_names, mappings))

        cols = len(result[0]) if len(result) > 0 else 0

        if group_by:
            result = sorted(result, key = lambda x: x[0]['value'])
    else:
        result = []
        headings = []
        cols = 0

    context = dict(
        results=result,
        headings=headings,
        body_id = "tools",
        page_title = "Engagement Review Criterias",
        form = form,
        group_by = group_by,
        cols = cols
    )

    return render_to_response("engagement/engagementreviewcriteria_list.html", context, context_instance=RequestContext(request))

@login_required
@permission_required("engagement_task.add_engagementreviewcriteria", return_403=True)
def engagementreviewcriteria_add(request):
    context = dict()

    if request.POST:
        criteriaform = EngagementReviewCriteriaForm(request.POST)
        criteriaroleform = EngagementReviewCriteriaRoleFormset(request.POST)

        if criteriaform.is_valid():
            criteria = criteriaform.save(commit=False)
            criteriaroleform.instance = criteria

            if criteriaroleform.is_valid():
                criteriaform.save()
                criteriaroleform.save()

                return HttpResponseRedirect(reverse('engagementreviewcriteria_list'))
    else:
        criteria = EngagementReviewCriteria()
        criteriaform = EngagementReviewCriteriaForm()
        criteriaroleform = EngagementReviewCriteriaRoleFormset(instance=criteria)

    context.update(
        criteriaform=criteriaform,
        criteriaroleform=criteriaroleform,
        empty_form=criteriaroleform.empty_form,
        page_title='Add Review Criteria'
    )

    return render(request, "engagement/engagement_review_criteria_form.html", context)

@login_required
@permission_required("engagement_task.change_engagementreviewcriteria", return_403=True)
def engagementreviewcriteria_edit(request, criteria_id=None):
    criteria = get_object_or_404(EngagementReviewCriteria, pk=criteria_id)
    context = dict()

    if request.POST:
        criteriaform = EngagementReviewCriteriaForm(request.POST, instance=criteria)
        criteriaroleform = EngagementReviewCriteriaRoleFormset(request.POST, instance=criteria)

        if criteriaform.is_valid():
            criteria = criteriaform.save(commit=False)
            criteriaroleform.instance = criteria

            if criteriaroleform.is_valid():
                criteriaform.save()
                criteriaroleform.save()

                return HttpResponseRedirect(reverse('engagementreviewcriteria_list'))
    else:
        criteriaform = EngagementReviewCriteriaForm(instance=criteria)
        criteriaroleform = EngagementReviewCriteriaRoleFormset(instance=criteria)

    context.update(
        criteriaform=criteriaform,
        criteriaroleform=criteriaroleform,
        empty_form=criteriaroleform.empty_form,
        mode="EDIT",
        page_title='Edit Review Criteria'
    )

    return render(request, "engagement/engagement_review_criteria_form.html", context)

@login_required
@permission_required("engagement_task.delete_engagementreviewcriteria", return_403=True)
def engagementreviewcriteria_delete(request, criteria_id=None):
    get_object_or_404(EngagementReviewCriteria, pk=criteria_id).delete()

    return HttpResponseRedirect(reverse('engagementreviewcriteria_list'))

@login_required
def templateattachment(request, file_path):
    # Get the object
    attachment = TemplateAttachment.objects.get(attached_file=file_path)

    # This is a requirement from the underlying Storage object
    attachment.attached_file.name += "/download"

    return redirect(attachment.attached_file.url)

@login_required
def taskattachment(request, file_path):
    # Get the object
    attachment = TaskAttachment.objects.get(attached_file=file_path)

    # This is a requirement from the underlying Storage object
    attachment.attached_file.name += "/download"

    return redirect(attachment.attached_file.url)

@login_required
def engagementtask_activation(request, engagementtask_id):
    client_message_form = None
    activation_packages = list()
    fund_with_activity = list()
    context = dict()

    try:
        engagementtask = EngagementTask.objects.select_related("engagement", "engagement__manager")\
                                               .prefetch_related("engagement__clients", "engagement__activities",
                                                                 "engagement__meetings",
                                                                 "engagement__meetings__ch_attendees",
                                                                 "engagement__meetings__manager_representatives",
                                                                 "engagement__meetings__other_representatives")\
                                               .get(pk=engagementtask_id)
    except EngagementTask.DoesNotExist:
        messages.add_message(request, messages.ERROR, "Engagement cannot be activated because the engagement task "
                                                      "(" + str(engagementtask_id) + ") does not exist.")

        return HttpResponseRedirect(reverse("engagement_index"))

    # Get the manager version
    try:
        manager_version = engagementtask.engagement.manager.versions.latest("version_date")
    except Manager.DoesNotExist:
        manager_version = None

    # Get all the fund versions that are covered under this engagement and activateable which means they have an
    # "activate" task
    fund_versions = Fund.objects.filter(
        Q(fund__in=engagementtask.engagement.funds.all()),
        ~Q(status__pk=6) | Q(status=None),
        Q(fund__relatedtasks_1__template__enable_activation=True)
    ).prefetch_related("fund__financial_statement").distinct()
    review_level_acts = engagementtask.engagement.activities.filter(
        Q(activity_type__review_level=True),
        Q(~Q(activity_status__pk=7) | Q(is_active_on_web_site=False))
    ).distinct()

    # https://cha.sifterapp.com/issues/537 1.
    if fund_versions.count() == 0:
        fund_versions = Fund.objects.filter(
            Q(fund__in=engagementtask.engagement.funds.all()),
            Q(version_date__gte=engagementtask.engagement.sub_start_date),
            Q(version_date__lte=engagementtask.engagement.sub_end_date)
        ).prefetch_related("fund__financial_statement").distinct()

    # If there is no fund_versions/manager_version/activities to activate, quit
    if fund_versions.count() == 0 or manager_version is None or not engagementtask.engagement.activities.exists():
        message = "Engagement cannot be activated because there is no:
"

        if fund_versions.count() == 0:
            message += "
fund version that has an activation task and is not activated
"

        if manager_version is None:
            message += "
manager version
"

        if not engagementtask.engagement.activities.exists():
            message += "
activity
"

        messages.add_message(request, messages.ERROR, message+"
")

        return HttpResponseRedirect(reverse("engagement_index"))

    if request.method == "POST":
        activation_form = EngagementTaskActivationForm(request.POST)
        selected_fund_ids = set([long(i) for i in request.POST.getlist("funds")])

        # Get the correspondence template
        try:
            correspondence_template = Correspondence.objects.filter(type=Correspondence.ACTIVATION)[0]
        except IndexError:
            correspondence_template = None

        # Early validation check to populate cleaned_data dictionary which will be then used for the intiation of
        # client_message_form
        is_a_valid = activation_form.is_valid()

        if correspondence_template is not None:
            client_message_form = EmailMessageForm(
                request.POST,
                clients=engagementtask.engagement.clients.all(),
                client_message=True,
                client_users_required=not activation_form.cleaned_data["no_client_notification"]
            )

        if client_message_form is not None:
            is_c_valid = client_message_form.is_valid()
        else:
            is_c_valid = True

        is_details_valid = True

        activity_other_form = ActivationActivityOtherDetailsForm(request.POST, prefix="activity_other")
        if not activity_other_form.is_valid():
            is_details_valid = False
            activity_formset = ActivationActivityFormset(request.POST, request.FILES, prefix="activity_formset")
        else:
            #Hijack the cleaned data from the form that is used for common data and pass it as initial to the formset
            initial = activity_other_form.cleaned_data
            ActivationActivityFormset.form = staticmethod(curry(ActivationActivityForm, extra_data = initial))
            activity_formset = ActivationActivityFormset(request.POST, request.FILES, prefix="activity_formset")

        if is_a_valid and is_c_valid and is_details_valid and activity_formset.is_valid():
            # The activation process happens here
            activity_formset.save()
            selected_funds = [f for f in fund_versions if f.id in selected_fund_ids]

            # One timers: Manager status update
            if manager_version is not None:
                Manager.objects.filter(pk=manager_version.id).update(status=6)

            for fund in selected_funds:
                # Update the fund
                fund.status_id = 6
                try:
                    fund.master_fund.status_id = 6
                except AttributeError:
                    pass

                fund.save()

                # Financial Statements are updated
                fund.fund.financial_statement.all().update(status=6)
                #status 6 means Active on website

                # Activities are updated
                # couldnt get update() work on the queryset so explicity setting the values.
                for a in engagementtask.engagement.activities.filter(fund=fund.fund).exclude(activity_type__code="BC"):
                    a.activity_status = ActivityStatus.objects.get(pk=7)
                    a.issued_internal_date = timezone.now()
                    a.client_activated_date = timezone.now()
                    a.is_active_on_web_site = True
                    a.save()

            if activation_form.cleaned_data["notify_analysts"]:
                # Analysts are notified using the message model
                notification = Message(
                    engagementtask=engagementtask,
                    creator=request.user,
                    type=Message.ISSUED_REPORT,
                    status=Message.NEW,
                    body=activation_form.cleaned_data.get("wiki_points", None)
                )

                notification.save()

                internal_recipients = User.objects.filter(
                    Q(user_permissions__codename="receive_issued_report_notifications") |
                    Q(groups__permissions__codename="receive_issued_report_notifications")
                ).distinct()

                for recipient in internal_recipients:
                    InternalRecipient.objects.create(
                        message=notification,
                        recipient=recipient
                    )

            if not activation_form.cleaned_data["no_client_notification"] and client_message_form is not None:
                # Client notification is flushed here
                c_m = client_message_form.save()
                c_m.status = Message.PENDING
                c_m.save()
                # Send the mail to the client users
                c_m.prep(request, funds=selected_funds)
                c_m.send()

            # If all the funds are activated under the same engagement, then set the engagement completed
            if set(fund_versions) == set(selected_funds):
                engagementtask.engagement.status = Engagement.COMPLETE
                engagementtask.activation = EngagementTask.ACTIVATED
                engagementtask.save()
                engagementtask.engagement.save()

            engagementtask.engagement.on_status_change()

            return HttpResponseRedirect(reverse("engagement_index"))

    else:
        # https://cha.sifterapp.com/issues/537
        meetings = engagementtask.engagement.meetings.filter(type__in=[Meeting.CCR, Meeting.ONSITE, Meeting.ADMIN_CALL])
        meeting = None
        meeting_admin = None

        for m in meetings:
            if m.type == Meeting.ONSITE:
                meeting = m
                break
            if m.type == Meeting.CCR:
                meeting = m
            if m.type == Meeting.ADMIN_CALL:
                meeting_admin = m

        # Initiate the activation form with parameters and activity formset
        activation_form = EngagementTaskActivationForm()
        activity_formset = ActivationActivityFormset(queryset=review_level_acts, prefix="activity_formset")
        activity_other_form = ActivationActivityOtherDetailsForm(
            initial={
                "castlehall_representative": meeting.ch_attendees.values_list("id", flat=True) if meeting else [],
                "manager_representative": meeting.manager_representatives.values_list("id", flat=True) if meeting else [],
                "other_representative": meeting.other_representatives.values_list("id", flat=True) if meeting else [],
                "location_of_meeting": engagementtask.engagement.manager.primary_city.id if engagementtask.engagement.manager.primary_city else "",
                "date": engagementtask.engagement.target_meeting_date,
                "administrator_completed_date": meeting_admin.time_start if meeting_admin else "",
                "administrator_contact_notes": meeting_admin.notes if meeting_admin else ""
            },
            prefix="activity_other"
        )

        # Get the correspondence template
        try:
            correspondence_template = Correspondence.objects.filter(type=Correspondence.ACTIVATION)[0]
        except IndexError:
            correspondence_template = None

        if correspondence_template is not None:
            client_message_form = EmailMessageForm(
                initial=dict(
                    template=correspondence_template,
                    engagementtask=engagementtask,
                    creator=request.user,
                    type=Message.EMAIL
                ),
                clients=engagementtask.engagement.clients.all(),
                client_message=True
            )

    # Bring Fund specific activation packages together to be showed in UI (easy to loop over)
    ui_id = -1
    for fund_version in fund_versions:
        ui_id += 1
        d = dict(
            name=fund_version.fund.fund_name,
            status="Status: Active on Website",
            fs=list(),
            act=list(),
            id=ui_id,
            update=True,
            master_fund_name =fund_version.fund.master_fund_index if fund_version.fund and fund_version.fund.master_fund_index else None
        )
        fund_with_activity.append([fund_version, None])

        if fund_version.status and fund_version.status.pk == 6:
            d.update(status="No Change")
            d.update(update=False)

        for fs in fund_version.fund.financial_statement.all():
            fs_status = "Status: Active on Website"
            update = True

            if fs.status.pk == 6:
                fs_status = "Already Active"
                update = False

            d["fs"].append(dict(year=fs.year, status=fs_status, update=update))

        if fund_version.fund.master_fund_index:
            for fs in fund_version.fund.master_fund_index.financial_statement.all():
                fs_master_status = "Status: Active on Website"
                update = True

                if fs.status.pk == 6:
                    fs_master_status = "Already Active"
                    update = False
                d["fs"].append(dict(master_year=fs.year,master_status=fs_master_status, master_update=update))

        for act in engagementtask.engagement.activities.filter(fund=fund_version.fund).exclude(activity_type__code="BC"):
            act_status = "Status: Active on Website"
            update = True

            if act.activity_status is not None and act.activity_status.pk == 7:
                act_status = "Already Active"
                update = False

            d["act"].append(dict(type=act.activity_type.name, status=act_status, update=update))

        activation_packages.append(d)

    if manager_version is not None:
        if manager_version.status.pk == 6:
            manager_status = "No Change"
        else:
            manager_status = "Status: Active on Website"
    else:
        manager_status = ""

    if manager_version is not None:
        manager_name = manager_version.manager.name + " (" + manager_version.version_date.strftime("%b %d %Y") + ")"
    else:
        manager_name = None

    # Bind the fund versions with the corresponding review level activity form
    for form in activity_formset.forms:
        for t in fund_with_activity:
            if form.instance.fund == t[0].fund:
                t[1] = form
                break

    context.update(
        engagementtask=engagementtask,
        fund_versions=fund_versions,
        activation_packages=activation_packages,
        manager_status=manager_status,
        manager_name=manager_name,
        manager_update=False if manager_version.status.pk == 6 else True,
        activation_form=activation_form,
        client_message_form=client_message_form,
        activity_formset=activity_formset,
        activity_other_form=activity_other_form,
        fund_with_activity=fund_with_activity,
        contact_form= AddContactForm(prefix="contact")
    )

    return render(request, "engagement/engagement_task_activation_form.html", context)

def __get_etask_list(e, field_names, mappings, assignees, funds, clients, filters, sdict):
    if len(filters) > 0:
        if sdict[e.status] in filters:
            return (e, format_list_view(e, {'assignees': assignees, 'clients': clients, 'funds': funds}, field_names, mappings))
        else:
            return (None, None)
    else:
        return (e, format_list_view(e, {'assignees': assignees, 'clients': clients, 'funds': funds}, field_names, mappings))

#Returns a properly formatted list of dictionaries for initializing a formset's data
def __format_initial_data(activities):
    i = {}
    seen = []

    for a in activities:
        if a.fund not in seen:
            i[a.fund] = {'clients': [], 'id': None, 'activity_type': [], 'activity_select': None}
            seen = seen + [a.fund]

        the_dict = i[a.fund]
        clients = a.client_concerned.all()
        for c in clients:
            if not c in the_dict['clients']:
                the_dict['clients'] = the_dict['clients'] + [c]

        if a.activity_type.review_level == 1:
            #Can only be one selected
            the_dict['id'] = a.pk
            the_dict['activity_select'] = a.activity_type

        else:
            #Composite key of activity_type, fund
            the_dict['activity_type'] = the_dict['activity_type'] + [a.activity_type]

    initial_data = []

    for f in seen:
        i[f]['clients'] = sorted(i[f]['clients'], key = lambda x: x.pk)
        initial_data = initial_data + [dict(fund= f, clients=i[f]['clients'], id=i[f]['id'], activity_type=i[f]['activity_type'], activity_select=i[f]['activity_select'])]

    return sorted(initial_data, key=lambda x: x['fund'].fund_name)

#Return an array of invoice items for this client, for engagements for which this client has not had a final invoice sent yet
def __engagement_for_client(client, fund_list = None):
    #Fund list: Tuple of (Fund.pk, Engagement.pk) of items already on this invoice

    q = Engagement.objects.filter(clients = client).exclude(status = Engagement.CANCELLED).select_related('manager').prefetch_related('funds', 'billings', 'clients', 'activities', 'activities__client_concerned').order_by('manager')

    #Engagements for this client that have had a final billing sent
    final = [x.billing.engagement.pk for x in InvoiceItem.objects.filter(billing__client = client, final = 1).select_related('billing', 'billing__engagement')]

    current_engagements = [r for r in q if r.pk not in final]
    b = dict([[b.engagement.pk, b.pk] for b in ClientBilling.objects.filter(client = client, engagement__in = current_engagements).select_related('engagement', 'engagement__manager')])

    data = []

    for e in current_engagements:
        tmp = {}
        tmp[e.manager.name] = []

        #Make sure this client has a subscription to this fund in this engagement
        funds = [x.fund for x in e.activities.all() if int(client.pk) in [int(j.pk) for j in x.client_concerned.all()]]
        funds = list(set(funds))
        if fund_list:
            funds = [f for f in funds if (f.pk, e.pk) not in fund_list]

        for f in funds:
            tmp_f = {}
            tmp_f['billing'] = b[e.pk]
            tmp_f['fund'] = f.pk
            tmp_f['fund_name'] = f.fund_name + " - " + e.status
            tmp_f['meeting_date'] = e.target_meeting_date.strftime("%Y-%m-%d")
            tmp_f['notes'] = linebreaksbr(e.notes)
            tmp[e.manager.name].append(tmp_f)
        data.append(tmp)

    data = sorted(data, key = lambda x: x.keys())
    return data

def __get_engagement_params(form, query, request):
    query = build_query_from_filter(form, request, [{'form_field': 'status', 'model_field': 'status'},
                                                    {'form_field': 'priority', 'model_field': 'priority'},
                                                    {'form_field': 'services', 'model_field': 'activities__activity_type_id'},
                                                    {'form_field': 'trip', 'model_field': 'trip'},
                                                    {'form_field': 'analyst', 'model_field': 'lead_analyst'}], query)

    if form.data.has_key('egmnt_meeting_month'):
        tmp = Q()
        meeting_months = request.POST.getlist('egmnt_meeting_month')
        for m in meeting_months:
            month_year = m.split("_")
            tmp |= Q(target_meeting_date__month = month_year[0], target_meeting_date__year = month_year[1])
        query.append(tmp)
    else:
        if form.data.has_key('egmnt_selected_tab'):
            tmp = Q()
            meeting_months = request.POST.getlist('egmnt_selected_tab')
            for m in meeting_months:
                if m != 'engmt_ALL' and m != '' and m != 'engmnt_prospects_tab' and m != 'engmnt_cancelled_tab':
                    month_year = m.split("_")
                    tmp |= Q(target_meeting_date__month = month_year[0], target_meeting_date__year = month_year[1])
            query.append(tmp)

    query = build_query_from_search_param(form, [{'form_field': 'clientSearchField', 'model_field': 'clients__id'},
                                                 {'form_field': 'managerSearchField', 'model_field': 'manager__id'},
                                                 {'form_field': 'fundSearchField', 'model_field': 'funds__id'},
                                                 {'form_field': 'managerLocationSearchField', 'model_field': 'manager__primary_city__id'},], query)
    if form.data.has_key('staffSearch') and form.data['staffSearch'] != '':
        staff_user_id = form.data['staffSearch']

        query.append(Q(desk_analyst__id = staff_user_id) | Q(reviewer_1__id = staff_user_id) | Q(reviewer_2__id = staff_user_id)|Q(data_analyst__id = staff_user_id)|Q(lead_analyst__id=staff_user_id))

    if exists_with_value(form.data, 'egmnt_meeting_date_from'):
        query.append(Q(target_meeting_date__gte = datetime.datetime.strptime(form.data['egmnt_meeting_date_from'], "%Y-%m-%d")))

    if exists_with_value(form.data, 'egmnt_meeting_date_to'):
        query.append(Q(target_meeting_date__lte = datetime.datetime.strptime(form.data['egmnt_meeting_date_to'], "%Y-%m-%d")))

    if exists_with_value(form.data, 'egmnt_delivery_date_from'):
        query.append(Q(target_delivery_date__gte = datetime.datetime.strptime(form.data['egmnt_delivery_date_from'], "%Y-%m-%d")))

    if exists_with_value(form.data, 'egmnt_delivery_date_to'):
        query.append(Q(target_delivery_date__lte = datetime.datetime.strptime(form.data['egmnt_delivery_date_to'], "%Y-%m-%d")))

    group_by = exists_with_value(form.data, 'group_by')
    return query, group_by


import pandas as pd

data=pd.read_excel('../data/meorient_data/付费买家数据翻译后打标.xlsx',encoding='gbk').rename(columns={'product':'PRODUCT_NAME'})

#tag_stand=pd.read_excel('../data/tagpack2/8.19 打标标签.xlsx',sheetname=0)
#tag_stand.columns=['PRODUCT_TAG_NAME','T1','T2']
#

#data=pd.merge(data,tag_stand[['T1']].drop_duplicates(),on=['T1'],how='inner')

md=data[['PRODUCT_NAME']].drop_duplicates()

md=md.rename(columns={'PRODUCT_NAME':'source_text'})
md['fromLang']='auto'
md['toLang']='en'

md.to_csv('../data/meorient_data/付费买家数据翻译后打标_need_transed.csv',index=False,encoding='utf8')


import os
import json
import boto3
import requests

# The Environment Tag is used by APM to filter Environments in UI
APM_ENVIRONMENT = os.environ['SIGNALFX_APM_ENVIRONMENT']
PRICE_URL       = os.environ['PRICE_URL']
ORDER_LINE      = os.environ['ORDER_LINE']

# Define the client to interact with AWS Lambda
client = boto3.client('lambda')

def lambda_handler(event,context):
    print("event received :", event)

    # Define / read input parameters from the event trigger
    Name         =  json.loads(event ['body']).get("ProductName")  # Value passed in from test case
    Quantity     =  json.loads(event ['body']).get("Quantity")     # Value passed in from test case
    CustomerType =  json.loads(event ['body']).get("CustomerType") # Value passed in from test case

    # Call Node-JS lambda via Api Gateway to get the Price

    payload = {'CustomerType': CustomerType}
    r = requests.post(PRICE_URL,  params=payload)
    print( "Price Url: ",r.url)
    print( "Price Payload: ",r.text)

    #Get Price from response
    Price =  json.loads(r.text).get('Price') # Get Value from the Price calculator
    print("Price: ",Price)

    # Define the input parameters that will be passed on to the child function
    inputParams = {
        "ProductName" : Name ,
        "Quantity"    : Quantity,
        "UnitPrice"   : Price
    }
    print (inputParams)
    # Invoking Lambda directly
    response = client.invoke(
        FunctionName = ORDER_LINE, # This could be set as a Lambda Environment Variable
        InvocationType = 'RequestResponse',
        Payload = json.dumps(inputParams)
    )
    responseCode = 200
    responseFromOrderLine = json.load(response['Payload'])
    print (responseFromOrderLine)
    newPrice = responseFromOrderLine.get('Amount')
    print ("Price:" , newPrice)
    transactionID =  responseFromOrderLine.get('TransactionID')
    print ("transactions id:",  transactionID)
    retval={'phoneType'     : Name,
            'quantity'      : Quantity,
            'customerType'  : CustomerType,
            'price'         : newPrice,
            'transaction'   : transactionID
            }
    return {
            'statusCode': responseCode,
            'body': json.dumps(retval)

        }


from xai.brain.wordbase.nouns._geographer import _GEOGRAPHER

#calss header
class _GEOGRAPHERS(_GEOGRAPHER, ):
	def __init__(self,):
		_GEOGRAPHER.__init__(self)
		self.name = "GEOGRAPHERS"
		self.specie = 'nouns'
		self.basic = "geographer"
		self.jsondata = {}


"""
Python wrappers for the DSP (Displacement Map) primitives of BRL-CAD.
"""
from base import Primitive
import os
import brlcad.exceptions as BRLCADException

class DSP(Primitive):

    def __init__(self, name, dsp_name, data_src="file", width=142, length=150, interpolation=False, cut_direction=1,
                 cell_size=1, unit_elevation=0.005, copy=False):
        """
        :param name:
        :param data_src: (1)file('f')/(2)object('o')
        :param dsp_name: name of file/object
        :param width: width of displacement-map
        :param length: length of displacement-map
        :param interpolation: True for normal interpolation
        :param cut_direction: (1,ad)ad(adaptive),(2,lR)llUR(lower left to upper right) or (3,Lr)ULlr(Upper Left to lower right)
        :return:
        """
        Primitive.__init__(self, name=name)
        if isinstance(data_src, str):
            if data_src[0] == "f" or data_src[0] == "F":
                self.data_src = 1
            elif data_src[0] == "o" or data_src[0] == "O":
                self.data_src = 2
            else:
                raise BRLCADException("Invalid Data")
        elif data_src == 1 or data_src == 2:
            self.data_src = data_src
        else:
            raise BRLCADException("Invalid Data")
        if self.data_src == 1:
            if not os.path.isfile(dsp_name):
                raise ValueError("File {} does not exist !".format(dsp_name))

        self.dsp_name = dsp_name
        self.width = width
        self.length = length
        self.interpolation = interpolation
        self.cell_size = cell_size
        self.unit_elevation = unit_elevation
        if isinstance(cut_direction, str):
            if cut_direction[0] == "a" or cut_direction[0] == "A":
                self.cut_direction = 1
            elif cut_direction == "llUR" or cut_direction == "l":
                self.cut_direction = 2
            elif cut_direction == "ULlr" or cut_direction == "L":
                self.cut_direction = 3
            else:
                raise BRLCADException("Invalid Data")
        elif cut_direction in range(1, 4):
            self.cut_direction = cut_direction
        else:
            raise BRLCADException("Invalid Data")

    def __repr__(self):
        result = "{}({}, dsp_name={}, data_src={}, width={}, length={}, interpolation={}, cut_direction={}, " \
                 "cell_size={}, unit_elevation={}"
        return result.format(
            self.__class__.__name__, self.name, self.dsp_name, self.get_data_src(), self.width, self.length,
            self.interpolation, self.get_cut_direction(), self.cell_size, self.unit_elevation
        )

    def get_cut_direction(self):
        if self.cut_direction == 1:
            return "Adaptive"
        elif self.cut_direction == 2:
            return "Lower Left Upper Right"
        elif self.cut_direction == 3:
            return "Upper Left Lower Right"
        else:
            return "Unknown"

    def get_data_src(self):
        if self.data_src == 1:
            return "File"
        elif self.data_src == 2:
            return "Object"
        else:
            return "Unknown"

    def update_params(self, params):
        params.update({
            "dsp_name": self.dsp_name,
            "data_src": self.data_src,
            "width": self.width,
            "length": self.length,
            "interpolation": self.interpolation,
            "cut_direction": self.cut_direction,
            "cell_size": self.cell_size,
            "unit_elevation": self.unit_elevation
        })

    def copy(self):
        return DSP(self.name, self.dsp_name, self.data_src, self.width, self.length, self.interpolation,
                   self.cut_direction, self.cell_size, self.unit_elevation, copy=True)

    def has_same_data(self, other):
        return self.dsp_name == other.dsp_name and \
               self.data_src == other.data_src and \
               self.width == other.width and  \
               self.length == other.length and \
               self.interpolation == other.interpolation and \
               self.cut_direction == other.cut_direction and \
               self.cell_size == other.cell_size and \
               self.unit_elevation == other.unit_elevation

    @staticmethod
    def from_wdb(name, data):
        if data.dsp_smooth == 1:
            interpolation = True
        else:
            interpolation = False
        if data.dsp_cuttype == 97:
            dsp_cuttype = 1
        if data.dsp_cuttype == 154:
            dsp_cuttype = 2
        if data.dsp_cuttype == 114:
            dsp_cuttype = 3
        return DSP(
            name=name,
            dsp_name= data.dsp_name.vls_str.data,
            data_src= data.dsp_datasrc,
            width=data.dsp_xcnt,
            length=data.dsp_ycnt,
            interpolation=interpolation,
            cut_direction=dsp_cuttype,
            cell_size=data.dsp_stom[5],
            unit_elevation=data.dsp_stom[10]
        )