from django.conf import settings
from django.conf.urls.static import static
from django.contrib import admin
from django.urls import include, path, re_path

app_name = "pages"

urlpatterns = [
    path( 'accounts/', include('django.contrib.auth.urls')),
    path( 'admin/doc/', include('django.contrib.admindocs.urls')),
    path( 'admin/', admin.site.urls),
]

if settings.DEBUG:
    import debug_toolbar
    urlpatterns += [
        path( '__debug__/', include( debug_toolbar.urls)),
    ]
    urlpatterns += static( settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)

urlpatterns += [
    path( '', include( 'pages.urls')),
]


import whois
import datetime
import requests
from requests_html import HTMLSession
import urllib.request as urllib2
import re
from urllib.parse import urlparse
from xml.dom import minidom
from lxml import html

def checkDash(url):
  if len(urlparse(url).netloc.split('-')) > 1:
    return 1
  else:
    return -1

def checkDot(url):
  dotPresent = urlparse(url).netloc.split('.')
  if len(dotPresent) > 3:
    return 1
  elif len(dotPresent) == 3:
    return 0
  else:
    return -1

def checkIfHTTPS(url):
  o = urlparse(url)
  hostname = o.netloc
  print(hostname)
  if o.scheme != "https":
    try:
      o=requests.get("https://"+hostname, verify=True, timeout=1)
      return -1
    except Exception as e:
      #print(e)
      if "SSL23_GET_SERVER_HELLO" in str(e) or "Failed to establish a new connection" in str(e):
        return 1
      else:
        return 0
  else:
    return -1

def checkLen(url):
  if len(url) < 54:
    return -1
  elif len(url) >= 54 and len(url) <= 75:
    return 0
  else:
    return 1

def ageOfTheDomainandDNS(url):
  try:
   details= whois.whois(url)
   now = datetime.datetime.now()
   diff = (now-details.creation_date[0]).days
   if diff >= 180:
    return -1
   return 1
  except:
    return 1

def find_ele_with_attribute(dom,ele,attribute):
  for subelement in dom.getElementsByTagName(ele):
    if subelement.hasAttribute(attribute):
        return subelement.attributes[attribute].value
  return None

def sitepopularity(host):
  xmlpath='http://data.alexa.com/data?cli=10&dat=snbamz&url='+host
  try:
    xml= urllib2.urlopen(xmlpath)
    dom =minidom.parse(xml)
    rank_host=find_ele_with_attribute(dom,'REACH','RANK')
    if int(rank_host) < 100000:
      return -1
    elif int(rank_host) > 100000:
      return 0
  except:
    return 1

def parseHTMLForRequestURL(url):
  page = requests.get(url)
  webpage = html.fromstring(page.content)
  div = webpage.xpath('//img/@src')
  host= urlparse(url).hostname
  imgcount = 0
  for k in range(0,len(div)):
    try:
    # print(div[k].attrs['src'])
      if div[k].startswith("/"):
        #print("Here")
        imgcount = imgcount+1
      else:
        o = urlparse(div[k])
        if o.hostname == host:
          imgcount = imgcount+1
    except Exception as e:
      print(e)
      continue

  if len(div) != 0:
    percent = ((len(div)-imgcount)/len(div))*100
    if percent < 22:
      return -1
    elif percent>=22 and percent<61:
      return 0
    else:
      return 1
  else:
    return 0

def parseHTMLForAnchors(url):
  page = requests.get(url)
  webpage = html.fromstring(page.content)
  anchor = webpage.xpath('//a/@href')
  host= urlparse(url).hostname
  anchorcount = 0
  for k in range(0,len(anchor)):
    try:
      if anchor[k].startswith("/"):
        anchorcount = anchorcount+1
      else:
        o = urlparse(anchor[k])
      if o.hostname == host:
        anchorcount = anchorcount+1
    except Exception as e:
      print(e)
      continue
  if len(anchor)!=0:
   percent = ((len(anchor)-anchorcount)/len(anchor))*100
   if percent < 31:
    return -1
   elif percent>=31 and percent<67:
    return 0
   else:
    return 1
  else:
    return 0

def extractandreturn(url):
  try:
    page = requests.get(url,timeout=1,verify=False)
    webpage = html.fromstring(page.content)
    cdash = checkDash(url)
    cdot = checkDot(url)
    chtps = checkIfHTTPS(url)
    clen = checkLen(url)
    creq = parseHTMLForRequestURL(url)
    canc = parseHTMLForAnchors(url)
    cage = ageOfTheDomainandDNS(urlparse(url).hostname)
    cpop = sitepopularity(urlparse(url).hostname)
    return([cdash, cdot, chtps, clen , creq, canc, cage, cpop])
  except Exception as e:
   print("Site down")


import numbertheory
import math
# (2x-N)**2 + (2y-N)**2 = 2*N**2
# N = q_1**1*q_2**2*q_3**3 where q is a prime congruent 1 mod 4
# N*prod(p**a_i) where p is a prime not congruent 1 mod 4 is also a solution
# f(N) = 420

def r(n):
    s = 0
    for d in numbertheory.divisorList(n):
        s += (d % 4 == 1) - (d % 4 == 3)
    return 4*s

def f(n):
    return r(n**2)

#q = [n for n in numbertheory.primeList(int(10**11/21125)) if n % 4 == 1]
#print(len(q))
base = []
limit = 10**11
for q3 in numbertheory.primeList(5+int((limit/(5**2*13))**(1/3))):
    if not q3 % 4 == 1:
        continue
    for q2 in numbertheory.primeList(5+int((limit/(q3**3*5))**(1/2))):
        if not q2 % 4 == 1:
            continue
        for q1 in numbertheory.primeList(5+int(limit/(q3**3*q2**2))):
            if not q1 % 4 == 1:
                continue
            if not (q1 == q2 or q2 == q3 or q1 == q3):
                n = q3**3*q2**2*q1
                base += [n]
base = sorted(base)

mult = [True]*(limit//base[0]+5)
for m in numbertheory.primeList(limit//base[0]+5):
    if not m % 4 == 1:
        continue
    mult[m::m] = [False]*len(mult[m::m])
mas = []
i = 0
for t in mult:
    if t:
        mas += [i]
    i += 1
mas = sorted(mas[1:])
print(len(base), len(mas), mas[0: 10])

s = 0
for b in base:
    for m in mas:
        if m*b > limit:
            break
        s += m*b

print(s)
#(x-N/2)**2 + (y-N/2)**2 = N**2/2
#(2*x-N)**2 + (2*y-N)**2 = 2*N**2 = a**2 + b**2

#div = [0]*(10**8+1)
#print('finished allocating space')
#for i in range(1, 10**8+1, 4):
#    div[i] += 1
#for i in range(3, 10**8+1, 4):
#    div[i] -= 1
#print('finished siving')
#for i, d in enumerate(div):
#    if d == 52:
#        print(i)


class TreeNode:
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None

class Solution:
    def findMode(self, root):
        """
        :type root: TreeNode
        :rtype: List[int]
        """
        def ct(node, m):
            if node:
                m[node.val] = m.get(node.val, 0) + 1
                ct(node.left, m)
                ct(node.right, m)

        res = []
        d = {}
        ct(root, d)
        maxCount = 0
        for k, v in d.items():
            if v > maxCount:
                maxCount = v
                res = [k]
            elif v == maxCount:
                res.append(k)
        return res


# per-publisher configuration for pubCrawl.py
from os.path import *
import logging, urllib2, urlparse, urllib, re
import pubConf, pubGeneric, maxCommon, html, maxCommon

# crawler delay config, values in seconds
# these overwrite the default set with the command line switch to pubCrawl
# special case is highwire, handled in the code:
# (all EST): mo-fri: 9-5pm: 120 sec, mo-fri 5pm-9am: 10 sec, sat-sun: 5 sec (no joke)
crawlDelays = {
    "www.nature.com"              : 10,
    "onlinelibrary.wiley.com" : 1,
    "dx.doi.org"              : 1,
    "ucelinks.cdlib.org"      : 20,
    "eutils.ncbi.nlm.nih.gov"      : 3,
    "sciencedirect.com"      : 15
}

def parseHighwire():
    """ create two dicts
    printIssn -> url to pmidlookup-cgi of highwire
    and
    publisherName -> top-level hostnames
    >>> temps, domains = parseHighwire()
    >>> temps['0270-6474']
    u'http://www.jneurosci.org/cgi/pmidlookup?view=long&pmid=%(pmid)s'
    >>> domains["Society for Neuroscience"]
    set([u'jneurosci'])
    """
    # highwire's publisher names are not resolved ("SAGE", "SAGE Pub", etc)
    # so: first get dict printIssn -> resolved publisherName from publishers.tab
    pubFname = join(pubConf.publisherDir, "publishers.tab")
    pIssnToPub = {}
    for row in maxCommon.iterTsvRows(pubFname):
        if not row.pubName.startswith("HIGHWIRE"):
            continue
        for issn in row.journalIssns.split("|"):
            issn = issn.rstrip(" ")
            pIssnToPub[issn] = row.pubName.replace("HIGHWIRE ","").strip()

    # go over highwire table and make dict pubName -> issn -> templates
    # and dict pubName -> domains
    fname = join(pubConf.journalListDir, "highwire.tab")
    templates = {}
    domains = {}
    for row in maxCommon.iterTsvRows(fname, encoding="latin1"):
        if row.eIssn.strip()=="Unknown":
            continue
        pubName = pIssnToPub[row.pIssn.strip()].strip()
        templates.setdefault(pubName, {})
        templates[row.pIssn.strip()] = row.urls.strip()+"/cgi/pmidlookup?view=long&pmid=%(pmid)s"

        host = urlparse.urlparse(row.urls).hostname
        domain = ".".join(host.split('.')[-2:]).strip()
        domains.setdefault(pubName, set()).add(domain)

    return templates, domains

def highwireConfigs():
    " return dict publisher name -> config for all highwire publishers "
    logging.info("Creating config for Highwire publishers")
    res = {}
    issnTemplates, pubDomains = parseHighwire()

    for pubName, domains in pubDomains.iteritems():
        templates = {}
        for issn, templUrl in issnTemplates.iteritems():
            for domain in domains:
                if domain in templUrl:
                    templates[issn]=templUrl
                    break

        res[pubName] = {
            "hostnames" : domains,
            "landingUrl_templates" : templates,
            "landingPage_errorKeywords" : "We are currently doing routine maintenance", # wait for 15 minutes and retry
            "doiUrl_replace" : {"$" : ".long"},
            "landingUrl_isFulltextKeyword" : ".long",
            "landingPage_ignoreMetaTag" : True,
            "landingUrl_fulltextUrl_replace" : {"long" : "full.pdf", "abstract" : "full.pdf" },
            "landingPage_suppFileList_urlREs" : [".*/content.*suppl/DC[0-9]"],
            "suppListPage_suppFile_urlREs" : [".*/content.*suppl/.*"],
        }
    return res

highwireCfg = highwireConfigs()

# crawl configuration: for each website, define how to crawl the pages
confDict = {
    "oup" :
        highwireCfg["Oxford University Press"],

    "pmc" :
    # not used at UCSC
    # caveats: we always keep html, can't distinguish between PDF-only and
    # and HTML/PDF articles
    # supplementals might not always work, tested only on PLOS
    # only gets the first supplemental file
    {
        "hostnames" : ["www.ncbi.nlm.nih.gov"],
        "landingUrl_fulltextUrl_append" : "pdf",
        "landingUrl_isSuppListUrl": True,
        "landingUrl_isFulltextKeyword" : "pmc", # always true -> always keep fulltext
        "suppListPage_suppFileTextREs" : ["Click here for additional data file.*"]
    },
    "springer" :
    # we don't use this at UCSC, we get springer directly
    {
        "hostnames" : ["link.springer.com"],
        #only pdfs "landingUrl_replaceREs" : {"$" : "?np=y"}, # switch sd to screen reader mode
        "landingPage_stopPhrases": ["make a payment", "purchase this article"],
        "landingPage_mainLinkTextREs" : [".*Download PDF +\([0-9]+ KB\)"],
        "suppListPage_suppFile_urlREs" : [".*/file/MediaObjects/.*"],
        "landingUrl_isSuppListUrl": True
    },

    "elsevier" :
    # at UCSC we don't use this, we get elsevier data via consyn.elsevier.dom
    # this is mostly for off-site use or for the odd project that doesn't
    # want to pull from consyn
    # caveats:
    # * we cannot download text html (no function to add np=y to landing url)
    # * we don't know if we actually have access to an article
    # * no supplemental files
    {
        "hostnames" : ["www.sciencedirect.com"],
        #only pdfs "landingUrl_replaceREs" : {"$" : "?np=y"}, # switch sd to screen reader mode
        "landingPage_stopPhrases": ["make a payment", "purchase this article", \
            "This is a one-page preview only"],
        "landingPage_mainLinkTextREs" : ["PDF  +\([0-9]+ K\)"],
    },

    "npg" :
    # http://www.nature.com/nature/journal/v463/n7279/suppinfo/nature08696.html
    # http://www.nature.com/pr/journal/v42/n4/abs/pr19972520a.html - has no pdf
    #
    {
        "hostnames" : ["www.nature.com"],
        "landingPage_stopPhrases": ["make a payment", "purchase this article"],
        "landingPage_acceptNoPdf": True,
        "landingUrl_isFulltextKeyword" : "full",
        "landingUrl_fulltextUrl_replace" : {"full" : "pdf", "html" : "pdf", "abs" : "pdf"},
        "landingPage_mainLinkTextREs" : ["Download PDF"],
        "landingUrl_suppListUrl_replace" : {"full" : "suppinfo", "abs" : "suppinfo"},
        "landingPage_suppListTextREs" : ["Supplementary information index", "[Ss]upplementary [iI]nfo", "[sS]upplementary [iI]nformation"],
        "suppListPage_suppFileTextREs" : ["[Ss]upplementary [dD]ata.*", "[Ss]upplementary [iI]nformation.*", "Supplementary [tT]able.*", "Supplementary [fF]ile.*", "Supplementary [Ff]ig.*", "Supplementary [lL]eg.*", "Download PDF file.*", "Supplementary [tT]ext.*", "Supplementary [mM]ethods.*", "Supplementary [mM]aterials.*", "Review Process File"]
    # Review process file for EMBO, see http://www.nature.com/emboj/journal/v30/n13/suppinfo/emboj2011171as1.html
    },

    # with suppl
    # PMID 22017543
    # http://online.liebertpub.com/doi/full/10.1089/nat.2011.0311
    # with html
    # PMID 22145933
    # http://online.liebertpub.com/doi/abs/10.1089/aid.2011.0232
    # no html
    # PMID 7632460
    # http://online.liebertpub.com/doi/abs/10.1089/aid.1995.11.443
    "mal" :
    {
        "hostnames" : ["online.liebertpub.com"],
        "landingUrl_templates" : {"anyIssn" : "http://online.liebertpub.com/doi/full/%(doi)s"},
        "landingUrl_isFulltextKeyword" : "/full/",
        "landingUrl_fulltextUrl_replace" : {"/abs/" : "/full/" },
        "landingPage_mainLinkTextREs" : ["Full Text PDF.*"],
        "landingPage_suppListTextREs" : ["Supplementary materials.*"]
    },

    # https://www.jstage.jst.go.jp/article/circj/75/4/75_CJ-10-0798/_article
    # suppl file download does NOT work: strange javascript links
    "jstage" :
    {
        "hostnames" : ["www.jstage.jst.go.jp"],
        "landingUrl_fulltextUrl_replace" : {"_article" : "_pdf" },
        "landingPage_mainLinkTextREs" : ["Full Text PDF.*"],
        "landingPage_suppListTextREs" : ["Supplementary materials.*"]
    },
    # rupress tests:
    # PMID 12515824 - with integrated suppl files into main PDF
    # PMID 15824131 - with separate suppl files
    # PMID 8636223  - landing page is full (via Pubmed), abstract via DOI
    # cannot do suppl zip files like this one http://jcb.rupress.org/content/169/1/35/suppl/DC1
    #
    "rupress" :
    {
        "hostnames" : ["rupress.org", "jcb.org"],
        "doiUrl_replace" : {"$" : ".long"},
        "landingUrl_isFulltextKeyword" : ".long",
        "landingPage_ignorePageWords" : ["From The Jcb"],
        "landingPage_ignoreMetaTag" : True,
        "landingUrl_fulltextUrl_replace" : {"long" : "full.pdf", "abstract" : "full.pdf"},
        "suppListPage_addSuppFileTypes" : ["html", "htm"], # pubConf does not include htm/html
        "landingPage_mainLinkTextREs" : ["Full Text (PDF)"],
        #"landingPage_suppListTextREs" : ["Supplemental [Mm]aterial [Iindex]", "Supplemental [Mm]aterial"],
        "landingUrl_suppListUrl_replace" : {".long" : "/suppl/DC1", ".abstract" : "/suppl/DC1"},
        "suppListPage_suppFileTextREs" : ["[Ss]upplementary [dD]ata.*", "[Ss]upplementary [iI]nformation.*", "Supplementary [tT]able.*", "Supplementary [fF]ile.*", "Supplementary [Ff]ig.*", "[ ]+Figure S[0-9]+.*", "Supplementary [lL]eg.*", "Download PDF file.*", "Supplementary [tT]ext.*", "Supplementary [mM]aterials and [mM]ethods.*", "Supplementary [mM]aterial \(.*"],
        "ignoreSuppFileLinkWords" : ["Video"],
        "ignoreSuppFileContentText" : ["Reprint (PDF) Version"],
        "suppListPage_suppFile_urlREs" : [".*/content/suppl/.*"]
    },
    # http://jb.asm.org/content/194/16/4161.abstract = PMID 22636775
    "asm" :
    {
        "hostnames" : ["asm.org"],
        "landingUrl_isFulltextKeyword" : ".long",
        "doiUrl_replace" : {"$" : ".long"},
        "landingPage_ignoreMetaTag" : True,
        "landingPage_errorKeywords" : "We are currently doing routine maintenance", # if found on landing page Url, wait for 15 minutes and retry
        "landingUrl_fulltextUrl_replace" : {"long" : "full.pdf", "abstract" : "full.pdf" },
        #"landingUrl_fulltextUrl_replace" : {"long" : "full.pdf?with-ds=yes", "abstract" : "full.pdf?with-ds=yes" },
        #"landingUrl_suppListUrl_replace" : {".long" : "/suppl/DCSupplemental", ".abstract" : "/suppl/DCSupplemental"},
        "landingPage_suppFileList_urlREs" : [".*suppl/DCSupplemental"],
        "suppListPage_suppFile_urlREs" : [".*/content/suppl/.*"],
    },
    #
    # 21159627 http://cancerres.aacrjournals.org/content/70/24/10024.abstract has suppl file
    "aacr" :
    {
        "hostnames" : ["aacrjournals.org"],
        "landingUrl_templates" : {"0008-5472" : "http://cancerres.aacrjournals.org/content/%(vol)s/%(issue)s/%(firstPage)s.long"},
        "landingPage_errorKeywords" : "We are currently doing routine maintenance", # wait for 15 minutes and retry
        "doiUrl_replace" : {"$" : ".long"},
        "landingUrl_isFulltextKeyword" : ".long",
        "landingPage_ignoreMetaTag" : True,
        "landingUrl_fulltextUrl_replace" : {"long" : "full.pdf", "abstract" : "full.pdf" },
        "landingPage_suppFileList_urlREs" : [".*/content[0-9/]*suppl/DC1"],
        "suppListPage_suppFile_urlREs" : [".*/content[0-9/]*suppl/.*"],
        "landingPage_stopPhrases" : ["Purchase Short-Term Access"]
    },
    # 1995 PMID 7816814
    # 2012 PMID 22847410 has one supplement, has suppl integrated in paper
    "cshlp" :
    {
        "hostnames" : ["cshlp.org"],
        "landingUrl_templates" : {"1355-8382" : "http://rnajournal.cshlp.org/content/%(vol)s/%(issue)s/%(firstPage)s.full"},
        "landingPage_errorKeywords" : "We are currently doing routine maintenance", # wait for 15 minutes and retry
        "doiUrl_replace" : {"$" : ".long"},
        "landingUrl_isFulltextKeyword" : ".long",
        "landingPage_ignoreMetaTag" : True,
        "landingUrl_fulltextUrl_replace" : {"long" : "full.pdf", "abstract" : "full.pdf" },
        "landingPage_suppFileList_urlREs" : [".*/content[0-9/]*suppl/DC1"],
        "suppListPage_suppFile_urlREs" : [".*/content[0-9/]*suppl/.*"],
        "landingPage_stopPhrases" : ["Purchase Short-Term Access"]
    },
    "pnas" :
    {
        "hostnames" : ["pnas.org"],
        "landingUrl_templates" : {"0027-8424" : "http://pnas.org/cgi/pmidlookup?view=long&pmid=%(pmid)s"},
        "landingPage_errorKeywords" : "We are currently doing routine maintenance", # wait for 15 minutes and retry
        "doiUrl_replace" : {"$" : ".long"},
        "landingUrl_isFulltextKeyword" : ".long",
        "landingPage_ignoreMetaTag" : True,
        "landingUrl_fulltextUrl_replace" : {"long" : "full.pdf", "abstract" : "full.pdf" },
        "landingPage_suppFileList_urlREs" : [".*suppl/DCSupplemental"],
        "suppListPage_suppFile_urlREs" : [".*/content/suppl/.*"],
    },
    "aspet" :
    {
        "hostnames" : ["aspetjournals.org"],
        "landingUrl_templates" : {"0022-3565" : "http://jpet.aspetjournals.org/cgi/pmidlookup?view=long&pmid=%(pmid)s"},
        "landingPage_errorKeywords" : "We are currently doing routine maintenance", # wait for 15 minutes and retry
        "doiUrl_replace" : {"$" : ".long"},
        "landingUrl_isFulltextKeyword" : ".long",
        "landingPage_ignoreMetaTag" : True,
        "landingUrl_fulltextUrl_replace" : {"long" : "full.pdf", "abstract" : "full.pdf" },
        "landingPage_suppFileList_urlREs" : [".*/content[0-9/]*suppl/DC1"],
        "suppListPage_suppFile_urlREs" : [".*/content[0-9/]*suppl/.*"],
    },
    "faseb" :
    {
        "hostnames" : ["fasebj.org"],
        "landingUrl_templates" : {"0892-6638" : "http://www.fasebj.org/cgi/pmidlookup?view=long&pmid=%(pmid)s"},
        "landingPage_errorKeywords" : "We are currently doing routine maintenance", # wait for 15 minutes and retry
        "doiUrl_replace" : {"$" : ".long"},
        "landingUrl_isFulltextKeyword" : ".long",
        "landingPage_ignoreMetaTag" : True,
        "landingUrl_fulltextUrl_replace" : {"long" : "full.pdf", "abstract" : "full.pdf" },
        "landingPage_suppFileList_urlREs" : [".*/content[0-9/]*suppl/DC1"],
        "suppListPage_suppFile_urlREs" : [".*/content[0-9/]*suppl/.*"],
    },
    # society of leukocyte biology
    # PMID 20971921
    "slb" :
    {
        "hostnames" : ["jleukbio.org"],
        "landingUrl_templates" : {"0741-5400" : "http://www.jleukbio.org/cgi/pmidlookup?view=long&pmid=%(pmid)s"},
        "landingPage_errorKeywords" : "We are currently doing routine maintenance", # wait for 15 minutes and retry
        "doiUrl_replace" : {"$" : ".long"},
        "landingUrl_isFulltextKeyword" : ".long",
        "landingPage_ignoreMetaTag" : True,
        "landingUrl_fulltextUrl_replace" : {"long" : "full.pdf", "abstract" : "full.pdf" },
        "landingPage_suppFileList_urlREs" : [".*/content[0-9/]*suppl/DC1"],
        "suppListPage_suppFile_urlREs" : [".*/content[0-9/]*suppl/.*"],
    },
    # Company of Biologists
    "cob" :
    {
        "hostnames" : ["biologists.org"],
        "landingUrl_templates" : {"0950-1991" : "http://dev.biologists.org/cgi/pmidlookup?view=long&pmid=%(pmid)s", "0022-0949" : "http://jcs.biologists.org/cgi/pmidlookup?view=long&pmid=%(pmid)s"},
        "landingPage_errorKeywords" : "We are currently doing routine maintenance", # wait for 15 minutes and retry
        "doiUrl_replace" : {"$" : ".long"},
        "landingUrl_isFulltextKeyword" : ".long",
        "landingPage_ignoreMetaTag" : True,
        "landingUrl_fulltextUrl_replace" : {"long" : "full.pdf", "abstract" : "full.pdf" },
        "landingPage_suppFileList_urlREs" : [".*/content[0-9/]*suppl/DC1"],
        "suppListPage_suppFile_urlREs" : [".*/content[0-9/]*suppl/.*"],
    },
    # Genetics Society of America
    # PMID 22714407
    "genetics" :
    {
        "hostnames" : ["genetics.org"],
        "landingUrl_templates" : {"0016-6731" : "http://genetics.org/cgi/pmidlookup?view=long&pmid=%(pmid)s"},
        "landingPage_errorKeywords" : "We are currently doing routine maintenance", # wait for 15 minutes and retry
        "doiUrl_replace" : {"$" : ".long"},
        "landingUrl_isFulltextKeyword" : ".long",
        "landingPage_ignoreMetaTag" : True,
        "landingUrl_fulltextUrl_replace" : {"long" : "full.pdf", "abstract" : "full.pdf" },
        "landingPage_suppFileList_urlREs" : [".*/content[0-9/]*suppl/DC1"],
        "suppListPage_suppFile_urlREs" : [".*/content[0-9/]*suppl/.*"],
    },
    # Society of General Microbiology
    # PMID 22956734
    # THEY USE DC1 AND DC2 !!! Currently we're missing the DC1 or DC2 files... :-(
    # todo: invert linkdict to link -> text and not text -> link
    # otherwise we miss one link if we see twice "supplemental table" (see example)
    "sgm" :
    {
        "hostnames" : ["sgmjournals.org"],
        "landingUrl_templates" : {\
            "1466-5026" : "http://ijs.sgmjournals.org/cgi/pmidlookup?view=long&pmid=%(pmid)s", \
            "1350-0872" : "http://mic.sgmjournals.org/cgi/pmidlookup?view=long&pmid=%(pmid)s", \
            "0022-2615" : "http://jmm.sgmjournals.org/cgi/pmidlookup?view=long&pmid=%(pmid)s", \
            "0022-1317" : "http://vir.sgmjournals.org/cgi/pmidlookup?view=long&pmid=%(pmid)s"},
        "landingPage_errorKeywords" : "We are currently doing routine maintenance", # wait for 15 minutes and retry
        "doiUrl_replace" : {"$" : ".long"},
        "landingUrl_isFulltextKeyword" : ".long",
        "landingPage_ignoreMetaTag" : True,
        "landingUrl_fulltextUrl_replace" : {"long" : "full.pdf", "abstract" : "full.pdf" },
        "landingPage_suppFileList_urlREs" : [".*/content.*suppl/DC[0-9]"],
        "suppListPage_suppFile_urlREs" : [".*/content.*suppl/.*"],
    },
    # SMBE - Soc of Mol Biol and Evol
    # part of OUP - careful, duplicates!
    # PMID 22956734
    "smbe" :
    {
        "hostnames" : ["mbe.oxfordjournals.org"],
        "landingUrl_templates" : \
            {"0737-4038" : "http://mbe.oxfordjournals.org/cgi/pmidlookup?view=long&pmid=%(pmid)s"},
        "landingPage_errorKeywords" : "We are currently doing routine maintenance", # wait for 15 minutes and retry
        "doiUrl_replace" : {"$" : ".long"},
        "landingUrl_isFulltextKeyword" : ".long",
        "landingPage_ignoreMetaTag" : True,
        "landingUrl_fulltextUrl_replace" : {"long" : "full.pdf", "abstract" : "full.pdf" },
        "landingPage_suppFileList_urlREs" : [".*/content.*suppl/DC[0-9]"],
        "suppListPage_suppFile_urlREs" : [".*/content.*suppl/.*"],
    },
    # http://www.jimmunol.org/content/189/11/5129/suppl/DC1
    # http://www.jimmunol.org/content/suppl/2012/10/25/jimmunol.1201570.DC1/12-01570_S1-4_ed10-24.pdf
    "aai" :
    {
        "hostnames" : ["jimmunol.org"],
        "landingUrl_templates" : {"0022-1767" : "http://www.jimmunol.org/cgi/pmidlookup?view=long&pmid=%(pmid)s"},
        "landingPage_errorKeywords" : "We are currently doing routine maintenance", # wait for 15 minutes and retry
        "doiUrl_replace" : {"$" : ".long"},
        "landingUrl_isFulltextKeyword" : ".long",
        "landingPage_ignoreMetaTag" : True,
        "landingUrl_fulltextUrl_replace" : {"long" : "full.pdf", "abstract" : "full.pdf" },
        "landingPage_suppFileList_urlREs" : [".*/content[0-9/]*suppl/DC1"],
        "suppListPage_suppFile_urlREs" : [".*/content/suppl/.*"],
    },
    # example suppinfo links 20967753 (major type of suppl, some also have "legacy" suppinfo
    # example spurious suppinfo link 8536951
    #
    "wiley" :
    {
        "hostnames" : ["onlinelibrary.wiley.com"],
        #"landingUrl_templates" : {None: "http://onlinelibrary.wiley.com/doi/%(doi)s/full"},
        "doiUrl_replace" : {"abstract" : "full"},
        "landingUrl_isFulltextKeyword" : "full",
        "landingUrl_fulltextUrl_replace" : {"full" : "pdf", "abstract" : "pdf"},
        "landingPage_suppListTextREs" : ["Supporting Information"],
        "suppListPage_suppFile_urlREs" : [".*/asset/supinfo/.*", ".*_s.pdf"],
        "suppFilesAreOffsite" : True,
        "landingPage_ignoreUrlREs"  : ["http://onlinelibrary.wiley.com/resolve/openurl.genre=journal&issn=[0-9-X]+/suppmat/"],
        "landingPage_stopPhrases" : ["You can purchase online access", "Registered Users please login"]
    },
    # http://www.futuremedicine.com/doi/abs/10.2217/epi.12.21
    "futureScience" :
    {
        "hostnames" : ["futuremedicine.com", "future-science.com", "expert-reviews.com", "future-drugs.com"],
        "landingUrl_fulltextUrl_replace" : {"abs" : "pdfplus"},
        "landingUrl_suppListUrl_replace" : {"abs" : "suppl"},
        "suppListPage_suppFile_urlREs" : [".*suppl_file.*"],
        "landingPage_stopPhrases" : ["single article purchase is required", "The page you have requested is unfortunately unavailable"]
    },

}

def compileRegexes():
    " compile regexes in confDict "
    ret = {}
    for pubId, crawlConfig in confDict.iteritems():
        ret[pubId] = {}
        for key, values in crawlConfig.iteritems():
            if key.endswith("REs"):
                newValues = []
                for regex in values:
                    newValues.append(re.compile(regex))
            else:
                newValues = values
            ret[pubId][key] = newValues
    return ret

def prepConfigIndexByHost():
    """ compile regexes in config and return dict publisherId -> config and hostname -> config
    these make it possible to get the config either by hostname (for general mode)
    or by publisher (for per-publisher mode)
    """
    compCfg = compileRegexes()
    byHost = {}
    for pubId, crawlConfig in compCfg.iteritems():
        for host in crawlConfig["hostnames"]:
            byHost[host] = crawlConfig
    return compCfg, byHost


#for managing the wikipedia requests
import requests
import json
#for parsing html tags
from lxml import etree
from io import StringIO
import lxml.html
# for the waiting
import time
#for managing command line arguments
from sys import argv

outputformat = "{id},{name},{release_year},[{developers}],[{publisher}],{img_url},{src_url},[{genres}],\"{description}\"\n"	#the format string for the output file writes
starter = "https://en.wikipedia.org/w/api.php?action=parse&page="	#used for all queries
URLstarter = "https://en.wikipedia.org/wiki/"	#used only for the file output
checkedPages = []

#inputManagement
#args = argv
def manageArgs():
	if (len(argv)!=4):
		print("wiki_to_csv.py [URL] [section #] [FILE]\n")
		return -1;
	else:
		global mainPage
		mainPage = argv[1]
		global sectionNumb
		sectionNumb = argv[2]
		global outputName
		outputName = argv[3]
		return 0;

def investigate_further(Name):
	prompt = starter+Name+"&prop=text&format=json&redirects"
	time.sleep(0.5)
	print("looking at ", Name)
	req = requests.get(prompt)
	Listfile = json.loads(req.text)
	if "error" in Listfile:
		print(Name+" is not on wikipedia")
		return "n/a","n/a","n/a","n/a"
	Text = Listfile["parse"]["text"]["*"]
	store = lxml.html.fromstring(Text)
	URL = URLstarter+Listfile["parse"]["title"]
	TempList = store.xpath("//a[@class='image']/@href")
	if len(TempList)>=1:
		Picture =  URLstarter+TempList[0]
	else:
		Picture = "n/a"
	Genres = store.xpath("//th/a[text()='Genre(s)']/../following::td[1]//text()")
	Genres = [item for item in Genres if item != ' ']
	Paragraphs = store.xpath("//h2/span[@id='Gameplay']/../following::p[position()<=2]")
	if len(Paragraphs)==0:#if no "gameplay" section just default to front
		Paragraphs = store.xpath("//div[@class='mw-parser-output']/p[position()<=2]")
	if len(Paragraphs)==1:#if only one paragraph, just use that
		Discription = Paragraphs[0].text_content().replace("\n", "")#no newlines!
	else:#default behavior
		Discription = Paragraphs[0].text_content().replace("\n", "")+" \\n "+Paragraphs[1].text_content().replace("\n", "")

	return URL, Picture, Genres, Discription

def Scrape(file):
	prompt = starter+mainPage+"&section="+sectionNumb+"&prop=text&format=json&redirects"
	print("asking "+prompt)
	#parser = etree.XMLParser(encoding='utf-8', recover=True)
	req = requests.get(prompt)
	Listfile = json.loads(req.text)
	if "error" in Listfile:
		print(mainPage+" is not on wikipedia")
		return
	MainText = Listfile["parse"]["text"]['*']
#	MainText = "
testing testing testing
"
#	MainText = "
Hello my name is Pedro, and I like crayons
"
	#store = etree.parse(StringIO(MainText), parser)
	store = lxml.html.fromstring(MainText)
	tableData = store.xpath("//table[@id='softwarelist']//td")
	# for item in tableData:
		# print(item.text_content())
	i=0
	j=0
	while (i<len(tableData)):
	#while (i<12):
		Name = tableData[i].text_content().replace("\n", "")
		#print("name:",tableData[i].text_content())
		Devs = tableData[i+1].text_content().replace("\n", "")
		#print("devs:",tableData[i+1].text_content())
		Publisher = tableData[i+2].text_content().replace("\n", "")
		#print("NApubs:",tableData[i+2].text_content())
		#print("EUpubs:",tableData[i+3].text_content())
		year = tableData[i+4].text_content().replace("\n", "")
		#print("NAdate",tableData[i+4].text_content())
		#print("EUdate",tableData[i+5].text_content())
		#print("rowend\n")
		URL, Picture, Genres, Discription = investigate_further(Name.replace(" ","_"))	#get image, genres, and descriptions
		i+=6;
		j+=1;
		outputString = outputformat.format(id = j,name = Name,release_year = year,developers = Devs,publisher = Publisher,img_url = Picture,genres = Genres, description = Discription, src_url=URL)
		file.write(outputString)

#the main function
if (manageArgs()==0):
	OutputFile = open(outputName,'a', encoding = "utf_16")
	OutputFile.write("id,name,release_year,publisher,image,src,genres,description\n")
	Scrape(OutputFile);
	OutputFile.close()
	#Picture, Genres, Description = investigate_further("The_3-D_Battles_of_WorldRunner")
	#print(Picture)
	#print(Genres)
	#print(Description)


# -*- coding: utf-8 -*-
from __future__ import division

import numpy as np
from numpy import dot, identity
from numpy.linalg import norm, inv

import basis_functions as bf
import source_distribution as sd
import potentials as pt
import dist_table_utils as dt
import plotting_utils as plut
import parameters_utils as parut

class KCSD3D(object):
    """
    3D variant of the kCSD method.
    It assumes sources are distributed in 3D space.

    **Parameters**

    elec_pos : numpy array
        positions of electrodes

    sampled_pots : numpy array
        potentials measured by electrodes

    params : set, optional
        configuration parameters, that may contain the following keys:

        'sigma' : float
            space conductance of the medium

        'n_sources' : int
            number of sources

        'source_type' : str
            basis function type ('gauss', 'step', 'gauss_lim')

        'R_init' : float
            demanded thickness of the basis element

        'h' : float
            cylinder radius

        'dist_density' : int
            resolution of the dist_table

        'x_min', 'x_max', 'y_min', 'y_max' : floats
            boundaries for CSD estimation space

        'ext' : float
            length of space extension: x_min-ext ... x_max+ext

        'gdX', 'gdY' : float
            space increments in the estimation space

        'lambd' : float
            regularization parameter for ridge regression
    """

    def __init__(self, elec_pos, sampled_pots, params={}):
        self.validate_parameters(elec_pos, sampled_pots)
        self.elec_pos = elec_pos
        self.sampled_pots = sampled_pots
        self.set_parameters(params)

    def validate_parameters(self, elec_pos, sampled_pots):
        if elec_pos.shape[0] != sampled_pots.shape[0]:
            raise Exception("Number of measured potentials is not equal \
                             to electrode number!")
        if elec_pos.shape[0] < 4:
            raise Exception("Number of electrodes must be at least 4!")
        if parut.check_for_duplicated_electrodes(elec_pos) is False:
            raise Exception("Error! Duplicated electrode!")

    def set_parameters(self, params):
        default_params = {
            'sigma': 1.0,
            'n_sources': 100,
            'xmin': np.min(self.elec_pos[:, 0]),
            'xmax': np.max(self.elec_pos[:, 0]),
            'ymin': np.min(self.elec_pos[:, 1]),
            'ymax': np.max(self.elec_pos[:, 1]),
            'zmin': np.min(self.elec_pos[:, 2]),
            'zmax': np.max(self.elec_pos[:, 2]),
            'dist_table_density': 100,
            'lambd': 0.0,
            'R_init': 2 * parut.min_dist(self.elec_pos),
            'ext_X': 0.0,
            'ext_Y': 0.0,
            'ext_Z': 0.0,
            'h': 1.0,
            'source_type': 'gauss'
        }
        for (prop, default) in default_params.iteritems():
            setattr(self, prop, params.get(prop, default))

        self.gdX = params.get('gdX', 0.05 * (self.xmax - self.xmin))
        self.gdY = params.get('gdY', 0.05 * (self.ymax - self.ymin))
        self.gdZ = params.get('gdZ', 0.05 * (self.zmax - self.zmin))
        self.dist_table_density = 100

        self.source_type = params.get('source_type', 'gauss')
        basis_types = {
            "step": bf.step_rescale_3D,
            "gauss": bf.gauss_rescale_3D,
            "gauss_lim": bf.gauss_rescale_lim_3D,
        }
        if self.source_type not in basis_types.keys():
            raise Exception("Incorrect source type!")
        else:
            self.basis = basis_types.get(self.source_type)

        nx = (self.xmax - self.xmin)/self.gdX + 1
        ny = (self.ymax - self.ymin)/self.gdY + 1
        nz = (self.zmax - self.zmin)/self.gdZ + 1
        lin_x = np.linspace(self.xmin, self.xmax, nx)
        lin_y = np.linspace(self.ymin, self.ymax, ny)
        lin_z = np.linspace(self.zmin, self.ymax, nz)
        self.space_X, self.space_Y, self.space_Z = np.meshgrid(lin_x,
                                                               lin_y,
                                                               lin_z)

        (self.X_src, self.Y_src, self.Z_src, self.R) = sd.make_src_3D(
            self.space_X, self.space_Y, self.space_Z,
            self.n_sources,
            self.ext_X, self.ext_Y, self.ext_Z,
            self.R_init
        )

        Lx = np.max(self.X_src) - np.min(self.X_src) + self.R
        Ly = np.max(self.Y_src) - np.min(self.Y_src) + self.R
        Lz = np.max(self.Z_src) - np.min(self.Z_src) + self.R
        self.dist_max = (Lx**2 + Ly**2 + Lz**2)**0.5

    def estimate_pots(self):
        """Calculates Local Field Potentials."""
        estimation_table = self.interp_pot
        self.estimated_pots = self.estimate(estimation_table)
        return self.estimated_pots

    def estimate_csd(self):
        """Calculates Current Source Density."""
        estimation_table = self.k_interp_cross
        self.estimated_csd = self.estimate(estimation_table)
        return self.estimated_csd

    def estimate(self, estimation_table):
        k_inv = inv(self.k_pot + self.lambd * identity(self.k_pot.shape[0]))
        nt = self.sampled_pots.shape[1]
        (nx, ny, nz) = self.space_X.shape
        estimation = np.zeros((nx * ny * nz, nt))

        for t in xrange(nt):
            beta = dot(k_inv, self.sampled_pots[:, t])
            for i in xrange(self.elec_pos.shape[0]):
                estimation[:, t] += beta[i] * estimation_table[:, i]

        estimation = estimation.reshape(nx, ny, nz, nt)
        return estimation

    def save(self, filename='result'):
        """Save results to file."""
        pass

    def __repr__(self):
        info = ''.join(self.__class__.__name__)
        for key in vars(self).keys():
            if not key.startswith('_'):
                info += '%s : %s\n' % (key, vars(self)[key])
        return info

    def plot_all(self):
        extent = [self.xmin, self.xmax,
                  self.ymin, self.ymax,
                  self.zmin, self.zmax]
        plut.plot_3D(self.elec_pos, self.estimated_pots,
                     self.estimated_csd, extent)

    #
    # subfunctions
    #

    def init_model(self):
        """
        Prepares all the required matrices to calculate CSD and potentials.
        """
        self.create_dist_table()

        self.calculate_b_pot_matrix()
        self.k_pot = dot(self.b_pot_matrix.T, self.b_pot_matrix)

        self.calculate_b_src_matrix()
        self.k_interp_cross = dot(self.b_src_matrix, self.b_pot_matrix)

        self.calculate_b_interp_pot_matrix()
        self.interp_pot = dot(self.b_interp_pot_matrix, self.b_pot_matrix)

    def create_dist_table(self):
        """
        Create table of a single source base element contribution
        to overall potential as a function of distance.
        The last record corresponds to the distance equal to the
        diagonal of the cuboid.
        """
        self.dist_table = dt.create_dist_table(self.basis,
                                               pt.b_pot_3d_mc,
                                               self.R,
                                               self.h,
                                               self.sigma,
                                               self.dist_max,
                                               self.dist_table_density
                                               )
        """self.dist_table = dt.create_dist_table(self.basis,
                                               pt.b_pot_3d_analytic,
                                               self.R,
                                               self.h,
                                               self.sigma,
                                               self.dist_max,
                                               self.dist_table_density
                                               )"""

    def calculate_b_pot_matrix(self):
        """
        Compute the matrix of potentials generated by every
        source basis function at every electrode position.
        """
        self.calculate_b_pot_matrix_3D()

    def calculate_b_pot_matrix_3D(self):
        """
        Calculates b_pot_matrix - matrix containing the values of all
        the potential basis functions in all the electrode
        positions (essential for calculating the cross_matrix)
        """
        n_obs = self.elec_pos.shape[0]
        (nx, ny, nz) = self.X_src.shape
        n = nx * ny * nz

        self.b_pot_matrix = np.zeros((n, n_obs))

        for i in xrange(0, n):
            # finding the coordinates of the i-th source
            i_x, i_y, i_z = np.unravel_index(i, (nx, ny, nz))
            src = [self.X_src[i_x, i_y, i_z],
                   self.Y_src[i_x, i_y, i_z],
                   self.Z_src[i_x, i_y, i_z]]

            for j in xrange(0, n_obs):
                # for all the observation points
                # checking the distance between the observation point and
                # the source,
                # calculating the base value
                dist = norm(self.elec_pos[j] - src)

                self.b_pot_matrix[i, j] = dt.generated_potential(
                    dist,
                    self.dist_max,
                    self.dist_table
                )

    def calculate_b_src_matrix(self):
        """
        Compute the matrix of basis sources.
        """
        self.make_b_src_matrix_3D()

    def make_b_src_matrix_3D(self):
        """
        Calculate b_src_matrix - matrix containing containing the values of
        all the source basis functions in all the points at which we want to
        calculate the solution (essential for calculating the cross_matrix)
        """
        (nsx, nsy, nsz) = self.X_src.shape
        n = nsy * nsx * nsz  # total number of sources
        (ngx, ngy, ngz) = self.space_X.shape
        ng = ngx * ngy * ngz

        self.b_src_matrix = np.zeros((self.space_X.shape[0],
                                     self.space_X.shape[1],
                                     self.space_X.shape[2],
                                     n))

        for i in xrange(n):
            # getting the coordinates of the i-th source
            (i_x, i_y, i_z) = np.unravel_index(i, (nsx, nsy, nsz), order='F')
            z_src = self.Z_src[i_x, i_y, i_z]
            y_src = self.Y_src[i_x, i_y, i_z]
            x_src = self.X_src[i_x, i_y, i_z]

            self.b_src_matrix[:, :, :, i] = self.basis(
                self.space_X,
                self.space_Y,
                self.space_Z,
                [x_src, y_src, z_src],
                self.R
            )

        self.b_src_matrix = self.b_src_matrix.reshape(ng, n)

    def calculate_b_interp_pot_matrix(self):
        """
        Compute the matrix of potentials generated by every source
        basis function at every position in the interpolated space.
        """
        self.make_b_interp_pot_matrix_3D()

    def make_b_interp_pot_matrix_3D(self):
        (ngx, ngy, ngz) = self.space_X.shape
        ng = ngx * ngy * ngz
        (nsx, nsy, nsz) = self.X_src.shape
        n_src = nsy * nsx * nsz

        self.b_interp_pot_matrix = np.zeros((ngx, ngy, ngz, n_src))

        for i in xrange(0, n_src):
            # getting the coordinates of the i-th source
            i_x, i_y, i_z = np.unravel_index(i, (nsx, nsy, nsz), order='F')
            z_src = self.Z_src[i_x, i_y, i_z]
            y_src = self.Y_src[i_x, i_y, i_z]
            x_src = self.X_src[i_x, i_y, i_z]
            norms = np.sqrt((self.space_X - x_src)**2
                            + (self.space_Y - y_src)**2
                            + (self.space_Z - z_src)**2)

            self.b_interp_pot_matrix[:, :, :, i] = dt.generated_potential(
                norms,
                self.dist_max,
                self.dist_table
            )

        self.b_interp_pot_matrix = self.b_interp_pot_matrix.reshape(ng, n_src)

def main():
    elec_pos = np.array([(0, 0, 0), (0, 0, 1), (0, 1, 0), (1, 0, 0),
                         (0, 1, 1), (1, 1, 0), (1, 0, 1), (1, 1, 1),
                         (0.5, 0.5, 0.5)])
    pots = np.array([[-0.5], [0], [-0.5], [0], [0], [0.2], [0], [0], [1]])
    params = {
        'gdX': 0.05,
        'gdY': 0.05,
        'gdZ': 0.05,
        'n_sources': 64,
    }
    k = KCSD3D(elec_pos, pots, params)
    k.init_model()

    k.estimate_pots()
    k.estimate_csd()

    k.plot_all()

if __name__ == '__main__':
    main()


from ftw.builder import Builder
from ftw.builder import create
from ftw.testbrowser import browsing
from opengever.testing import FunctionalTestCase
from plone.app.testing import TEST_USER_ID
import datetime

class TestDelegateTaskForm(FunctionalTestCase):

    @browsing
    def test_delegate(self, browser):
        task = create(Builder('task')
                      .titled('A Task')
                      .having(issuer=TEST_USER_ID,
                              deadline=datetime.date(2013, 1, 1))
                      .in_state('task-state-in-progress'))

        # pre-fill responsibles
        selected_user = '{}:{}'.format(self.org_unit.id(), self.user.userid)
        url = "{}/@@delegate_recipients?form.widgets.responsibles={}".format(
            task.absolute_url(), selected_user)
        browser.login().open(url)
        browser.css('#form-buttons-save').first.click()  # can't use submit()

        browser.fill({'Issuer': self.user.userid})
        browser.css('#form-buttons-save').first.click()  # can't use submit()