import os
from app import app, lm
from flask import render_template, flash, redirect, request, url_for
from flask.ext.login import login_user, login_required, logout_user, current_user
from app.models import User, Book, Author
from database import db_session
from forms import LoginForm, BookForm, AuthorForm, RegistrationForm, SearchForm
import random

#                       User/login
@lm.user_loader
def load_user(id):
  return User.query.get(int(id))

@lm.unauthorized_handler
def unauthorized():
  flash("Login required.", "danger")
  return redirect(url_for('index'))

@app.errorhandler(404)
def page_not_found(e):
    return render_template('404.html'), 404

@app.errorhandler(500)
def page_not_found(e):
    return render_template('500.html'), 500

@app.route('/register/', methods=('GET', 'POST'))
def register_view():
  if current_user.is_authenticated():
    flash('sorry, registration is available only for not-authorised users')
    return redirect(url_for('index'))
  form_r = RegistrationForm()
  form=LoginForm()
  if form_r.validate_on_submit() and form_r.validate_login():
      user = User()
      user.name = form_r.name.data
      user.email = form_r.email.data
      user.set_password(form_r.password.data)
      db_session.add(user)
      db_session.commit()
      flash('User %s created succesfully' %(user.email),'success')
      if current_user.is_authenticated():
          logout_user()
      login_user(user)
      return redirect(url_for('index'))
  return render_template('register.html',form = form, form_r=form_r, user=current_user, is_authenticated=current_user.is_authenticated())

@app.route('/login/', methods=['GET', 'POST'])
def login():
  form = LoginForm()
  if form.validate_on_submit():
      user = form.get_user()
      if user and user.check_password(form.password.data):
        login_user(user)
        flash('Welcome ' + user.email, 'success')
        return redirect(request.args.get("next") or url_for("index"))
      else:
        flash('wrong username or password', 'danger')
        return redirect(url_for("login"))
  return render_template("login.html", form=form)

@login_required
@app.route('/logout/')
def logout():
  logout_user()
  flash("You have logged out.", 'info')
  return redirect(url_for('index'))

##                                   index/Search
@app.route('/',methods=('GET', 'POST'))
def index():

    form_s = SearchForm()
    books = db_session.query(Book).all()
    authors = db_session.query(Author).all()

    return render_template("index.html", authors = authors, user=current_user, is_authenticated=current_user.is_authenticated(), books=books, form_s=form_s)

@app.route("/search/", methods=('GET', 'POST'))
def search():

  form_s = SearchForm()
  if form_s.validate_on_submit():
    books = Book.query.all()
    authors = Author.query.all()
    query = form_s.search.data.lower()
    result_books = []
    result_authors = []
    for book in books:
      if query in book.title.lower():
        if not book in result_books:
          result_books.append(book)
    for author in authors:
      if query in author.name.lower():
        result_authors.append(author)
        for b in author.books:
          if not b in result_books:
            result_books.append(b)
    flash("found %s book(s) and %s author(s)" %(len(result_books),len(result_authors)), 'info')
    return render_template('index.html', books=result_books, authors = result_authors, user=current_user,
                         is_authenticated=current_user.is_authenticated(), form_s=form_s)

  return redirect(url_for('index'))

##              BOOKS

@app.route('/books/')
@login_required
def book():
  form = BookForm(request.form)
  books = db_session.query(Book).all()
  return render_template("books.html", user=current_user, books=books,is_authenticated=current_user.is_authenticated())

@app.route('/book_add/', methods=['GET', 'POST'])
@login_required
def book_add():

  book_form = BookForm(request.form)
  book_form.authors.choices = [(p.id, p.name) for p in db_session.query(Author).order_by('id')]

  if book_form.validate_on_submit():
      book = Book()
      book.title = book_form.title.data
      book.authors = [db_session.query(Author).get(o) for o in book_form.authors.data]
      db_session.add(book)
      db_session.commit()
      flash('Successfully added.', 'success')
      return redirect(url_for('index'))

  return render_template("book_create.html", bform=book_form, user=current_user, is_authenticated=current_user.is_authenticated())

@app.route('/book/<id>', methods=['GET', 'POST'])
@login_required
def book_edit(id):

  book = db_session.query(Book).get(id)
  book_form = BookForm(request.form, obj=book)
  book_form.authors.choices = [(p.id, p.name) for p in db_session.query(Author).order_by('id')]

  if book_form.validate_on_submit():
      book = db_session.query(Book).get(id)
      book.title = book_form.title.data
      book.authors = [db_session.query(Author).get(o) for o in book_form.authors.data]
      db_session.commit()
      flash('Changes saved', 'info')
      return redirect(url_for('index'))

  book_form.authors.data = [p.id for p in book.authors]
  return render_template("book.html", bform=book_form, book=book, user=current_user, is_authenticated=current_user.is_authenticated())

@app.route('/book_rm/<id>', methods=['GET', 'POST'])
@login_required
def book_rm(id):
  book = db_session.query(Book).get(id)
  if book:
      db_session.delete(book)
      db_session.commit()
      flash('Deleted successfully.', 'warning')
  return redirect(url_for('index'))

##            AUTHORS
@app.route('/authors/')
@login_required
def author():

  authors = db_session.query(Author).all()
  return render_template("authors.html", user=current_user, is_authenticated=current_user.is_authenticated(), authors=authors)

@app.route('/author/<id>', methods=['GET', 'POST'])
@login_required
def author_edit(id):

  author = db_session.query(Author).get(id)
  author_form = AuthorForm(request.form, obj=author)
  author_form.books.choices = [(b.id, b.title) for b in db_session.query(Book).order_by('id')]

  if author_form.validate_on_submit():
      author = db_session.query(Author).get(id)
      author.name = author_form.name.data
      author.books = [db_session.query(Book).get(b) for b in author_form.books.data]
      db_session.commit()
      flash('Changes saved', 'info')
      return redirect(url_for('index'))

  author_form.books.data = [auth.id for auth in author.books]
  return render_template("author.html", bform=author_form, author=author, user=current_user, is_authenticated=current_user.is_authenticated())

@app.route('/author_add/', methods=['GET', 'POST'])
@login_required
def author_add():
  author_form = AuthorForm(request.form)
  author_form.books.choices = [(p.id, p.title) for p in db_session.query(Book).order_by('id')]

  if author_form.validate_on_submit():
      author = Author()
      author.name = author_form.name.data
      author.books = [db_session.query(Book).get(o) for o in author_form.books.data]
      db_session.add(author)
      db_session.commit()
      flash('Successfully added.', 'success')
      return redirect(url_for('index'))

  return render_template("author_create.html", bform=author_form, user=current_user, is_authenticated=current_user.is_authenticated())

@app.route('/author_rm/<id>', methods=['GET', 'POST'])
@login_required
def author_rm(id):
  author = db_session.query(Author).get(id)
  if author:
      db_session.delete(author)
      db_session.commit()
      flash('Deleted.', 'warning')
  return redirect(url_for('index'))

# random page
@app.route('/random', methods=['GET', 'POST'])
def random_page():
  boa=random.randint(1,2)   #boa = book(1) or author(2)
  id_list=[]
  if boa == 1:
    authors = db_session.query(Author).all()
    for i in authors: id_list.append(i.id)
    randid= random.choice(id_list)
    return redirect('/author/'+str(randid))
  books = db_session.query(Book).all()
  for i in books: id_list.append(i.id)
  randid= random.choice(id_list)
  return redirect('/book/'+str(randid))


from db.models.movies import Movie
from schemas.movies import MovieCreate,FilterMovie
from .movie_genres import create_movie_genres,delete_movie_genres
from sqlalchemy.orm import Session
from db.models.genres import Genre
from db.models.reviews import Review
from sqlalchemy import or_,and_

# create movie
def create_new_movie(movie: MovieCreate,db: Session):
    # obtengo el dict; le separo los ids de los géneros y luego inserto apropiadamente
    data = movie.dict()
    genres = data.pop('genres')

    # agrego la peli
    movie_object = Movie(**data)
    db.add(movie_object)
    db.flush()

    # los géneros
    create_movie_genres(movie_object.id,genres,db)

    # commit
    db.commit()
    db.refresh(movie_object)
    return movie_object

# retrieve movie by id
def retreive_movie(id: int, db: Session):
    return db.query(Movie).filter(Movie.id == id).first()

#http://127.0.0.1:8000/movies/?date__lte=218-07-03&date__lte=2018-07-03
#http://127.0.0.1:8000/movies/?date__gte=2022-09-21&date__lte=2022-09-21#

def list_movies(db: Session,f:FilterMovie):

    filters = [Movie.id>0]

    if f.title:
        filters.append(Movie.title==f.title)

    if f.title__contains:
        filters.append(Movie.title.contains(f.title__contains))

    if f.genres:
        generos = db.query(Genre.id).filter(Genre.id.in_(f.genres))
        filters.append(Genre.id.in_(generos))

    # rango de fechas
    if f.date__gte:
        filters.append( Movie.date >=  f.date__gte)
    if f.date__lte:
        filters.append( Movie.date <=  f.date__lte)

    # junto todo
    filters = and_(*filters)

    query = db.query(Movie).join(Movie.genres).filter(filters).limit(f.limit).offset(f.offset-1).all()
     # le agrego count, page, limit
    count = db.query(Movie).filter(filters).count() # join en la cuenta??

    #
    data = [
        {
            'id':d.id,
            'title': d.title,
            'date': d.date,
            'description': d.description,
            'image_link':d.image_link,
            'genres': [{'description':t.description,'id':t.id} for t in d.genres]
        }
        for d in query
    ]

    # respuesta
    response = {
        'data': data,
        'count': count,
        'limit':f.limit,
        'offset':f.offset
    }
    return response

# update by id
def update_movie_by_id(id: int, movie: MovieCreate, db: Session):

    existing_movie = db.query(Movie).filter(Movie.id == id)
    if not existing_movie.first():
        return 0

    # actualizo la peli y sus géneros
    data = movie.dict(exclude_unset=True) # ipte
    genres = data.pop('genres')

    # géneros: borro los viejos y agrego los nuevos
    delete_movie_genres(id,db)
    create_movie_genres(id,genres,db)

    # la peli
    existing_movie.update(data)
    #existing_movie.update(movie.__dict__)
    db.commit()
    return 1

def delete_movie_by_id(id: int, db: Session):
    existing_movie = db.query(Movie).filter(Movie.id == id)
    if not existing_movie.first():
        return 0
    # borro sus géneros
    delete_movie_genres(movie_id=id,db=db)
    # borro la peli
    existing_movie.delete(synchronize_session=False) # ver esto
    db.commit()
    return 1

#movie_object = Movie(**movie.dict()); antes, sin tener que agregar en la otra tabla
#db.add(movie_object)


import json
import requests
import trafilatura
from nltk.tokenize import word_tokenize
from nltk.probability import FreqDist
import concurrent.futures
import string
from bs4 import BeautifulSoup

from const import API_KEY, CONTENT_TYPE, PATTERNS

def fetch_google_search_results(query: str, gl: str, hl: str, num: int = 10) -> dict:
    url = "https://google.serper.dev/search"
    headers = {'X-API-KEY': API_KEY, 'Content-Type': CONTENT_TYPE}
    payload = json.dumps({"q": query, "gl": gl, "hl": hl, "num": num, "autocorrect": False})
    response = requests.post(url, headers=headers, data=payload)
    response = json.loads(response.text)
    return {
        "organic": [item for item in response.get('organic', [])],
        "paa": [item for item in response.get('peopleAlsoAsk', [])],
        "related": [item for item in response.get('relatedSearches', [])],
    }

def get_content(url: str) -> str:
    try:
        return trafilatura.fetch_url(url)
    except:
        print(f"can't get content from url: {url}")
        return None

def get_url_content(url:str) -> str:
    with concurrent.futures.ThreadPoolExecutor() as executor:
        future = executor.submit(get_content, url)
        try:
            content = future.result()
            if content is not None:
                return {
                    "content": content,
                    "url": url,
                    "text": trafilatura.extract(content, include_images=False),
                    "html": trafilatura.extract(content),
                    "title": extract_title_from_html(content),
                    "descr": extract_descr_from_html(content),
                    "headings": extract_headings_from_html(content),
                }
            else:
                print(f"can't get content from url")
                return ""
        except Exception as e:
            print(e)
            print(f"can't get content from url")
            return ""

def get_content_list(urls: list) -> list:
    contents = []
    with concurrent.futures.ThreadPoolExecutor() as executor:
        future_to_url = {executor.submit(get_content, url): url for url in urls}
        for future in concurrent.futures.as_completed(future_to_url):
            url = future_to_url[future]
            try:
                content = future.result()
                if content is not None:
                    contents.append({
                        "content": content,
                        "url": url,
                        "text": trafilatura.extract(content, include_images=False),
                        "html": trafilatura.extract(content),
                        "title": extract_title_from_html(content),
                        "descr": extract_descr_from_html(content),
                        "headings": extract_headings_from_html(content),
                    })
                else:
                    print(f"can't get content from url: {url}")
            except Exception:
                print(f"can't get content from url: {url}")
    return contents

def filter_backlinks(backlinks: list, patterns: list = PATTERNS, offset: int = 0) -> list:
    return [{"index": index + offset + 1, "url": backlinks[index]} for index in range(0, len(backlinks)) if any(pattern in backlinks[index] for pattern in patterns)]

def preprocess_content(contents: list, stop_words: set) -> FreqDist:
    filtered_tokens = [
        token.lower() for token in word_tokenize(contents)
        if token.lower() not in stop_words and token.isalpha()
    ]

    return FreqDist(filtered_tokens)

def remove_punctuation_and_numerics(word_frequencies: FreqDist) -> str:
    for word in list(word_frequencies):
        if word in string.punctuation or not word.isalpha():
            del word_frequencies[word]
    return word_frequencies

def density(most_common_keywords: any, content: str) -> dict:
    return {word: float(content.count(word)/len(content.split(' '))) for word, freq in most_common_keywords}

def extract_content_from_html(html: str) -> str:
    soup = BeautifulSoup(html, 'html.parser')
    return soup.get_text()

def extract_title_from_html(html: str) -> str:
    soup = BeautifulSoup(html, 'html.parser')
    return soup.title.string

def extract_descr_from_html(html: str) -> str:
    try:
        soup = BeautifulSoup(html, 'html.parser')
        return soup.find("meta",  property="og:description").get("content", None)
    except Exception as e:
        print(e)
        return "Pas de description"

def extract_headings_from_html(html: str) -> list:
    soup = BeautifulSoup(html, 'html.parser')
    return [{"type": heading.name, "text": heading.text} for heading in soup.find_all(["h1", "h2", "h3", "h4", "h5", "h6"])]


import os
import random
import numpy as np
import scipy.io as sio
from PIL import Image
from glob import glob
from natsort import natsorted

import torch
import torchvision
from torchvision import transforms
import torchvision.transforms.functional as F
from torch.utils.data import Dataset, DataLoader

from .utils import *

import ipdb

# This class is for SIDD_Medium train on sRGB patches
from natsort import natsorted
from glob import glob
class SIDD_sRGB_Train_DataLoader(Dataset):
    def __init__(self, path, length, patch_size, patched_input=False):
        super(SIDD_sRGB_Train_DataLoader, self).__init__()
        assert length % 160 == 0, 'You should specify a proper length.'

        self.len            = length
        self.patch_size     = patch_size
        self.patched_input  = patched_input
        self.noisy_imgs, self.clean_imgs = [], []
        self.transform      = torchvision.transforms.ToTensor()
        self.return_name    = False

        if not self.patched_input:
            # get file names from original dataset
            imgs = natsorted(glob(os.path.join(path, '*', '*.PNG')))
            for img in imgs:
                img_name = os.path.split(img)[-1]
                if 'GT' in img_name:
                    self.clean_imgs.append(img)
                if 'NOISY' in img_name:
                    self.noisy_imgs.append(img)
        else:
            # get file names from cropped dataset
            self.noisy_imgs = natsorted(glob(os.path.join(path, 'input', '*.png')))
            self.clean_imgs = natsorted(glob(os.path.join(path, 'target', '*.png')))

    def __len__(self):
        return self.len if self.len else len(self.noisy_imgs)

    def __getitem__(self, index):
        index = index % len(self.noisy_imgs) # in case out of range.
        aug   = random.randint(0, 7)

        # read image
        noisy_img   = np.array(Image.open(self.noisy_imgs[index]))
        clean_img   = np.array(Image.open(self.clean_imgs[index]))

        # random crop
        if self.patch_size:
            noisy_patch, clean_patch = crop_patches(noisy_img, clean_img,
                                                    self.patch_size)

        # random augmentation
        noisy_patch = Data_Augmentation(self.transform(noisy_patch), aug)
        clean_patch = Data_Augmentation(self.transform(clean_patch), aug)

        filename = os.path.splitext(os.path.split(self.clean_imgs[index])[-1])[0]
        if self.return_name:
            return noisy_patch, clean_patch, filename
        else:
            return noisy_patch, clean_patch

# this class is for SIDD val on sRGB patches.
# could download patched val data from https://drive.google.com/drive/folders/1S44fHXaVxAYW3KLNxK41NYCnyX9S79su
# If the full dataset is found, this class could be extended to support full graph/any patched size input.
class SIDD_sRGB_Val_DataLoader(Dataset):
    def __init__(self, path, patch_size=None):
        super(SIDD_sRGB_Val_DataLoader, self).__init__()
        self.noisy_imgs = natsorted(glob(os.path.join(path, 'input', '*.png')))
        self.clean_imgs = natsorted(glob(os.path.join(path, 'target', '*.png')))
        self.length     = len(self.noisy_imgs)
        self.patch_size = patch_size
        self.transform  = torchvision.transforms.ToTensor()
        self.return_name= False

    def __len__(self):
        return self.length

    def __getitem__(self, index):
        index %= self.length

        noisy_patch = np.array(Image.open(self.noisy_imgs[index]))
        clean_patch = np.array(Image.open(self.clean_imgs[index]))

        noisy_patch, clean_patch = self.transform(noisy_patch), \
                                   self.transform(clean_patch)

        if self.patch_size:
            noisy_patch = F.center_crop(noisy_patch, (self.patch_size, self.patch_size))
            clean_patch = F.center_crop(clean_patch, (self.patch_size, self.patch_size))

        filename = os.path.splitext(os.path.split(self.clean_imgs[index])[-1])[0]

        if self.return_name:
            return noisy_patch, clean_patch, filename
        else:
            return noisy_patch, clean_patch

# this class is for SIDD test on sRGB patches stored in .mat format
class SIDD_sRGB_mat_Test_DataLoader(Dataset):
    """
    Args:
        path - the dir to '*.mat' file
    """
    def __init__(self, path):
        super(SIDD_sRGB_mat_Test_DataLoader, self).__init__()
        self.len = 1280
        noisy_patch  = sio.loadmat(os.path.join(path, 'ValidationNoisyBlocksSrgb.mat'))
        target_patch = sio.loadmat(os.path.join(path, 'ValidationGtBlocksSrgb.mat'))
        noisy_array  = np.float32(np.array(noisy_patch['ValidationNoisyBlocksSrgb']))
        assert noisy_array.shape == (40, 32, 256, 256, 3)
        noisy_array  = np.resize(noisy_array / 255., (self.len, 256, 256, 3))
        target_array = np.float32(np.array(target_patch['ValidationGtBlocksSrgb']))
        target_array = np.resize(target_array / 255., (self.len, 256, 256, 3))

        self.transform    = torchvision.transforms.ToTensor()
        self.noise_patch  = noisy_array
        self.target_patch = target_array

    def __getitem__(self, index):
        input_patch = self.noise_patch[index]
        target_patch = self.target_patch[index]

        return self.transform(input_patch), self.transform(target_patch)

    def __len__(self):
        return self.len


#!/usr/bin/env python3

import torch
import argparse

from nnpf.problems import HeatProblem
from nnpf.nn import FFTConvolutionArray
from nnpf.functional import norm, heat_kernel_spatial
from nnpf.utils import get_default_args
from nnpf.trainer import Trainer

__all__ = [
    "HeatArray",
]

class HeatArray(HeatProblem):
    """
    Heat problem model using a discretized kernel.

    Parameters
    ----------
    kernel_size: int or list
        Size of the convolution kernel (a list for 2D and 3D convolutions)
    padding_mode: string
        'zeros', 'reflect', 'replicate' or 'circular'
    bias: bool
        If True, adds a learnable bias to the output.
    init: ['zeros', 'random', 'solution']
        Initialization of the convolution kernel:
        - 'random' for the default from PyTorch.
        - 'zeros' for zero-initialization.
        - 'solution' to initialize with the truncated heat kernel.
    kernel_norms: list of pair (p, weight)
        Compose kernel penalization term as sum of weight * kernel.norm(p).exp(e). Exponent e is defined with kernel_power parameter.
    kernel_power: int
        Power applied to the kernel penalization term (for regularization purpose).

    Examples
    --------

    Training:
    >>> from nnpf.trainer import Trainer
    >>> trainer = Trainer(default_root_dir="logs_doctest", name="HeatArray", version="test0", max_epochs=1, log_every_n_steps=1)
    >>> model = HeatArray(N=64, train_N=10, val_N=20, num_workers=4)
    >>> import contextlib, io
    >>> with contextlib.redirect_stdout(io.StringIO()):
    ...     with contextlib.redirect_stderr(io.StringIO()):
    ...         trainer.fit(model)
    >>> trainer.global_step > 0
    True

    Loading from checkpoint:
    >>> import os
    >>> from nnpf.problems import Problem
    >>> model = Problem.load_from_checkpoint(os.path.join('logs_doctest', 'HeatArray', 'test0'))
    >>> type(model).__name__
    'HeatArray'
    """

    def __init__(self, kernel_size=17, padding_mode='circular', bias=False, init='zeros', kernel_norms=[], kernel_power=2, **kwargs):
        super().__init__(**kwargs)

        # Fix kernel size to match domain dimension
        if isinstance(kernel_size, int):
            kernel_size = [kernel_size]
        else:
            kernel_size = list(kernel_size)
        if len(kernel_size) == 1:
            kernel_size = kernel_size * self.domain.dim

        # Hyper-parameters (used for saving/loading the module)
        self.save_hyperparameters('kernel_size', 'padding_mode', 'bias', 'init', 'kernel_norms', 'kernel_power')

        # Model
        self.model = FFTConvolutionArray(
            self.hparams.kernel_size,
            padding='center',
            padding_mode=self.hparams.padding_mode,
            bias=self.hparams.bias,
        )

        # Kernel initialization (FIXME: ugly)
        if not self.is_loaded:
            if self.hparams.init != 'random':
                with torch.no_grad():
                    self.model.weight[:] *= 0
                    if self.hparams.init == 'solution':
                        self.model.weight[:] += heat_kernel_spatial(self.domain, self.hparams.dt, self.hparams.kernel_size)

    @property
    def weight(self):
        return self.model.weight

    # TODO: auto reshape when no batch nor channel are included
    def forward(self, x):
        return self.model(x)

    def loss(self, output, target):
        dim = tuple(range(2, 2 + self.domain.dim))
        return super().loss(output, target) + sum(
            w * norm(self.model.weight, p, dim).squeeze().pow(self.hparams.kernel_power)
            for p, w in self.hparams.kernel_norms)

    @staticmethod
    def add_model_specific_args(parent_parser, defaults={}):

        from distutils.util import strtobool
        # Parser for loss definition
        def float_or_str(v):
            try:
                return float(v)
            except ValueError:
                return v

        parser = HeatProblem.add_model_specific_args(parent_parser, defaults)
        group = parser.add_argument_group("Heat equation by convolution array", "Options specific to this model")
        group.add_argument('--kernel_size', type=int, nargs='+', help='Size of the kernel (nD)')
        group.add_argument('--padding_mode', choices=['zeros', 'reflect', 'replicate', 'circular'], help="Padding mode for the convolution")
        group.add_argument('--bias', type=lambda s:bool(strtobool(s)), nargs='?', const=True, help="Add a bias to the convolution")
        group.add_argument('--init', choices=['zeros', 'random', 'solution'], help="Initialization of the convolution kernel")
        group.add_argument('--kernel_norms', type=float_or_str, nargs=2, action='append', help="List of (p, weight). Compose the kernel penalization term as sum of weight * kernel.norm(p).pow(e). Exponent e is defined with --kernel_power option.")
        group.add_argument('--kernel_power', type=float, help="Power applied to each penalization term (for regularization purpose)")
        group.set_defaults(**{**get_default_args(HeatArray), **defaults})
        return parser


# PYTHON_ARGCOMPLETE_OK

import argcomplete
import argparse

from . import io
from . import cfg
from . import utils

@utils.logExceptionsAndRaise
def main():
  # show startup message
  io.startup()

  # options used by multiple subparsers
  def addVerboseArg(parser):
    parser.add_argument('-v', '--verbose', action='store_true',
                        help='print more detailed output')
  def addTexFileArg(parser):
    parser.add_argument('tex_file', default='', nargs='?',
                        help='toplevel tex file of the document')

  # setup argparser and global arguments
  p = argparse.ArgumentParser()
  addVerboseArg(p)
  p.add_argument('--version', action='store_true',
                 help='print installed paperman version')

  sub = p.add_subparsers(metavar='subcommands', dest='command')

  # img command
  s = sub.add_parser('img', help='image helper: check if unused or missing '
                                 'images exist or import missing images')
  addTexFileArg(s)
  addVerboseArg(s)
  s.add_argument('-i', '--import', action='store_true',
                 help='try importing missing images from image search path')
  #s.add_argument('-c', '--clean', action='store_true',
  #               help='remove unused images')

  # bib command
  s = sub.add_parser('bib', help='bibliography helper: check if unused or '
                                 'missing bib entries exist or import '
                                 'missing entries')
  addTexFileArg(s)
  addVerboseArg(s)
  s.add_argument('-p', '--print', action='store_true',
                 help='try to find missing citations in existing bibliographies '
                      'and show them in the shell, conflicts with --import, '
                      '--rewrite, and --sort')
  s.add_argument('-i', '--import', action='store_true',
                 help='try importing missing citations from existing '
                      'bibliographies')
  #s.add_argument('-c', '--clean', action='store_true',
  #               help='remove unused citations from bibliography, implies '
  #                    '--rewrite')
  s.add_argument('-r', '--rewrite', action='store_true',
                 help='rewrite and repair entire bibliography')
  s.add_argument('-s', '--sort', nargs='?', default=False,
                 help='sort bibliography, sort order can be one of "date", '
                      '"author" or "key", default is "key", implies --rewrite')

  # input command
  s = sub.add_parser('input',
                     help=r'input helper: check if all \input{} files exist '
                          r'or import files')
  addVerboseArg(s)
  s.add_argument('-i', '--import', action='store_true',
                 help=r'try importing missing \input{} files from input '
                      r'search path')

  # call img, bib and input command with --import flag set
  s = sub.add_parser('import-all',
                     help='shortcut to run img, bib and input subcommands '
                          'with --import option enabled')
  addVerboseArg(s)

  # sort authors command
  s = sub.add_parser('sort-authors',
                     help='alphabetic sorting and formatting of author lists '
                          'for scientific publications and presentations')
  addVerboseArg(s)
  s.add_argument('authors', nargs='+',
                 help='comma separated list of authors seperated. Use -s '
                      'option for a different separator')
  s.add_argument('-s', '--separator', default=',',
                 help='separation character in author list')
  s.add_argument('-f', '--full-name', action='store_true',
                 help='output full author names.')
  s.add_argument('--keep-first', action='store_true',
                 help='keep position of first author in list.')
  s.add_argument('--keep-last', action='store_true',
                 help='keep position of last author in list.')
  s.add_argument('-k', '--keep-first-and-last', action='store_true',
                 help='keep position of first and last author in list.')
  s.add_argument('-q', '--quiet', action='store_true',
                 help='just output the sorted author list, nothing else.')
  s.add_argument('--no-and', action='store_true',
                 help='use the separator instead of "and" before the final '
                      'name and ignore ands in input.')

  # collect bib/pdf pairs from specified collect directories
  s = sub.add_parser('collect',
                     help='rename and move bib-pdf file pairs from specified '
                          'folders to library')
  addVerboseArg(s)
  s.add_argument('paths', nargs='*',
                 help='specify paths to import bib-pdf file pairs from, '
                      'defaults to paths given in config')
  s.add_argument('-w', '--watch', action='store_true',
                 help='never exit and keep watching the collect directories')
  s.add_argument('-e', '--err-to-file', action='store_true',
                 help='write errors not only to stdout, but also to a txt file '
                      'located in the same position as the bib-pdf file pair '
                      'that caused the error')

  # library subcommand
  s = sub.add_parser('lib',
                     help='search library, check library health, detect '
                          'corrupt bib files, duplicates and possibly '
                          'broken pdfs')
  addVerboseArg(s)
  s.add_argument('-f', '--find', nargs='+',
                 help='find library entries that contain given words in '
                      'bibtex fields')
  s.add_argument('-F', '--find-fulltext', nargs='+',
                 help='find library entries that contain given words in '
                      'full manuscript text (requires pdf2txt program to '
                      'to be installed on your $PATH)')
  s.add_argument('-k', '--key', action='store_true',
                 help='print citation key instead of path')
  s.add_argument('-l', '--long', action='store_true',
                 help='print full bibtex entries instead of path only')
  #s.add_argument('-L', '--extra-long', action='store_true',
  #               help='print full pdf2txt result instead of path only')

  # sync subcommand
  s = sub.add_parser('sync',
                     help='sync library pdfs to a tablet or other device')
  addVerboseArg(s)
  s.add_argument('-p', '--sync-path',
                 help='path to the device root that is to be synced.')

  # diff subcommand
  s = sub.add_parser('diff',
                     help='build pdfs that show changes between file versions')
  addVerboseArg(s)
  s.add_argument('-t' , '--old-is-tag', action='store_true',
                 help='do not treat "old" argument as filename, '
                      'but as git tag name (requires git to be installed '
                      'in your $PATH, requires current directory to be '
                      'git repository)')
  s.add_argument('-T' , '--new-is-tag', action='store_true',
                 help='do not treat "new" argument as filename, '
                      'but as git tag name (requires git to be installed '
                      'in your $PATH, requires current directory to be '
                      'git repository)')
  s.add_argument('-o', '--outfile', nargs='?', default='diff.pdf',
                 help='name of the pdf file to be built')
  s.add_argument('-c', '--clean', action='store_true',
                 help='remove all build files except for output pdf')
  s.add_argument('old', help='filename of old version')
  s.add_argument('new', help='filename of new version')
  s.add_argument('filename', nargs='?',
                 help='only used if both --old-is-tag and '
                      '--new-is-tag are set, specifies filename')

  # make-diff subcommand
  s = sub.add_parser('journal',
                     help='find full journal names and abbreviations')
  addVerboseArg(s)
  s.add_argument('-f', '--full', action='store_true',
                 help='print full names of found journals')
  s.add_argument('-a', '--abbreviated', action='store_true',
                 help='print abbreviated names of found journals ')
  s.add_argument('--list-suspicious', action='store_true',
                 help='print all journals in database that might by errornous, '
                      'i.e., abbreviations that do not end with a dot')
  s.add_argument('name', nargs='*',
                 help='journal name, abbreviation or partial name')

  # lint subcommand
  s = sub.add_parser('lint',
                     help='search latex project for potential errors')
  addVerboseArg(s)
  addTexFileArg(s)

  # clean subcommand
  s = sub.add_parser('clean',
                     help='recurse into subfolders of current directory and '
                          'remove latex build files')
  addVerboseArg(s)

  # config subcommand
  s = sub.add_parser('config',
                     help='print location of config file and exit')
  s.add_argument('-o', '--open', action='store_true',
                 help='open config file in vim')
  addVerboseArg(s)

  # enable autocompletion and parse args
  argcomplete.autocomplete(p)
  args = p.parse_args()

  # apply verbosity setting
  io.isVerbose = args.verbose

  # if version is requested, print version and exit
  if args.version:
    io.info(f'paperman version {io.__version__}')
    return

  # select submodule for subcommands
  cmd, cmds = None, None
  if not args.command:
    io.err('subcommand is required')
    return

  elif args.command == 'img':
    from .subcommands import img as cmd

  elif args.command == 'bib':
    from .subcommands import bib as cmd

  elif args.command == 'input':
    from .subcommands import inp as cmd

  elif args.command == 'import-all':
    from .subcommands import inp as cmd1
    from .subcommands import img as cmd2
    from .subcommands import bib as cmd3
    setattr(args, 'import', True)
    args.clean = False
    args.rewrite = False
    args.sort = False
    args.print = False
    cmds = [cmd1, cmd2, cmd3]

  elif args.command == 'sort-authors':
    from .subcommands import sort_authors as cmd

  elif args.command == 'collect':
    from .subcommands import collect as cmd

  elif args.command == 'lib':
    from .subcommands import lib as cmd

  elif args.command == 'sync':
    from .subcommands import sync as cmd

  elif args.command == 'diff':
    from .subcommands import diff as cmd

  elif args.command == 'journal':
    from .subcommands import journal as cmd

  elif args.command == 'lint':
    from .subcommands import lint as cmd

  elif args.command == 'clean':
    from .subcommands import clean as cmd

  elif args.command == 'config':
    from .subcommands import config as cmd

  else:
    io.dbg(f'args={args}')
    raise ValueError(f'args.command has unexpected value')

  # raise error if required config keys are missing
  cfg.testIfRequiredExist()

  # run subcommand module
  cmdStr = str(cmd or '\n'+'\n'.join([str(c) for c in cmds]))
  io.verb(f'selected subcommand(s): {cmdStr}')
  try:
    if cmd:
      cmds = [cmd]
    if cmds:
      for cmd in cmds:
        cmd.main(args)
  except KeyboardInterrupt:
    raise
  except RuntimeError as e:
    io.err(str(e))
  except Exception as e:
    if cfg.get('debug'):
      raise
    io.err(str(e))

if __name__ == '__main__':
  main()


#!/usr/bin/env python3.3

"""The main socrates script (i.e., the one users should invoke from
the command line).
"""

import os
import sys
import datetime

import cmdline
import util
import criteria
import config
import hooks

# conditions that are deemed "okay" to be returned by a subprocess when
# socrates is run in batch mode
OKAY_CONDITIONS = [util.EXIT_WITH_MISSING, util.ERR_GRADE_FILE_EXISTS,
                   util.EXIT_WITH_DEFER]

CRITERIA_FILE_PATTERN = r'^[a-z]+\d+[a-z]$'

def main(args):
    """The function invoked when socrates starts from the command line."""
    if args.mode == 'config':
        _config()

    if args.mode == 'edit':
        _edit(args)

    if args.mode in ['grade', 'submit', 'batch']:
        try:
            sname, group = _parse_assignment_name(args.assignment_with_group)
        except ValueError as err:
            util.error(str(err))
            util.exit(util.ERR_BAD_ASSIGNMENT, traceback=False)

        criteria_path = _form_criteria_path(sname, group)

        if not os.path.isfile(criteria_path):
            message = "could not find criteria file for {}"
            message = message.format(sname)

            if group:
                message += ", group " + group

            util.error(message)
            util.exit(util.ERR_CRITERIA_MISSING)

        criteria_object = _create_criteria_object(criteria_path)

        assignment_name, group = criteria_object.name, criteria_object.group
        grade_filename = assignment_name
        if group:
            grade_filename += group
        grade_filename += '-grade.txt'

        if args.mode == 'grade':
            _grade(args, criteria_object, grade_filename)

        elif args.mode == 'submit':
            _submit(args, criteria_object, grade_filename)

        elif args.mode == 'batch':
            _batch(args)

def _config():
    """Handles 'config' mode."""
    from inspect import getmembers, ismodule

    for member_name, member_val in getmembers(config):
        if member_name[0] != '_' and not ismodule(member_val):
            util.print("{}: {}".format(member_name, member_val))

def _edit(args):
    """Handles 'edit' mode. This function creates a temporary file and
    tries to use the user's default editor to edit the specified criteria
    file. If the new file has a syntax error or is missing essential
    components, the user is prompted to edit again or stop.
    """
    import tempfile
    import shutil
    import filecmp
    import subprocess

    import criteria
    from prompt import prompt

    isfile = os.path.isfile

    try:
        short_name, group = _parse_assignment_name(args.assignment_with_group)
    except ValueError as err:
        util.error(str(err))
        util.exit(util.ERR_BAD_ASSIGNMENT, traceback=False)

    existing_path = _form_criteria_path(short_name, group)

    if not isfile(existing_path):
        util.warning("the file '{}' does not exist".format(existing_path))

        choices = ["create and edit this criteria file now",
                   "exit without creating anything"]

        selections = prompt(choices, mode='1')
        if 1 in selections:
            util.info("exiting without changing anything")
            return

    temp_file, temp_path = tempfile.mkstemp(suffix='.yml')

    # copy contents of existing file into temp file
    if isfile(existing_path):
        shutil.copyfile(existing_path, temp_path)

    while True:
        # try to open editor with file
        _edit_file(temp_path)

        try:
            _ = criteria.Criteria.from_yaml(temp_path)

        except Exception as e:
            util.error("proposed criteria file produced an error")
            util.print_traceback()

            choices = ["return to the editor to fix the problem",
                       "abort the change (existing file will not change)"]

            selections = prompt(choices, mode='1')

            if 0 in selections:
                continue
            else:
                util.info("exiting without changing anything")
                break

        else:
            # temporary file contains valid, new version of the criteria file;
            # if the temporary file is any different, we will copy it over
            # the old file

            if isfile(existing_path) and \
               filecmp.cmp(temp_path, existing_path, shallow=False):
                # files seem to be the same
                util.info("no changes made; exiting without changing original")
                os.remove(temp_path)

            else:
                # files are different; copy over the old file
                shutil.copyfile(temp_path, existing_path)

                # remove temporary file
                os.remove(temp_path)

                util.info("{} updated".format(existing_path))

            break

def _grade(args, criteria_object, grade_filename):
    """Handles 'grade' mode. Most of the work is handed off
    to the 'grader' module, which runs interactively.
    """
    import grader
    from prompt import prompt

    if not args.overwrite and os.path.isfile(grade_filename):
        util.error("refusing to overwrite existing grade file")
        util.exit(util.ERR_GRADE_FILE_EXISTS, hooks=False)

    util.ui_start()

    if not args.submission_files:
        util.warning("no submission files specified")

    hooks.run_hooks_for('before_file_search')

    any_missing = grader.grade(criteria_object, args.submission_files,
                               grade_filename,
                               assume_missing=args.assume_missing,
                               late_check=False if args.no_late else True)

    if not args.no_edit:
        util.info("please review the following grade file ({}) "
                  "for issues".format(grade_filename))

        with open(grade_filename) as f:
            util.print(f.read())

        choices = ["edit the grade file now", "do not edit the grade file"]
        selections = prompt(choices, mode='1')

        if 0 in selections:
            _edit_file(grade_filename)

    hooks.run_hooks_for('before_exit')

    util.ui_stop()

    util.info("grading completed")

    if any_missing:
        util.exit(util.EXIT_WITH_MISSING)

def _submit(args, criteria_object, grade_filename, umask=0o002):
    """Handles 'submit' mode. Allows a grader to send completed grade files
    to the "dropbox" directory.
    """
    import shutil

    os.umask(umask)

    num_submitted = 0

    # args.submission_dirs should contain a list of directories whose names
    # are student user names
    for username in args.submission_dirs:
        if username[-1] == os.sep:
            username = username[:-1]            # removes ending '/'

        # check if specified file is a directory
        if not os.path.isdir(username):
            util.error("'{}' is not a directory".format(username))
            continue

        grade_file_path = username + os.sep + grade_filename
        if not os.path.isfile(grade_file_path):
            util.error("not submitting '{}': directory has no "
                       "grade file".format(username))
            continue

        # TODO check if directory name is a valid student name

        dest_path = config.dropbox_dir + os.sep + username

        # if this is the first time this user's grades are being submitted,
        # we may need to create the directory in the dropbox
        try:
            os.umask(0o000)
            util.makedirs(dest_path)
        except OSError:
            util.error("error making user directory in dropbox")
            util.exit(util.ERR_DROPBOX_MAKEDIRS)

        dest_path += os.sep + grade_filename
        shutil.copyfile(grade_file_path, dest_path)
        os.chmod(dest_path, 0o666)

        num_submitted += 1

        util.info("wrote '{}' to dropbox".format(dest_path))

    util.info("submitted {} {}".format(num_submitted,
                                       util.plural('grade', num_submitted)))

def _batch(args):
    """Handles 'batch' mode."""
    import inspect
    import subprocess

    # absolute path of the current running Python script
    proc = os.path.abspath(inspect.getfile(inspect.currentframe()))

    if not args.submission_dirs:
        util.warning("no submissions specified")

    for subdir in args.submission_dirs:
        if not os.path.isdir(subdir):
            util.error("invalid submission directory '{}'".format(subdir))
            continue

        os.chdir(subdir)

        # this will simulate a user executing socrates grade * at a shell
        files_here = os.listdir(os.curdir)

        util.info("running socrates in '{}'".format(subdir))

        sub_args = [proc, 'grade']

        if args.no_edit:
            sub_args.append("--no-edit")

        if args.assume_missing:
            sub_args.append("--assume-missing")

        if args.no_late:
            sub_args.append("--no-late")

        sub_args.append(args.assignment_with_group)

        try:
            return_val = subprocess.call(sub_args + files_here)

        except KeyboardInterrupt:
            util.warning("parent stopping (received interrupt)")
            util.warning("stopped while grading '{}'".format(subdir))
            os.chdir(os.pardir)
            util.exit(util.ERR_INTERRUPTED)

        if return_val != 0 and return_val not in OKAY_CONDITIONS:
            util.error("child process encountered an error")
            os.chdir(os.pardir)
            util.exit(return_val)

        util.info("completed subdirectory '{}'".format(subdir))
        os.chdir(os.pardir)

def _parse_assignment_name(short_name_with_group):
    """Given a short assignment name with a group (e.g., "ps4a"),
    return the assignment's short name ("ps4") and the group
    ("a") in a tuple.
    """
    from re import match

    sng = short_name_with_group

    m = match(CRITERIA_FILE_PATTERN, sng)
    if not m:
        raise ValueError("invalid assignment name '{}'".format(sng))

    return short_name_with_group[:-1], short_name_with_group[-1]

def _form_criteria_path(short_name, group):
    """Given an assignment's short name (e.g., "ps4"), and optionally a
    group, use the configuration file data to form a path to the appropriate
    criteria file for the assignment. The path returned by this function
    starts with config.criteria_dir, which is given by the socrates
    configuration file.
    """
    criteria_path = config.criteria_dir + os.sep + short_name
    if group:
        criteria_path += os.sep + short_name + group + '.yml'
    else:
        criteria_path += '.yml'

    return criteria_path

def _create_criteria_object(criteria_path):
    """Given a path to a valid criteria file, return a Criteria object
    using that criteria file as a source. This function relies on the
    'criteria' module and its Criteria class. Calling that class'
    constructor is pretty costly.
    """
    try:
        criteria_object = criteria.Criteria.from_yaml(criteria_path)

    except FileNotFoundError:
        util.error("criteria file does not exist")
        util.exit(util.ERR_CRITERIA_MISSING)

    except IsADirectoryError:
        util.error("specified criteria is a directory")
        util.exit(util.ERR_CRITERIA_IMPORT)

    return criteria_object

def _get_dropbox_path(args):
    """Given valid arguments from a user, use the configuration file data
    and the assignment specified on the command line to form a path to the
    grading "dropbox", where final grade files should be saved. The path
    returned by this function starts with config.dropbox_dir, which is given
    by the socrates configuration file.
    """
    short_name = args.assignment[0]
    dropbox_path = config.dropbox_dir + os.sep + short_name

    if not os.path.isdir(dropbox_path):
        util.error("no dropbox for assignment '{}'".format(short_name))
        util.exit(util.ERR_BAD_DROPBOX)

    return dropbox_path

def _edit_file(path):
    """Attempts to consult the $EDITOR environment variable for the user's
    favorite text editor (otherwise trying to use vi) to edit the file
    specfied by the given path. socrates will wait for the editor to exit
    before continuing.
    """
    import os
    import subprocess
    import shutil

    if 'EDITOR' in os.environ:
        # run $EDITOR with the grade file
        subprocess.call([os.environ['EDITOR'], path])

    elif shutil.which('nano') is not None:
        # try to run nano
        subprocess.call(['nano', path])
        util.info("your $EDITOR environment variable is not set")
        util.info("if you set $EDITOR, socrates will use that editor")

    else:
        util.error("could not open your favorite text editor or nano")
        util.exit(util.ERR_NO_EDITOR)

if __name__ == '__main__':
    main(cmdline.get_args())


import time
from datetime import datetime, timedelta
from urllib.parse import urlparse
from typing import Callable, Dict
import pandas as pd
import requests
from bs4 import BeautifulSoup
from lib.helper import configs, setup_logger

logger = setup_logger(' SCRAPER:PY ', 'lib/log.xml', level="INFO")
header = configs.header
timeout = configs.timeout

def get_request(url, proxies=None):
    try:
        response = requests.get(url, headers=header, timeout=timeout, proxies=proxies)  # TODO Proxy ayarları eklenecek
        response.raise_for_status()
        return response

    except requests.exceptions.HTTPError as errh:
        logger.error("HTTP Hatası: %s. Proxy: %s", errh, proxies)
        raise errh
    except requests.exceptions.ConnectionError as errc:
        logger.error("Bağlantı Hatası: %s.  Proxy: %s", errc, proxies)
        raise errc
    except requests.exceptions.Timeout as errt:
        logger.error("Zaman Aşımı Hatası: %s.  Proxy: %s", errt, proxies)
        raise errt
    except requests.exceptions.RequestException as err:
        logger.error("Bir Hata Oluştu: %s.  Proxy: %s", err, proxies)
        raise err

def amazon(url):
    response = get_request(url)
    try:

        runtime = datetime.now()
        soup = BeautifulSoup(response.text, 'html.parser')
        title = soup.select_one('#productTitle').getText().strip()
        try:
            price = soup.select_one(
                '#corePriceDisplay_desktop_feature_div > div.a-section.a-spacing-none.aok-align-center > span > span:nth-child(2)').getText()
        except:
            price = soup.find(id="price").getText()

        ean = soup.find(id="ASIN").get("value")
        stock = False if soup.find('input', value="Add to Basket") == None else True

        results = {"Product ID": "", "Runtime DateTime": runtime, "Product Title": title, "Price": price,
                   "EAN": ean,
                   "Stock Availability": stock}

        return results
    except (AttributeError, TypeError, ValueError) as err:
        logger.error("Amazon BeautifulSoup Hatası: %s. URL: %s", err, url)
        raise
    except Exception as e:
        raise e

def argos(url):
    response = get_request(url)
    try:

        soup = BeautifulSoup(response.text, 'html.parser')
        runtime = datetime.now()
        title = soup.select_one(
            '#content > main > div:nth-child(2) > div.xs-block > div:nth-child(1) > section.xs-12--none.md-7.xl-8.pdp-core.bolt-v2 > div.Namestyles__ProductName-sc-269llv-0.kEQsqD.bolt-v2 > h1 > span:nth-child(1)'
        ).getText().strip()

        price = soup.select_one(
            '#content > main > div:nth-child(2) > div.xs-block > div:nth-child(1) > section.xs-12--none.md-5--none.xl-4--none.pdp-right > section > ul > li > h2'
        ).getText()
        ean = soup.select_one(
            '#content > main > div:nth-child(2) > div.xs-block > div:nth-child(1) > section.xs-12--none.md-7.xl-8.pdp-core.bolt-v2 > div.Namestyles__ProductName-sc-269llv-0.kEQsqD.bolt-v2 > h1 > span.Namestyles__CatNumber-sc-269llv-2.lbKeLk'
        ).getText()

        stock = False if soup.find('button', {"data-test": "add-to-trolley-button-button"}) == None else True

        results = {"Product ID": "", "Runtime DateTime": runtime, "Product Title": title, "Price": price,
                   "EAN": ean,
                   "Stock Availability": stock}

        return results
    except (AttributeError, TypeError, ValueError) as err:
        logger.error("Argos BeautifulSoup Hatası: %s. URL: %s", err, url)
        raise
    except Exception:
        pass

def bestwaywholesale(url):
    response = get_request(url)
    try:

        soup = BeautifulSoup(response.text, 'html.parser')
        runtime = datetime.now()
        title = soup.select_one(
            '#shop-products > div.productpagedetail-inner > div.right > h1'
        ).getText().strip()
        table = soup.find('table', {'class': 'prodtable'})
        price = ""
        ean = ""
        for tr in table.find_all('tr'):
            th = tr.find('th').text
            td = tr.find('td').text
            if ("RSP" in th):
                price = td
            if ("EAN" in th):
                ean = td

        stock = False if soup.find('span', {"class": "must"}) == None else True
        results = {"Product ID": "", "Runtime DateTime": runtime, "Product Title": title, "Price": price,
                   "EAN": ean, "Stock Availability": stock}

        return results
    except (AttributeError, TypeError, ValueError) as err:
        logger.error("Bestwaywholesale BeautifulSoup Hatası: %s. URL: %s", err, url)
        raise
    except Exception:
        raise

def read_data():
    # Son bir saat içindeki okunmamış dataları DataFrame olarak döndürür.
    try:
        df = pd.read_csv(configs.input_file)
    except Exception as e:
        logger.error("İnput file okuma hatası ERR: {}".format(e))
        raise e
    else:
        df.dropna(subset=['Products'], inplace=True)
        try:
            df['LastRead'] = pd.to_datetime(df['LastRead'])  # TODO Dt atama işlemi gereksiz olabilir bak!
            df['LastScrape'] = pd.to_datetime(df['LastScrape'])
        except Exception as e:
            logger.error("İnput file sütun isimleri hatalı ERR: {}".format(e))
            raise e
        return df

scrapers: Dict[str, Callable[[str], dict]] = {
    "www.amazon.co.uk": amazon,
    "www.argos.co.uk": argos,
    "www.bestwaywholesale.co.uk": bestwaywholesale

}

def run_scraper():
    try:
        df = read_data()
        last_read = datetime.now() - timedelta(minutes=configs.run_period)
        selected_product_df = df[df['LastRead'] < last_read][:configs.max_product]
        selected_product_df['LastRead'] = datetime.now()

        scraped_product_list = []

        if selected_product_df is not None:
            for index, row in selected_product_df.iterrows():

                for url in row['Products'].split(';'):  # TODO Loglama ve hata handle
                    try:
                        scraper = scrapers.get(urlparse(url).netloc)
                        print(url)

                        result = scraper(url)
                        if result is not None:
                            dict_data = result
                            dict_data['Product ID'] = str(index)
                            dict_data['Runtime DateTime'] = str(datetime.now())
                            scraped_product_list.append(dict_data)

                            time.sleep(configs.frequency)
                    except TypeError:
                        logger.error("Scraper Bulunamadı : {} ".format(url))
                    except Exception as e:
                        # logger.error(e)
                        pass
                selected_product_df.at[
                    index, 'LastScrape'] = datetime.now()  # TODO işlem başarıyla tamamlanınca değişmeli
            df.update(selected_product_df)
            df.to_csv(configs.input_file, header=True, mode='w', index=False)

            output_data = pd.DataFrame(scraped_product_list,
                                       columns=["Product ID", "Runtime DateTime", "Product Title", "Price",
                                                "EAN", "Stock Availability"])
            output_data.to_csv(configs.output_file, header=False, mode='a', index=False)

    except Exception as e:
        logger.error(e)
        # raise e

if __name__ == "__main__":
    while True:
        run_scraper()
        print("Scraper çalışıyor : ", datetime.now())
        time.sleep(5)


import math
import logging

import numpy as np
from scipy import integrate, interpolate, constants

from hyperspy.misc.eels.base_gos import GOSBase

_logger = logging.getLogger(__name__)

XU = [
    .82, .52, .52, .42, .30, .29, .22, .30, .22, .16, .12, .13, .13, .14, .16,
    .18, .19, .22, .14, .11, .12, .12, .12, .10, .10, .10]
# IE3=[73,99,135,164,200,245,294,347,402,455,513,575,641,710,
# 779,855,931,1021,1115,1217,1323,1436,1550,1675]

# IE1=[118,149,189,229,270,320,377,438,500,564,628,695,769,846,
# 926,1008,1096,1194,1142,1248,1359,1476,1596,1727]

R = constants.value("Rydberg constant times hc in eV")

class HydrogenicGOS(GOSBase):

    """Computes the K and L GOS using R. Egerton's  routines.

    Parameters
    ----------
    element_subshell : str
        For example, 'Ti_L3' for the GOS of the titanium L3 subshell

    Methods
    -------
    parametrize_GOS()
        Parametrize the GOS to speed up the calculation.
    get_qaxis_and_gos(ienergy, qmin, qmax)
        Given the energy axis index and qmin and qmax values returns
        the qaxis and gos between qmin and qmax using linear
        interpolation to include qmin and qmax in the range.

    Attributes
    ----------
    energy_axis : array
        The tabulated energy axis
    qaxis : array
        The tabulated qaxis
    energy_onset: float
        The energy onset for the given element subshell as obtained
        from iternal tables.

    Notes
    -----
    The Hydrogeninc GOS are calculated using R. Egerton's SIGMAK3 and
    SIGMAL3 routines that has been translated from Matlab to Python by
    I. Iyengar. See http://www.tem-eels.ca/ for the original code.

    """
    _name = 'hydrogenic'

    def __init__(self, element_subshell):
        """
        Parameters
        ----------

        element_subshell : str
            For example, 'Ti_L3' for the GOS of the titanium L3 subshell

        """
        # Check if the Peter Rez's Hartree Slater GOS distributed by
        # Gatan are available. Otherwise exit

        self.element, self.subshell = element_subshell.split('_')
        self.read_elements()
        self.energy_shift = 0

        if self.subshell[:1] == 'K':
            self.gosfunc = self.gosfuncK
            self.rel_energy_axis = self.get_parametrized_energy_axis(
                50, 3, 50)
        elif self.subshell[:1] == 'L':
            self.gosfunc = self.gosfuncL
            self.onset_energy_L3 = self.element_dict['Atomic_properties'][
                'Binding_energies']['L3']['onset_energy (eV)']
            self.onset_energy_L1 = self.element_dict['Atomic_properties'][
                'Binding_energies']['L1']['onset_energy (eV)']
            self.onset_energy = self.onset_energy_L3
            relative_axis = self.get_parametrized_energy_axis(
                50, 3, 50)
            dL3L2 = self.onset_energy_L1 - self.onset_energy_L3
            self.rel_energy_axis = np.hstack((
                relative_axis[:relative_axis.searchsorted(dL3L2)],
                relative_axis + dL3L2))
        else:
            raise ValueError(
                'The Hydrogenic GOS currently can only'
                'compute K or L shells. Try using Hartree-Slater GOS')

        self.energy_axis = self.rel_energy_axis + self.onset_energy

        info_str = (
            "\nHydrogenic GOS\n" +
            ("\tElement: %s " % self.element) +
            ("\tSubshell: %s " % self.subshell) +
            ("\tOnset Energy = %s " % self.onset_energy))
        _logger.info(info_str)

    def integrateq(self, onset_energy, angle, E0):
        energy_shift = onset_energy - self.onset_energy
        self.energy_shift = energy_shift
        gamma = 1 + E0 / 511.06
        T = 511060 * (1 - 1 / gamma ** 2) / 2
        qint = np.zeros((self.energy_axis.shape[0]))
        for i, E in enumerate(self.energy_axis + energy_shift):
            qa0sqmin = (E ** 2) / (4 * R * T) + (E ** 3) / (
                8 * gamma ** 3 * R * T ** 2)
            p02 = T / (R * (1 - 2 * T / 511060))
            pp2 = p02 - E / R * (gamma - E / 1022120)
            qa0sqmax = qa0sqmin + 4 * np.sqrt(p02 * pp2) * \
                (math.sin(angle / 2)) ** 2

            # dsbyde IS THE ENERGY-DIFFERENTIAL X-SECN (barn/eV/atom)
            qint[i] = 3.5166e8 * (R / T) * (R / E) * (
                integrate.quad(
                    lambda x: self.gosfunc(E, np.exp(x)),
                    math.log(qa0sqmin), math.log(qa0sqmax))[0])
        self.qint = qint
        return interpolate.interp1d(self.energy_axis + energy_shift, qint)

    def gosfuncK(self, E, qa02):
        # gosfunc calculates (=DF/DE) which IS PER EV AND PER ATOM
        z = self.Z
        r = 13.606
        zs = 1.0
        rnk = 1
        if z != 1:
            zs = z - 0.5
            rnk = 2

        q = qa02 / zs ** 2
        kh2 = E / (r * zs ** 2) - 1
        akh = np.sqrt(abs(kh2))
        if akh < 0.01:
            akh = 0.01
        if kh2 >= 0.0:
            d = 1 - np.e ** (-2 * np.pi / akh)
            bp = np.arctan(2 * akh / (q - kh2 + 1))
            if bp < 0:
                bp = bp + np.pi
            c = np.e ** ((-2 / akh) * bp)
        else:
            d = 1
            y = -1 / akh * np.log((q + 1 - kh2 + 2 * akh) / (
                q + 1 - kh2 - 2 * akh))
            c = np.e ** y
        a = ((q - kh2 + 1) ** 2 + 4 * kh2) ** 3
        return 128 * rnk * E / (
            r * zs ** 4) * c / d * (q + kh2 / 3 + 1 / 3) / (a * r)

    def gosfuncL(self, E, qa02):
        # gosfunc calculates (=DF/DE) which IS PER EV AND PER ATOM
        # Note: quad function only works with qa02 due to IF statements in
        # function

        z = self.Z
        r = 13.606
        zs = z - 0.35 * (8 - 1) - 1.7
        iz = z - 11
        if iz >= len(XU):
            # Egerton does not tabulate the correction for Z>36.
            # This produces XSs that are within 10% of Hartree-Slater XSs
            # for these elements.
            u = .1
        else:
            # Egerton's correction to the Hydrogenic XS
            u = XU[int(iz)]
        el3 = self.onset_energy_L3 + self.energy_shift
        el1 = self.onset_energy_L1 + self.energy_shift

        q = qa02 / zs ** 2
        kh2 = E / (r * zs ** 2) - 0.25
        akh = np.sqrt(abs(kh2))
        if kh2 >= 0.0:
            d = 1 - np.exp(-2 * np.pi / akh)
            bp = np.arctan(akh / (q - kh2 + 0.25))
            if bp < 0:
                bp = bp + np.pi
            c = np.exp((-2 / akh) * bp)
        else:
            d = 1
            y = -1 / akh * \
                np.log((q + 0.25 - kh2 + akh) / (q + 0.25 - kh2 - akh))
            c = np.exp(y)

        if E - el1 <= 0:
            g = 2.25 * q ** 4 - (0.75 + 3 * kh2) * q ** 3 + (
                0.59375 - 0.75 * kh2 - 0.5 * kh2 ** 2) * q * q + (
                0.11146 + 0.85417 * kh2 + 1.8833 * kh2 * kh2 + kh2 ** 3) * \
                q + 0.0035807 + kh2 / 21.333 + kh2 * kh2 / 4.5714 + kh2 ** 3 \
                / 2.4 + kh2 ** 4 / 4

            a = ((q - kh2 + 0.25) ** 2 + kh2) ** 5
        else:
            g = q ** 3 - (5 / 3 * kh2 + 11 / 12) * q ** 2 + (
                kh2 * kh2 / 3 + 1.5 * kh2 + 65 / 48) * q + kh2 ** 3 / 3 + \
                0.75 * kh2 * kh2 + 23 / 48 * kh2 + 5 / 64
            a = ((q - kh2 + 0.25) ** 2 + kh2) ** 4
        rf = ((E + 0.1 - el3) / 1.8 / z / z) ** u
        # The following commented lines are to give a more accurate GOS
        # for edges presenting white lines. However, this is not relevant
        # for quantification by curve fitting.
        # if abs(iz - 11) <= 5 and E - el3 <= 20:
        #     rf = 1
        return rf * 32 * g * c / a / d * E / r / r / zs ** 4


#
# @lc app=leetcode.cn id=154 lang=python3
#
# [154] 寻找旋转排序数组中的最小值 II
#
from typing import List
import unittest
# @lc code=start

class Solution:
    def findMin(self, nums: List[int]) -> int:
        left, right = 0, len(nums)-1
        while left < right:
            mid = (left + right) // 2
            if nums[mid] > nums[right]:
                left = mid + 1
            elif nums[mid] < nums[right]:
                right = mid
            else:  # nums[mid] == nums[right]
                right -= 1

        return nums[left]

# @lc code=end

class Test(unittest.TestCase):
    def test(self):
        def f(arr, ans):
            self.assertEqual(Solution().findMin(arr), ans)
        f([3, 4, 5, 1, 2], 1)
        f([2, 2, 2, 0, 1], 0)
        f([1], 1)
        f([1, 1], 1)
        f([1, 1, 1], 1)
        f([1, 1, 1, 1], 1)
        f([1, 1, 2, 1, 1], 1)
        f([1, 1, 0, 1, 1], 0)
        f([1, 2, 3, 4, 5], 1)
        f([0, 0, 0, -1, 0], -1)
        f([3, 1], 1)
        f([5, 1, 1, 1, 1, 1, 1], 1)

if __name__ == "__main__":
    unittest.main()

154.寻找旋转排序数组中的最小值-ii.py


import serial.tools.list_ports as port_list
import serial
from time import sleep

def idn_scan(dump_yml=False, baud_rates=(9600,)):
    """
    Sends "*IDN?" command to all COM ports where hardware is present.
    Commands are sent blindly with EOL (\r\n) termination and at baud_rates specified.
    Parameters
    ----------
    dump_yml - enables (True) or disables (False) dumping of .yaml file containing the mapping of
                IDN string to COM and BAUD
    baud_rates - tuple of baud rates at which the IDN query sequence will be repeated

    Returns - idn_to_com dictionary mapping IDN string (key) to a dictionary of (PORT) and (BAUD)
    This dictionary is also output to a .yaml file if dump_yaml argument is set to true.
    -------

    """
    TIMEOUT = 0.5  # If a large number of serial devices are present you might wish to shorten the timeout time to
    # speed up the query rounds however this may be at the cost of some devices not having enough time to respond

    idn_to_com = {}

    for baud in baud_rates:
        print(f'\n<NOTE>: *IDN? requests are sent blindly as:\n ASCII, EOL terminated,\n baudrate={str(baud)}, '
              f'bytesize=EIGHTBITS, \n parity=PARITY_NONE, stopbits=STOPBITS_ONE \n Device may not respond with IDN '
              f'if:\n  -device uses other baudrate or encoding, \n  -device is non SCPI compliant.\n\n')

        for port, desc, hwid in port_list.comports():

            try:
                ser = serial.Serial(port, timeout=TIMEOUT, baudrate=baud, writeTimeout=0)
                sleep(0.5)
                ser.write(b'*IDN?\r\n')
                idn = ser.readline()
                print(f'PORT: {port}\n IDN: {idn}\n DESC: {desc} \n HWID: {hwid}\n')
                ser.close()
                if idn != b'':
                    idn_to_com.update({idn.strip(): {'PORT': port, 'BAUD': baud}})
            except serial.SerialException as exc:
                print(f'PORT: {port}\n IDN: ?\n DESC: {exc} \n HWID: ?\n')

    if dump_yml:
        import yaml
        with open('idn_to_com.yml', 'w') as f:
            f.write(yaml.dump(idn_to_com, default_flow_style=False))

    return idn_to_com

if __name__ == "__main__":
    import sys

    if len(sys.argv) == 1:
        idn_scan()
    elif len(sys.argv) == 2:
        idn_scan(dump_yml=(sys.argv[1] == 'dump_yml=True'))
    else:
        idn_scan(dump_yml=(sys.argv[1] == 'dump_yml=True'), baud_rates=tuple(sys.argv[2:]))