text:
export CircularArrayBuffer, capacity, isfull, consecutive_view, select_frame

mutable struct CircularArrayBuffer{T,N} <: AbstractArray{T,N}
    buffer::Array{T,N}
    first::Int
    length::Int
    step_size::Int

    function CircularArrayBuffer{T}(d::Integer...) where {T}
        N = length(d)
        N > 0 || throw(ArgumentError("dimension must be greater than 0"))
        new{T,N}(Array{T}(undef, d...), 1, 0, N == 1 ? 1 : *(d[1:end-1]...))
    end
end

Base.IndexStyle(::CircularArrayBuffer) = IndexLinear()
Base.size(cb::CircularArrayBuffer{T,N}) where {T,N} = (size(cb.buffer)[1:N-1]..., cb.length)
Base.getindex(cb::CircularArrayBuffer{T, N}, i::Int) where {T, N} = getindex(cb.buffer, _buffer_index(cb, i))
Base.setindex!(cb::CircularArrayBuffer{T, N}, v, i::Int) where {T, N} = setindex!(cb.buffer, v, _buffer_index(cb, i))
capacity(cb::CircularArrayBuffer{T, N}) where {T, N} = size(cb.buffer, N)
isfull(cb::CircularArrayBuffer) = cb.length == capacity(cb)
Base.isempty(cb::CircularArrayBuffer) = cb.length == 0

@inline function _buffer_index(cb::CircularArrayBuffer, i::Int)
    ind = (cb.first - 1) * cb.step_size + i
    if ind > length(cb.buffer)
        ind - length(cb.buffer)
    else
        ind
    end
end

@inline function _buffer_frame(cb::CircularArrayBuffer, i::Int)
    n = capacity(cb)
    idx = cb.first + i - 1
    if idx > n
        idx - n
    else
        idx
    end
end

_buffer_frame(cb::CircularArrayBuffer, I::Vector{Int}) = map(i -> _buffer_frame(cb, i), I)

function Base.empty!(cb::CircularArrayBuffer)
    cb.first = 1
    cb.length = 0
    cb
end

function Base.push!(cb::CircularArrayBuffer{T,N}, data) where {T,N}
    # length(data) == cb.step_size || throw(ArgumentError("length of , $(cb.step_size) != $(length(data))"))
    if cb.length == capacity(cb)
        cb.first = (cb.first == capacity(cb) ? 1 : cb.first + 1)
    else
        cb.length += 1
    end
    selectdim(cb.buffer, N, _buffer_frame(cb, cb.length)) .= data
    cb
end

Base.push!(cb::CircularArrayBuffer{T, N}, data::CircularArrayBuffer{T, N}) where {T, N} = push!(cb, select_frame(data, data.length))

select_frame(cb::CircularArrayBuffer{T, 1}, i) where {T} = getindex(cb.buffer, _buffer_frame(cb, i))
select_frame(cb::CircularArrayBuffer{T, 2}, i) where {T} = view(cb.buffer, :, _buffer_frame(cb, i))
select_frame(cb::CircularArrayBuffer{T, 3}, i) where {T} = view(cb.buffer, :, :, _buffer_frame(cb, i))
select_frame(cb::CircularArrayBuffer{T, 4}, i) where {T} = view(cb.buffer, :, :, :, _buffer_frame(cb, i))

select_frame(cb::Array{T, 1}, i::Int) where {T} = getindex(cb, i)
select_frame(cb::Array{T, N}, i::Int) where {T, N} = selectdim(cb, N, i)

consecutive_view(cb::CircularArrayBuffer, inds::Vector{Int}) = select_frame(cb, inds)

consecutive_view(cb::CircularArrayBuffer{T,1}, inds::Vector{Int}, n::Int) where {T} = reshape(view(cb.buffer, [_buffer_frame(cb, i) for x in inds for i in x:x+n-1]), n, length(inds))
consecutive_view(cb::CircularArrayBuffer{T,2}, inds::Vector{Int}, n::Int) where {T} = reshape(view(cb.buffer, :, [_buffer_frame(cb, i) for x in inds for i in x:x+n-1]), size(cb.buffer, 1), n, length(inds))
consecutive_view(cb::CircularArrayBuffer{T,3}, inds::Vector{Int}, n::Int) where {T} = reshape(view(cb.buffer, :, :, [_buffer_frame(cb, i) for x in inds for i in x:x+n-1]), size(cb.buffer, 1), size(cb.buffer, 2), n, length(inds))
consecutive_view(cb::CircularArrayBuffer{T,4}, inds::Vector{Int}, n::Int) where {T} = reshape(view(cb.buffer, :, :, :, [_buffer_frame(cb, i) for x in inds for i in x:x+n-1]), size(cb.buffer, 1), size(cb.buffer, 2), size(cb.buffer, 3), n, length(inds))

consecutive_view(cb::CircularArrayBuffer{T,1}, inds::Vector{Int}, n::Int, c::Int) where {T} = reshape(view(cb.buffer, [_buffer_frame(cb, c) for x in inds for i in x:x+n-1 for c in i-c+1:i]), c, n, length(inds))
consecutive_view(cb::CircularArrayBuffer{T,2}, inds::Vector{Int}, n::Int, c::Int) where {T} = reshape(view(cb.buffer, :, [_buffer_frame(cb, c) for x in inds for i in x:x+n-1 for c in i-c+1:i]), size(cb.buffer, 1), c, n, length(inds))
consecutive_view(cb::CircularArrayBuffer{T,3}, inds::Vector{Int}, n::Int, c::Int) where {T} = reshape(view(cb.buffer, :, :, [_buffer_frame(cb, c) for x in inds for i in x:x+n-1 for c in i-c+1:i]), size(cb.buffer, 1), size(cb.buffer, 2), c, n, length(inds))
consecutive_view(cb::CircularArrayBuffer{T,4}, inds::Vector{Int}, n::Int, c::Int) where {T} = reshape(view(cb.buffer, :, :, :, [_buffer_frame(cb, c) for x in inds for i in x:x+n-1 for c in i-c+1:i]), size(cb.buffer, 1), size(cb.buffer, 2), size(cb.buffer, 3), c, n, length(inds))

consecutive_view(cb::CircularArrayBuffer, inds::Vector{Int}, n::Int, ::Nothing) = consecutive_view(cb, inds, n)
consecutive_view(cb::CircularArrayBuffer, inds::Vector{Int}, ::Nothing, ::Nothing) = consecutive_view(cb, inds)

Base.getindex(cb::CircularArrayBuffer{T, 2}, i1::Int, i2::Int) where T = cb.buffer[i1, _buffer_frame(cb, i2)]
Base.getindex(cb::CircularArrayBuffer{T, 3}, i1::Int, i2::Int, i3::Int) where T = cb.buffer[i1, i2, _buffer_frame(cb, i3)]
Base.getindex(cb::CircularArrayBuffer{T, 4}, i1::Int, i2::Int, i3::Int, i4::Int) where T = cb.buffer[i1, i2, i3, _buffer_frame(cb, i4)]


text:
mutable struct QueryResult
    handle::Ref{duckdb_result}
    df::DataFrame

    function QueryResult(handle::Ref{duckdb_result})
        df = toDataFrame(handle)

        result = new(handle, df)
        finalizer(_close_result, result)
        return result
    end
end

function _close_result(result::QueryResult)
    return duckdb_destroy_result(result.handle)
end

mutable struct ColumnConversionData
    chunks::Vector{DataChunk}
    col_idx::Int64
    logical_type::LogicalType
    conversion_data::Any
end

mutable struct ListConversionData
    conversion_func::Function
    conversion_loop_func::Function
    child_type::LogicalType
    internal_type::Type
    target_type::Type
    child_conversion_data::Any
end

mutable struct StructConversionData
    tuple_type::Any
    child_conversion_data::Vector{ListConversionData}
end

function nop_convert(column_data::ColumnConversionData, val)
    return val
end

function convert_string(column_data::ColumnConversionData, val::Ptr{Cvoid}, idx::UInt64)
    base_ptr = val + (idx - 1) * sizeof(duckdb_string_t)
    length_ptr = Base.unsafe_convert(Ptr{Int32}, base_ptr)
    length = unsafe_load(length_ptr)
    if length <= STRING_INLINE_LENGTH
        prefix_ptr = Base.unsafe_convert(Ptr{UInt8}, base_ptr + sizeof(Int32))
        return unsafe_string(prefix_ptr, length)
    else
        ptr_ptr = Base.unsafe_convert(Ptr{Ptr{UInt8}}, base_ptr + sizeof(Int32) * 2)
        data_ptr = Base.unsafe_load(ptr_ptr)
        return unsafe_string(data_ptr, length)
    end
end

function convert_blob(column_data::ColumnConversionData, val::Ptr{Cvoid}, idx::UInt64)::Base.CodeUnits{UInt8, String}
    return Base.codeunits(convert_string(column_data, val, idx))
end

function convert_date(column_data::ColumnConversionData, val::Int32)::Date
    return Dates.epochdays2date(val + ROUNDING_EPOCH_TO_UNIX_EPOCH_DAYS)
end

function convert_time(column_data::ColumnConversionData, val::Int64)::Time
    return Dates.Time(Dates.Nanosecond(val * 1000))
end

function convert_timestamp(column_data::ColumnConversionData, val::Int64)::DateTime
    return Dates.epochms2datetime((val ÷ 1000) + ROUNDING_EPOCH_TO_UNIX_EPOCH_MS)
end

function convert_timestamp_s(column_data::ColumnConversionData, val::Int64)::DateTime
    return Dates.epochms2datetime((val * 1000) + ROUNDING_EPOCH_TO_UNIX_EPOCH_MS)
end

function convert_timestamp_ms(column_data::ColumnConversionData, val::Int64)::DateTime
    return Dates.epochms2datetime((val) + ROUNDING_EPOCH_TO_UNIX_EPOCH_MS)
end

function convert_timestamp_ns(column_data::ColumnConversionData, val::Int64)::DateTime
    return Dates.epochms2datetime((val ÷ 1000000) + ROUNDING_EPOCH_TO_UNIX_EPOCH_MS)
end

function convert_interval(column_data::ColumnConversionData, val::duckdb_interval)::Dates.CompoundPeriod
    return Dates.CompoundPeriod(Dates.Month(val.months), Dates.Day(val.days), Dates.Microsecond(val.micros))
end

function convert_hugeint(column_data::ColumnConversionData, val::duckdb_hugeint)::Int128
    return Int128(val.lower) + Int128(val.upper) << 64
end

function convert_uuid(column_data::ColumnConversionData, val::duckdb_hugeint)::UUID
    hugeint = convert_hugeint(column_data, val)
    base_value = Int128(170141183460469231731687303715884105727)
    if hugeint < 0
        return UUID(UInt128(hugeint + base_value) + 1)
    else
        return UUID(UInt128(hugeint) + base_value + 1)
    end
end

function convert_enum(column_data::ColumnConversionData, val)::String
    return column_data.conversion_data[val + 1]
end

function convert_decimal_hugeint(column_data::ColumnConversionData, val::duckdb_hugeint)
    return Base.reinterpret(column_data.conversion_data, convert_hugeint(column_data, val))
end

function convert_decimal(column_data::ColumnConversionData, val)
    return Base.reinterpret(column_data.conversion_data, val)
end

function convert_vector(
    column_data::ColumnConversionData,
    vector::Vec,
    size::UInt64,
    convert_func::Function,
    result,
    position,
    all_valid,
    ::Type{SRC},
    ::Type{DST}
) where {SRC, DST}
    array = get_array(vector, SRC)
    if !all_valid
        validity = get_validity(vector)
    end
    for i in 1:size
        if all_valid || isvalid(validity, i)
            result[position] = convert_func(column_data, array[i])
        end
        position += 1
    end
    return size
end

function convert_vector_string(
    column_data::ColumnConversionData,
    vector::Vec,
    size::UInt64,
    convert_func::Function,
    result,
    position,
    all_valid,
    ::Type{SRC},
    ::Type{DST}
) where {SRC, DST}
    raw_ptr = duckdb_vector_get_data(vector.handle)
    ptr = Base.unsafe_convert(Ptr{duckdb_string_t}, raw_ptr)
    if !all_valid
        validity = get_validity(vector)
    end
    for i in 1:size
        if all_valid || isvalid(validity, i)
            result[position] = convert_func(column_data, raw_ptr, i)
        end
        position += 1
    end
    return size
end

function convert_vector_list(
    column_data::ColumnConversionData,
    vector::Vec,
    size::UInt64,
    convert_func::Function,
    result,
    position,
    all_valid,
    ::Type{SRC},
    ::Type{DST}
) where {SRC, DST}
    child_vector = list_child(vector)
    lsize = list_size(vector)

    # convert the child vector
    ldata = column_data.conversion_data

    child_column_data =
        ColumnConversionData(column_data.chunks, column_data.col_idx, ldata.child_type, ldata.child_conversion_data)
    child_array = Array{Union{Missing, ldata.target_type}}(missing, lsize)
    ldata.conversion_loop_func(
        child_column_data,
        child_vector,
        lsize,
        ldata.conversion_func,
        child_array,
        1,
        false,
        ldata.internal_type,
        ldata.target_type
    )

    array = get_array(vector, SRC)
    if !all_valid
        validity = get_validity(vector)
    end
    for i in 1:size
        if all_valid || isvalid(validity, i)
            start_offset::UInt64 = array[i].offset + 1
            end_offset::UInt64 = array[i].offset + array[i].length
            result[position] = child_array[start_offset:end_offset]
        end
        position += 1
    end
    return size
end

function convert_struct_children(column_data::ColumnConversionData, vector::Vec, size::UInt64)
    # convert the child vectors of the struct
    child_count = get_struct_child_count(column_data.logical_type)
    child_arrays = Vector()
    for i in 1:child_count
        child_vector = struct_child(vector, i)
        ldata = column_data.conversion_data.child_conversion_data[i]

        child_column_data =
            ColumnConversionData(column_data.chunks, column_data.col_idx, ldata.child_type, ldata.child_conversion_data)
        child_array = Array{Union{Missing, ldata.target_type}}(missing, size)
        ldata.conversion_loop_func(
            child_column_data,
            child_vector,
            size,
            ldata.conversion_func,
            child_array,
            1,
            false,
            ldata.internal_type,
            ldata.target_type
        )
        push!(child_arrays, child_array)
    end
    return child_arrays
end

function convert_vector_struct(
    column_data::ColumnConversionData,
    vector::Vec,
    size::UInt64,
    convert_func::Function,
    result,
    position,
    all_valid,
    ::Type{SRC},
    ::Type{DST}
) where {SRC, DST}
    child_count = get_struct_child_count(column_data.logical_type)
    child_arrays = convert_struct_children(column_data, vector, size)

    if !all_valid
        validity = get_validity(vector)
    end
    for i in 1:size
        if all_valid || isvalid(validity, i)
            result_tuple = Vector()
            for child_idx in 1:child_count
                push!(result_tuple, child_arrays[child_idx][i])
            end
            result[position] = NamedTuple{column_data.conversion_data.tuple_type}(result_tuple)
        end
        position += 1
    end
    return size
end

function convert_vector_map(
    column_data::ColumnConversionData,
    vector::Vec,
    size::UInt64,
    convert_func::Function,
    result,
    position,
    all_valid,
    ::Type{SRC},
    ::Type{DST}
) where {SRC, DST}
    child_arrays = convert_struct_children(column_data, vector, size)
    keys = child_arrays[1]
    values = child_arrays[2]

    if !all_valid
        validity = get_validity(vector)
    end
    for i in 1:size
        if all_valid || isvalid(validity, i)
            result_dict = Dict()
            key_count = length(keys[i])
            for key_idx in 1:key_count
                result_dict[keys[i][key_idx]] = values[i][key_idx]
            end
            result[position] = result_dict
        end
        position += 1
    end
    return size
end

function convert_column_loop(
    column_data::ColumnConversionData,
    convert_func::Function,
    ::Type{SRC},
    ::Type{DST},
    convert_vector_func::Function
) where {SRC, DST}
    # first check if there are null values in any chunks
    has_missing = false
    row_count = 0
    for chunk in column_data.chunks
        if !all_valid(chunk, column_data.col_idx)
            has_missing = true
        end
        row_count += get_size(chunk)
    end
    if has_missing
        # missing values
        result = Array{Union{Missing, DST}}(missing, row_count)
        position = 1
        for chunk in column_data.chunks
            position += convert_vector_func(
                column_data,
                get_vector(chunk, column_data.col_idx),
                get_size(chunk),
                convert_func,
                result,
                position,
                all_valid(chunk, column_data.col_idx),
                SRC,
                DST
            )
        end
    else
        # no missing values
        result = Array{DST}(undef, row_count)
        position = 1
        for chunk in column_data.chunks
            position += convert_vector_func(
                column_data,
                get_vector(chunk, column_data.col_idx),
                get_size(chunk),
                convert_func,
                result,
                position,
                true,
                SRC,
                DST
            )
        end
    end
    return result
end

function create_child_conversion_data(child_type::LogicalType)
    internal_type_id = get_internal_type_id(child_type)
    internal_type = duckdb_type_to_internal_type(internal_type_id)
    target_type = duckdb_type_to_julia_type(child_type)

    conversion_func = get_conversion_function(child_type)
    conversion_loop_func = get_conversion_loop_function(child_type)
    child_conversion_data = init_conversion_loop(child_type)
    return ListConversionData(
        conversion_func,
        conversion_loop_func,
        child_type,
        internal_type,
        target_type,
        child_conversion_data
    )
end

function init_conversion_loop(logical_type::LogicalType)
    type = get_type_id(logical_type)
    if type == DUCKDB_TYPE_DECIMAL
        return duckdb_type_to_julia_type(logical_type)
    elseif type == DUCKDB_TYPE_ENUM
        return get_enum_dictionary(logical_type)
    elseif type == DUCKDB_TYPE_LIST
        child_type = get_list_child_type(logical_type)
        return create_child_conversion_data(child_type)
    elseif type == DUCKDB_TYPE_STRUCT || type == DUCKDB_TYPE_MAP
        child_count = get_struct_child_count(logical_type)
        child_symbols::Vector{Symbol} = Vector()
        child_data::Vector{ListConversionData} = Vector()
        for i in 1:child_count
            child_symbol = Symbol(get_struct_child_name(logical_type, i))
            child_type = get_struct_child_type(logical_type, i)
            child_conv_data = create_child_conversion_data(child_type)
            push!(child_symbols, child_symbol)
            push!(child_data, child_conv_data)
        end
        return StructConversionData(Tuple(x for x in child_symbols), child_data)
    else
        return nothing
    end
end

function get_conversion_function(logical_type::LogicalType)::Function
    type = get_type_id(logical_type)
    if type == DUCKDB_TYPE_VARCHAR || type == DUCKDB_TYPE_JSON
        return convert_string
    elseif type == DUCKDB_TYPE_BLOB
        return convert_blob
    elseif type == DUCKDB_TYPE_DATE
        return convert_date
    elseif type == DUCKDB_TYPE_TIME
        return convert_time
    elseif type == DUCKDB_TYPE_TIMESTAMP
        return convert_timestamp
    elseif type == DUCKDB_TYPE_TIMESTAMP_S
        return convert_timestamp_s
    elseif type == DUCKDB_TYPE_TIMESTAMP_MS
        return convert_timestamp_ms
    elseif type == DUCKDB_TYPE_TIMESTAMP_NS
        return convert_timestamp_ns
    elseif type == DUCKDB_TYPE_INTERVAL
        return convert_interval
    elseif type == DUCKDB_TYPE_HUGEINT
        return convert_hugeint
    elseif type == DUCKDB_TYPE_UUID
        return convert_uuid
    elseif type == DUCKDB_TYPE_DECIMAL
        internal_type_id = get_internal_type_id(logical_type)
        if internal_type_id == DUCKDB_TYPE_HUGEINT
            return convert_decimal_hugeint
        else
            return convert_decimal
        end
    elseif type == DUCKDB_TYPE_ENUM
        return convert_enum
    else
        return nop_convert
    end
end

function get_conversion_loop_function(logical_type::LogicalType)::Function
    type = get_type_id(logical_type)
    if type == DUCKDB_TYPE_VARCHAR || type == DUCKDB_TYPE_BLOB || type == DUCKDB_TYPE_JSON
        return convert_vector_string
    elseif type == DUCKDB_TYPE_LIST
        return convert_vector_list
    elseif type == DUCKDB_TYPE_STRUCT
        return convert_vector_struct
    elseif type == DUCKDB_TYPE_MAP
        return convert_vector_map
    else
        return convert_vector
    end
end

function convert_column(column_data::ColumnConversionData)
    internal_type_id = get_internal_type_id(column_data.logical_type)
    internal_type = duckdb_type_to_internal_type(internal_type_id)
    target_type = duckdb_type_to_julia_type(column_data.logical_type)

    conversion_func = get_conversion_function(column_data.logical_type)
    conversion_loop_func = get_conversion_loop_function(column_data.logical_type)

    column_data.conversion_data = init_conversion_loop(column_data.logical_type)
    return convert_column_loop(column_data, conversion_func, internal_type, target_type, conversion_loop_func)
end

function toDataFrame(result::Ref{duckdb_result})::DataFrame
    column_count = duckdb_column_count(result)
    # duplicate eliminate the names
    names = Vector{Symbol}(undef, column_count)
    for i in 1:column_count
        name = sym(duckdb_column_name(result, i))
        if name in view(names, 1:(i - 1))
            j = 1
            new_name = Symbol(name, :_, j)
            while new_name in view(names, 1:(i - 1))
                j += 1
                new_name = Symbol(name, :_, j)
            end
            name = new_name
        end
        names[i] = name
    end
    # gather all the data chunks
    chunk_count = duckdb_result_chunk_count(result[])
    chunks::Vector{DataChunk} = []
    for i in 1:chunk_count
        push!(chunks, DataChunk(duckdb_result_get_chunk(result[], i), true))
    end

    df = DataFrame()
    for i in 1:column_count
        name = names[i]
        logical_type = LogicalType(duckdb_column_logical_type(result, i))
        column_data = ColumnConversionData(chunks, i, logical_type, nothing)
        df[!, name] = convert_column(column_data)
    end
    return df
end

function execute(stmt::Stmt, params::DBInterface.StatementParams = ())
    bind_parameters(stmt, params)

    handle = Ref{duckdb_result}()
    if duckdb_execute_prepared(stmt.handle, handle) != DuckDBSuccess
        error_ptr = duckdb_result_error(handle)
        if error_ptr == C_NULL
            error_message = string("Execute of query \"", stmt.sql, "\" failed: unknown error")
        else
            error_message = string("Execute of query \"", stmt.sql, "\" failed: ", unsafe_string(error_ptr))
        end
        duckdb_destroy_result(handle)
        throw(QueryException(error_message))
    end
    return QueryResult(handle)
end

# explicitly close prepared statement
function DBInterface.close!(stmt::Stmt)
    return _close_stmt(stmt)
end

function execute(con::Connection, sql::AbstractString, params::DBInterface.StatementParams)
    stmt = Stmt(con, sql)
    try
        return execute(stmt, params)
    finally
        _close_stmt(stmt) # immediately close, don't wait for GC
    end
end

execute(con::Connection, sql::AbstractString; kwargs...) = execute(con, sql, values(kwargs))
execute(db::DB, sql::AbstractString, params::DBInterface.StatementParams) = execute(db.main_connection, sql, params)
execute(db::DB, sql::AbstractString; kwargs...) = execute(db.main_connection, sql, values(kwargs))

Tables.isrowtable(::Type{QueryResult}) = true
Tables.columnnames(q::QueryResult) = Tables.columnnames(q.df)

function Tables.schema(q::QueryResult)
    return Tables.schema(q.df)
end

Base.IteratorSize(::Type{QueryResult}) = Base.SizeUnknown()
Base.eltype(q::QueryResult) = Any

function DBInterface.close!(q::QueryResult)
    return _close_result(q)
end

function Base.iterate(q::QueryResult)
    return Base.iterate(eachrow(q.df))
end

function Base.iterate(q::QueryResult, state)
    return Base.iterate(eachrow(q.df), state)
end

DataFrames.DataFrame(q::QueryResult) = DataFrame(q.df)

"Return the last row insert id from the executed statement"
function DBInterface.lastrowid(con::Connection)
    throw(NotImplementedException("Unimplemented: lastrowid"))
end

DBInterface.lastrowid(db::DB) = DBInterface.lastrowid(db.main_connection)

"""
    DBInterface.prepare(db::DuckDB.DB, sql::AbstractString)

Prepare an SQL statement given as a string in the DuckDB database; returns a `DuckDB.Stmt` object.
See `DBInterface.execute`(@ref) for information on executing a prepared statement and passing parameters to bind.
A `DuckDB.Stmt` object can be closed (resources freed) using `DBInterface.close!`(@ref).
"""
DBInterface.prepare(con::Connection, sql::AbstractString) = Stmt(con, sql)
DBInterface.prepare(db::DB, sql::AbstractString) = DBInterface.prepare(db.main_connection, sql)

"""
    DBInterface.execute(db::DuckDB.DB, sql::String, [params])
    DBInterface.execute(stmt::SQLite.Stmt, [params])

Bind any positional (`params` as `Vector` or `Tuple`) or named (`params` as `NamedTuple` or `Dict`) parameters to an SQL statement, given by `db` and `sql` or
as an already prepared statement `stmt`, execute the query and return an iterator of result rows.

Note that the returned result row iterator only supports a single-pass, forward-only iteration of the result rows.
Calling `SQLite.reset!(result)` will re-execute the query and reset the iterator back to the beginning.

The resultset iterator supports the [Tables.jl](https://github.com/JuliaData/Tables.jl) interface, so results can be collected in any Tables.jl-compatible sink,
like `DataFrame(results)`, `CSV.write("results.csv", results)`, etc.
"""
function DBInterface.execute(stmt::Stmt, params::DBInterface.StatementParams)
    return execute(stmt, params)
end

function DBInterface.execute(con::Connection, sql::AbstractString)
    return execute(Stmt(con, sql))
end

Base.show(io::IO, result::DuckDB.QueryResult) = print(io, result.df)

ext: jl


text:
# Autogenerated wrapper script for QPALM_jll for i686-w64-mingw32
export libladel, libqpalm

using OpenBLAS32_jll
using SuiteSparse_jll
JLLWrappers.@generate_wrapper_header("QPALM")
JLLWrappers.@declare_library_product(libladel, "libladel.dll")
JLLWrappers.@declare_library_product(libqpalm, "libqpalm.dll")
function __init__()
    JLLWrappers.@generate_init_header(OpenBLAS32_jll, SuiteSparse_jll)
    JLLWrappers.@init_library_product(
        libladel,
        "bin\\libladel.dll",
        RTLD_LAZY | RTLD_DEEPBIND,
    )

    JLLWrappers.@init_library_product(
        libqpalm,
        "bin\\libqpalm.dll",
        RTLD_LAZY | RTLD_DEEPBIND,
    )

    JLLWrappers.@generate_init_footer()
end  # __init__()


text:
@testset "Differentials on Sn(2) and SPD(2)" begin
x = SnPoint([1.,0.,0.])
y = SnPoint([0.,1.,0.])
M = Sphere(2)
ξ = log(M,x,y)
# Text differentials (1) Dx of Log_xy
@test DxLog(M,x,x,ξ) == -ξ
@test DxLog(M,x,y,ξ) == -ξ
@test DyLog(M,x,x,ξ) == ξ
@test DyLog(M,x,y,ξ) == zeroTVector(M,y)
@test DxExp(M,x,zeroTVector(M,x),ξ) == ξ
@test norm(M,y, DxExp(M,x,ξ,ξ) - SnTVector([-π/2, 0., 0.])) ≈ 0 atol=2*10^(-16)
@test DξExp(M,x,zeroTVector(M,x),ξ) == ξ
@test norm(M,y,DξExp(M,x,ξ,zeroTVector(M,x))) ≈ 0
for t in [0,0.15,0.33,0.66,0.9]
    @test DxGeo(M,x,x,t,ξ) == (1-t)*ξ
    @test norm(M,x,DyGeo(M,x,x,t,ξ) - t*ξ)  ≈ 0 atol=10.0^(-16)
end
Mp = Power(M,2)
xP = PowPoint([x,y,x])
yP = PowPoint([x,x,y])
ξP = PowTVector([ξ, zeroTVector(M,x), -ξ])
@test norm(Mp,xP, DforwardLogs(Mp,xP,ξP)
     - PowTVector([-ξ, SnTVector([π/2, 0., 0.]),zeroTVector(M,x)])  )  ≈ 0 atol=3*10.0^(-16)
#
# Single differentials on Hn
M2 = SymmetricPositiveDefinite(2)
x2 = SPDPoint([1. 0.; 0. 1.])
ξ2 = SPDTVector([0.5 1.;1. 0.5])
y2 = exp(M2,x2,ξ2)
# Text differentials (1) Dx of Log_xy
@test norm(M2, x2, DxLog(M2, x2, x2, ξ2) + ξ2) ≈ 0 atol=4*10^(-16)
@test norm(M2, y2, DxLog(M2, x2, y2, ξ2) + ξ2) ≈ 0 atol=4*10^(-16)
@test norm(M2, x2, DyLog(M2, x2, x2, ξ2) - ξ2) ≈ 0 atol=4*10^(-16)
@test norm(M2, y2, DyLog(M2, x2, y2, zeroTVector(M2,x2))) ≈ 0 atol=4*10^(-16)
@test norm(M2, x2, DxExp(M2, x2, zeroTVector(M2,x2), ξ2) - ξ2) ≈ 0 atol=4*10^(-16)
@test norm(M2, x2, DξExp(M2,x2,zeroTVector(M2,x2),ξ2) - ξ2) ≈ 0 atol=4*10^(-16)
for t in [0,0.15,0.33,0.66,0.9]
    @test norm(M2, x2, DxGeo(M2, x2, x2, t, ξ2) - (1-t)*ξ2 ) ≈ 0 atol=4*10^(-16)
    @test norm(M2, x2, DyGeo(M2, x2, x2, t, ξ2) - t*ξ2) ≈ 0 atol=4*10.0^(-16)
end
@test norm(M2, y2, DxGeo(M2, x2, y2, 1., ξ2)) ≈ 0 atol=4*10.0^(-16)
@test norm(M2, y2, DxExp(M2, x2, ξ2, zeroTVector(M2,x2) )) ≈ 0 atol=4*10.0^(-16)
@test norm(M2,y2,DξExp(M2,x2,ξ2,zeroTVector(M2,x2))) ≈ 0 atol=4*10.0^(-16)
end
#
# And a final Rn
#
M3 = Euclidean(2)
x3 = RnPoint([1., 2.])
ξ3 = RnTVector([1.,0.])
@test norm(M3, x3, DxExp(M3, x3, ξ3, ξ3) - ξ3) ≈ 0 atol=4*10.0^(-16)


text:
@testset "Logbook count 64" begin

    n = 100

    # A SINGLE ENTRY IN THE LOGBOOK
    reset_logbook()
    Sherlog64(0)+Sherlog64(0)
    lb = get_logbook()
    @test 1 == sum(lb)

    # N ENTRIES IN THE LOGBOOK
    reset_logbook()
    a = Sherlog64(1.0)
    for i in 1:n
        a + randn(Float64)         # promotion included
    end
    lb = get_logbook()
    @test n == sum(lb)
end

@testset "Logbook count 32" begin

    n = 100

    # A SINGLE ENTRY IN THE LOGBOOK
    reset_logbook()
    Sherlog32(0)+Sherlog32(0)
    lb = get_logbook()
    @test 1 == sum(lb)

    # N ENTRIES IN THE LOGBOOK
    reset_logbook()
    a = Sherlog32(1.0)
    for i in 1:n
        a + randn(Float32)         # promotion included
    end
    lb = get_logbook()
    @test n == sum(lb)
end

@testset "Logbook count 16" begin

    n = 100

    # A SINGLE ENTRY IN THE LOGBOOK
    reset_logbook()
    Sherlog16(0)+Sherlog16(0)
    lb = get_logbook()
    @test 1 == sum(lb)

    # N ENTRIES IN THE LOGBOOK
    reset_logbook()
    a = Sherlog16(1.0)
    for i in 1:n
        a + randn(Float16)         # promotion included
    end
    lb = get_logbook()
    @test n == sum(lb)
end

@testset "Minimum, maximum logbook Float16" begin

    n = 100

    for T in [Float16,Float32,Float64]

        # MAXIMUM
        for _ in 1:n
            i = rand(1:32)          # pick a random logbook
            reset_logbook(i)        # set it to zero
            x = rand(T)
            Sherlogs.log_it(Float16,x,i)     # log a random Float16
            logbook = get_logbook(i)
            @test Float16(x) == maximum(logbook)
        end

        # MINIMUM
        for _ in 1:n
            i = rand(1:32)          # pick a random logbook
            reset_logbook(i)        # set it to zero
            x = -rand(Float16)
            Sherlogs.log_it(Float16,x,i)     # log a random Float16
            logbook = get_logbook(i)
            @test Float16(x) == minimum(logbook)
        end
    end
end


text:
texpyplot()
pyplot(size=(400,300))
names = ["", "", "AlexNet", "", "VGG", "GoogLeNet", "ResNet", ""]
years = ["2010", "2011", "2012", "2013", "2014", "2014", "2015", "2016"]

ϵ = 0.75

y = [28.2, 25.8, 16.4, 11.7, 7.3, 6.7, 3.57, 2.99]
bar(y, leg = false, fillcolor = :MidnightBlue, fillopacity = 0.5, ygrid = true, xaxis = nothing, ylabel = "top-5 error rate", guidefontsize=12)
plot!(ann=[(k,-ϵ,text(years[k],pointsize=10)) for k in eachindex(years)]);
plot!(ann=[(k,y[k]+ϵ,text(s,pointsize=10)) for (k,s) in enumerate(names)]);
savefig("imagenet-results.pdf")
run(`pdf2svg imagenet-results.pdf ../svg/imagenet-results.svg`)
run(`open imagenet-results.pdf`)

max_stars_repo_name: seanrattana/courses

max_issues_repo_name: seanrattana/courses

max_forks_repo_name: seanrattana/courses


text:
#
# Correctness Tests
#

require("Calculus")
using Calculus

my_tests = ["test/finite_difference.jl",
            "test/derivative.jl",
            "test/check_derivative.jl",
            "test/integrate.jl",
            "test/symbolic.jl",
            "test/deparse.jl"]

println("Running tests:")

for my_test in my_tests
    println(" * $(my_test)")
    include(my_test)
end


text:
__precompile__()

module AWS
using Requests
using JSON
using LightXML
using Compat

const EP_US_EAST_NORTHERN_VIRGINIA     = "ec2.us-east-1.amazonaws.com"
const EP_US_WEST_OREGON                = "ec2.us-west-2.amazonaws.com"
const EP_US_WEST_NORTHERN_CALIFORNIA   = "ec2.us-west-1.amazonaws.com"
const EP_EU_IRELAND                    = "ec2.eu-west-1.amazonaws.com"
const EP_AP_SINGAPORE                  = "ec2.ap-southeast-1.amazonaws.com"
const EP_AP_SYDNEY                     = "ec2.ap-southeast-2.amazonaws.com"
const EP_AP_TOKYO                      = "ec2.ap-northeast-1.amazonaws.com"
const EP_SA_SAO_PAULO                  = "ec2.sa-east-1.amazonaws.com"

export EP_US_EAST_NORTHERN_VIRGINIA, EP_US_WEST_OREGON, EP_US_WEST_NORTHERN_CALIFORNIA
export EP_EU_IRELAND, EP_AP_SINGAPORE, EP_AP_SYDNEY, EP_AP_TOKYO, EP_SA_SAO_PAULO

export AWSEnv

const AP_NORTHEAST_1 = "ap-northeast-1" # Asia Pacific (Tokyo) Region
const AP_SOUTHEAST_1 = "ap-southeast-1" # Asia Pacific (Singapore) Region
const AP_SOUTHEAST_2 = "ap-southeast-2" # Asia Pacific (Sydney) Region
const EU_WEST_1 = "eu-west-1" # EU (Ireland) Region
const SA_EAST_1 = "sa-east-1" # South America (Sao Paulo) Region
const US_EAST_1 = "us-east-1" # US East (Northern Virginia) Region
const US_WEST_1 = "us-west-1" # US West (Northern California) Region
const US_WEST_2 = "us-west-2" # US West (Oregon) Region

function read_aws_config(configFileName::String = joinpath(homedir(), ".aws/config"))
    global AWS_ID
    global AWS_SECKEY
    global AWS_REGION

    isfile(configFileName) || return
    for line in readlines(configFileName)
        line = replace(line, " ", "")
        line = replace(line, "\n", "")
        segs = split(line, "=")
        if ismatch(r"^aws_secret_access_key", segs[1])
            AWS_SECKEY = segs[2]
        elseif ismatch(r"^aws_access_key_id", segs[1])
            AWS_ID = segs[2]
        elseif ismatch(r"^region", segs[1])
            AWS_REGION = segs[2]
        end
    end
end

function __init__()
    config_file_base = Sys.KERNEL == :Windows ? ENV["APPDATA"] : homedir()

    # Search for default AWS_ID and AWS_SECKEY
    global AWS_ID = ""
    global AWS_SECKEY = ""
    global AWS_REGION = US_EAST_1
    global AWS_TOKEN = ""

    # read the default configuration file first
    read_aws_config()

    # override specific options in the same precedence as specified in
    # http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#config-settings-and-precedence
    credentials_path = joinpath(homedir(), ".aws/credentials")
    if haskey(ENV, "AWS_ID") && haskey(ENV, "AWS_SECKEY")
        AWS_ID = ENV["AWS_ID"]
        AWS_SECKEY = ENV["AWS_SECKEY"]
    elseif haskey(ENV, "AWS_ACCESS_KEY_ID") && haskey(ENV, "AWS_SECRET_ACCESS_KEY")
        AWS_ID = ENV["AWS_ACCESS_KEY_ID"]
        AWS_SECKEY = ENV["AWS_SECRET_ACCESS_KEY"]
    elseif isfile(credentials_path)
        read_aws_config(credentials_path)
    else
        secret_path = joinpath(config_file_base, ".awssecret")
        if isfile(secret_path)
            AWS_ID, AWS_SECKEY = split(readchomp(secret_path))
        end
    end

    # Search for default AWS_REGION
    if haskey(ENV, "AWS_REGION")
        AWS_REGION = ENV["AWS_REGION"]
    elseif haskey(ENV, "AWS_DEFAULT_REGION")
        AWS_REGION = ENV["AWS_DEFAULT_REGION"]
    elseif isfile( joinpath(config_file_base, ".awsregion") )
        region_path = joinpath(config_file_base, ".awsregion")
        AWS_REGION = readchomp(region_path)
    else
        info("using default region: ", AWS_REGION)
    end
end

type AWSEnv
    aws_id::String         # AWS Access Key id
    aws_seckey::String     # AWS Secret key for signing requests
    aws_token::String      # AWS Security Token for temporary credentials
    region::String      # region name
	ep_scheme::String      # URL scheme: http or https
    ep_host::String     # region endpoint (host)
    ep_path::String     # region endpoint (path)
    sig_ver::Int                # AWS signature version (2 or 4)
    timeout::Float64            # request timeout in seconds, default is no timeout.
    dry_run::Bool               # If true, no actual request will be made - implies dbg flag below
    dbg::Bool                   # print request to screen

    function AWSEnv(; id=AWS_ID, key=AWS_SECKEY, token=AWS_TOKEN, ec2_creds=false, scheme="https", region=AWS_REGION, ep="", sig_ver=4, timeout=0.0, dr=false, dbg=false)
        if ec2_creds
            creds = get_instance_credentials()
            if creds != nothing
                id = creds["AccessKeyId"]
                key = creds["SecretAccessKey"]
                token = creds["Token"]
            end
        end

        if (id == "") || (key == "")
            error("Invalid AWS security credentials provided")
        end

		#=
        s = search(ep,"/")
        if length(s) == 0
            ep_host = ep
            ep_path = "/"
        else
            ep_host = ep[1:(first(s)-1)]
            ep_path = ep[first(s):end]
        end
		=#
		ep_scheme, ep_host, ep_path = parse_endpoint(ep, scheme)

        # host portion of ep overrides region
        if length(ep_host) > 19 && ep_host[1:4] == "ec2." && ep_host[(end-13):end] == ".amazonaws.com"
            warn("AWSEnv: ep keyword argument has been deprecated for AWS native services. Use region keyword argument instead.")
            region = ep_host[5:end-14]
            ep_host = ""
        end
        if ep_host != ""
            region = ""
        end
        if (region == "" && ep == "")
            error("No region or endpoint provided")
        end

        if sig_ver != 2 && sig_ver != 4
          error("Invalid signature version number")
        end

        if dr
            new(id, key, token, region, ep_scheme, ep_host, ep_path, sig_ver, timeout, dr, true)
        else
            new(id, key, token, region, ep_scheme, ep_host, ep_path, sig_ver, timeout, false, dbg)
        end
    end

	function AWSEnv(env::AWSEnv; ep="")
        ep_scheme, ep_host, ep_path = parse_endpoint(ep, env.ep_scheme)

        new(env.aws_id, env.aws_seckey, env.aws_token, env.region, ep_scheme,
            ep_host, ep_path, env.sig_ver, env.timeout, env.dry_run, env.dbg)
    end
end

function parse_endpoint(ep, default_scheme)
    s = search(ep,"://")
    if length(s) == 0
        ep_scheme = default_scheme
        ephp = ep
    else
        ep_scheme = ep[1:(first(s)-1)]
        ephp = ep[first(s)+3:end]
    end
    s = search(ephp,"/")
    if length(s) == 0
        ep_host = ephp
        ep_path = "/"
    else
        ep_host = ephp[1:(first(s)-1)]
        ep_path = ephp[first(s):end]
    end
    return (ep_scheme, ep_host, ep_path)
end

ep_host(env::AWSEnv, service) = lowercase(env.ep_host=="" ? "$service.$(env.region).amazonaws.com" : env.ep_host)
export ep_host

function get_instance_credentials()
    try
        #if getaddrinfo("instance-data.ec2.internal.") != ip"169.254.169.254"
        #    return nothing
        #end

        url = "http://169.254.169.254/2014-11-05/meta-data/iam/security-credentials/"
        resp = Requests.get(url)
        if resp.status != 200
            return nothing
        end

        iam = split(String(copy(resp.data)))
        if length(iam) == 0
            return nothing
        end

        url *= iam[1]
        resp = Requests.get(url)
        if resp.status != 200
            return nothing
        end

        return JSON.Parser.parse(String(copy(resp.data)))
    catch
        return nothing
    end
end

# trying to get elements from nothing should result in nothing
elements_by_tagname(::Void, s) = nothing
elements_by_tagname(node, s) = LightXML.get_elements_by_tagname(node, s)

include("codegen.jl")
include("aws_utils.jl")
include("crypto.jl")
include("sign.jl")
include("ec2.jl")
include("S3.jl")
include("sqs.jl")
include("autoscaling.jl")

include("show.jl")

end


text:
## ---------------  GroupStats

"""
	$(SIGNATURES)

Container that holds model statistics (arrays or scalars) and meta information.
For each variable, a `VarInfo` object holds info needed to display and check the data.

The `T` parameter is not used anywhere, but it enables the user to construct different parametric `ModelStats{T}` and dispatch on those.
"""
struct ModelStats{T}
    meta :: Dict{Symbol, Any}
    varMeta :: Dict{Symbol, VarInfo}
    values :: Dict{Symbol, Any}
end

# Blank
ModelStats{T}() where T = ModelStats{T}(Dict{Symbol, Any}(),
    Dict{Symbol, VarInfo}(), Dict{Symbol, Any}());

# Meta only
ModelStats{T}(meta :: Dict{Symbol, Any}) where T = ModelStats{T}(meta,
    Dict{Symbol, VarInfo}(), Dict{Symbol, Any}());

# Meta and Vector{VarInfo}. Easier to provide than `Dict`
ModelStats{T}(meta :: Dict{Symbol, Any},
    viV :: AbstractVector{VT}) where {T, VT <: VarInfo} =
    ModelStats{T}(meta, dict_from_vector(viV));

function dict_from_vector(viV :: AbstractVector{VT}) where VT <: VarInfo
    d = Dict{Symbol, VarInfo}();
    for vi in viV
        d[var_name(vi)] = vi;
    end
    return d
end

"""
    $(SIGNATURES)

Constructor. Values are not provided. They are initialized with default values (usually zeros).

# Examples
```
meta = Dict{Symbol,Any}([:group => :test]);
T = (:x, :y);
s = ModelStats{T}(meta);
s = ModelStats{T}(meta, [VarInfo(:test, "Test", Float64)]);
s = ModelStats{T}(meta, Dict{Symbol, VarInfo}([:test => VarInfo(:test, "Test", Float64)]));
```
"""
function ModelStats{T}(meta :: Dict{Symbol, Any},
    varMeta :: Dict{Symbol, VarInfo}) where T

    values = init_values(varMeta);
    return ModelStats{T}(meta, varMeta, values)
end

"""
	$(SIGNATURES)

Initialize all arrays for which size info is known.
"""
function init_values(varMeta :: Dict{Symbol, VarInfo})
    values = Dict{Symbol, Any}();
    for (varName, vInfo) in varMeta
        values[varName] = init_values(vInfo);
    end
    return values
end

Base.show(io :: IO, g :: ModelStats{T}) where T =
    print(io, "ModelStats{$T} with $(n_vars(g)) variables.");

## -----------  Access

"""
	$(SIGNATURES)

This allows the user to access values using dot notation:
`g.x == get_values(g, :x)`
"""
function Base.getproperty(g :: ModelStats{T}, v :: Symbol) where T
    # Spelling out the fields gives type stability.
    if v === :meta
        return getfield(g, :meta);
    elseif v === :varMeta
        return getfield(g, :varMeta);
    elseif v === :values
        return getfield(g, :values);
    else
        return get_values(g, v);
    end
end

# This allows functions that operate on properties to work.
Base.propertynames(g :: ModelStats{T}) where T =
    var_names(g);

Base.hasproperty(g :: ModelStats{T}, v :: Symbol) where T =
    has_variable(g, v);

"""
	$(SIGNATURES)

Retrieve selected elements of a variable.

# Example
```
z = g.x[1:2, 3, 4]
```
"""
Base.getindex(g :: ModelStats{T}, v :: Symbol, idx...) where T =
    get_values(g, v)[idx...];

"""
	$(SIGNATURES)

Set values of a variable.

# Example
```
g.x = [1,2,3];
```
"""
function Base.setproperty!(g :: ModelStats{T}, v :: Symbol, values) where T
    set_values!(g, v, values)
end

"""
	$(SIGNATURES)

# Example
```
g.x[1:2, 3:5] = rand(2,3);
g.x[1:2, 3:5] .= 0.5;
```
"""
Base.setindex!(g :: ModelStats{T}, v :: Symbol,
    newValue :: Number, idx :: Integer) where T =
    get_values(g, v)[idx] = newValue;

Base.setindex!(g :: ModelStats{T}, v :: Symbol, newValueV, idx...) where T =
    get_values(g, v)[idx...] .= newValueV;

# """
#     $(SIGNATURES)

# Number of groups in `ModelStats`.
# """
# n_groups(g :: ModelStats) = length(group_names(g));

# """
# 	$(SIGNATURES)

# Group names as Symbols.
# """
# group_names(g :: ModelStats) = g.groupNames;

"""
	$(SIGNATURES)

Number of variables.
"""
n_vars(g :: ModelStats{T}) where T = length(g.varMeta);

"""
	$(SIGNATURES)

Return all variable names.
"""
var_names(g :: ModelStats{T}) where T = keys(g.varMeta);

Base.eltype(g :: ModelStats{T}, vName :: Symbol) where T =
    eltype(var_meta(g, vName));

## ----------  Validation

"""
	$(SIGNATURES)

Check that
- all data are inside bounds
"""
function validate_stats(g :: ModelStats{T}; silent :: Bool = true) where T
    isValid = true;
    if n_vars(g) > 0
        for vName in var_names(g)
            isValid = isValid  &&  validate_variable(g, vName; silent = silent);
        end
    end
    return isValid
end

"""
	$(SIGNATURES)

Validate one variable.
"""
function validate_variable(g :: ModelStats{T}, vName :: Symbol;
    silent :: Bool = true) where T

    @assert haskey(g.values, vName)  "$vName not found"
    valueV = get_values(g, vName);
    isValid = check_var(var_meta(g, vName), valueV; silent = false);
    return isValid
end

## ----------  Retrieve

"""
	$(SIGNATURES)

Retrieve meta info (not for a variable).
"""
get_meta(g :: ModelStats{T}, mName :: Symbol) where T =
    g.meta[mName];

"""
    $(SIGNATURES)

Return meta info for a variable. Errors if not found.
"""
function var_meta(g :: ModelStats, vName :: Symbol)
    vMeta = _var_meta(g, vName);
    @assert !isnothing(vMeta)  "$vName not found"
    return vMeta
end

# Returns nothing if not found.
function _var_meta(g :: ModelStats{T}, vName :: Symbol) where T
    if haskey(g.varMeta, vName)
        return g.varMeta[vName];
    else
        return nothing
    end
end

"""
	$(SIGNATURES)

Do `GroupStats` contain variable `vName`?
"""
has_variable(g :: ModelStats{T}, vName :: Symbol) where T =
    haskey(g.varMeta, vName);

"""
    $(SIGNATURES)

Return values for a variable. Errors if not found, unless `defaultValue` is provided.
"""
function get_values(g :: ModelStats{T}, vName :: Symbol;
    defaultValue = :error) where T
    if has_variable(g, vName)
        x = getfield(g, :values)[vName];
    elseif isnothing(defaultValue)
        x = nothing;
    elseif ismissing(defaultValue)
        x = missing;
    elseif !isequal(defaultValue, :error)
        x = defaultValue;
    else
        error("$vName not found in $g");
    end
    return x
end

"""
	$(SIGNATURES)

Set values for a variable. Errors if variable does not exist.
Optional: validates values against meta info.

# Example
```julia
set_values!(g, :x, [1,2,3]);
# This can also be called with dot notation
g.x = [1,2,3];
```
"""
function set_values!(g :: ModelStats{T}, vName :: Symbol, newValues;
    validate :: Bool = true, silent :: Bool = false)  where T
    vMeta = var_meta(g, vName);
    g.values[vName] = promote_new_values(vMeta, newValues);
    validate  &&  validate_variable(g, vName; silent = silent);
end

# Promote new values when possible, so that any Integer can be passed to, say, a `UInt8` field.
promote_new_values(vi :: VarInfo{T}, newValues) where T = newValues;
promote_new_values(vi :: VarInfo{T1}, newValues :: T2) where
    {T1 <: Number, T2 <: Number} = convert(T1, newValues);
promote_new_values(vi :: VarInfo{<:AbstractArray{T1}},
    newValues :: AbstractArray{T2}) where {T1, T2} =
    convert.(T1, newValues);

"""
	$(SIGNATURES)

Add a variable.

# Arguments
- `vInfo`: Can be a `VarInfo` or a `Tuple` of values that can be fed into the `VarInfo` constructor.

# Example
```julia
add_variable!(g, (:vName, 0.0, 1.0), [1,2,3]);
```
"""
add_variable!(g :: ModelStats{T}, vInfo, newValues) where T =
    add_variable!(g, VarInfo(vInfo...), newValues);

function add_variable!(g :: ModelStats{T}, vInfo :: VarInfo, newValues) where T
    vName = var_name(vInfo);
    @assert !has_variable(g, vName)  "$vName already exists"
    g.varMeta[vName] = vInfo;
    g.values[vName] = newValues;
end

"""
	$(SIGNATURES)

Delete a variable. Errors if it does not exist.
"""
function delete_variable!(g :: ModelStats{T}, vName :: Symbol) where T
    @assert has_variable(g, vName)  "$vName not found"
    delete!(g.varMeta, vName);
    delete!(g.values, vName);
end

# """
# 	$(SIGNATURES)

# Show a summary table for selected variables and groups. Because each variable can be of a different type, there is no good way of returning a single table as values (without introducing more dependencies that seems optimal).
# """
# function data_table(g :: ModelStats{T}; vars = var_names(g))
#     tbM = fill("", n_groups(g), length(vars) + 1);

#     for ig = 1 : n_groups(g)
#         tbM[ig,1] = string(g.groupNames[ig]);
#     end

#     for (j, varName) in enumerate(vars)
#         tbM[:, j+1] = format_values(get_values(g, varName));
#     end
#     return tbM
# end

## ---------  Testing

function make_test_model_stats(T, nVars, rng :: AbstractRNG)
    mMeta = Dict{Symbol, Any}([:test => true]);
    varMeta = make_test_var_infos(nVars, rng);
    values = make_test_values(varMeta, rng);
    gs = ModelStats{T}(mMeta, varMeta, values);
    @assert validate_stats(gs);
    return gs
end

# -------------------


text:
"""
    struct ObsScheme{T}
        obs::T
        pattern::Vector{Int64}
        full_pattern::Vector{Int64}
        mode::Val
    end

A struct `ObsScheme` acts as a template for reading in multiple observations.
- `obs`: contains a list of all possible observation schemes according to which
         the data (that are supposed to be loaded in) were recorded
- `pattern`: specifies the order in which the list of `obs` is supposed to be
             iterated through when reading in successive observations. Usually
             the number of datapoint is larger than the length of `pattern` in
             which case the `pattern` is cycled through repeatedly.
- `full_pattern`: as pattern, but must be of the same length as the loaded in
                  data.
- `mode`: if set to Val{:simple}(), then cycles through `pattern`. Otherwise
          cycles through `full_pattern`.

    ObsScheme(
        obs...;
        pattern=[1:length(obs)...],
        full_pattern=[],
        mode=Val(:simple)
    )

Base constructor.
"""
struct ObsScheme{T}
    obs::T
    pattern::Vector{Int64}
    full_pattern::Vector{Int64}
    mode::Val
end

ObsScheme(obs::T) where T = ObsScheme{Vector{T}}([obs], [1], [], Val(:simple))

function ObsScheme(
        obs...;
        pattern=[1:length(obs)...],
        full_pattern=[],
        mode=Val(:simple)
    )
    obs_v = [obs...]
    T = typeof(obs_v)
    ObsScheme{T}(obs_v, pattern, full_pattern, mode)
end

"""
    load_data(os::ObsScheme, tt, xx)

Decorate the data in `xx` and times of recordings in `tt` according to an
observation scheme template stored in `os`.
"""
load_data(os::ObsScheme, tt, xx) = load_data(os, collect(zip(tt, xx)))

"""
    load_data(os::ObsScheme, tt_xx)

Same as `load_data(os::ObsScheme, tt, xx)`, but `tt_xx` is a vector of tuples
that pair the observations with their recorded times.

    load_data(os::ObsScheme, tt_xx_filename::String)

Same as above, but `tt_xx` are stored in a .csv file named `tt_xx_filename`,
with each row containing time `t` and its respective observation `x` stored
back-to-back.
"""
load_data(os::ObsScheme, tt_xx) = load_data(
    _pattern_iter(os.mode, os),
    os, tt_xx
)

function _pattern_iter(::Val{:simple}, os::ObsScheme)
    N = length(os.pattern)
    λ(i) = os.pattern[mod1(i, N)]
    λ
end

function _pattern_iter(::Any, os::ObsScheme)
    λ(i) = os.full_pattern[i]
    λ
end

function load_data(λ::Function, os::ObsScheme, tt_xx::Vector)
    observs = [
        obs_from_template(tt_xx[i], os.obs[λ(i)])
        for i in eachindex(tt_xx)
    ]
end

function load_data(λ::Function, os::ObsScheme, tt_xx_filename::String)
    i = 0
    observs = map(eachline(tt_xx_filename)) do line
        i += 1
        tt_xx = parse.(Float64, split(line, ","))
        obs_from_template(
            (tt_xx[1], tt_xx[2:end]),
            os.obs[λ(i)]
        )
    end
    observs
end

function obs_from_template(obs, tmp::LinearGsnObs)
    o = (
        typeof(obs[2]) <: Number && typeof(tmp.obs) <: AbstractArray ?
        [obs[2]] :
        obs[2]
    )
    LinearGsnObs(
        obs[1],
        convert(typeof(tmp.obs), o),
        deepcopy(tmp.L),
        deepcopy(tmp.Σ),
        deepcopy(tmp.μ),
        fpt_info(tmp)(),
        tmp.full_obs,
        deepcopy(tmp.θ),
        get_tag(tmp)
    )
end

function obs_from_template(obs, tmp::GeneralObs)
    GeneralObs(
        obs[1],
        convert(typeof(tmp.obs), obs[2]),
        obs_from_template(obs, tmp.lin_obs),
        fpt_info(tmp)(),
        tmp.g,
        tmp.dist,
        deepcopy(tmp.θ),
        get_tag(tmp)
    )
end

"""
    Base.collect(os::ObsScheme, path, step=1, record_start_pt=false)

Record observations from a raw trajectory `path` according to the observation
scheme `os`. Collect 1 observation for every `step` many points in the raw data.
If `record_start_pt`, then the starting is recorded, otherwise it is omitted.
`path` should have fields `.x` and `.t` with the underlying process and times
respectively.
"""
function Base.collect(os::ObsScheme, path, step=1, omit_first=true)
    i1 = (1+omit_first)
    raw_xx = _prepare_xx(os.mode, os, path.x[1:step:end][i1:end])

    load_data(os, path.t[1:step:end][i1:end], raw_xx)
end

"""
    _prepare_xx(::Val, os::ObsScheme, X)

Cycle through a collection `X` and transform it according to the observation
schemes specified in `os`.
"""
function _prepare_xx(::Val{:simple}, os::ObsScheme, X)
    N = length(os.pattern)
    raw_xx = map(1:length(X)) do i
        rand(os.obs[os.pattern[mod1(i, N)]], X[i])
    end
end

function _prepare_xx(::Any, os::ObsScheme, X)
    @assert length(full_pattern) == length(X)
    raw_xx = map(1:length(X)) do i
        rand(os.obs[os.full_pattern[i]], X[i])
    end
end


text:
Installed ForwardDiff ──────────── v0.10.19
   Installed SortingAlgorithms ────── v1.0.1
   Installed Missings ─────────────── v1.0.1
   Installed Requires ─────────────── v1.1.3
   Installed Static ───────────────── v0.3.1
   Installed StatsBase ────────────── v0.33.10
   Installed DiffRules ────────────── v1.3.0
   Installed CommonSubexpressions ─── v0.3.0
   Installed DataStructures ───────── v0.18.10
   Installed Compat ───────────────── v3.35.0
   Installed JLLWrappers ──────────── v1.3.0
   Installed PositiveFactorizations ─ v0.2.4
   Installed DiffResults ──────────── v1.0.3
   Installed SpecialFunctions ─────── v1.6.1
   Installed UnPack ───────────────── v1.0.2
  Downloaded artifact: OpenSpecFun
    Updating `C:\Users\DELL\.julia\environments\v1.6\Project.toml`
  [429524aa] + Optim v1.4.1
    Updating `C:\Users\DELL\.julia\environments\v1.6\Manifest.toml`
  [4fba245c] + ArrayInterface v3.1.31
  [d360d2e6] + ChainRulesCore v1.3.1
  [bbf7d656] + CommonSubexpressions v0.3.0
  [34da2185] + Compat v3.35.0
  [9a962f9c] + DataAPI v1.8.0
  [864edb3b] + DataStructures v0.18.10
  [163ba53b] + DiffResults v1.0.3
  [b552c78f] + DiffRules v1.3.0
  [ffbed154] + DocStringExtensions v0.8.5
  [1a297f60] + FillArrays v0.12.3
  [6a86dc24] + FiniteDiff v2.8.1
  [f6369f11] + ForwardDiff v0.10.19
  [615f187c] + IfElse v0.1.0
  [92d709cd] + IrrationalConstants v0.1.0
  [692b3bcd] + JLLWrappers v1.3.0
  [d3d80556] + LineSearches v7.1.1
  [2ab3a3ac] + LogExpFunctions v0.3.2
  [1914dd2f] + MacroTools v0.5.7
  [e1d29d7a] + Missings v1.0.1
  [d41bc354] + NLSolversBase v7.8.1
  [77ba4419] + NaNMath v0.3.5
  [429524aa] + Optim v1.4.1
  [bac558e1] + OrderedCollections v1.4.1
  [d96e819e] + Parameters v0.12.2
  [85a6dd25] + PositiveFactorizations v0.2.4
  [21216c6a] + Preferences v1.2.2
  [ae029012] + Requires v1.1.3
  [a2af1166] + SortingAlgorithms v1.0.1
  [276daf66] + SpecialFunctions v1.6.1
  [aedffcd0] + Static v0.3.1
  [90137ffa] + StaticArrays v1.2.12
  [82ae8749] + StatsAPI v1.0.0
  [2913bbd2] + StatsBase v0.33.10
  [3a884ed6] + UnPack v1.0.2
  [efe28fd5] + OpenSpecFun_jll v0.5.5+0
  [0dad84c5] + ArgTools
  [56f22d72] + Artifacts
  [2a0f44e3] + Base64
  [ade2ca70] + Dates
  [8bb1440f] + DelimitedFiles
  [8ba89e20] + Distributed
  [f43a241f] + Downloads
  [b77e0a4c] + InteractiveUtils
  [b27032c2] + LibCURL
  [76f85450] + LibGit2
  [8f399da3] + Libdl
  [37e2e46d] + LinearAlgebra
  [56ddb016] + Logging
  [d6f4376e] + Markdown
  [a63ad114] + Mmap
  [ca575930] + NetworkOptions
  [44cfe95a] + Pkg
  [de0858da] + Printf
  [3fa0cd96] + REPL
  [9a3f8284] + Random
  [ea8e919c] + SHA
  [9e88b42a] + Serialization
  [1a1011a3] + SharedArrays
  [6462fe0b] + Sockets
  [2f01184e] + SparseArrays
  [10745b16] + Statistics
  [fa267f1f] + TOML
  [a4e569a6] + Tar
  [8dfed614] + Test
  [cf7118a7] + UUIDs
  [c8ffd9c3] + MbedTLS_jll
  [14a3606d] + MozillaCACerts_jll
  [83775a58] + Zlib_jll
  [8e850ede] + nghttp2_jll
  [3f19e933] + p7zip_jll
Precompiling project...
  39 dependencies successfully precompiled in 105 seconds

julia> Using HTTP
ERROR: syntax: extra token "HTTP" after end of expression
Stacktrace:
 [1] top-level scope
  [cd3eb016] + HTTP v0.9.14
    Updating `C:\Users\DELL\.julia\environments\v1.6\Manifest.toml`
  [cd3eb016] + HTTP v0.9.14
  [83e8ac13] + IniFile v0.5.0
  [739be429] + MbedTLS v1.0.3
  [5c2747f8] + URIs v1.3.0
Precompiling project...
  4 dependencies successfully precompiled in 23 seconds (39 already precompiled)

julia> Pkg.add("GLM")
   Resolving package versions...
   Installed IteratorInterfaceExtensions ─ v1.0.0
   Installed Rmath ─────────────────────── v0.7.0
   Installed ShiftedArrays ─────────────── v1.0.0
   Installed TableTraits ───────────────── v1.0.1
   Installed QuadGK ────────────────────── v2.4.1
   Installed Tables ────────────────────── v1.5.1
   Installed StatsModels ───────────────── v0.6.25
   Installed GLM ───────────────────────── v1.5.1
   Installed Reexport ──────────────────── v1.2.2
   Installed DataValueInterfaces ───────── v1.0.0
   Installed Rmath_jll ─────────────────── v0.3.0+0
   Installed StatsFuns ─────────────────── v0.9.10
   Installed PDMats ────────────────────── v0.11.1
   Installed Distributions ─────────────── v0.25.16
  Downloaded artifact: Rmath
    Updating `C:\Users\DELL\.julia\environments\v1.6\Project.toml`
  [38e38edf] + GLM v1.5.1
    Updating `C:\Users\DELL\.julia\environments\v1.6\Manifest.toml`
  [e2d170a0] + DataValueInterfaces v1.0.0
  [31c24e10] + Distributions v0.25.16
  [38e38edf] + GLM v1.5.1
  [82899510] + IteratorInterfaceExtensions v1.0.0
  [90014a1f] + PDMats v0.11.1
  [1fd47b50] + QuadGK v2.4.1
  [189a3867] + Reexport v1.2.2
  [79098fc4] + Rmath v0.7.0
  [1277b4bf] + ShiftedArrays v1.0.0
  [4c63d2b9] + StatsFuns v0.9.10
  [3eaba693] + StatsModels v0.6.25
  [3783bdb8] + TableTraits v1.0.1
  [bd369af6] + Tables v1.5.1
  [f50d1b31] + Rmath_jll v0.3.0+0
  [4607b0f0] + SuiteSparse
Precompiling project...
  14 dependencies successfully precompiled in 35 seconds (43 already precompiled)

julia> Using LinearAlgebra
ERROR: syntax: extra token "LinearAlgebra" after end of expression
Stacktrace:
 [1] top-level scope
   @ none:1

julia> Pkg.add("Linear Algebra")
ERROR: `Linear Algebra` is not a valid package name
Stacktrace:
  [1] pkgerror(msg::String)
    @ Pkg.Types C:\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.6\Pkg\src\Types.jl:55
  [2] check_package_name
    @ C:\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.6\Pkg\src\API.jl:62 [inlined]
  [3] #96
    @ C:\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.6\Pkg\src\API.jl:156 [inlined]
  [4] foreach(f::Pkg.API.var"#96#101", itr::Vector{Pkg.Types.PackageSpec})
    @ Base .\abstractarray.jl:2141
  [5] add(ctx::Pkg.Types.Context, pkgs::Vector{Pkg.Types.PackageSpec}; preserve::Pkg.Types.PreserveLevel, platform::Base.BinaryPlatforms.Platform, kwargs::Base.Iterators.Pairs{Symbol, Base.TTY, Tuple{Symbol}, NamedTuple{(:io,), Tuple{Base.TTY}}})
    @ Pkg.API C:\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.6\Pkg\src\API.jl:156
  [6] add(pkgs::Vector{Pkg.Types.PackageSpec}; io::Base.TTY, kwargs::Base.Iterators.Pairs{Union{}, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ Pkg.API C:\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.6\Pkg\src\API.jl:80
  [7] add(pkgs::Vector{Pkg.Types.PackageSpec})
    @ Pkg.API C:\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.6\Pkg\src\API.jl:78
  [8] #add#23
    @ C:\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.6\Pkg\src\API.jl:76 [inlined]
  [9] add
    @ C:\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.6\Pkg\src\API.jl:76 [inlined]
 [10] #add#22
    @ C:\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.6\Pkg\src\API.jl:75 [inlined]
 [11] add(pkg::String)
    @ Pkg.API C:\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.6\Pkg\src\API.jl:75
 [12] top-level scope
    @ REPL[8]:1

julia> using DataFrames
ERROR: ArgumentError: Package DataFrames not found in current path:
julia> Pkg.add("LinearAlgebra")
   Resolving package versions...
    Updating `C:\Users\DELL\.julia\environments\v1.6\Project.toml`
  [37e2e46d] + LinearAlgebra
  No Changes to `C:\Users\DELL\.julia\environments\v1.6\Manifest.toml`

julia> Pkg.add("Random")
   Resolving package versions...
    Updating `C:\Users\DELL\.julia\environments\v1.6\Project.toml`
  [9a3f8284] + Random
  No Changes to `C:\Users\DELL\.julia\environments\v1.6\Manifest.toml`

julia> Pkg.add("Statistics")
   Resolving package versions...
    Updating `C:\Users\DELL\.julia\environments\v1.6\Project.toml`
  [10745b16] + Statistics
  No Changes to `C:\Users\DELL\.julia\environments\v1.6\Manifest.toml`
   Installed DataFrames ────── v1.2.2
   Installed PrettyTables ──── v1.1.0
    Updating `C:\Users\DELL\.julia\environments\v1.6\Project.toml`
  [a93c6f00] + DataFrames v1.2.2
    Updating `C:\Users\DELL\.julia\environments\v1.6\Manifest.toml`
  [a8cc5b0e] + Crayons v4.0.4
  [a93c6f00] + DataFrames v1.2.2
  [59287772] + Formatting v0.4.2
  [41ab1584] + InvertedIndices v1.1.0
  [2dfb63ee] + PooledArrays v1.3.0
  6 dependencies successfully precompiled in 74 seconds (57 already precompiled)
   Installed Parsers ──────────── v2.0.3
   Installed TranscodingStreams ─ v0.9.6
   Installed CSV ──────────────── v0.9.0
    Updating `C:\Users\DELL\.julia\environments\v1.6\Project.toml`
  [336ed68f] + CSV v0.9.0
    Updating `C:\Users\DELL\.julia\environments\v1.6\Manifest.toml`
  [336ed68f] + CSV v0.9.0
  [944b1d66] + CodecZlib v0.7.0
  [48062228] + FilePathsBase v0.9.10
  [69de0a69] + Parsers v2.0.3
  [91c51154] + SentinelArrays v1.3.7
  [3bb67fe8] + TranscodingStreams v0.9.6
  [ea10d353] + WeakRefStrings v1.2.2
Precompiling project...
  7 dependencies successfully precompiled in 30 seconds (63 already precompiled)

julia> Pkg.add("FreqTables")
   Resolving package versions...
   Installed FreqTables ──────── v0.4.4
   Installed Combinatorics ───── v1.0.2
   Installed NamedArrays ─────── v0.9.6
   Installed StructTypes ─────── v1.7.3
   Installed RecipesBase ─────── v1.1.2
   Installed JSON ────────────── v0.21.2
   Installed CategoricalArrays ─ v0.10.0
    Updating `C:\Users\DELL\.julia\environments\v1.6\Project.toml`
  [da1fdf0e] + FreqTables v0.4.4
    Updating `C:\Users\DELL\.julia\environments\v1.6\Manifest.toml`
  [324d7699] + CategoricalArrays v0.10.0
  [861a8166] + Combinatorics v1.0.2
  [da1fdf0e] + FreqTables v0.4.4
  [682c06a0] + JSON v0.21.2
  [86f7a689] + NamedArrays v0.9.6
  [3cdcf5f2] + RecipesBase v1.1.2
  [856f2bd8] + StructTypes v1.7.3
Precompiling project...
  7 dependencies successfully precompiled in 12 seconds (70 already precompiled)

julia> #:::::::::::::::::::::::::::::::::::::::::::::::::::

julia> # question 1

julia> #:::::::::::::::::::::::::::::::::::::::::::::::::::

julia> using Optim

julia> f(x) = -x[1]^4-10x[1]^3-2x[1]^2-3x[1]-2
f (generic function with 1 method)

julia> minusf(x) = x[1]^4+10x[1]^3+2x[1]^2+3x[1]+2
minusf (generic function with 1 method)

julia> startval = rand(1)   # random starting value
1-element Vector{Float64}:
 0.6433707274778036

julia> result = optimize(minusf, startval, BFGS())
 * Status: success

 * Candidate solution
    Final objective value:     -9.643134e+02

 * Found with
    Algorithm:     BFGS

 * Convergence measures
    |x - x'|               = 1.17e-05 ≰ 0.0e+00
    |x - x'|/|x'|          = 1.59e-06 ≰ 0.0e+00
    |f(x) - f(x')|         = 1.47e-08 ≰ 0.0e+00
    |f(x) - f(x')|/|f(x')| = 1.53e-11 ≰ 0.0e+00
    |g(x)|                 = 5.09e-09 ≤ 1.0e-08

 * Work counters
    Seconds run:   1  (vs limit Inf)
    Iterations:    5
    f(x) calls:    25
    ∇f(x) calls:   25

julia> println(result)
 * Status: success

 * Candidate solution
    Final objective value:     -9.643134e+02

 * Found with
    Algorithm:     BFGS

 * Convergence measures
    |x - x'|               = 1.17e-05 ≰ 0.0e+00
    |x - x'|/|x'|          = 1.59e-06 ≰ 0.0e+00
    |f(x) - f(x')|         = 1.47e-08 ≰ 0.0e+00
    |f(x) - f(x')|/|f(x')| = 1.53e-11 ≰ 0.0e+00
    |g(x)|                 = 5.09e-09 ≤ 1.0e-08

 * Work counters
    Seconds run:   1  (vs limit Inf)
    Iterations:    5
    f(x) calls:    25
    ∇f(x) calls:   25

julia> #:::::::::::::::::::::::::::::::::::::::::::::::::::

julia> # question 2

julia> #:::::::::::::::::::::::::::::::::::::::::::::::::::

julia> using DataFrames

julia> using CSV

julia> using HTTP
julia> url = "https://raw.githubusercontent.com/OU-PhD-Econometrics/fall-2020/master/ProblemSets/PS1-julia-intro/nlsw88.csv"
"https://raw.githubusercontent.com/OU-PhD-Econometrics/fall-2020/master/ProblemSets/PS1-julia-intro/nlsw88.csv"

julia> df = CSV.read(HTTP.get(url).body)
ERROR: ArgumentError: provide a valid sink argument, like `using DataFrames; CSV.read(source, DataFrame)`
Stacktrace:
 [1] read(source::Vector{UInt8}, sink::Nothing; copycols::Bool, kwargs::Base.Iterators.Pairs{Union{}, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
   @ CSV C:\Users\DELL\.julia\packages\CSV\eB98W\src\CSV.jl:64
 [2] read(source::Vector{UInt8}, sink::Nothing) (repeats 2 times)
   @ CSV C:\Users\DELL\.julia\packages\CSV\eB98W\src\CSV.jl:63
 [3] top-level scope
   @ REPL[32]:1

julia> X = [ones(size(df,1),1) df.age df.race.==1 df.collgrad.==1]
ERROR: UndefVarError: df not defined
Stacktrace:
 [1] top-level scope
   @ REPL[33]:1

julia> y = df.married.==1
ERROR: UndefVarError: df not defined
Stacktrace:
 [1] top-level scope
   @ REPL[34]:1

julia>

julia> function ols(beta, X, y)
           ssr = (y.-X*beta)'*(y.-X*beta)
               return ssr
       end
ols (generic function with 1 method)

julia>
julia> beta_hat_ols = optimize(b -> ols(b, X, y), rand(size(X,2)), LBFGS(), Optim.Options(g_tol=1e-6, iterations=100_000, show_trace=true))
ERROR: UndefVarError: X not defined
Stacktrace:
 [1] top-level scope
   @ REPL[36]:1

julia> println(beta_hat_ols.minimizer)
ERROR: UndefVarError: beta_hat_ols not defined
Stacktrace:
 [1] top-level scope
   @ REPL[37]:1

julia>

julia> using GLM

julia> bols = inv(X'*X)*X'*y
ERROR: UndefVarError: X not defined
Stacktrace:
 [1] top-level scope
   @ REPL[39]:1

julia> df.white = df.race.==1
ERROR: UndefVarError: df not defined
Stacktrace:
 [1] top-level scope
   @ REPL[40]:1

julia> bols_lm = lm(@formula(married ~ age + white + collgrad), df)
ERROR: UndefVarError: df not defined
Stacktrace:
 [1] top-level scope
   @ REPL[41]:1

julia> using DataFrames; CSV.read (source, DataFrame)
ERROR: syntax: space before "(" not allowed in "CSV.read (" at REPL[42]:1
Stacktrace:
 [1] top-level scope
   @ none:1

julia> using DataFrames;CSV.read(source,DataFrame)
ERROR: UndefVarError: source not defined
Stacktrace:
 [1] top-level scope
   @ REPL[43]:1

julia> using DataFrames

julia> using CSV

julia> using HTTP
julia> url = "https://raw.githubusercontent.com/OU-PhD-Econometrics/fall-2020/master/ProblemSets/PS1-julia-intro/nlsw88.csv"
"https://raw.githubusercontent.com/OU-PhD-Econometrics/fall-2020/master/ProblemSets/PS1-julia-intro/nlsw88.csv"

julia> df=CSV.read(HTTP.get(url).body,Dataframe)
ERROR: UndefVarError: Dataframe not defined
Stacktrace:
 [1] top-level scope
   @ REPL[48]:1

julia> df=CSV.read(HTTP.get(url).body,DataFrame)
2246×17 DataFrame
  Row │ idcode  age    race   married  never_married  grade   collgrad  south  smsa   c_city  industry  occupation  un ⋯
      │ Int64   Int64  Int64  Int64    Int64          Int64?  Int64     Int64  Int64  Int64   Int64?    Int64?      In ⋯
──────┼─────────────────────────────────────────────────────────────────────────────────────────────────────────────────
    1 │      1     37      2        0              0      12         0      0      1       0         5           6     ⋯
    2 │      2     37      2        0              0      12         0      0      1       1         4           5
    3 │      3     42      2        0              1      12         0      0      1       1         4           3  mi
    4 │      4     43      1        1              0      17         1      0      1       0        11          13
    5 │      6     42      1        1              0      12         0      0      1       0         4           6     ⋯
    6 │      7     39      1        1              0      12         0      0      1       0        11           3
    7 │      9     37      1        0              0      12         0      0      1       1         5           2
    8 │     12     40      1        1              0      18         1      0      1       0        11           2
    9 │     13     40      1        1              0      14         0      0      1       0        11           3     ⋯
  ⋮   │   ⋮       ⋮      ⋮       ⋮           ⋮          ⋮        ⋮        ⋮      ⋮      ⋮        ⋮          ⋮          ⋱
 2238 │   5134     40      2        0              0      17         1      0      1       1        11          13
 2239 │   5143     38      2        1              0      12         0      1      1       0         9           6
 2240 │   5148     35      1        1              0      12         0      1      0       0        11           3
 2241 │   5152     35      1        1              0      13         0      1      0       0        11           3     ⋯
 2242 │   5153     35      1        0              1      12         0      1      1       0        11           8  mi
 2243 │   5154     44      1        1              0      16         1      1      1       0        11           1
 2244 │   5156     42      1        1              0      12         0      1      0       0         8           3
 2245 │   5157     38      2        1              0      12         0      1      0       0        11           8     ⋯
 2246 │   5159     43      2        0              0      12         0      1      1       1        11           3
                                                                                5 columns and 2228 rows omitted

julia> X = [ones(size(df,1),1) df.age df.race.==1 df.collgrad.==1]
2246×4 Matrix{Float64}:
 1.0  37.0  0.0  0.0
 1.0  37.0  0.0  0.0
 1.0  42.0  0.0  0.0
 1.0  43.0  1.0  1.0
 1.0  42.0  1.0  0.0
 1.0  39.0  1.0  0.0
 1.0  37.0  1.0  0.0
 1.0  40.0  1.0  1.0
 1.0  40.0  1.0  0.0
 1.0  40.0  1.0  0.0
 1.0  39.0  1.0  1.0
 ⋮
 1.0  41.0  0.0  0.0
 1.0  36.0  1.0  1.0
 1.0  40.0  0.0  1.0
 1.0  38.0  0.0  0.0
 1.0  35.0  1.0  0.0
 1.0  35.0  1.0  0.0
 1.0  35.0  1.0  0.0
 1.0  44.0  1.0  1.0
 1.0  42.0  1.0  0.0
 1.0  38.0  0.0  0.0
 1.0  43.0  0.0  0.0

julia> y = df.married.==1
2246-element BitVector:
 0
 0
 0
 1
 1
 1
 0
 1
 1
 1
 1
 ⋮
 1
 1
 0
 1
 1
 1
 0
 1
 1
 1
 0

julia>

julia> function ols(beta, X, y)
           ssr = (y.-X*beta)'*(y.-X*beta)
           return ssr
       end
ols (generic function with 1 method)

julia>
julia> beta_hat_ols = optimize(b -> ols(b, X, y), rand(size(X,2)), LBFGS(), Optim.Options(g_tol=1e-6, iterations=100_000, show_trace=true))
Iter     Function value   Gradient norm
     0     8.072236e+05     3.342437e+06
 * time: 0.0
     1     9.471005e+02     6.502574e+02
 * time: 1.11899995803833
     2     4.953501e+02     2.496294e+01
 * time: 1.1339998245239258
     3     4.944945e+02     6.924277e+00
 * time: 1.1339998245239258
     4     4.936093e+02     6.939238e+00
 * time: 1.1339998245239258
     5     4.936093e+02     7.040357e-08
 * time: 1.1499998569488525
 * Status: success

 * Candidate solution
    Final objective value:     4.936093e+02

 * Found with
    Algorithm:     L-BFGS

 * Convergence measures
    |x - x'|               = 1.03e-06 ≰ 0.0e+00
    |x - x'|/|x'|          = 1.55e-06 ≰ 0.0e+00
    |f(x) - f(x')|         = 3.48e-06 ≰ 0.0e+00
    |f(x) - f(x')|/|f(x')| = 7.04e-09 ≰ 0.0e+00
    |g(x)|                 = 7.04e-08 ≤ 1.0e-06

 * Work counters
    Seconds run:   1  (vs limit Inf)
    Iterations:    5
    f(x) calls:    21
    ∇f(x) calls:   21

julia> println(beta_hat_ols.minimizer)
[0.6613509507510695, -0.004625908024220015, 0.2259504322727182, -0.012184197726049258]

julia>

julia> using GLM

julia> bols = inv(X'*X)*X'*y
4-element Vector{Float64}:
  0.6613509511877042
 -0.004625908035375747
  0.22595043227530237
 -0.01218419772065558

julia> df.white = df.race.==1
2246-element BitVector:
 0
 0
 0
 1
 1
 1
 1
 1
 1
 1
 1
 ⋮
 0
 1
 0
 0
 1
 1
 1
 1
 1
 0
 0

julia> bols_lm = lm(@formula(married ~ age + white + collgrad), df)
StatsModels.TableRegressionModel{LinearModel{GLM.LmResp{Vector{Float64}}, GLM.DensePredChol{Float64, LinearAlgebra.CholeskyPivoted{Float64, Matrix{Float64}}}}, Matrix{Float64}}

married ~ 1 + age + white + collgrad

Coefficients:
─────────────────────────────────────────────────────────────────────────────
                   Coef.  Std. Error      t  Pr(>|t|)   Lower 95%   Upper 95%
─────────────────────────────────────────────────────────────────────────────
(Intercept)   0.661351    0.127491     5.19    <1e-06   0.411338   0.911364
age          -0.00462591  0.00324314  -1.43    0.1539  -0.0109858  0.00173397
white         0.22595     0.0223827   10.09    <1e-22   0.182057   0.269843
collgrad     -0.0121842   0.0233565   -0.52    0.6020  -0.0579868  0.0336184
─────────────────────────────────────────────────────────────────────────────

julia> prin
ERROR: UndefVarError: prin not defined

julia>

julia> println(result)
 * Status: success

 * Candidate solution
    Final objective value:     -9.643134e+02

 * Found with
    Algorithm:     BFGS

 * Convergence measures
    |x - x'|               = 1.17e-05 ≰ 0.0e+00
    |x - x'|/|x'|          = 1.59e-06 ≰ 0.0e+00
    |f(x) - f(x')|         = 1.47e-08 ≰ 0.0e+00
    |f(x) - f(x')|/|f(x')| = 1.53e-11 ≰ 0.0e+00
    |g(x)|                 = 5.09e-09 ≤ 1.0e-08

 * Work counters
    Seconds run:   1  (vs limit Inf)
    Iterations:    5
    f(x) calls:    25
    ∇f(x) calls:   25

julia> println()

julia> #:::::::::::::::::::::::::::::::::::::::::::::::::::

julia> # question 3

julia> #:::::::::::::::::::::::::::::::::::::::::::::::::::

julia> function logit(alpha, X, d)

ERROR: syntax: incomplete: premature end of input
Stacktrace:
 [1] top-level scope
   @ none:1

julia> log_mat=[d[i]*log((1+exp(-(X*alpha)[i]))^-1)+(1-d[i])*log(1-(1+exp(-(X*alpha)[i]))^-1) for i in 1:size(d,1)]
ERROR: UndefVarError: d not defined
Stacktrace:
 [1] top-level scope
   @ REPL[66]:1

julia> function logit(alpha,X,d)
       ^C

julia> function logit(alpha,X,d)
       log_likeli=[d[i]*log((1+exp(-(X*alpha)[i]))^-1)+(1-d[i])*log(1-(1+exp(-(X*alpha)[i]))^-1) for i in 1:size(d,1)]
       loglike=-sum(log_likeli)
       return loglike
       end
logit (generic function with 1 method)

julia>

julia>
julia> beta_loglike=optimize(b-> logit(b,X,y),rand(size(X,2)),LBFGS(),Optim.Options(g_tol=1e-6, iterations=100_00, show_trace=true))
Iter     Function value   Gradient norm
     0     2.390411e+04     2.209590e+04
 * time: 0.0
     1     8.207293e+03     5.608561e+04
 * time: 8.032999992370605
     2     3.791609e+03     3.072234e+04
 * time: 11.203999996185303
     3     2.013047e+03     3.017661e+04
 * time: 14.683000087738037
     4     1.461702e+03     4.237664e+03
 * time: 16.865000009536743
     5     1.449865e+03     7.917543e+01
 * time: 19.850000143051147
     6     1.449741e+03     4.280369e+02
 * time: 25.036999940872192
     7     1.417416e+03     3.579350e+02
 * time: 31.608999967575073
     8     1.416994e+03     1.350456e+02
 * time: 35.08400011062622
     9     1.416951e+03     1.630266e+01
 * time: 38.41600012779236
    10     1.416951e+03     3.043366e-02
 * time: 41.980000019073486
    11     1.416951e+03     3.101587e-02
 * time: 45.27800011634827
    12     1.416951e+03     1.062688e+00
 * time: 54.5220000743866
    13     1.416951e+03     7.143616e-05
 * time: 57.53800010681152
    14     1.416951e+03     2.252914e-07
 * time: 59.42799997329712
 * Status: success

 * Candidate solution
    Final objective value:     1.416951e+03

 * Found with
    Algorithm:     L-BFGS

 * Convergence measures
    |x - x'|               = 3.13e-08 ≰ 0.0e+00
    |x - x'|/|x'|          = 3.27e-08 ≰ 0.0e+00
    |f(x) - f(x')|         = 0.00e+00 ≤ 0.0e+00
    |f(x) - f(x')|/|f(x')| = 0.00e+00 ≤ 0.0e+00
    |g(x)|                 = 2.25e-07 ≤ 1.0e-06

 * Work counters
    Seconds run:   59  (vs limit Inf)
    Iterations:    14
    f(x) calls:    57
    ∇f(x) calls:   57

julia> println(beta_loglike.minimizer)
[0.7465534179105563, -0.02107775597442363, 0.9558065344475369, -0.05597400464143933]

julia>

julia> #:::::::::::::::::::

julia> question 4
ERROR: syntax: extra token "4" after end of expression
Stacktrace:
 [1] top-level scope
   @ none:1

julia> #question4

julia> #::::::::::::::::::::

julia> df.white=df.race.==1
2246-element BitVector:
 0
 0
 0
 1
 1
 1
 1
 1
 1
 1
 1
 ⋮
 0
 1
 0
 0
 1
 1
 1
 1
 1
 0
 0

julia> a=glm(@formula(married~age+white+collgrad),df,Binomial(),LogitLink()
       println(a)
ERROR: syntax: missing comma or ) in argument list
Stacktrace:
 [1] top-level scope
   @ none:1

julia> a=glm(@formula(married~age+white+collgrad),df,Binomial(),LogitLink())
StatsModels.TableRegressionModel{GeneralizedLinearModel{GLM.GlmResp{Vector{Float64}, Binomial{Float64}, LogitLink}, GLM.DensePredChol{Float64, LinearAlgebra.Cholesky{Float64, Matrix{Float64}}}}, Matrix{Float64}}

married ~ 1 + age + white + collgrad

Coefficients:
────────────────────────────────────────────────────────────────────────────
                  Coef.  Std. Error      z  Pr(>|z|)   Lower 95%   Upper 95%
────────────────────────────────────────────────────────────────────────────
(Intercept)   0.746554    0.578983    1.29    0.1973  -0.388231   1.88134
age          -0.0210778   0.0147497  -1.43    0.1530  -0.0499867  0.00783116
white         0.955807    0.0981953   9.73    <1e-21   0.763347   1.14827
collgrad     -0.055974    0.106218   -0.53    0.5982  -0.264157   0.152209
────────────────────────────────────────────────────────────────────────────

julia> println(a)
StatsModels.TableRegressionModel{GeneralizedLinearModel{GLM.GlmResp{Vector{Float64}, Binomial{Float64}, LogitLink}, GLM.DensePredChol{Float64, LinearAlgebra.Cholesky{Float64, Matrix{Float64}}}}, Matrix{Float64}}

married ~ 1 + age + white + collgrad

Coefficients:
────────────────────────────────────────────────────────────────────────────
                  Coef.  Std. Error      z  Pr(>|z|)   Lower 95%   Upper 95%
────────────────────────────────────────────────────────────────────────────
(Intercept)   0.746554    0.578983    1.29    0.1973  -0.388231   1.88134
age          -0.0210778   0.0147497  -1.43    0.1530  -0.0499867  0.00783116
white         0.955807    0.0981953   9.73    <1e-21   0.763347   1.14827
collgrad     -0.055974    0.106218   -0.53    0.5982  -0.264157   0.152209
────────────────────────────────────────────────────────────────────────────

julia> #::::::::::::::::::::::::

julia> #question5

julia> #::::::::::::::::::::::::

julia> freqtable(df,:occupation)
ERROR: UndefVarError: freqtable not defined
Stacktrace:
 [1] top-level scope
   @ REPL[81]:1

julia> FreqTable(df,:occupation)
ERROR: UndefVarError: FreqTable not defined
Stacktrace:
 [1] top-level scope
   @ REPL[82]:1

julia> FreqTables(df,:occupation)
ERROR: UndefVarError: FreqTables not defined
Stacktrace:
 [1] top-level scope
   @ REPL[83]:1

julia> freqtable(df,:occupation)
ERROR: UndefVarError: freqtable not defined
Stacktrace:
 [1] top-level scope
   @ REPL[84]:1

julia> using Pkg

julia> pkg.add("FreqTables")
ERROR: UndefVarError: pkg not defined
Stacktrace:
 [1] top-level scope
   @ REPL[86]:1

julia> Pkg.add("FreqTables")
   Resolving package versions...
  No Changes to `C:\Users\DELL\.julia\environments\v1.6\Project.toml`
  No Changes to `C:\Users\DELL\.julia\environments\v1.6\Manifest.toml`

julia> freqtable(df,:occupation)
ERROR: UndefVarError: freqtable not defined
Stacktrace:
 [1] top-level scope
   @ REPL[88]:1

julia> df=dropmissing(df,:occupation)
2237×18 DataFrame
  Row │ idcode  age    race   married  never_married  grade   collgrad  south  smsa   c_city  industry  occupation  un ⋯
      │ Int64   Int64  Int64  Int64    Int64          Int64?  Int64     Int64  Int64  Int64   Int64?    Int64       In ⋯
──────┼─────────────────────────────────────────────────────────────────────────────────────────────────────────────────
    1 │      1     37      2        0              0      12         0      0      1       0         5           6     ⋯
    2 │      2     37      2        0              0      12         0      0      1       1         4           5
    3 │      3     42      2        0              1      12         0      0      1       1         4           3  mi
    4 │      4     43      1        1              0      17         1      0      1       0        11          13
    5 │      6     42      1        1              0      12         0      0      1       0         4           6     ⋯
    6 │      7     39      1        1              0      12         0      0      1       0        11           3
    7 │      9     37      1        0              0      12         0      0      1       1         5           2
    8 │     12     40      1        1              0      18         1      0      1       0        11           2
    9 │     13     40      1        1              0      14         0      0      1       0        11           3     ⋯
  ⋮   │   ⋮       ⋮      ⋮       ⋮           ⋮          ⋮        ⋮        ⋮      ⋮      ⋮        ⋮          ⋮          ⋱
 2229 │   5134     40      2        0              0      17         1      0      1       1        11          13
 2230 │   5143     38      2        1              0      12         0      1      1       0         9           6
 2231 │   5148     35      1        1              0      12         0      1      0       0        11           3
 2232 │   5152     35      1        1              0      13         0      1      0       0        11           3     ⋯
 2233 │   5153     35      1        0              1      12         0      1      1       0        11           8  mi
 2234 │   5154     44      1        1              0      16         1      1      1       0        11           1
 2235 │   5156     42      1        1              0      12         0      1      0       0         8           3
 2236 │   5157     38      2        1              0      12         0      1      0       0        11           8     ⋯
 2237 │   5159     43      2        0              0      12         0      1      1       1        11           3
                                                                                6 columns and 2219 rows omitted

julia> df[df.occupation.==10,:occupation].=9
9-element view(::Vector{Int64}, [676, 1627, 1908, 1918, 1919, 1920, 1921, 2088, 2162]) with eltype Int64:
 9
 9
 9
 9
 9
 9
 9
 9
 9

julia> df[df.occupation.==11,:occupation].=9
16-element view(::Vector{Int64}, [319, 412, 608, 764, 1071, 1134, 1301, 1459, 1484, 1525, 1680, 1952, 1999, 2082, 2094, 2211]) with eltype Int64:
 9
 9
 9
 9
 9
 9
 9
 9
 9
 9
 9
 9
 9
 9
 9
 9

julia> df[df.occupation.==12,:occupation].=9
2-element view(::Vector{Int64}, [385, 1156]) with eltype Int64:
 9
 9

julia> df[df.occupation.==12,:occupation].=9
0-element view(::Vector{Int64}, Int64[]) with eltype Int64

julia> freqtable(df,:occupation)
ERROR: UndefVarError: freqtable not defined
Stacktrace:
 [1] top-level scope
   @ REPL[93]:1

julia> X=[ones(size(df,1),1)df.age df.race.==1 df.collgrad.==1]
ERROR: DimensionMismatch("second dimension of A, 1, does not match length of x, 2237")
Stacktrace:
 [1] gemv!(y::Vector{Float64}, tA::Char, A::Matrix{Float64}, x::Vector{Float64}, α::Bool, β::Bool)
   @ LinearAlgebra C:\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.6\LinearAlgebra\src\matmul.jl:530
 [2] mul!
   @ C:\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.6\LinearAlgebra\src\matmul.jl:66 [inlined]
 [3] mul!
   @ C:\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.6\LinearAlgebra\src\matmul.jl:275 [inlined]
 [4] *(A::Matrix{Float64}, x::Vector{Int64})
   @ LinearAlgebra C:\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.6\LinearAlgebra\src\matmul.jl:47
 [5] top-level scope
   @ REPL[94]:1

julia> y=df.occupation
2237-element Vector{Int64}:
  6
  5
  3
 13
  6
  3
  2
  2
  3
  1
  1
  ⋮
  6
  1
 13
  6
  3
  3
  8
  1
  3
  8
  3

julia> function mlogit(alpha,X,d)
       end
mlogit (generic function with 1 method)

julia> ps2()
ERROR: UndefVarError: ps2 not defined
Stacktrace:
 [1] top-level scope
   @ REPL[97]:1

julia> include("pset2aleeze.jl")
ERROR: SystemError: opening file "C:\\Users\\DELL\\pset2aleeze.jl": No such file or directory
Stacktrace:
  [1] systemerror(p::String, errno::Int32; extrainfo::Nothing)
    @ Base .\error.jl:168
  [2] #systemerror#62
    @ .\error.jl:167 [inlined]
  [3] systemerror
    @ .\error.jl:167 [inlined]
  [4] open(fname::String; lock::Bool, read::Nothing, write::Nothing, create::Nothing, truncate::Nothing, append::Nothing)
    @ Base .\iostream.jl:293
  [5] open
    @ .\iostream.jl:282 [inlined]
  [6] open(f::Base.var"#326#327"{String}, args::String; kwargs::Base.Iterators.Pairs{Union{}, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ Base .\io.jl:328
  [7] open
    @ .\io.jl:328 [inlined]
  [8] read
    @ .\io.jl:434 [inlined]
  [9] _include(mapexpr::Function, mod::Module, _path::String)
    @ Base .\loading.jl:1166
 [10] include(fname::String)
    @ Base.MainInclude .\client.jl:444
 [11] top-level scope
    @ REPL[98]:1

julia>


text:
using DistanceAndSimilarity
using Test

@testset "Function test" begin

    @test euclidianDistance([1,1],[1,3]) == 2
    @test manhattanDistance([3,45,7,2],[2,54,13,15]) == 29
    @test cosineDistance([3,45,7,2],[2,54,13,15]) == 0.02771574828765011
    @test minkowskiDistance([1,2,3],[1,2,3],2) == 0
    @test jaccardDistance([1,2,3],[3,4,5]) == (1- 1/5)
    @test jaccardSimilarity([1,2,3],[3,4,5]) == 1/5

    @test errorCheck([0,0],[1,1,1],1) == -1
    @test errorCheck([1,1],[1,1],0) == -1

    end

max_stars_repo_name: nicolasmagalhaes/DistanceAndSimilarity

max_issues_repo_name: nicolasmagalhaes/DistanceAndSimilarity

max_forks_repo_name: nicolasmagalhaes/DistanceAndSimilarity


text:
module AddToField

using MacroTools

export @addnt, @adddict, @addto!

is_add(x::Expr) = x.head == :macrocall && x.args[1] == Symbol("@add")
is_add(x) = false

is_interp_string(x) = false
is_interp_string(x::String) = true
is_interp_string(x::Expr) = x.head == :string
is_interp_string(x::QuoteNode) = true

function add_property_or_index!(d::AbstractDict, name, value)
	setindex!(d, value, Symbol(name))
end

function add_property_or_index!(d::AbstractDict{<:AbstractString}, name, value)
	setindex!(d, value, string(name))
end

function add_property_or_index!(d, name, value)
	setproperty!(d, Symbol(name), value)
end

function add_to_props_vec!(props_vec, arg)
	if arg.args[3] isa Symbol
		v = gensym()
		n = QuoteNode(arg.args[3])
		push!(props_vec, :($n => $v))
		Expr(:block, Expr(:(=), v, arg.args[3]))
	elseif is_interp_string(arg.args[3])
		v = gensym()
		v_n = gensym()
		n = arg.args[3]
		push!(props_vec, :(Symbol($n) => $v))
		Expr(:block, Expr(:(=), v, arg.args[4]))
	elseif arg.args[3].head == :(=)
		v = gensym()
		n = QuoteNode(arg.args[3].args[1])
		push!(props_vec, :($n => $v))
		Expr(:block, Expr(:(=), v, arg.args[3]))
	elseif arg.args[3].head === :tuple
		equals_vec = []
		for s in arg.args[3].args
			v = gensym()
			@assert s isa Symbol "Ony Symbols allowed in tuples with @add"
			n = QuoteNode(s)
			push!(equals_vec, Expr(:block, Expr(:(=), v, s)))
			push!(props_vec, :($n => $v))
		end
		Expr(:block, equals_vec...)
	else
		throw(ArgumentError("Malformed @add expression"))
	end
end

function addto_nt_helper(body)
	body = MacroTools.unblock(body)

	props_vec = Expr[]

	newbody = MacroTools.postwalk(body) do x
		if is_add(x)
			add_to_props_vec!(props_vec, x)
		else
			x
		end
	end

	nt_expr = Expr(:tuple, Expr(:parameters, props_vec...))

	if body isa Expr && body.head == :let
		newbody.args[2] = Expr(:block, newbody.args[2], nt_expr)
		return newbody
	else
   	return Expr(:block, newbody, nt_expr)
   end
end

"""
	@addnt(body)

Create a named tuple using `@add` statments inside `body`.

### Usage

Within `body`, you can create a namedtuple with the following
syntaxes.

- `@add x`: The named tuple will contain the field `:x` with the value of `x`.
- `@add x, y, z`: The named tuple will contain the fields `x`, `y`, and `z` with
   their corresponding values.
- `@add x = expr`: The named tuple will contain the field `:x` with the value of
   `expr`. The variable `x` will still be created.
- `@add "My value" x`: The named tuple will contain the field `Symbol("My value)`
   with the value of `x`. You can also interpolate values inside the `String`
   name.
- `@add "My value" x = expr`: The named tuple will contain the field
   `Symbol("My value")` with the value of `expr`. The variable `x` will still be created.

`@addnt begin ... end` does not create a new scope, meaning changes
all variable assignments in the inside the expression modify the
existing scope. To create a new scope, use `@addnt let ... end`.

### Example:

```jldoctest
julia> s = "My long name";

julia> res = @addnt begin
		a = 1
		@add a

		f, g, h = 6, 7, 8
		@add f, g, h

		@add b = 2
		@add :c1 c = 3
		@add "d1" d = 4
		@add "My long name" e = 5
	end
end
(a = 1, f = 6, g = 7, h = 8, b = 2, c1 = 3, d1 = 4, My long name = 5)

julia> @addnt let
           @add local_var = 500
       end
(local_var = 500,)

julia> isdefined(Main, :local_var)
false
```

!!! note
    `You` cannot use `@add` in new scopes created with
    `body`. The following will fail

    ```julia
    @addnt begin
    	let
    	    a = 1
    	    @add a
    	end
    end
    ```

    This is because `@addnt` creates anonymous variables,
    then constructs the named tuple at the end of the
    expression. The same applies for `for` loops and `function`s
    inside the `@addnt` and `@addto!` blocks.
"""
macro addnt(body)
	esc(addto_nt_helper(body))
end

function addto_dict_helper(body)
	body = MacroTools.unblock(body)

	props_vec = Expr[]

	newbody = MacroTools.postwalk(body) do x
		if is_add(x)
			add_to_props_vec!(props_vec, x)
		else
			x
		end
	end
	dict_expr = :(Dict([$(props_vec...)]))

	if body isa Expr && body.head == :let
		newbody.args[2] = Expr(:block, newbody.args[2], dict_expr)
		return newbody
	else
   	return Expr(:block, newbody, dict_expr)
   end
end

"""
	@adddict(body)

Create a `Dict` with `Symbol`s as keys using `@add` statments inside `body`.

### Usage

Within `body`, you can create a `Dict` with the following
syntaxes.

- `@add x`: The `Dict` will contain the field `:x` with the value of `x`.
- `@add x, y, z`: The `Dict` will contain the fields `x`, `y`, and `z` with
   their corresponding values.
- `@add x = expr`: The `Dict` will contain the field `:x` with the value of
   `expr`. The variable `x` will still be created.
- `@add "My value" x`: The `Dict` will contain the field `Symbol("My value)`
   with the value of `x`. You can also interpolate values inside the `String`
   name.
- `@add "My value" x = expr`: The `Dict` will contain the field
   `Symbol("My value")` with the value of `expr`. The variable `x` will still be created.

`@addict begin ... end` does not create a new scope, meaning changes
all variable assignments in the inside the expression modify the
existing scope. To create a new scope, use `@adddict let ... end`.

### Example:

```jldoctest
julia> s = "My long name";

julia> res = @adddict begin
		a = 1
		@add a

		f, g, h = 6, 7, 8
		@add f, g, h

		@add b = 2
		@add :c1 c = 3
		@add "d1" d = 4
		@add "My long name" e = 5
	end
end
Dict{Symbol, Int64} with 8 entries:
  :a                     => 1
  :f                     => 6
  :b                     => 2
  :h                     => 8
  :g                     => 7
  :d1                    => 4
  :c1                    => 3
  Symbol("My long name") => 5

julia> @adddict let
           @add local_var = 500
       end
Dict{Symbol, Int64} with 1 entry:
  :local_var => 500

julia> isdefined(Main, :local_var)
false
```

!!! note
    `You` cannot use `@add` in new scopes created with
    `body`. The following will fail

    ```julia
    @adddict begin
    	let
    	    a = 1
    	    @add a
    	end
    end
    ```

    This is because `@adddict` creates anonymous variables,
    then constructs the `Dict` at the end of the
    expression. The same applies for `for` loops and `function`s
    inside the `@addict` block.
   """
macro adddict(body)
	esc(addto_dict_helper(body))
end

function addto_two_arg_helper(x, body)
	body = MacroTools.unblock(body)

	props_vec = []

	newbody = MacroTools.postwalk(body) do x
		if is_add(x)
			add_to_props_vec!(props_vec, x)
		else
			x
		end
	end

	anon = gensym()
	assign_to_anon = :($anon = $x)

	addproperties = map(props_vec) do p
		name = p.args[2]
		value = p.args[3]
		:(AddToField.add_property_or_index!($anon, $name, $value))
	end
	push!(addproperties, quote $anon end)

	if body isa Expr && body.head == :let
		# The first argument of a let block is a block of
		# assignments for the variable captures if
		# there are multiple captures. But if there
		# is only one, it is not, so we put it in
		# a block.
		initial_let_assignments = MacroTools.block(newbody.args[1])
		push!(initial_let_assignments.args, assign_to_anon)
		newbody.args[1] = initial_let_assignments
		newbody.args[2] = Expr(:block, newbody.args[2], addproperties...)
		return newbody
	else
   	return Expr(:block, assign_to_anon, newbody, addproperties...)
   end
end

"""
	@addto!(x, body)

Set properties or indices of `x` using `@add` statments inside `body`.

### Usage

Within `body`, you can add properies or indices of `x` with the following.

- `@add x`: The named tuple will contain the field `:x` with the value of `x`.
- `@add x, y, z`: The named tuple will contain the fields `x`, `y`, and `z`
   with their corresponding values.
- `@add x = expr`: The named tuple will contain the field `:x` with the value of
   `expr`. The variable `x` will still be created.
- `@add "My value" x`: The named tuple will contain the field `Symbol("My value)`
   with the value of `x`. You can also interpolate values inside the `String`
   name.
- `@add "My value" x = expr`: The named tuple will contain the field
   `Symbol("My value")` with the value of `expr`. The variable `x` will still be created.

`@addto!` defaults to calling `setproperty!` on `x`. However
with `AbstractDict`, it calls `setindex!` with `Symbol`s.
Wieh `AbstractDict{<:AbstractString}` is calls `setindex!` with `String`s.

`@addto! d begin ... end` does not create a new scope, meaning changes
all variable assignments in the inside the expression modify the
existing scope. To create a new scope, use `@addto! d let ... end`.

### Example:

```jldoctest
julia> D = Dict();

julia> s = "My long name";

julia> res = @addto! D begin
		a = 1
		@add a

		f, g, h = 6, 7, 8
		@add f, g, h

		@add b = 2
		@add :c1 c = 3
		@add "d1" d = 4
		@add "My long name" e = 5
	end
end
Dict{Any,Any} with 8 entries:
  :a                     => 1
  :f                     => 6
  :b                     => 2
  :h                     => 8
  :g                     => 7
  :d1                    => 4
  :c1                    => 3
  Symbol("My long name") => 5

julia> @addto! d let
           @add local_var = 500
       end;

julia> isdefined(Main, :local_var)
false
```

!!! note
    `You` cannot use `@addto!` in new scopes created with
    `body`. The following will fail

    ```julia
    @addto! D begin
    	let
    	    a = 1
    	    @add a
    	end
    end
    ```

    This is because `@addto!` creates anonymous variables,
    then constructs the `setproperty!` or `setindex!` calls
    at the end of the expression.
"""
macro addto!(x, body)
	esc(addto_two_arg_helper(x, body))
end

end # module