text:
import Base.==, Base.!=

"""

A multidimensional array whose elements are ordered dictionaries with common keys.
Internally, it is represented as an ordered dictionary from keys to multidimensional arrays.
Note that most functions return a new `DictArray` rather than modify the existing one.
However, the new `DictArray` just shallow copies the key vector and the value vector
of the underlying `LDict`. Therefore, creating a new `DictArray` is cheap, but you have to be
careful when you modify the underlying array elements directly.

Because a `DictArray` can be multidimensional we will call the keys in the key vector of the underlying `LDict` the *field names*.
The values in the value vector will be called *fields*. With a slight bit of abuse of notation, we sometimes call a field name and a field tuple collectively just a field.

Use the function `darr` to construct a `DictArray`.

##### Constructors
DictArraay is internally just a wrapper of `LDict`. Therefore, the constructors takes the same kind of arguments:

```julia
DictArray(data::LDict{K,V})
DictArray{K,V}(dict::Dict{K,V})
DictArray{K,V}(dict::Dict{K,V}, ks)
DictArray{K}(ks::Vector{K}, vs::Vector)
DictArray(ps::Pair...)
DictArray(tuples::Tuple...)
DictArray(;kwargs...)
```

"""
immutable DictArray{K,N,VS,SV} <: AbstractDictArray{K,N,VS,SV}
  # each element is of type LDictt{K,SV}. The values are combined as a tuple of type V (V is taken out for now).
  # the number of dimensions is N.
  # The values of the array have type VS as a vector of arrays.
  data::LDict{K,VS}
  DictArray{K,N,VS,SV}(data::LDict{K,VS}) where {K,N,VS,SV} = begin
    dictsize = length(data)
    if dictsize > 0
      commonlen = size(data.values[1])
    end
    for i in 2:dictsize
      len = size(data.values[i])
      if len != commonlen
        throw(ArgumentError("field lengths are not the same: common length is $commonlen while the length for this particular column is $len."))
      end
    end
    new(data)
  end
end

(==)(arr1::DictArray, arr2::DictArray) = arr1.data == arr2.data

type ZeroNumberOfFieldsException <: Exception end
type KeysDoNotMatchException <: Exception
  keys1::AbstractVector
  keys2::AbstractVector
end

DictArray{K,VS}(data::LDict{K,VS}) = begin
  wrapped_dict = mapvalues(simplify_array, data)
  DictArray{K,
            ndims(data.values[1]),
            eltype(wrapped_dict.values),
            #during v0.4 to v0.5 change promote_type does not work anymore.
            # e.g. promote_type(map(eltype,Any[[Nullable(1)],[Nullable("a")]])...)
            # it is Nullable{Any} in v0.6 but Nullable{T} in v0.4.
            # but the elements of darr(a=[1], b=["a"]) at (1,1) is
            # LDict{Symbol, Nullable} in v0.6 and LDict{Symbol, Nullable{T}} in v0.6.
            # That is, there is an inconsistent in v0.6.
            # To fix it, let's use the second line eltype(...) instead of promote_type(...)..
            # NOT SO FAST YET...
            promote_type([eltype(v) for v in wrapped_dict.values]...)}(wrapped_dict)
            #eltype(cat(1,map(x->collect(take(x,0)), wrapped_dict.values)...))}(wrapped_dict)
end
create_dictarray_nocheck{K,VS}(data::LDict{K,VS}) = begin
  DictArray{K,
            ndims(data.values[1]),
            eltype(data.values),
            promote_type([eltype(v) for v in data.values]...)}(data)
            #eltype(cat(1,map(x->collect(take(x,0)), data.values)...))}(data)
end
DictArray(dict::Associative) = DictArray(LDict(dict))
DictArray(dict::Associative, ks) = DictArray(LDict(dict, ks))
DictArray(ks::Vector, vs::Vector) = DictArray(LDict(ks, vs))
DictArray(p1::Pair, ps::Pair...) = DictArray(LDict(p1, ps...))
DictArray(t1::Tuple, tuples::Tuple...) = DictArray(LDict(t1, tuples...))
DictArray(;kwargs...) = DictArray(kwargs...)
DictArray{K,V,N}(arr::AbstractArray{Nullable{LDict{K,V}},N}) = DictArray(map(x->mapvalues(apply_nullable, x.value), arr))
DictArray{T<:LDict,N}(arr::AbstractArray{T,N}) = DictArray(map(x->map(apply_nullable,x), arr))
DictArray{K,V<:Nullable,N}(arr::AbstractArray{LDict{K,V},N}) = begin
  if isempty(arr)
    error("the size of the input array should at least be 1.")
  end
  # first check if all keys of the elements in arr match with each other.
  common_keys = keys(first(arr))
  for elem in arr
    if common_keys != keys(elem)
      throw(KeysDoNotMatchException(common_keys, keys(elem)))
    end
  end
  # now we know all keys are the same.
  # first, let's figure out the element types.
  eltypes = map(typeof, values(first(arr)))
  # then, create result arrays to fill in later.
  results = [similar(arr, tp) for tp in eltypes]
  # let's fill in.
  for i in 1:length(common_keys)
    map!(x->x.values[i], results[i], arr)
  end
  DictArray(common_keys, results)
end
convert_to_dictarray_if_possible{K,V,N}(arr::AbstractArray{Nullable{LDict{K,V}},N}) = if any(isnull, arr)
  arr
else
  # needed to use map! instead of map to avoid some "type A is not equal to type A" error in Julia.
  # map works fine finally in v0.5.
  #nullablearr = similar(arr, LDict{K,Nullable})
  #map!(x->mapvalues(apply_nullable, x.value), nullablearr, arr)
  nullablearr = map(x->mapvalues(apply_nullable, x.value), arr)
  convert_to_dictarray_if_possible(nullablearr)
end
convert_to_dictarray_if_possible{K,V<:Nullable,N}(arr::AbstractArray{Nullable{LDict{K,V}},N}) = if any(isnull, arr)
  arr
else
  convert_to_dictarray_if_possible(map(x->x.value, arr))
end
convert_to_dictarray_if_possible{K,V<:Nullable,N}(arr::AbstractArray{LDict{K,V},N}) = begin
  if isempty(arr)
    arr
  end
  # first check if all keys of the elements in arr match with each other.
  common_keys = keys(first(arr))
  for elem in arr
    if common_keys != keys(elem)
      return arr
    end
  end
  # now we know all keys are the same.
  # first, let's figure out the element types.
  eltypes = map(typeof, values(first(arr)))
  # then, create result arrays to fill in later.
  results = [similar(arr, tp) for tp in eltypes]
  # let's fill in.
  for i in 1:length(common_keys)
    map!(x->x.values[i], results[i], arr)
  end
  DictArray(common_keys, results)
end
Base.convert(::Type{DictArray}, arr::DictArray) = arr
Base.convert{K,N,VS,SV<:Nullable}(::Type{DictArray}, arr::DictArray{K,N,VS,SV}) = DictArray(arr)
Base.convert{K,V,N}(::Type{DictArray}, arr::AbstractArray{Nullable{LDict{K,V}},N}) = DictArray(arr)
Base.convert{K,V<:Nullable,N}(::Type{DictArray}, arr::AbstractArray{LDict{K,V},N}) = DictArray(arr)

Base.getindex(arr::DictArray, arg::Symbol) = selectfield(arr, arg)
Base.getindex(arr::DictArray, arg::Symbol, args::Symbol...) = [selectfield(arr, a) for a in [arg;args...]]
Base.getindex{N}(arr::DictArray, args::Tuple{N,Symbol}) = map(a->selectfield(arr, a), args)
Base.getindex(arr::DictArray, args::AbstractVector{Symbol}) = selectfields(arr, args...)

Base.getindex(arr::DictArray, indices::CartesianIndex) = #getindex(arr, indices.I...)
  create_ldict_nocheck(arr.data.keys, map(x->getindex(x, indices), arr.data.values))

# some standard implementations to make a DictArray an AbstractArray.
Base.getindex(arr::DictArray, args...) = begin
  res = LDict(arr.data.keys, map(x->getindex(x, args...), arr.data.values))
  create_dictarray_nocheck(res)
end
Base.getindex{K,N,VS,SV}(arr::DictArray{K,N,VS,SV}, args::Int...) = begin
  create_ldict_nocheck(arr.data.keys, map(x->getindex(x, args...), arr.data.values))
end
# used internally to skip key check.
setindex_nocheck!(arr::DictArray, v::DictArray, args...) = begin
  for (tgt,src) in zip(arr.data.values, v.data.values)
    setindex!(tgt, src, args...)
  end
  arr
end
setindex_nocheck!(arr::AbstractArray, v::LDict, args...) = setindex!(arr, v, args...)
setindex_nocheck!(arr::AbstractArray, v, args...) = setindex!(arr, v, args...)

Base.view(arr::DictArray, args::Union{Colon, Int64, AbstractArray}...) =
  create_dictarray_nocheck(create_ldict_nocheck(arr.data.keys, map(x->view(x, args...), arr.data.values)))
Base.view(arr::DictArray, args::Tuple{Vararg{Union{Colon, Int64, AbstractArray}}}) = view(arr, args...)

"""

`getindexvalue(arr::DictArray, args...)`

Return the value tuple of `arr` at index `args`.

"""
getindexvalue(arr::DictArray, args...) = ntuple(length(arr.data)) do i
  #arr.data.values[i][args...]
  getindexvalue(arr.data.values[i], args...)
end
#getindexvalue{T}(arr::DictArray, ::Type{T}, args...) = error("need more specifics")
"""

`getindexvalue(arr::AbstractArray, args...)`

Return `arr[args...]`.

"""
getindexvalue(arr::AbstractArray, args...) = getindex(arr, args...)
#getindexvalue{T}(arr::AbstractArray, ::Type{T}, args...) = getindex(arr, args...)::T
# for internal use to impose type constraints. Similarly for other getindexvalue methods below.
# note that these versions are used only to access one element, and not a range.
getindexvalue{T1}(arr::DictArray, ::Type{Tuple{T1}}, args...) =
  (getindexvalue(arr.data.values[1],args...)::T1,)
  #(arr.data.values[1][args...]::T1,)
getindexvalue{T1,T2}(arr::DictArray, ::Type{Tuple{T1,T2}}, args...) =
  (getindexvalue(arr.data.values[1],args...)::T1,
    getindexvalue(arr.data.values[2],args...)::T2)
getindexvalue{T1,T2,T3}(arr::DictArray, ::Type{Tuple{T1,T2,T3}}, args...) =
  (getindexvalue(arr.data.values[1],args...)::T1,
    getindexvalue(arr.data.values[2],args...)::T2,
    getindexvalue(arr.data.values[3],args...)::T3)
getindexvalue{T1,T2,T3,T4}(arr::DictArray, ::Type{Tuple{T1,T2,T3,T4}}, args...) =
  (getindexvalue(arr.data.values[1],args...)::T1,
    getindexvalue(arr.data.values[2],args...)::T2,
    getindexvalue(arr.data.values[3],args...)::T3,
    getindexvalue(arr.data.values[4],args...)::T4)
getindexvalue{T1,T2,T3,T4,T5}(arr::DictArray, ::Type{Tuple{T1,T2,T3,T4,T5}}, args...) =
  (getindexvalue(arr.data.values[1],args...)::T1,
    getindexvalue(arr.data.values[2],args...)::T2,
    getindexvalue(arr.data.values[3],args...)::T3,
    getindexvalue(arr.data.values[4],args...)::T4,
    getindexvalue(arr.data.values[5],args...)::T5)
getindexvalue(arr::DictArray, ::Type, args...) = getindexvalue(arr, args...)

Base.setindex!(arr::DictArray, v::Tuple, args::Int...) = begin
  for (tgt, sr) in zip(arr.data.values, v)
    setindex!(tgt, sr, args...)
  end
  arr
end
Base.setindex!(arr::DictArray, v::LDict, args::Int...) = begin
  if arr.data.keys != v.keys
    throw(KeysDoNotMatchException(arr.data.keys, v.keys))
  end
  for (tgt, sr) in zip(arr.data.values, v.values)
    setindex!(tgt, sr, args...)
  end
  arr
end
Base.setindex!(arr::DictArray, v0::Associative, args::Int...) = begin
  v = LDict(v0, arr.data.keys)
  if arr.data.keys != v.keys
    throw(KeysDoNotMatchException(arr.data.keys, v.keys))
  end
  for (tgt, sr) in zip(arr.data.values, v.values)
    setindex!(tgt, sr, args...)
  end
  arr
end
Base.setindex!(arr::DictArray, src::DictArray, args...) = begin
  if arr.data.keys != src.data.keys
    throw(KeysDoNotMatchException(arr.data.keys, v.keys))
  end
  for (tgt, sr) in zip(arr.data.values, src.data.values)
    setindex!(tgt, sr, args...)
  end
  arr
end
# this is internally used when setting a DictArray using the value parts only.
Base.setindex!(arr::DictArray, src::Tuple, args...) = begin
  for (tgt, sr) in zip(arr.data.values, src)
    setindex!(tgt, sr, args...)
  end
  arr
end

Base.size{K,N}(arr::DictArray{K,N}) = if isempty(arr.data)
  throw(ZeroNumberOfFieldsException())
else
  size(arr.data.values[1])::NTuple{N,Int}
end

# assume that each column has the same linearindexing..
Base.IndexStyle(arr::DictArray) = if isempty(arr.data)
  throw(ZeroNumberOfFieldsExceptin())
else
  Base.IndexStyle(typeof(arr.data.values[1]))
end

# assume all arrays in DictArray are of the same form.
Base.eachindex(arr::DictArray) = eachindex(arr.data.values[1])
Base.start(arr::DictArray) = (iter=eachindex(arr.data.values[1]);(iter, start(iter)))
Base.next(arr::DictArray, state) = begin
  nextelem,nextstate=next(state[1],state[2])
  arr[nextelem], (state[1], nextstate)
end
Base.done(arr::DictArray, state) = done(state[1], state[2])
Base.length(arr::DictArray) = if isempty(arr.data)
  throw(ZeroNumberOfFieldsException())
else
  length(arr.data.values[1])
end
Base.eltype{K,N,VS,SV}(::Type{DictArray{K,N,VS,SV}}) = LDict{K,SV}
Base.endof(arr::DictArray) = length(arr)
Base.findfirst(arr::DictArray, v::Tuple) = findfirst(arr, LDict(arr.data.keys, [v...]))
Base.transpose{T}(arr::DictArray{T,2}) = mapvalues(transpose, arr)
Base.permutedims(arr::DictArray, perm) = mapvalues(v->permutedims(v,perm), arr)
Base.reshape(arr::DictArray, dims::Int...) = mapvalues(x->reshape(x, dims...), arr)
Base.reshape(arr::DictArray, dims::Tuple{Vararg{Int}}) = reshape(arr, dims...)

arrayadd(arr1::DictArray, arr2::DictArray) = DictArray(merge(arr1.data, arr2.data))
arrayadd(arr1::DictArray, arr2::DictArray, args...;kwargs...) = arrayadd(arr1, darr(arr2, args...;kwargs...))
arrayadd(arr1::DictArray, args...;kwargs...) = arrayadd(arr1, darr(TableSize(arr1), args...;kwargs...))
arrayadd(arr1::DictArray, args::AbstractArray) = throw(ArgumentError("cannot implement."))
arrayadd(arr1::AbstractArray, arr2::AbstractArray) = mapna((x,y)->(x,y), arr1, arr2)

"""

merge(::DictArray, ::DictArray)`

Merge the two `DictArray`s. A duplicate field in the second `DictArray` will override that in the first one. Otherwise, the new field in the second `DictArray` will be appened after the first `DictArray` fields.
If the first is `DictArray` and the remaining arguments are used to construct a `DictArray` and then the two are merged.

##### Example

```julia
julia> merge(darr(a=[1,2,3], b=[4,5,6]), darr(b=[:x,:y,:z], c=["A","B","C"]))
3 DictArray

a b c
------
1 x A
2 y B
3 z C
```
"""
Base.merge(arr1::DictArray, arr2::DictArray) = arrayadd(arr1, arr2)
"""

`merge(::DictArray, args...)`

Construct a `DictArray` using `args...`, and merges the two `DictArray`s together.

##### Example

```julia
julia> merge(darr(a=[1,2,3], b=[4,5,6]), b=[:x,:y,:z], :c=>["A","B","C"])
3 DictArray

a b c
------
1 x A
2 y B
3 z C
```

"""
Base.merge(arr1::DictArray, args...;kwargs...) = arrayadd(arr1, args...;kwargs...)

# select and delete fields from DictArray and return a new DictArray.
deletefields(arr::DictArray, fields...) = DictArray(deletekeys(arr.data, fields...))
selectfields(arr::DictArray, fields...) = DictArray(selectkeys(arr.data, fields...))
selectfield(arr::DictArray, name) = arr.data[name]

"""

`show(io::IO, arr::DictArray [; height::Int=..., width::Int=..., alongorow::Bool=true])`

Show a `DictArray` in `io` in a square box of given `height` and `width`. If not provided, the current terminal's size is used to get the default `height` and `weight`. `alongrow` determines whether to display field names along row or columns.

##### Examples
```julia
julia> show(STDOUT, darr(a=[1,2,3], b=[:x,:y,:z]))
3 DictArray

a b
----
1 x
2 y
3 z

julia> show(STDOUT, darr(a=[1,2,3], b=[:x,:y,:z]), alongrow=false)
3 DictArray

a |1
b |x
--+--
a |2
b |y
--+--
a |3
b |z
```

"""
Base.show{K,N}(io::IO, arr::DictArray{K,N}, indent=0; height::Int=show_size()[1], width::Int=show_size()[2], alongrow=toshow_alongrow) = begin
  # show is implemented for a DictArray with dimesion 1 and 2 separately, and
  # show for a DictArray with dimension greater than 2 calls the 1 and 2 dimensional versions recursively.
  print(io, join(size(arr), " x "))
  print(io, " DictArray")
  print(io, '\n')
  ndimstable = ndims(arr)
  for index=1:last(size(arr))
    coords = [fill(:,ndimstable-1);index]
    print(io, '\n')
    show_indent(io, indent)
    print(io, '[')
    for i in 1:length(coords)-1
      print(io, ":,")
    end
    print(io, index)
    print(io, ']')
    print(io, '\n')
    show(io, getindex(arr, coords...), indent+4;height = height, width=width, alongrow=alongrow)
  end
end

Base.show{K}(io::IO, arr::DictArray{K,0}, indent=0; height::Int=show_size()[1], width::Int=show_size()[2], alongrow=toshow_alongrow) = begin
  println(io, "0 dimensional DictArray")
  show(io, mapvalues(x->x[1], arr.data))
end

Base.show{K}(io::IO, arr::DictArray{K,1}, indent=0; height::Int=show_size()[1], width::Int=show_size()[2], alongrow=toshow_alongrow) = begin
  print(io, join(size(arr), " x "))
  print(io, " DictArray")
  print(io, '\n')
  (result, hlines, vlines) = if alongrow
    create_string_reprmat_alongrow(arr; height=height, width=width)
  else
    create_string_reprmat_alongcol(arr; height=height, width=width)
  end
  show_string_matrix(io, result, height, width, indent, hlines, vlines)
end

create_string_reprmat_alongrow{K}(arr::DictArray{K,1}; height::Int=show_size()[1], width::Int=show_size()[2]) = begin
  show_height,show_width = (height, width)
  tabledata = arr
  nrows = min(show_height, length(arr) + 1)
  nkeys = 0
  ncols = max(nkeys, min(show_width, nkeys + arr_width(tabledata)))
  result = fill("", nrows, ncols)
  result[1,nkeys+1:ncols] = map(string, tabledata.data.keys[1:ncols-nkeys])
  for i in nkeys+1:ncols
    result[2:nrows,i] = map(cell_to_string, tabledata.data.values[i-nkeys][1:nrows-1])
  end
  (result, [2], [])
end

create_string_reprmat_alongcol{K}(arr::DictArray{K,1}; height::Int=show_size()[1], width::Int=show_size()[2]) = begin
  show_height,show_width = (height, width)
  tabledata = arr
  widthtabledata = arr_width(tabledata)
  nrows = min(show_height, widthtabledata*length(arr))
  nkeys = 0
  ncols = max(nkeys, min(show_width, nkeys + 2))
  result = fill("", nrows, ncols)
  for i in 1:nrows
    result[i,nkeys+1] = string(tabledata.data.keys[1+(i-1) % widthtabledata]) #string(data.keys[i loop_key_index])
    result[i,nkeys+2] = cell_to_string(tabledata.data.values[1+(i-1) % widthtabledata][1+div(i-1,widthtabledata)])
  end
  (result, collect(1+widthtabledata*(1:1+fld(nrows,widthtabledata))), [2])
end

Base.show{K}(io::IO, arr::DictArray{K,2}, indent; height::Int=show_size()[1], width::Int=show_size()[2], alongrow=toshow_alongrow) = begin
  print(io, join(size(arr), " x "))
  print(io, " DictArray")
  print(io, '\n')
  (result, hlines, vlines) = if alongrow
    create_string_reprmat_alongrow(arr; height=height, width=width)
  else
    create_string_reprmat_alongcol(arr; height=height, width=width)
  end
  show_string_matrix(io, result, height, width, indent, hlines, vlines)
end

create_string_reprmat_alongrow{K}(arr::DictArray{K,2}; height::Int=show_size()[1], width::Int=show_size()[2]) = begin
  show_height,show_width = (height, width)
  tabledata = arr
  sizearr2 = size(arr, 2)
  (tableheight,tablewidth) = size(tabledata)
  widthtabledata = arr_width(tabledata)
  nrows = min(show_height, tableheight + 1)
  nkeys = 0
  ncols = max(nkeys, min(show_width, nkeys + sizearr2*widthtabledata))
  result = fill("", nrows, ncols)
  # fill in the key columns.
  if isa(tabledata, DictArray)
    if 1 <= nrows
      for i in 1:sizearr2
        for j in 1:widthtabledata
          ind = nkeys+widthtabledata*(i-1)+j
          if ind>ncols break end
          result[1,ind] = string(tabledata.data.keys[j])
        end
      end
      for i in 2:nrows
        for j in 1:sizearr2
          for k in 1:widthtabledata
            ind = nkeys+widthtabledata*(j-1)+k
            if ind>ncols break end
            result[i,ind] = cell_to_string(tabledata.data.values[k][i-1,j])
          end
        end
      end
    end
  else
    result[2:nrows,nkeys+1:ncols] = map(cell_to_string, tabledata[1:nrows-1,1:ncols-nkeys])
  end
  (result, [2], map(x->1+widthtabledata*(x-1),2:ncols))
end

create_string_reprmat_alongcol{K}(arr::DictArray{K,2}; height::Int=show_size()[1], width::Int=show_size()[2]) = begin
  show_height,show_width = (height, width)
  tabledata = arr
  (tableheight,tablewidth) = size(tabledata)
  widthtabledata = arr_width(tabledata)
  nrows = min(show_height, widthtabledata*tableheight)
  nkeys = 0
  ncols = max(nkeys, min(show_width, nkeys + tablewidth + 1))
  result = fill("", nrows, ncols)
  # fill in the key columns.
  if isa(tabledata, DictArray)
    for j in 1:nrows
      result[j,nkeys+1] = string(tabledata.data.keys[1+(j-1) % widthtabledata])
    end
    for i in 1:nrows
      for j in 1:tablewidth
        ind = nkeys + 1 + j
        if ind>ncols break end
        result[i,ind] = cell_to_string(tabledata.data.values[1+(i-1) % widthtabledata][div(i-1, widthtabledata)+1,j])
      end
    end
  else
    result[1:nrows,nkeys+2:ncols] = map(cell_to_string, tabledata[1:nrows,1:ncols-nkeys-1])
  end
  (result, collect(1+widthtabledata*(1:1+fld(nrows-1,widthtabledata))), [2])
end

"""

`allfieldnames(::DictArray)`

Return all field names in the input `DictArray`, which are just the keys in the underlying `LDict`.

##### Examples

```julia
julia> allfieldnames(darr(a=reshape(1:6,3,2),b=rand(3,2)))
2-element Array{Symbol,1}:
 :a
 :b
```

"""
allfieldnames(arr::DictArray) = arr.data.keys
Base.show(io::IO, ::MIME"text/plain", arr::DictArray) = show(io, arr)
Base.similar(arr::DictArray) = similar(arr, size(arr))
Base.similar{K,N,VS,SV}(arr::DictArray{K,N,VS,SV}, dims::Int...) = similar(arr, dims)
Base.similar{K,N,VS,SV,M}(arr::DictArray{K,N,VS,SV}, dims::NTuple{M,Int}) = begin
  newvals = map(arr.data.values) do elemarr
    similar(elemarr, dims)
  end
  create_dictarray_nocheck(LDict(arr.data.keys, newvals))
end
# use the first column as a representative.
Base.similar{T,M}(arr::DictArray, ::Type{T}, dims::NTuple{M,Int}) = similar(arr.data.values[1], T, dims)

isna(arr::DictArray) = isna(arr.data)
isna(arr::DictArray, indices...) = isna(arr[indices...])

Base.copy(arr::DictArray) = DictArray(copy_arrayvalues(arr.data))
Base.copy!(tgt::DictArray, src::DictArray) = begin
  tgtvalues = values(tgt)
  srcvalues = values(src)
  assert(keys(tgt) == keys(src))
  for i in 1:length(tgtvalues)
    copy!(tgtvalues[i], srcvalues[i])
  end
end

# expand DictArrays so that they have common field names.
expand_array_fields(arrs::DictArray...) = begin
  eltypemap = LDict{Any,Any}()
  for arr in arrs
    for (k,v) in arr.data
      if haskey(eltypemap, k)
        # 2 eltypes because it is AbstractArray{Nullable{?}}.
        eltypemap[k] = (k, promote_type(eltype(eltype(v)), eltypemap[k][2]))
      else
        eltypemap[k] = (k, eltype(eltype(v)))
      end
    end
  end
  expanded_arrays = map(arrs) do arr
    create_dictarray_nocheck(create_ldict_nocheck(mapvalues(eltypemap) do kv
      k = kv[1]
      v = kv[2]
      if haskey(arr.data, k)
        arrdatak = arr.data[k]
        if eltype(eltype(arrdatak)) === v
          arrdatak
        else
          # copy if type promotion is necessary.
          copied = similar(arrdatak, Nullable{v})
          copy!(copied, arrdatak)
          copied
        end
      else
        fill(Nullable{v}(), size(arr))
      end
    end...))
  end
end

"""

`cat(catdim::Integer, arrs::DictArray...)`

Concatenate the `DictArray`s `arrs` along the `catdim` direction.
The common fields are concatenated field by field.
If a field name does not exist in all of `arrs`, a null field with that field name will be added to those `DictArray`s with that missing field name, and then the arrays will be concatenated field by field.

##### Examples
```julia
julia> cat(1, darr(a=[1 2 3], b=[:x :y :z]),
              darr(c=[3 2 1], b=[:m :n :p]))
2 x 3 DictArray

a b c |a b c |a b c
------+------+------
1 x   |2 y   |3 z
  m 3 |  n 2 |  p 1

julia> cat(2, darr(a=[1 2 3], b=[:x :y :z]),
              darr(c=[3 2 1], b=[:m :n :p]))
1 x 6 DictArray

a b c |a b c |a b c |a b c |a b c |a b c
------+------+------+------+------+------
1 x   |2 y   |3 z   |  m 3 |  n 2 |  p 1
```

"""
Base.cat(catdim::Integer, arrs::DictArray...) = begin
  if length(arrs) == 1
    arrs[1]
  else
    common_keys = arrs[1].data.keys
    common_value_eltypes = map(eltype, arrs[1].data.values)
    for i in 2:length(arrs)
      if arrs[i].data.keys != common_keys || map(eltype, arrs[i].data.values) != common_value_eltypes
        return cat(catdim, expand_array_fields(arrs...)...)
      end
    end
    newvalues = map(map(x->x.data.values, arrs)...) do values...
      cat(catdim, values...)
    end
    DictArray(LDict(common_keys, newvalues))
  end
end
Base.vcat(arrs::DictArray...) = cat(1, arrs...)
Base.hcat(arrs::DictArray...) = cat(2, arrs...)

"""

`repeat(arr::DictArray [; inner=..., outer=...])`

Apply `repeat` field by field to the `DictArray` `arr`.

"""
Base.repeat(arr::DictArray; inner::Array{Int}=ones(Int,ndims(arr)), outer::Array{Int}=ones(Int,ndims(arr))) =
  DictArray(LDict(arr.data.keys, map(v->repeat(v, inner=inner, outer=outer), arr.data.values)))
# cannot define this convert because LabeledArray has not been defined yet.
# move this to LabeledArray.jl
#Base.convert(::Type{DictArray}, larr::LabeledArray) = selectfields(larr, allfieldnames(larr)...)

Base.repmat{T}(arr::DictArray{T,1}, n::Int) = create_dictarray_nocheck(mapvalues(x->repmat(x,n), arr.data))
Base.repmat{T}(arr::DictArray{T,2}, n::Int) = create_dictarray_nocheck(mapvalues(x->repmat(x,n), arr.data))
Base.repmat{T}(arr::DictArray{T,1}, m::Int, n::Int) = create_dictarray_nocheck(mapvalues(x->repmat(x,m,n), arr.data))
Base.repmat{T}(arr::DictArray{T,2}, m::Int, n::Int) = create_dictarray_nocheck(mapvalues(x->repmat(x,m,n), arr.data))

Base.show{T,N}(io::IO,
                ::MIME"text/html",
                arr::DictArray{T,N};
                height::Int=dispsize()[1],
                width::Int=dispsize()[2],
                alongrow::Bool=todisp_alongrow) = begin
  print(io, join(size(arr), " x "))
  print(io, " DictArray")
  print(io, '\n')
  ndimstable = ndims(arr)
  print(io, "<ul>")
  for index=1:last(size(arr))
    coords = [fill(:,ndimstable-1);index]
    print(io, "<li>")
    print(io, '[')
    for i in 1:length(coords)-1
      print(io, ":,")
    end
    print(io, index)
    print(io, ']')
    print(io, '\n')
    show(io, MIME("text/html"), getindex(arr, coords...); height=height, width=width, alongrow=alongrow)
    print(io, "</li>")
  end
  print(io, "</ul>")
end

Base.show{T}(io::IO, ::MIME"text/html", table::DictArray{T,0}; height::Int=dispsize()[1], width::Int=dispsize()[2], alongrow::Bool=todisp_alongrow) = begin
  print(io, "0 dimensional DictArray")
end

Base.show{T}(io::IO, ::MIME"text/html", table::DictArray{T,1}; height=dispsize()[1], width=dispsize()[2], alongrow::Bool=todisp_alongrow) = showDictArray12(io, table; height=height, width=width, alongrow=alongrow)
Base.show{T}(io::IO, ::MIME"text/html", table::DictArray{T,2}; height=dispsize()[1], width=dispsize()[2], alongrow::Bool=todisp_alongrow) = showDictArray12(io, table; height=height, width=width, alongrow=alongrow)

showDictArray12(io::IO, table; height=dispsize()[1], width=dispsize()[2], alongrow::Bool=todisp_alongrow) = begin
  print(io, join(size(table), " x "))
  print(io, " DictArray")
  print(io, '\n')
  (result, hlines, vlines) = if alongrow
    create_string_reprmat_alongrow(table; height=height, width=width)
  else
    create_string_reprmat_alongcol(table; height=height, width=width)
  end
  print_string_reprmat_to_html_table(io, result, height, width, hlines, vlines)
end

# This is a generic verion of map. Need to develop an optimized version later.
"""

`map(f::Function, arr::DictArray)`

Apply the function `f` to each element of `arr`.
`f` will take an `LDict` and produces a value of type, say `T`.
The return value will have the same size as `arr` and its elements have type `T`.
If the return element type `T` is not nullable, the result elements are wrapped by `Nullable`.
If the return element type `T` is `LDict`, the result will be again a `DictArray`.
However, in this case, the `LDict` should be of the type `LDict{K,Nullable{V}}`.

##### Examples

```julia
julia> map(x->x[:a].value + x[:b].value, darr(a=[1 2;3 4], b=[1.0 2.0;3.0 4.0]))
2x2 DataCubes.AbstractArrayWrapper{Nullable{Float64},2,DataCubes.FloatNAArray{Float64,2,Array{Float64,2}}}:
 Nullable(2.0)  Nullable(4.0)
 Nullable(6.0)  Nullable(8.0)

julia> map(x->LDict(:c=>Nullable(x[:a].value + x[:b].value)), darr(a=[1 2;3 4], b=[1.0 2.0;3.0 4.0]))
2 x 2 DictArray

c   |c
----+----
2.0 |4.0
6.0 |8.0
```
"""
Base.map(f::Function, arr::DictArray) = mapslices(f, arr, [])

"""

`reducedim(f::Function, arr::DictArray, dims [, initial])`

Reduce a two argument function `f` along dimensions of `arr`. `dims` is a vector specifying the dimensions to reduce, and `initial` is the initial value to use in the reduction.

* If `dims` includes all dimensions, `reduce` will be applied to the whole `arr` with initial value `initial`.
* Otherwise, `reduce` is applied with the function `f` to each slice spanned by the directions with initial value `initial`.
`initial` can be omitted if the underlying `reduce` does not require it.

```julia
julia> reducedim((acc,d)->acc+d[:a].value, darr(a=[1 2 3;4 5 6]), [1], 0)
3-element DataCubes.AbstractArrayWrapper{Nullable{Int64},1,Array{Nullable{Int64},1}}:
 Nullable(5)
 Nullable(7)
 Nullable(9)

julia> reducedim((acc,d)->acc+d[:a].value, darr(a=[1 2 3;4 5 6]), [2], 0)
2-element DataCubes.AbstractArrayWrapper{Nullable{Int64},1,Array{Nullable{Int64},1}}:
 Nullable(6)
 Nullable(15)

julia> reducedim((acc,d)->acc+d[:a].value, darr(a=[1 2 3;4 5 6]), [1,2], 0)
Nullable(21)
```

"""
Base.reducedim(f::Function, arr::DictArray, dims, initial) = begin
  mapslices(arr, dims) do slice
    reduce(f, initial, slice)
  end
end

Base.reducedim(f::Function, arr::DictArray, dims) = begin
  mapslices(arr, dims) do slice
    reduce(f, slice)
  end
end

"""

`mapslices(f::Function, arr::DictArray, dims)`

Apply the function `f` to each slice of `arr` specified by `dims`. `dims` is a vector of integers along which direction to reduce.

* If `dims` includes all dimensions, `f` will be applied to the whole `arr`.
* If `dims` is empty, `mapslices` is the same as `map`.
* Otherwise, `f` is applied to each slice spanned by the directions.

##### Return

Return a dimensionally reduced array along the directions in `dims`.
If the return value of `f` is an `LDict`, the return value of the corresponding `mapslices` is a `DictArray`.
If the return valud of `f` is an AbstractArray, the return value of the corresponding `mapslices` will be an array of the same type, but the dimensions have been extended. If the original dimensions are `N1 x N2 x ... x Nn` and `f` returns an array of dimensions `M1 x ... x Mm`, the return array will be of dimensions `M1 x ... x Mm x N1 x ... x Nn` with those `Ni`s omitted that belong to `dims`. You can think of this as the usual mapslices, followed by `cat`. See the last two examples below.
Otherwise, the return value is an `Array`.

```julia
julia> mapslices(d->d[:a] .* 2, darr(a=[1 2 3;4 5 6], b=[10 11 12;13 14 15]), [1])
3-element DataCubes.AbstractArrayWrapper{Nullable{DataCubes.AbstractArrayWrapper{Nullable{Int64},1,DataCubes.AbstractArrayWrapper{Nullable{Int64},1,Array{Nullable{Int64},1}}}},1,Array{Nullable{DataCubes.AbstractArrayWrapper{Nullable{Int64},1,DataCubes.AbstractArrayWrapper{Nullable{Int64},1,Array{Nullable{Int64},1}}}},1}}:
 Nullable([Nullable(2),Nullable(8)])
 Nullable([Nullable(4),Nullable(10)])
 Nullable([Nullable(6),Nullable(12)])

julia> mapslices(d->d[:a] .* 2, darr(a=[1 2 3;4 5 6], b=[10 11 12;13 14 15]), [2])
2-element DataCubes.AbstractArrayWrapper{Nullable{DataCubes.AbstractArrayWrapper{Nullable{Int64},1,DataCubes.AbstractArrayWrapper{Nullable{Int64},1,Array{Nullable{Int64},1}}}},1,Array{Nullable{DataCubes.AbstractArrayWrapper{Nullable{Int64},1,DataCubes.AbstractArrayWrapper{Nullable{Int64},1,Array{Nullable{Int64},1}}}},1}}:
 Nullable([Nullable(2),Nullable(4),Nullable(6)])
 Nullable([Nullable(8),Nullable(10),Nullable(12)])

julia> mapslices(d->LDict(:c=>sum(d[:a]), :d=>sum(d[:b] .* 3)), darr(a=[1 2 3;4 5 6], b=[10 11 12;13 14 15]), [2])
2 DictArray

c  d
-------
6  99
15 126

julia> mapslices(x->msum(x), darr(a=reshape(1:15,5,3), b=1.0*reshape(1:15,5,3)), [1])
reseltype = DataCubes.DictArray{Symbol,1,DataCubes.AbstractArrayWrapper{T,1,A<:AbstractArray{T,N}},Nullable{T}}
5 x 3 DictArray

a  b    |a  b    |a  b
--------+--------+--------
1  1.0  |6  6.0  |11 11.0
3  3.0  |13 13.0 |23 23.0
6  6.0  |21 21.0 |36 36.0
10 10.0 |30 30.0 |50 50.0
15 15.0 |40 40.0 |65 65.0

# note how the order of axes changed. the newly generated directions always preced the existing ones.
julia> mapslices(x->msum(x), darr(a=reshape(1:15,5,3), b=1.0*reshape(1:15,5,3)), [2])
reseltype = DataCubes.DictArray{Symbol,1,DataCubes.AbstractArrayWrapper{T,1,A<:AbstractArray{T,N}},Nullable{T}}
3 x 5 DictArray

a  b    |a  b    |a  b    |a  b    |a  b
--------+--------+--------+--------+--------
1  1.0  |2  2.0  |3  3.0  |4  4.0  |5  5.0
7  7.0  |9  9.0  |11 11.0 |13 13.0 |15 15.0
18 18.0 |21 21.0 |24 24.0 |27 27.0 |30 30.0

```

"""
Base.mapslices(f::Function, arr::DictArray, dims::AbstractVector) = mapslices_darr_larr(f, arr, dims)
Base.mapslices(f::Function, arr::DictArray, dims::Int...) = mapslices_darr_larr(f, arr, [dims...])

# if dims spans all dimensions, f! cannot be inplace. It will be f!::U->T. Otherwise, f will be f!(AbstractArray{T,N}, AbstractArray{U,N}).
map_array_preserve_shape!{T,U,N}(f!::Function, tgt::AbstractArray{T,N}, src::AbstractArray{U,N}, dims::Int...;rev=false) = begin
  if length(dims) != length(unique(dims))
    throw(ArgumentError("the dims argument should be distinct."))
  end
  dimtypes = ntuple(ndims(src)) do d
    if d in dims
      Colon()
    else
      0
    end
  end
  map_array_preserve_shape!(f!, tgt, src, dimtypes;rev=rev)
end

@generated map_array_preserve_shape!{T,U,V,N}(f!::Function, tgt::AbstractArray{T,N}, src::AbstractArray{U,N}, dims::V;rev=false) = begin
  dimtypes = dims.types
  # slice_ndims is N if all dims are numbers, i.e., if applying f to each element. tgt will be map(f, src). f::U->T, except that f is inplace.
  # slice_ndims is 0 if applying f to the entire array. i.e. tgt will be f(src). f::AbsractArray{U,N}->AbstractArray{T,N}, except that f is inplace.
  slice_ndims = foldl((acc,x)->acc + (x==Int), 0, dimtypes)
  slice_indices = Int[map(it->it[1], Iterators.filter(it->it[2]==Int, enumerate(dimtypes)))...]
  tgt_slice_exp = if slice_ndims == N
    :(tgt[coords...])
  else
    :(reverse_if_required(rev, view(tgt, coords...)))
  end
  src_slice_exp = if slice_ndims == N
    :(src[coords...]) # no need to reverse.
  else
    :(reverse_if_required(rev, view(src, coords...)))
  end
  apply_f_exp = if slice_ndims == N
    quote
      $tgt_slice_exp = f!($src_slice_exp)
    end
  else
    quote
      f!($tgt_slice_exp, $src_slice_exp)
    end
  end
  quote
    coords = Array{Any}($N)
    # assume that the types are all the same.
    @nloops $slice_ndims i j->1:size(src,$slice_indices[j]) begin
      fill!(coords, Colon())
      @nexprs $slice_ndims j->(coords[$slice_indices[j]] = i_j)
      $apply_f_exp
    end
    tgt
  end
end

reverse_if_required(rev::Bool, arr::AbstractArray) = if rev
  ranges = ntuple(ndims(arr)) do d
    size(arr,d):-1:1
  end
  view(arr, ranges...)
else
  arr
end

"""

`keys(::DictArray)`

Return the field name vector of the input `DictArray`, which are the keys of the underlying `LDict`.

##### Examples

```julia
julia> keys(darr(a=[1,2,3], b=[:x,:y,:z]))
2-element Array{Symbol,1}:
 :a
 :b
```

"""
Base.keys(arr::DictArray) = keys(peel(arr))

"""

`values(::DictArray)`

Return the vector of field arrays of the input `DictArray`, which are the values of the underlying `LDict`.

##### Examples

```julia
julia> values(darr(a=[1,2,3], b=[:x,:y,:z]))
2-element Array{DataCubes.AbstractArrayWrapper{T,1,A<:AbstractArray{T,N}},1}:
 [Nullable(1),Nullable(2),Nullable(3)]
 [Nullable(:x),Nullable(:y),Nullable(:z)]
```

"""
Base.values(arr::DictArray) = values(peel(arr))

# to avoid ambiguity with conventional use of reverse, specifically prohobit the integer argument.
Base.reverse(arr::DictArray, dummy::Integer) = error("not yet implemented.")
Base.reverse(arr::DictArray) = reverse(arr, [1])
# any iterable dims can be an argument.

"""

`reverse(arr::DictArray, dims)`

Reverse a `DictArray` `arr` using an iterable variable `dims`.

##### Arguments

* `arr` is an input `DictArray`.
* `dims` is an iterable variable of `Int`s.

##### Return

A `DictArray` whose elements along any directions belonging to `dims` are reversed.

##### Examples

```julia
julia> t = darr(a=[1 2 3;4 5 6], b=[:x :y :z;:u :v :w])
2 x 3 DictArray

a b |a b |a b
----+----+----
1 x |2 y |3 z
4 u |5 v |6 w

julia> reverse(t, [1])
2 x 3 DictArray

a b |a b |a b
----+----+----
4 u |5 v |6 w
1 x |2 y |3 z

julia> reverse(t, 1:2)
2 x 3 DictArray

a b |a b |a b
----+----+----
6 w |5 v |4 u
3 z |2 y |1 x
```

"""
Base.reverse(arr::DictArray, dims) = begin
  coords = ntuple(ndims(arr)) do d
    if d in dims
      size(arr, d):-1:1
    else
      Colon()
    end
  end
  getindex(arr, coords...)
end

"""

`flipdim(arr::DictArray, dims...)`

Flip a `DictArray` `arr` using an iterable variable `dims`. The same method as `reverse(arr::DictArray, dims)`.

##### Arguments

* `arr` is an input `DictArray`.
* `dims` is an iterable variable of `Int`s.

##### Return

A `DictArray` whose elements along any directions belonging to `dims` are fliped.

##### Examples

```julia
julia> t = darr(a=[1 2 3;4 5 6], b=[:x :y :z;:u :v :w])
2 x 3 DictArray

a b |a b |a b
----+----+----
1 x |2 y |3 z
4 u |5 v |6 w

julia> flipdim(t, 1)
2 x 3 DictArray

a b |a b |a b
----+----+----
4 u |5 v |6 w
1 x |2 y |3 z

julia> flipdim(t, 1, 2)
2 x 3 DictArray

a b |a b |a b
----+----+----
6 w |5 v |4 u
3 z |2 y |1 x
```

"""
Base.flipdim(arr::DictArray, dims::Integer...) = reverse(arr, dims)

"""

`fill(ldict::LDict, dims...)`

Fill a `DictArray` with `ldict`.

##### Return
A new `DictArray` whose elements are `ldict` and whose dimensions are `dims...`.

"""
Base.fill(ldict::LDict, dims::Integer...) = DictArray(mapvalues(v->fill(apply_nullable(v), dims...), ldict))
Base.fill(ldict::LDict, dims::Tuple{Vararg{Int}}) = fill(ldict, dims...)

"""

`reorder(arr::DictArray, ks...)`

Reorder the field names so that the first few field names are `ks`.

##### Return
A new `DictArray` whose fields are shuffled from `arr` so that the first few field names are `ks`.

"""
function reorder end

reorder(arr::DictArray, ks...) = DictArray(reorder(arr.data, ks...))

"""

`rename(arr::DictArray, ks...)`

Rename the field names so that the first few field names are `ks`.

##### Return
A new `DictArray` whose first few field names are `ks`.

"""
function rename end

rename{K}(arr::DictArray{K}, ks::K...) = DictArray(rename(arr.data, ks...))

"""

`@darr(...)`

Create a `DictArray`. The arguments `...` can be one of the following:

##### Arguments

* `k=>v` creates a field using an array `v` with field name `k`. `k` can be an arbitrary type. If the element type of `v` is not `Nullable`, each element will be wrapped by `Nullable`. If `NA` is provided as an element, it is translated as `Nullable{T}()` for an appropriate type `T`.
* `k=v` creates a field using an array `v` with field name `:k`.
* There can be at most one non pair type argument, which will be converted into a `DictArray` and other pair arguments will update it.

##### Examples

```julia
julia> t = @darr(a=[1 2;NA 4;5 NA],b=["abc" NA;1 2;:m "xyz"],:c=>[NA 1.5;:sym 'a';"X" "Y"])
3 x 2 DictArray

a b   c   |a b   c
----------+----------
1 abc     |2     1.5
  1   sym |4 2   a
5 m   X   |  xyz Y

julia> @darr(t, c=[1 2;3 4;5 6], "d"=>map(Nullable, [1 2;3 4;5 6]))
3 x 2 DictArray

a b   c d |a b   c d
----------+----------
1 abc 1 1 |2     2 2
  1   3 3 |4 2   4 4
5 m   5 5 |  xyz 6 6
```

"""
macro darr(args...)
  data_pairs = Any[]
  template = Nullable()
  for arg in args
    if :head in fieldnames(arg) && (arg.head == :kw || arg.head == :(=) || arg.head == :(=>))
      key = arg.head==:kw || arg.head==:(=) ? quote_symbol(arg.args[1]) : arg.args[1]
      value = arg.args[2]
      push!(data_pairs, quote
        $key => @nalift($(esc(value)))
      end)
    elseif :head in fieldnames(arg) && arg.head == :call && arg.args[1] == :(=>)
      key = arg.args[2]
      value = arg.args[3]
      push!(data_pairs, quote
        $key => @nalift($(esc(value)))
      end)
    elseif :head in fieldnames(arg) && (arg.head == :vect || arg.head == :dict)
      throw(ArgumentError("an array/dictionary expression is not allowed."))
    elseif :head in fieldnames(arg) && arg.head == :call && arg.args[1] == :Dict
      throw(ArgumentError("an array/dictionary expression is not allowed."))
    elseif isnull(template)
      template = Nullable(arg)
    else
      throw(ArgumentError("not a key=value type argument or an array argument $arg, or two or more base DictArrays are provided."))
    end
  end
  dataexp = if isnull(template) && isempty(data_pairs)
    throw(ArgumentError("neither a base DictArray or a fieldname=>array pair provided."))
  elseif isempty(data_pairs)
    nothing
  else
    Expr(:call, :darr, Expr(:..., Expr(:ref, :Any, data_pairs...)))
  end
  if isnull(template)
    dataexp
  else
    Expr(:call, :update_darr, esc(template.value), dataexp)
  end
end

"""

`darr(...)`

Create a `DictArray`. The arguments `...` can be one of the following:

##### Arguments

* `k=>v` creates a field using an array `v` with field name `k`. `k` can be an arbitrary type. If the element type of `v` is not `Nullable`, each element will be wrapped by `Nullable`. If you want to manually provide a `Nullable` array with `Nullable{T}()` elements in it, the macro version `@darr` may be more convenient to use. Note that this type of argument precedes the keyword type argument in the return `DictArray`, as shown in Examples below.
* `k=v` creates a field using an array `v` with field name `:k`.
* There can be at most one non pair type argument, which will be converted into a `DictArray` and other pair arguments will update it.
Especially, if the non pair type argument is an array of `LDict`, it will be converted into a `DictArray`.

##### Examples

```julia
julia> t = darr(a=[1 2;3 4;5 6],b=["abc" 'a';1 2;:m "xyz"],:c=>[1.0 1.5;:sym 'a';"X" "Y"])
3 x 2 DictArray

c   a b   |c   a b
----------+----------
1.0 1 abc |1.5 2 a
sym 3 1   |a   4 2
X   5 m   |Y   6 xyz

julia> darr(t, c=[1 2;3 4;5 6], :d=>map(Nullable, [1 2;3 4;5 6]))
3 x 2 DictArray

c a b   d |c a b   d
----------+----------
1 1 abc 1 |2 2 a   2
3 3 1   3 |4 4 2   4
5 5 m   5 |6 6 xyz 6

julia> darr(Any[LDict(:a => Nullable(1),:b => Nullable{Int}()),LDict(:a => Nullable(3),:b => Nullable(4))])
2 DictArray

a b
----
1
3 4
```

"""
function darr end

darr_inner(tblsize::TableSize, kwargs...) = begin
  sz = tblsize.size
  for (k,v) in kwargs
    if isa(v, AbstractArray) && sz != size(v)
      throw(ArgumentError("the result sizes for different fields are inconsistent: $(string([k=>size(v) for (k,v) in kwargs]))"))
    end
  end
  arraylifted_kwargs = Any[k=>isa(v, AbstractArray) ? v : fill(v, sz) for (k,v) in kwargs]
  nalifted_kwargs = Any[k=>nalift(v) for (k,v) in arraylifted_kwargs]
  DictArray(nalifted_kwargs...)
end

darr_inner(kwargs...) = begin
  common_array_size = Nullable()
  for (k,v) in kwargs
    if isa(v, AbstractArray)
      if isnull(common_array_size)
        common_array_size = Nullable(size(v))
      elseif common_array_size.value != size(v)
        throw(ArgumentError("the result sizes for different fields are inconsistent: $(string([k=>size(v) for (k,v) in kwargs]))"))
      end
    end
  end
  if isnull(common_array_size)
    throw(ArgumentError("at least one argument has to be an array."))
  end
  arraylifted_kwargs = Any[k=>isa(v, AbstractArray) ? v : fill(v, common_array_size.value) for (k,v) in kwargs]
  nalifted_kwargs = Any[k=>nalift(v) for (k,v) in arraylifted_kwargs]
  DictArray(nalifted_kwargs...)
end

darr(arr::AbstractArray) = convert(DictArray, nalift(arr))
darr(arr::DictArray, pairs...; kwargs...) = update_darr(arr, darr(pairs...; kwargs...))
darr(arr::DictArray) = arr
darr(args...;kwargs...) = darr_inner(args..., kwargs...)

# an internal function to update the base DictArray using data.
update_darr(base, data) = update_darr(convert(DictArray, base), data)
update_darr(base::DictArray, data) = begin
  if data == nothing
    base
  else
    darr(merge(base.data, data.data)...)
  end
end


text:
############################################################
# judiPDEfull ################################################
############################################################

# Authors: Philipp Witte (pwitte@eos.ubc.ca), Henryk Modzelewski (hmodzelewski@eos.ubc.ca)
# Date: January 2017

export judiPDEfull, judiPDEfullException, subsample

############################################################

# Type for linear operator representing  Pr*A(m)^-1*Ps,
# i.e. it includes source and receiver projections

struct judiPDEfull{DDT<:Number,RDT<:Number} <: joAbstractLinearOperator{DDT,RDT}
    name::String
    m::Integer
    n::Integer
    info::Info
    model::Model
    srcGeometry::Geometry
    recGeometry::Geometry
    options::Options
    fop::Function              # forward
    fop_T::Union{Function, Nothing}  # transpose
end

mutable struct judiPDEfullException <: Exception
    msg :: String
end

############################################################
## Constructor

function judiModeling(info::Info,model::Model, srcGeometry::Geometry, recGeometry::Geometry; options=Options(), DDT::DataType=Float32, RDT::DataType=DDT)
# JOLI wrapper for nonlinear forward modeling
    (DDT == Float32 && RDT == Float32) || throw(judiPDEfullException("Domain and range types not supported"))

    # Determine dimensions
    if typeof(recGeometry) == GeometryOOC
        m = sum(recGeometry.nsamples)
    else
        m = 0
        for j=1:info.nsrc m += length(recGeometry.xloc[j])*recGeometry.nt[j] end

    end
    if typeof(srcGeometry) == GeometryOOC
        n = sum(srcGeometry.nsamples)
    else
        n = 0
        for j=1:info.nsrc n += length(srcGeometry.xloc[j])*srcGeometry.nt[j] end
    end

    if info.nsrc > 1
        srcnum = 1:info.nsrc
    else
        srcnum = 1
    end

    return F = judiPDEfull{Float32,Float32}("Proj*F*Proj'", m, n, info, model, srcGeometry, recGeometry, options,
                              src -> time_modeling(model, srcGeometry, src.data, recGeometry, nothing, nothing, srcnum, 'F', 1, options),
                              rec -> time_modeling(model, srcGeometry, nothing, recGeometry, rec.data, nothing, srcnum, 'F', -1, options),
                              )
end

############################################################
## overloaded Base functions

# conj(judiPDEfull)
conj(A::judiPDEfull{DDT,RDT}) where {DDT,RDT} =
    judiPDEfull{DDT,RDT}("conj("*A.name*")",A.m,A.n,A.info,A.model,A.srcGeometry,A.recGeometry,A.options,
        A.fop,
        A.fop_T
        )

# transpose(judiPDEfull)
transpose(A::judiPDEfull{DDT,RDT}) where {DDT,RDT} =
    judiPDEfull{DDT,RDT}("Proj*F'*Proj'",A.n,A.m,A.info,A.model,A.srcGeometry,A.recGeometry,A.options,
        A.fop_T,
        A.fop
        )

# adjoint(judiPDEfull)
adjoint(A::judiPDEfull{DDT,RDT}) where {DDT,RDT} =
    judiPDEfull{DDT,RDT}("Proj*F'*Proj'",A.n,A.m,A.info,A.model,A.srcGeometry,A.recGeometry,A.options,
        A.fop_T,
        A.fop
        )

############################################################
## overloaded Base *(...judiPDEfull...)

# *(judiPDEfull,judiVector)
function *(A::judiPDEfull{ADDT,ARDT},v::judiVector{vDT}) where {ADDT,ARDT,vDT}
    A.n == size(v,1) || throw(judiPDEfullException("shape mismatch"))
    if compareGeometry(A.srcGeometry,v.geometry) == false && compareGeometry(A.recGeometry,v.geometry) == false
        throw(judiPDEfullException("geometry mismatch"))
    end
    jo_check_type_match(ADDT,vDT,join(["DDT for *(judiPDEfull,judiVector):",A.name,typeof(A),vDT]," / "))
    V = A.fop(v)
    jo_check_type_match(ARDT,eltype(V),join(["RDT from *(judiPDEfull,judiVector):",A.name,typeof(A),eltype(V)]," / "))
    return V
end

# *(num,judiPDEfull)
function *(a::Number,A::judiPDEfull{ADDT,ARDT}) where {ADDT,ARDT}
    return judiPDEfull{ADDT,ARDT}("(N*"*A.name*")",A.m,A.n,A.info,A.model,A.srcGeometry,A.recGeometry,A.options,
        v1 -> jo_convert(ARDT,a*A.fop(v1),false),
        v2 -> jo_convert(ADDT,a*A.fop_T(v2),false)
        )
end

function A_mul_B!(x::judiVector,A::judiPDEfull,y::judiVector)
    z = A*y
    x.data = z.data
end

function Ac_mul_B!(x::judiVector,A::judiPDEfull,y::judiVector)
    z = adjoint(A)*y
    x.data = z.data
end

############################################################
## overloaded Bases +(...judiPDEfull...), -(...judiPDEfull...)

# +(judiPDEfull,num)
function +(A::judiPDEfull{ADDT,ARDT},b::Number) where {ADDT,ARDT}
    return judiPDE{ADDT,ARDT}("("*A.name*"+N)",A.m,A.n,A.info,A.model,A.srcGeometry,A.recGeometry,A.options,
        v1 -> A.fop(v1)+joConstants(A.m,A.n,b;DDT=ADDT,RDT=ARDT)*v1,
        v2 -> A.fop_T(v2)+joConstants(A.m,A.n,b;DDT=ADDT,RDT=ARDT)*v2
        )
end

# -(judiPDEfull,num)
function -(A::judiPDEfull{ADDT,ARDT},b::Number) where {ADDT,ARDT}
    return judiPDE{ADDT,ARDT}("("*A.name*"-N)",A.m,A.n,A.info,A.model,A.srcGeometry,A.recGeometry,A.options,
        v1 -> A.fop(v1)-joConstants(A.m,A.n,b;DDT=ADDT,RDT=ARDT)*v1,
        v2 -> A.fop_T(v2)-joConstants(A.m,A.n,b;DDT=ADDT,RDT=ARDT)*v2
        )
end

# -(judiPDEfull)
-(A::judiPDEfull{DDT,RDT}) where {DDT,RDT} =
    judiPDEfull{DDT,RDT}("(-"*A.name*")",A.m,A.n,A.info,A.model,A.srcGeometry,A.recGeometry,A.options,
        v1 -> -A.fop(v1),
        v2 -> -A.fop_T(v2)
        )

############################################################
## Additional overloaded functions

# Subsample Modeling operator
function subsample(F::judiPDEfull{ADDT,ARDT}, srcnum) where {ADDT,ARDT}

    srcGeometry = subsample(F.srcGeometry,srcnum)       # Geometry of subsampled data container
    recGeometry = subsample(F.recGeometry,srcnum)

    info = Info(F.info.n, length(srcnum), F.info.nt[srcnum])
    return judiModeling(info, F.model, srcGeometry, recGeometry; options=F.options, DDT=ADDT, RDT=ARDT)
end

getindex(F::judiPDEfull,a) = subsample(F,a)

ext: jl


text:
@testset "Structured Matrices" begin
    @testset "/ and \\ on Square Matrixes" begin
        @testset "//, $T on the RHS" for T in (Diagonal, UpperTriangular, LowerTriangular)
            RHS = T(randn(T == Diagonal ? 10 : (10, 10)))
            test_rrule(/, randn(5, 10), RHS)
        end

        @testset "\\ $T on LHS" for T in (Diagonal, UpperTriangular, LowerTriangular)
            LHS = T(randn(T == Diagonal ? 10 : (10, 10)))
            test_rrule(\, LHS, randn(10))
            test_rrule(\, LHS, randn(10, 10))
        end
    end

    @testset "Diagonal" begin
        N = 3
        test_rrule(Diagonal, randn(N); output_tangent=randn(N, N))
        D = Diagonal(randn(N))
        test_rrule(Diagonal, randn(N); output_tangent=D)
        # Concrete type instead of UnionAll
        test_rrule(typeof(D), randn(N); output_tangent=D)

        # TODO: replace this with a `rrule_test` once we have that working
        # see https://github.com/JuliaDiff/ChainRulesTestUtils.jl/issues/24
        res, pb = rrule(Diagonal, [1, 4])
        @test pb(10*res) == (NO_FIELDS, [10, 40])
        comp = Composite{typeof(res)}(; diag=10*res.diag)  # this is the structure of Diagonal
        @test pb(comp) == (NO_FIELDS, [10, 40])
    end
    @testset "dot(x, ::Diagonal, y)" begin
        N = 4
        test_rrule(dot, randn(ComplexF64, N), Diagonal(randn(ComplexF64, N)), randn(ComplexF64, N))
    end
    @testset "::Diagonal * ::AbstractVector" begin
        N = 3
        test_rrule(*, Diagonal(randn(N)), randn(N))
    end
    @testset "diag" begin
        N = 7
        test_rrule(diag, randn(N, N))
        test_rrule(diag, Diagonal(randn(N)))
        test_rrule(diag, randn(N, N) ⊢ Diagonal(randn(N)))
        test_rrule(diag, Diagonal(randn(N)) ⊢ Diagonal(randn(N)))
        VERSION ≥ v"1.3" && @testset "k=$k" for k in (-1, 0, 2)
            test_rrule(diag, randn(N, N), k ⊢ nothing)
        end
    end
    @testset "diagm" begin
        @testset "without size" begin
            M, N = 7, 9
            s = (8, 8)
            a, ā = randn(M), randn(M)
            b, b̄ = randn(M), randn(M)
            c, c̄ = randn(M - 1), randn(M - 1)
            ȳ = randn(s)
            ps = (0 => a, 1 => b, 0 => c)
            y, back = rrule(diagm, ps...)
            @test y == diagm(ps...)
            ∂self, ∂pa, ∂pb, ∂pc = back(ȳ)
            @test ∂self === NO_FIELDS
            ∂a_fd, ∂b_fd, ∂c_fd = j′vp(_fdm, (a, b, c) -> diagm(0 => a, 1 => b, 0 => c), ȳ, a, b, c)
            for (p, ∂px, ∂x_fd) in zip(ps, (∂pa, ∂pb, ∂pc), (∂a_fd, ∂b_fd, ∂c_fd))
                ∂px = unthunk(∂px)
                @test ∂px isa Composite{typeof(p)}
                @test ∂px.first isa AbstractZero
                @test ∂px.second ≈ ∂x_fd
            end
        end
        VERSION ≥ v"1.3" && @testset "with size" begin
            M, N = 7, 9
            a, ā = randn(M), randn(M)
            b, b̄ = randn(M), randn(M)
            c, c̄ = randn(M - 1), randn(M - 1)
            ȳ = randn(M, N)
            ps = (0 => a, 1 => b, 0 => c)
            y, back = rrule(diagm, M, N, ps...)
            @test y == diagm(M, N, ps...)
            ∂self, ∂M, ∂N, ∂pa, ∂pb, ∂pc = back(ȳ)
            @test ∂self === NO_FIELDS
            @test ∂M === DoesNotExist()
            @test ∂N === DoesNotExist()
            ∂a_fd, ∂b_fd, ∂c_fd = j′vp(_fdm, (a, b, c) -> diagm(M, N, 0 => a, 1 => b, 0 => c), ȳ, a, b, c)
            for (p, ∂px, ∂x_fd) in zip(ps, (∂pa, ∂pb, ∂pc), (∂a_fd, ∂b_fd, ∂c_fd))
                ∂px = unthunk(∂px)
                @test ∂px isa Composite{typeof(p)}
                @test ∂px.first isa AbstractZero
                @test ∂px.second ≈ ∂x_fd
            end
        end
    end
    @testset "$f, $T" for
        f in (Adjoint, adjoint, Transpose, transpose),
        T in (Float64, ComplexF64)

        n = 5
        m = 3
        @testset "$f(::Matrix{$T})" begin
            A = randn(T, n, m)
            Y = f(A)
            Ȳ_mat = randn(T, m, n)
            Ȳ_composite = Composite{typeof(Y)}(parent=collect(f(Ȳ_mat)))

            test_rrule(f, A; output_tangent=Ȳ_mat)

            _, pb = rrule(f, A)
            @test pb(Ȳ_mat) == pb(Ȳ_composite)
        end

        @testset "$f(::Vector{$T})" begin
            a = randn(T, n)
            y = f(a)
            ȳ_mat = randn(T, 1, n)
            ȳ_composite = Composite{typeof(y)}(parent=collect(f(ȳ_mat)))

            test_rrule(f, a; output_tangent=ȳ_mat)

            _, pb = rrule(f, a)
            @test pb(ȳ_mat) == pb(ȳ_composite)
        end

        @testset "$f(::Adjoint{$T, Vector{$T})" begin
            a = randn(T, n)'
            ā = randn(T, n)'
            test_rrule(f, a ⊢ ā; output_tangent=randn(T, n))
        end

        @testset "$f(::Transpose{$T, Vector{$T})" begin
            a = transpose(randn(T, n))
            ā = transpose(randn(T, n))
            test_rrule(f, a ⊢ ā; output_tangent=randn(T, n))
        end
    end
    @testset "$T" for T in (UpperTriangular, LowerTriangular)
        n = 5
        test_rrule(T, randn(n, n); output_tangent=T(randn(n, n)))
    end
    @testset "$Op" for Op in (triu, tril)
        n = 7
        test_rrule(Op, randn(n, n))
        @testset "k=$k" for k in -2:2
            test_rrule(Op, randn(n, n), k ⊢ nothing)
        end
    end

    @testset "det and logdet $S" for S in (Diagonal, UpperTriangular, LowerTriangular)
        @testset "$op" for op in (det, logdet)
            @testset "$T" for T in (Float64, ComplexF64)
                n = 5
                # rand (not randn) so det will be postive, so logdet will be defined
                X = S(3*rand(T, (n, n)) .+ 1)
                X̄_acc = Diagonal(rand(T, (n, n)))  # sensitivity is always a diagonal for these types
                test_rrule(op, X ⊢ X̄_acc)
            end
            @testset "return type" begin
                X = S(3*rand(6, 6) .+ 1)
                _, op_pullback = rrule(op, X)
                X̄ = op_pullback(2.7)[2]
                @test X̄ isa Diagonal
            end
        end
    end
end


text:
### A Pluto.jl notebook ###
# v0.12.14

using Markdown
using InteractiveUtils

# ╔═╡ d16393e0-26de-11eb-17a4-e74d7e9015ae
begin
	using Pkg; Pkg.activate(".") #sets the Julia package environment
	using CSV #For processing csv files
	using DataStructures #Expands on standard Dict
	using DataFrames
	using StatsBase
	using StatPlots
	using MLDataUtils #Tools for splitting data
	using ScikitLearn #A Julia implementation of Python's ScikitLearn
	using ScikitLearn: fit!, predict
	using Plots
	using LsqFit # Simple library for least-squares

	path = pwd();
	data_path = path*"/../data/cp_data_cleaned_jl.csv"

	#Set random seed for reproducibility
	using Random
	const rng_seed = 42
	Random.seed!(rng_seed)
end;

# ╔═╡ d1e158da-278c-11eb-05ec-b1787dedc72f
using PyCall

# ╔═╡ c785b672-26df-11eb-229e-c904ef98c0ac
md"
# Data Featurization
The original python jupyter noteboook can be found [here](https://github.com/anthony-wang/BestPractices/blob/master/notebooks/3-modeling_classic_models.ipynb). This version was put together by [Stefan Bringuier](mailto:stefanbringuier@gmail.com)

The task at hand now is to featurize the composition input (e.g. 'BaO3') into a material descriptor/feature vector. This enables representing the string as a vector chemical properties based on the constituent atoms. For more detailed discussion refer to the manuscript. There are many different ways to do featurization in materials informatics and will depend on the type of model or analysis one is trying to achieve [^a]."

# ╔═╡ 8f9e387e-26df-11eb-06ea-313299966ce4
md"
Let us load the data sets corresponding to training, validation, and testing. One may also be interested in sub-sampling from these data sets if the size of the datasets is to large to enable easy prototyping and assesments. However, it is usually the case that materials data is to precious to do so. The steps aren't shown in this notebook but see the [original jupyter notebook](https://github.com/anthony-wang/BestPractices/blob/master/notebooks/3-modeling_classic_models.ipynb) if interested.

"

# ╔═╡ f6285478-26e0-11eb-2878-57a500d3f3b3
md"
## Generating features using the CBFV package

The composition-based feature vector(CBFV) utlizes exisiting material property databases (e.g. JARVIS) to determine what the features/descriptors should be given a specific chemical formula.

At the moment I'll make use of modified CBFV package[^1] in this directory and use pandas via PyCall.jl. Eventually it will be more desirable to have a native Julia CBFV package however this is somewhat duplicative.

"

# ╔═╡ 63254334-27d7-11eb-0252-b1e321613d49
begin
	py"""
	import sys
	sys.path.insert(0,".")
	"""
	generate_features = pyimport("CBFV.cbfv.composition")["generate_features"]
end;

# ╔═╡ e0be63ee-27cc-11eb-0ed9-3d933555f0b1
begin
	df_train = CSV.File("../data/cp_train.csv") |> DataFrame
	df_val = CSV.File("../data/cp_val.csv") |> DataFrame
	df_test = CSV.File("../data/cp_val.csv") |> DataFrame
end;

# ╔═╡ 2970a1d4-27cc-11eb-2ffc-114c91fe46aa
md"
Now lets read in the datasets that were generated from [2-data_splitting](https://github.com/anthony-wang/BestPractices/blob/master/notebooks/2-data_splitting.ipynb) and use the CBFV `generate_function` call to generate a new input features. The way CBFV works requires the dataframe have a column called `target`, in this cases this is the heat capcity $\text{C}_p$.

Lets first read in the datasets...

* Training dataset has $(size(df_train))
* Validation dataset has $(size(df_val))
* Testing dataset has $(size(df_test))
"

# ╔═╡ 5697ea04-27cd-11eb-28cf-07ccf586bb6b
md"
Now renaming $\text{C}_p$ to `target`.[^2]
"

# ╔═╡ c96b6ab8-27cd-11eb-2aed-c760d176ce4b
begin
	df_train_target = rename(df_train, :Cp => "target");
	df_train_val = rename(df_val, :Cp => "target");
	df_train_test = rename(df_test, :Cp => "target");
end

# ╔═╡ 37077d50-2938-11eb-018c-83e72c7374e5
md"
#### Using Pandas via PyCall.jl instead of DataFrames.jl
"

# ╔═╡ 68926d84-282b-11eb-3923-85ff978fffdf
pandas = pyimport("pandas");

# ╔═╡ abbf1bde-282b-11eb-3aab-afa2182c6e99
md"
Since this excersise interfaces with the python CBFV version at the momement, the steps above are repeated using PyCall.jl and pandas. This gives a PyObject that references the pandas DataFrame object that can then be passed to the `generate_features` call.
"

# ╔═╡ 75610794-282b-11eb-1464-078644009df6
begin
	pdf_train = pandas.read_csv("../data/cp_train.csv")
	pdf_val = pandas.read_csv("../data/cp_val.csv")
	pdf_test = pandas.read_csv("../data/cp_test.csv")

	dict_rename = Dict("Cp" => "target")
	pdf_train_raw= pdf_train.rename(columns=dict_rename)
	pdf_val_raw = pdf_val.rename(columns=dict_rename)
	pdf_test_raw = pdf_test.rename(columns=dict_rename)
end;

# ╔═╡ ab92c33a-38f3-11eb-29ea-393ca4cbb2cb
md"
### Sub-sampling Data (Optional)
If your trying to do some model assesement or prototyping it maybe more practical to train which a smaller dataset. Usually this isn't a luxury we have when working materials or chemistry datasets.
"

# ╔═╡ 12c2ede6-38f4-11eb-107b-f702ab0d48ee
begin
	pdf_train_subsampled = pdf_train_raw.sample(n=2000,random_state=rng_seed)
	pdf_val_subsampled = pdf_val_raw.sample(n=200,random_state=rng_seed)
	pdf_test_subsampled = pdf_val_raw.sample(n=200,random_state=rng_seed)
end;

# ╔═╡ 7c596716-27cf-11eb-25dc-b13f0a3618ba
md"
Now that we are ready to use the `generate_features` function we need to specify some key arguments about what is to be done. There following are:

* `elem_prop`: specifies which CBFV featurization scheme to use. This depends on the chemica/material feature database one wants to use.
* `drop_duplicates`: parameter that determines whether to drop duplicate formulae during featurization. Since there are multiple occurances in the data being used (i.e., multiple tempeatures for a given composition) this needs to be `false`.
* `extend_features`: adds the CBFV features onto existing features. Since we have temperature as a feature this will be `true`.
* `sum_feat`: uses/calculates the sum of constituent features.

The settings we will use are:

* `elem_prop=\"oliynyk\"`
* `drop_duplicates=false`
* `extend_features=true`
* `sum_feat=True`

"

# ╔═╡ 81fe1a6c-2834-11eb-0314-91345a00e98d
begin
	#df --> refers to dataframe, pdf --> refers to pandas dataframe
	cbfv_file = "CBFV/element_properties/oliynyk.csv"

	Xtrain_pdf,ytrain_pdf,formulae_train_pdf,skipped_train_pdf =
					  generate_features(pdf_train_subsampled,
					  	elem_prop="oliynyk",
					  	drop_duplicates=false,
			          	extend_features=true,
			          	sum_feat=true,
					  	file_provided=cbfv_file)
	Xval_pdf,yval_pdf,formulae_val_pdf,skipped_val_pdf =
					  generate_features(pdf_val_subsampled,
					  	elem_prop="oliynyk",
					  	drop_duplicates=false,
			          	extend_features=true,
			          	sum_feat=true,
					  	file_provided=cbfv_file)
	Xtest_pdf,ytest_pdf,formulae_test_pdf,skipped_test_pdf =
					  generate_features(pdf_test_subsampled,
					  	elem_prop="oliynyk",
					  	drop_duplicates=false,
			          	extend_features=true,
			          	sum_feat=true,
					  	file_provided=cbfv_file)
end;

# ╔═╡ ec4e091a-2837-11eb-083b-4f47556f7da4
md"
#### Optional conversion
Now moving back from pandas to DataFrame.jl to continue working with a Julia native environment. We just need a little wrapper function.

However its probably just best to convert everything to Julia arrays as this is what ScikitLearn.jl will use.

"

# ╔═╡ 894ecc98-2838-11eb-32af-ef0fcbb950cd
begin
	function pdf_to_df(df_pd)
    	df= DataFrame()
    	for col in df_pd.columns
        	df[!, col] = get(df_pd,col).values
    	end
    	df
	end
	function ps_to_df(s_pd)
		df = DataFrame();
		cname = s_pd.name;
		df[!,cname] = s_pd.values
		return df
	end
end

# ╔═╡ 6e25ba5e-2839-11eb-2efc-a512b21749ad
begin
	Xtrain,ytrain= pdf_to_df(Xtrain_pdf),ps_to_df(ytrain_pdf)
	Xval,yval = pdf_to_df(Xval_pdf),ps_to_df(yval_pdf)
	Xtest,ytest = pdf_to_df(Xtest_pdf),ps_to_df(ytest_pdf)
end;

# ╔═╡ c963e8d0-283b-11eb-0dc8-d9ea1da79741
md"

$(first(Xtrain,25))

Note that the entries are different from the orignal python jupyter notebook since no subsampling was done in this notebook.

"

# ╔═╡ 92a28dea-2842-11eb-2d3f-4bf55b271864
md"
## Data normalization and scaling

The next step that is usually key to improve model training performance is to scale and normalize the input features.

The scaling helps to address the order-of-magnitude spread that can occur between input features, this is a very common occurance in chemical and material data.

Normalization is the rescaling of values within an input feature to improve model training/performance order-of-magnitude spread exist within the feature, this is also common for many chemical and material properties.
"

# ╔═╡ 48baf3fe-3fec-11eb-38d3-7bac96226b3d
function normalize_scale(data_train::DataFrame,data_to_scale::DataFrame)
	# The scaling and normalize should be based on the training statistics fit.
	data_train_arry = convert(Array,data_train);
	centerscale = fit(ZScoreTransform,data_train_arry,dims=1)
	data_arry = convert(Array,data_to_scale);
	scaled_data = StatsBase.transform(centerscale,data_arry)
	return scaled_data
end;

# ╔═╡ a0d7c2fa-2843-11eb-2a33-5baa0ccd294e
begin
	Xtrain_f,Xval_f,Xtest_f = [normalize_scale(Xtrain,d) for d in (Xtrain,Xval,Xtest)]
	ytrain_f,yval_f, ytest_f = map(x->convert(Array,x),[ytrain,yval,ytest])
end;

# ╔═╡ 644a69ba-293a-11eb-15ef-87dc7794c14b
md"
## ScikitLearn.jl
This is an implementation of scikit learn but in Julia. It offers much of the same model functionlity as in the python implementation. We first use the macro to import linear, ensemble, and support vector machine algorithms/models.

The goal now is to assess different machine learning models performance for the data at hand. You can think of the models of different types of parameterization of the data, this can using arguments that statistical, categorical, or clustering, etc. by nature.

* Ridge regression
* Support vector machine
* Linear support vector machine
* Random forest
* Extra trees
* Adaptive boosting
* Gradient boosting
* k-nearest neighbors
* Dummy (if you can't beat this, something is wrong.)

Unhide below to see the macro for imported models.
"

# ╔═╡ 41831116-293a-11eb-1f82-c9629ca42334
begin
	@sk_import dummy: DummyRegressor
	@sk_import linear_model: Ridge
	@sk_import ensemble: AdaBoostRegressor
	@sk_import ensemble: GradientBoostingRegressor
	@sk_import ensemble: ExtraTreesRegressor
	@sk_import ensemble: RandomForestRegressor
	@sk_import neighbors: KNeighborsRegressor
	@sk_import svm: SVR
	@sk_import svm: LinearSVR
end;

# ╔═╡ fa45ba9e-2b6c-11eb-273a-c9e60f3c940d
md"""
Now creating a function call that instantiates the scikit model and then fits it to the provided training data
"""

# ╔═╡ 814d33f4-2966-11eb-3f4f-09450d8f3d32
function fit_model!(scikit_model,Xtrain::Array,ytrain::Array)
	tᵢ = time();
	model = scikit_model();
	fitmodel = fit!(model,Xtrain,ytrain);
	Δt = time() - tᵢ
	return fitmodel, Δt
end;

# ╔═╡ 299d79be-2b6d-11eb-372f-9b6d14ee8477
md"""
Next is a function to evaluate the performance of the fit. The metrics we will us are the coefficient of determination regression score r$^2$, mean absolute error, and mean-squared error.

To do this we will import the functions from ScikitLearn.jl
"""

# ╔═╡ 9fe0f246-2b71-11eb-3aa2-ffb5f71894d6
begin
	@sk_import metrics: r2_score
	@sk_import metrics: mean_absolute_error
	@sk_import metrics: mean_squared_error
end;

# ╔═╡ ce3a928c-2b6f-11eb-2f60-7bd5e039006b
function evaluate_model(fitmodel,X::Array,y::Array)
	ypredict = predict(fitmodel,X)
	r² = r2_score(y,ypredict)
	mae = mean_absolute_error(y,ypredict)
	rmse = mean_squared_error(y,ypredict,squared=false)
	return (r²,mae,rmse)
end

# ╔═╡ ae2c0eee-2b74-11eb-2ec4-335f23618d58
md"

The next step is to write a function that returns a model ScikitLearn.jl PyObject and the name of it.

Then writing a function to train and evaluate the training set and validation set and then sore the results in a dictionary that will be used to in the `model_results` dataframe.
"

# ╔═╡ 0cfaf15a-2b7b-11eb-10d7-51182f193565
begin
	function get_scikit_model(name_abbrv)
		model_names = dict_scikit_models()
		return model_names[name_abbrv],eval(model_names[name_abbrv])
	end

	function dict_scikit_models()
		Dict(
    	:dumr => :DummyRegressor,
    	:rr => :Ridge,
    	:abr => :AdaBoostRegressor,
    	:gbr => :GradientBoostingRegressor,
    	:rfr => :RandomForestRegressor,
    	:etr => :ExtraTreesRegressor,
    	:svr => :SVR,
    	:lsvr => :LinearSVR,
    	:knr => :KNeighborsRegressor
		);
	end
end

# ╔═╡ 46e78d20-2b75-11eb-24cb-531a9b80a9af
function fit_evaluate_model(scikit_model_abbrv,Xtrain,ytrain,Xval,yval)
	results = Dict() #store results;

	#Model type
	results[:model_abbrv] = String(scikit_model_abbrv);
	model_name,scikit_model = get_scikit_model(scikit_model_abbrv);
	results[:model_name] = String(model_name);

	#Fit model
	fitmodel,results[:fit_time] = fit_model!(scikit_model,Xtrain,ytrain);
	results[:model_params] = fitmodel.get_params();

	#Evaluate model
	begin
		results[:r2_train],
		results[:mae_train],
		results[:rmse_train] = evaluate_model(fitmodel,Xtrain,ytrain);

		results[:r2_val],
		results[:mae_val],
		results[:rmse_val] = evaluate_model(fitmodel,Xval,yval);
	end
	return fitmodel,results
end

# ╔═╡ 3fa9d45a-2b89-11eb-3162-4701219093b0
md"
A convience function to append and soft the model result database
"

# ╔═╡ 4fe5a448-2b89-11eb-2ef9-ff04a24f9451
function append_and_sort!(dataframe::DataFrame,data::Dict,col::Symbol=:r2_val)
	append!(dataframe,data)
	sort!(dataframe,col)
end

# ╔═╡ 6cdb025a-2940-11eb-1866-312d59c524b7
md"
To make the assesment of models easier we will create a julia DataFrame to store the results.
"

# ╔═╡ 959afe70-2b89-11eb-11fa-f3af9f1ef4cc
md"

The final step is to loop through each model, train it, evaluate it, and store the results.
"

# ╔═╡ b7525c34-2b89-11eb-0af6-6515f965d4a9
begin
	model_results = DataFrame(model_abbrv = String[],
						  model_name = String[],
						  model_params = Dict[],
						  fit_time = Float64[],
						  r2_train = Float64[],
						  mae_train = Float64[],
						  rmse_train = Float64[],
						  r2_val = Float64[],
						  mae_val = Float64[],
						  rmse_val = Float64[]
						  );

	stored_models = Dict();

	model_names = keys(dict_scikit_models())

	for m in model_names
		m_fitmodel,m_results = fit_evaluate_model(m,Xtrain_f,ytrain_f,Xval_f,yval_f)
		append_and_sort!(model_results,m_results)
		stored_models[m] = m_fitmodel;
	end
end

# ╔═╡ 28b419ac-2b8b-11eb-1304-af16c78b7e26
md"
Now the peformance of the models can be explored.
"

# ╔═╡ 4c42b16e-2b8e-11eb-1b7b-6158fbae63f0
model_results

# ╔═╡ d506fdfe-2b8e-11eb-355e-4f52e4b5093a
md"

Ultimately we want something a bit more easy to assess the performance of a model on the validation data. The most common way to do this is to create a scatter plot of predicted vs actual target values. A perfect agreement would be a straight line with slope 1.

Below is a function that plots the model prediction results vs. the actual results and also does a simple linear fit. The r$^2$ score is indicated in t
"

# ╔═╡ b027f326-2b8f-11eb-26a0-d1edcbfff4b6
function plot_performance!(model_name,ypredict,yactual;size=(300,300))

	#r2 score for how well variance is captured.
	r² = r2_score(yactual,ypredict)

	#GR plot default settings
	gr(size = size,
		linewidth= 1,
		markersize= 3,
		markeralpha= 0.4,
		markercolor=:silver,
		foreground_color_legend=nothing,
		xlabel="Actual Cp [J/mol K]",
		ylabel="Predicted Cp [J/mol K]",
		tickfont = (8, "arial"),
		guidefont = (8, "arial"),
		title="$(model_name), r2: $(round(r²,sigdigits=3))",
		titlefont = (6,"arial"),
		legendfont = (5,"arial"),
		legend=:topleft);

	maxbound = maximum(max(yactual,ypredict));
	lims = range(0,maxbound,length=100)
	p = scatter(yactual,ypredict,label=nothing)
	p = plot!(lims,lims,label="ideal",linecolor=:black,
		linestyle=:dash)

	#Least-squares linear fit linear algebra approach, not working
	# using LsqFit.jl package
	#coeff = (yactual' * yactual) \ (yactual' * ypredict)
	coeffs = [1.0, 0.0]
	lf = curve_fit((x,p)->p[1].*x .+ p[2], yactual, ypredict, coeffs)
	p = plot!(x->x,x->lf.param[1].*x .+ lf.param[2],yactual,
		label="linear fit",linealpha=0.8,linecolor=:orange)

	#p = annotate!(300,50,text("r2 score\n $(round(r²,sigdigits=3))",8,"arial"))

	return p
end

# ╔═╡ 0a11a784-2b91-11eb-093a-71d0e1e731f1
begin
mp = Plots.Plot{Plots.GRBackend}[]
for m in keys(stored_models)
		trained_model = stored_models[m]
		ypredict_val = predict(trained_model,Xval_f)
		model_name = String(dict_scikit_models()[m])
		p =plot_performance!(model_name,vcat(ypredict_val...),vcat(yval_f...),size=(700,700))
		push!(mp,p)
end
end

# ╔═╡ 3ab4364a-3212-11eb-3fcb-db53d1d87e03
md"""
Now plotting all the actual vs. prediction for all the trained models.
"""

# ╔═╡ 068af642-2cf1-11eb-2520-b95506a8cff4
plot(mp...)

# ╔═╡ 459f4870-2d48-11eb-031a-1b2093391a55
md"
Upon inspection of the plots we get a sense for which models perform well, that is, the model predictions which show a general behavior scattered around the ideal line. The linear least-squares fit shows how the model prediction deviates (in an average sense) and the R$^2$ score indicates how the variance or data spread is captured by the model prediction.

## Re-training the best performing model
The next step is to select the best performing machine learning model and use the validation data set to train the hyperparameters.

"

# ╔═╡ eaa5d594-2868-11eb-0a45-3bb4e8ed7921
best_model = last(model_results)

# ╔═╡ 5f379a00-3214-11eb-3716-652b4f197e3a
begin
	best_model_name = best_model[:model_abbrv]
	best_model_params = best_model[:model_params]
end;

# ╔═╡ ad1a1b00-3262-11eb-28b1-719264e3d7dd
trained_model = stored_models[Symbol(best_model_name)]

# ╔═╡ 8b8b6df8-3263-11eb-2faa-a55675ae8c1f
md"
The next step is to combine the training and validation data sets and then retrain the `GradientBoostingRegressor`
"

# ╔═╡ a887ec3a-3263-11eb-14c0-fdb6366ee22c
begin
	X_train_val = vcat(Xtrain_f,Xval_f);
	y_train_val = vcat(ytrain_f,yval_f);
end;

# ╔═╡ 771bc308-335b-11eb-3497-41115ad8b1f1
md"

The optimal parametes obtained from the initial training should be used as the starting point. Here the `stored_models` dictionary object retains the best parameters for the trained model, but you could set them using a function call `set_params!(model;params...)`

Then the model is retrained using the combined traning and validation datasets.
"

# ╔═╡ 22e193b4-3264-11eb-06d7-71a9ea777b2d
begin
	#The stored model retains the optimal parameters, but they can be assigned via:
	#set_params!(trained_model;best_model_params...)
	trained_model.fit(X_train_val,y_train_val)
end

# ╔═╡ 05d481f2-335c-11eb-2850-b543ac5ddb4b
md"
The final step is to see how the trained model now performs on the test data, which was never seen by the model.
"

# ╔═╡ 207d9b40-335c-11eb-229f-5916852a6800
begin
	y_pred_test = trained_model.predict(Xtest_f)
	r2,mae,rmse = evaluate_model(trained_model,Xtest_f,ytest_f)
	md"""
	Fit quality metrics for test data

	r$^2$: $(r2)

	mae: $(mae)

	mse: $(rmse)

	The model performs remarkable well, with metrics indicating that it captures the data variance and average error to a good degree.
	"""
end

# ╔═╡ 14359dc2-4178-11eb-0435-0ffb7c4a5814
# Write R2 value to a file for test, see runtest.jl
begin
	fitoutput = open("../data/R2_GradientBoostingRegressor.fit","w")
	write(fitoutput,"$(r2)");
	close(fitoutput);
end

# ╔═╡ 9d7f2590-335d-11eb-2ff8-97f1302243f9
plot_performance!(best_model_name,y_pred_test,vcat(ytest_f...),size=(500,500))

# ╔═╡ a5200be6-3371-11eb-30af-6dea1964901f
md"""
# Impact on splitting of dataset

This section will show how creating different splits of the datasets impacts the model training and predictive quality.
"""

# ╔═╡ b1d67f78-38f5-11eb-1a52-218b9349201e
begin
	#df --> refers to dataframe, pdf --> refers to pandas dataframe
	Xtrain_unscaled_pdf,ytrain_unscaled_pdf,_,_ =
					  generate_features(pdf_train_subsampled,
					  	elem_prop="oliynyk",
					  	drop_duplicates=false,
			          	extend_features=true,
			          	sum_feat=true,
					  	file_provided=cbfv_file)
	Xval_unscaled_pdf,yval_unscaled_pdf,_,_=
					  generate_features(pdf_val_subsampled,
					  	elem_prop="oliynyk",
					  	drop_duplicates=false,
			          	extend_features=true,
			          	sum_feat=true,
					  	file_provided=cbfv_file)
	Xtest_unscaled_pdf,ytest_unscaled_pdf,_,_ =
					  generate_features(pdf_test_subsampled,
					  	elem_prop="oliynyk",
					  	drop_duplicates=false,
			          	extend_features=true,
			          	sum_feat=true,
					  	file_provided=cbfv_file)
end;

# ╔═╡ f2388d08-3f36-11eb-01af-2b4453c7e98c
md"
now converting the pandas database to DataFrames.jl for convience
"

# ╔═╡ 0181745a-3f37-11eb-3135-015de061d70c
begin
	Xtrain_unscaled,ytrain_unscaled= pdf_to_df(Xtrain_pdf),ps_to_df(ytrain_pdf)
	Xval_unscaled,yval_unscaled = pdf_to_df(Xval_pdf),ps_to_df(yval_pdf)
	Xtest_unscaled,ytest_unscaled = pdf_to_df(Xtest_pdf),ps_to_df(ytest_pdf)
end;

# ╔═╡ 5b05ac3a-3f37-11eb-1c4b-c9c0015760be
md"
making initial copies of the data so that we can sample and then reassign during the splitting. A total of 10 different dataset splits will be performed (randomly) to show how it impacts training and performance outcomes.
"

# ╔═╡ eda4edbc-3f37-11eb-24e8-c95498dc9629
begin
	Xtrain_org = copy(Xtrain_unscaled);
	ytrain_org = copy(ytrain_unscaled);
	Xval_org = copy(Xval_unscaled);
	Xtest_org = copy(Xtest_unscaled);
end;

# ╔═╡ 5e3c7162-3f38-11eb-27d1-b5ee205371e3
nsplits = 1:10;

# ╔═╡ 3fd375b8-3f38-11eb-195b-e36da010b17c
split_results = DataFrame(split=Int[],
						  r2_train = Float64[],
						  mae_train = Float64[],
						  rmse_train = Float64[],
						  r2_val = Float64[],
						  mae_val = Float64[],
						  rmse_val = Float64[]);

# ╔═╡ 9719d72a-4179-11eb-3a53-cdb597966d83
md"Now set the fraction to 70% for data to split

# ╔═╡ c239a238-3fd7-11eb-0b3f-a9c0e1638772
frac = Int(0.7*nrow(Xtrain_org));

# ╔═╡ 043ad70c-3f39-11eb-2fd5-436ca790fcdb
let
	model_abbrv = :abr #AdaBoostRegressor
	for split in nsplits
		md"Fitting and evaluating random split $(split)"
		xindices = StatsBase.sample(1:nrow(Xtrain_org),frac,replace=false);
    	Xtrain = normalize_scale(Xtrain_org[xindices,:],Xtrain_org[xindices,:]);
    	ytrain = convert(Array,ytrain_org[xindices,:]);

	  	Xval = normalize_scale(Xtrain_org[xindices,:],Xval_org);
		yval = convert(Array,yval_unscaled);
		Xtest = normalize_scale(Xtrain_org[xindices,:],Xtest_org);
		ytest = convert(Array,ytest_unscaled);

		model_name,scikit_model = get_scikit_model(model_abbrv);
		fitmodel,time = fit_model!(scikit_model,Xtrain,ytrain);
		ypred_val = predict(fitmodel,Xval);

		r2_train, mae_train, rmse_train = evaluate_model(fitmodel, Xtrain, ytrain)
    	r2_val, mae_val, rmse_val = evaluate_model(fitmodel, Xval, yval)

		split_result = Dict(:split => split,
							:r2_train => r2_train,
							:mae_train => mae_train,
						    :rmse_train => rmse_train,
							:r2_val => r2_val,
							:mae_val => mae_val,
						    :rmse_val => rmse_val);

		append_and_sort!(split_results,split_result)
	end
end;

# ╔═╡ fc21a804-417a-11eb-09f2-1976176f5b78
split_results

# ╔═╡ 13641844-417b-11eb-013e-399c2f1a4fb9
md"
To visual inspect the r$^2$ values for each split, lets plot them. To faciliate statistical orientied plots via the StatPlots.jl package. This provies we can use functions and macros that interface with Plots.jl.

At first glance the variation in r$^2$ values seems more prevalent in the validation data set.

This can be further seen by looking at the mean absolute error

"

# ╔═╡ 2fb6b226-417c-11eb-1829-f31ba635c495
begin
	gr(size = (600,400),
		linewidth= 2,
		title = "Split fitting of AdaBoostRegressor",
		tickfont = (10, "arial"),
		guidefont = (10, "arial"),
		titlefont = (10,"arial"),
		legendfont = (10,"arial"),
		legend=:topright);

		@df split_results groupedbar(:split,	[:r2_train :r2_val],xlabel="Split",ylabel="r2",ylim=(0.0,1.15))
end

# ╔═╡ 74e9d29c-417c-11eb-27a1-dfeaa2573d18
@df split_results groupedbar(:split,[:mae_train :mae_val],xlabel="Split",ylabel="r2",ylim=(0.0,30.0))

# ╔═╡ 06073c08-4180-11eb-27ea-9154bb3ecbae
md"

Finally, it is common to report the statistical averages of the scores across the splits

Average validation r$^2$: $(mean(split_results[!,:r2_val]))

Average validation MAE: $(mean(split_results[!,:mae_val]))
"

# ╔═╡ 1476bd9c-27cf-11eb-0f1e-7d42c5a8df3e
md"

#### Footnotes

[^a]: Pluto currently doesn't display standard IO, so the progress bar associated with CBFV package won't be shown in the notebook but rather shown in the Julia session that launched the Pluto server. Similar behavior occurs for other IO (e.x. timing).

[^1]: The modification to the CBFV package in pluto_notebooks adds an additional kwarg to `generate_features\(...,file_provided=\"database_file_path\"\)` this was done because the existing composition.py uses the current working directory to find the database file, however, I believe PyCall exist within Julia's OS environment and therefore the two can't be changed indepedently.

[^2]: Note that we don't really need to create a new variable assignment, I could have used the `rename!()` call, but this wasn't choosen because of the way Pluto notebooks are reactive.
"

# ╔═╡ Cell order:
# ╟─c785b672-26df-11eb-229e-c904ef98c0ac
# ╠═d16393e0-26de-11eb-17a4-e74d7e9015ae
# ╟─8f9e387e-26df-11eb-06ea-313299966ce4
# ╟─f6285478-26e0-11eb-2878-57a500d3f3b3
# ╠═d1e158da-278c-11eb-05ec-b1787dedc72f
# ╠═63254334-27d7-11eb-0252-b1e321613d49
# ╟─2970a1d4-27cc-11eb-2ffc-114c91fe46aa
# ╠═e0be63ee-27cc-11eb-0ed9-3d933555f0b1
# ╟─5697ea04-27cd-11eb-28cf-07ccf586bb6b
# ╠═c96b6ab8-27cd-11eb-2aed-c760d176ce4b
# ╟─37077d50-2938-11eb-018c-83e72c7374e5
# ╠═68926d84-282b-11eb-3923-85ff978fffdf
# ╟─abbf1bde-282b-11eb-3aab-afa2182c6e99
# ╠═75610794-282b-11eb-1464-078644009df6
# ╟─ab92c33a-38f3-11eb-29ea-393ca4cbb2cb
# ╠═12c2ede6-38f4-11eb-107b-f702ab0d48ee
# ╟─7c596716-27cf-11eb-25dc-b13f0a3618ba
# ╠═81fe1a6c-2834-11eb-0314-91345a00e98d
# ╟─ec4e091a-2837-11eb-083b-4f47556f7da4
# ╠═894ecc98-2838-11eb-32af-ef0fcbb950cd
# ╠═6e25ba5e-2839-11eb-2efc-a512b21749ad
# ╟─c963e8d0-283b-11eb-0dc8-d9ea1da79741
# ╟─92a28dea-2842-11eb-2d3f-4bf55b271864
# ╠═48baf3fe-3fec-11eb-38d3-7bac96226b3d
# ╠═a0d7c2fa-2843-11eb-2a33-5baa0ccd294e
# ╟─644a69ba-293a-11eb-15ef-87dc7794c14b
# ╠═41831116-293a-11eb-1f82-c9629ca42334
# ╟─fa45ba9e-2b6c-11eb-273a-c9e60f3c940d
# ╠═814d33f4-2966-11eb-3f4f-09450d8f3d32
# ╟─299d79be-2b6d-11eb-372f-9b6d14ee8477
# ╠═9fe0f246-2b71-11eb-3aa2-ffb5f71894d6
# ╠═ce3a928c-2b6f-11eb-2f60-7bd5e039006b
# ╟─ae2c0eee-2b74-11eb-2ec4-335f23618d58
# ╠═0cfaf15a-2b7b-11eb-10d7-51182f193565
# ╠═46e78d20-2b75-11eb-24cb-531a9b80a9af
# ╟─3fa9d45a-2b89-11eb-3162-4701219093b0
# ╠═4fe5a448-2b89-11eb-2ef9-ff04a24f9451
# ╟─6cdb025a-2940-11eb-1866-312d59c524b7
# ╟─959afe70-2b89-11eb-11fa-f3af9f1ef4cc
# ╠═b7525c34-2b89-11eb-0af6-6515f965d4a9
# ╟─28b419ac-2b8b-11eb-1304-af16c78b7e26
# ╠═4c42b16e-2b8e-11eb-1b7b-6158fbae63f0
# ╟─d506fdfe-2b8e-11eb-355e-4f52e4b5093a
# ╠═b027f326-2b8f-11eb-26a0-d1edcbfff4b6
# ╠═0a11a784-2b91-11eb-093a-71d0e1e731f1
# ╟─3ab4364a-3212-11eb-3fcb-db53d1d87e03
# ╠═068af642-2cf1-11eb-2520-b95506a8cff4
# ╟─459f4870-2d48-11eb-031a-1b2093391a55
# ╠═eaa5d594-2868-11eb-0a45-3bb4e8ed7921
# ╠═5f379a00-3214-11eb-3716-652b4f197e3a
# ╠═ad1a1b00-3262-11eb-28b1-719264e3d7dd
# ╟─8b8b6df8-3263-11eb-2faa-a55675ae8c1f
# ╠═a887ec3a-3263-11eb-14c0-fdb6366ee22c
# ╟─771bc308-335b-11eb-3497-41115ad8b1f1
# ╠═22e193b4-3264-11eb-06d7-71a9ea777b2d
# ╟─05d481f2-335c-11eb-2850-b543ac5ddb4b
# ╟─207d9b40-335c-11eb-229f-5916852a6800
# ╟─14359dc2-4178-11eb-0435-0ffb7c4a5814
# ╠═9d7f2590-335d-11eb-2ff8-97f1302243f9
# ╟─a5200be6-3371-11eb-30af-6dea1964901f
# ╠═b1d67f78-38f5-11eb-1a52-218b9349201e
# ╟─f2388d08-3f36-11eb-01af-2b4453c7e98c
# ╠═0181745a-3f37-11eb-3135-015de061d70c
# ╟─5b05ac3a-3f37-11eb-1c4b-c9c0015760be
# ╠═eda4edbc-3f37-11eb-24e8-c95498dc9629
# ╠═5e3c7162-3f38-11eb-27d1-b5ee205371e3
# ╠═3fd375b8-3f38-11eb-195b-e36da010b17c
# ╠═9719d72a-4179-11eb-3a53-cdb597966d83
# ╠═c239a238-3fd7-11eb-0b3f-a9c0e1638772
# ╠═043ad70c-3f39-11eb-2fd5-436ca790fcdb
# ╠═fc21a804-417a-11eb-09f2-1976176f5b78
# ╟─13641844-417b-11eb-013e-399c2f1a4fb9
# ╠═2fb6b226-417c-11eb-1829-f31ba635c495
# ╠═74e9d29c-417c-11eb-27a1-dfeaa2573d18
# ╟─06073c08-4180-11eb-27ea-9154bb3ecbae
# ╟─1476bd9c-27cf-11eb-0f1e-7d42c5a8df3e