Memory Footprint Resolved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, division
-
# standard library dependencies
import io
import json
from json.encoder import JSONEncoder
-from petl.compat import PY2
-
+from petl.compat import PY2
+from petl.io.sources import read_source_from_arg, write_source_from_arg
# internal dependencies
from petl.util.base import data, Table, dicts as _dicts, iterpeek
-from petl.io.sources import read_source_from_arg, write_source_from_arg
def fromjson(source, *args, **kwargs):
@@ -91,7 +89,6 @@ def fromjson(source, *args, **kwargs):
class JsonView(Table):
-
def __init__(self, source, *args, **kwargs):
self.source = source
self.missing = kwargs.pop('missing', None)
@@ -109,7 +106,8 @@ class JsonView(Table):
write_through=True)
try:
if self.lines:
- dicts = [json.loads(jline) for jline in f.read().splitlines()]
+ for row in iterjlines(f, self.header, self.missing):
+ yield row
else:
dicts = json.load(f, *self.args, **self.kwargs)
for row in iterdicts(dicts, self.header, self.sample,
@@ -173,6 +171,22 @@ class DictsView(Table):
return iterdicts(self.dicts, self.header, self.sample, self.missing)
+def iterjlines(f, header, missing):
+ it = iter(f)
+
+ if header is None:
+ header = list()
+ peek, it = iterpeek(it, 1)
+ json_obj = json.loads(peek)
+ if hasattr(json_obj, 'keys'):
+ header += [k for k in json_obj.keys() if k not in header]
+ yield tuple(header)
+
+ for o in it:
+ json_obj = json.loads(o)
+ yield tuple(json_obj[f] if f in json_obj else missing for f in header)
+
+
def iterdicts(dicts, header, sample, missing):
it = iter(dicts)
