Commit 0994807d authored by Hippalectryon's avatar Hippalectryon

json load fix

parent 1923a361
import json
import ijson
import os
import pathlib
from typing import List, Union
......@@ -10,6 +9,14 @@ from God.Bird import Bird
from God.Pandora import NumpyEncoder
from God.Sky import Sky
def read_in_chunks(file_object, chunk_size=1024):
"""Lazy function (generator) to read a file piece by piece.
Default chunk size: 1k."""
while True:
data = file_object.read(chunk_size)
if not data:
break
yield data
def make_path_available(path: str) -> None:
pathlib.Path(os.path.dirname(path)).mkdir(parents=True, exist_ok=True)
......@@ -30,10 +37,11 @@ def load_data_dirname(input_dir: str, input_file: str) -> Union[list, dict]:
def load_data(input_file: str) -> Union[list, dict]:
with open(input_file, 'r') as f:
objects = ijson.items(f, 'meta.view.columns.item')
data = list(objects)
return data
f = open(input_file, "r")
content = ""
for piece in read_in_chunks(f):
content += piece
return json.loads(content)
def recreate_frame(frame: List[List], L: float, grid_step: float) -> Sky:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment