Commit 962bb158 authored by Hippalectryon's avatar Hippalectryon

dropbox save/load

parent 6b29cce1
import json
import os
import pathlib
import io
from typing import List, Union, Iterable
import dropbox
import numpy as np
from God.Bird import Bird
from God.Pandora import NumpyEncoder
from God.Sky import Sky
# init dropbox
dbx = dropbox.Dropbox('token ici')
dbx.users_get_current_account()
def read_in_chunks(file_object, chunk_size=1024 * 1000 * 100) -> Iterable:
"""Lazy function (generator) to read a file piece by piece.
......@@ -32,39 +38,72 @@ def make_path_available(path: str) -> None:
pathlib.Path(os.path.dirname(path)).mkdir(parents=True, exist_ok=True)
def save_data_dirname(data: Union[list, dict], output_dir: str, output_file: str) -> None:
def save_data_dirname(data: Union[list, dict], output_dir: str, output_file: str, use_dropbox: bool = True) -> None:
"""
Save the data in the directory output_dir, with the name output_file
"""
return save_data(data, os.path.join(output_dir, output_file))
return save_data(data, os.path.join(output_dir, output_file), use_dropbox)
def save_data(data: Union[list, dict], output_file: str) -> None:
def save_data(data: Union[list, dict], output_file: str, use_dropbox: bool = True) -> None:
"""
Save the data in the path output_file
"""
make_path_available(output_file)
with open(output_file, "w") as f:
json.dump(data, f, cls=NumpyEncoder)
def load_data_dirname(input_dir: str, input_file: str) -> Union[list, dict]:
if not use_dropbox:
make_path_available(output_file)
with open(output_file, "w") as f:
json.dump(data, f, cls=NumpyEncoder)
else:
if output_file[0] != "/":
output_file = "/" + output_file
CHUNK_SIZE = 1024 * 1024 * 100 # 100Mb
data_json = json.dumps(data, cls=NumpyEncoder).encode()
file_size = len(data_json)
with io.BytesIO(data_json) as f:
if file_size <= CHUNK_SIZE:
print(dbx.files_upload(f.read(), output_file, mode=dropbox.files.WriteMode('overwrite')))
else:
upload_session_start_result = dbx.files_upload_session_start(f.read(CHUNK_SIZE))
cursor = dropbox.files.UploadSessionCursor(session_id=upload_session_start_result.session_id,
offset=f.tell())
commit = dropbox.files.CommitInfo(path=output_file, mode=dropbox.files.WriteMode('overwrite'))
while f.tell() < file_size:
if (file_size - f.tell()) <= CHUNK_SIZE:
print(dbx.files_upload_session_finish(f.read(CHUNK_SIZE), cursor, commit))
else:
dbx.files_upload_session_append_v2(f.read(CHUNK_SIZE), cursor)
cursor.offset = f.tell()
def load_data_dirname(input_dir: str, input_file: str, use_dropbox: bool = True) -> Union[list, dict]:
"""
Load the data in the path input_dir, with the name input_file
:return: the data as python types
"""
return load_data(os.path.join(input_dir, input_file))
return load_data(os.path.join(input_dir, input_file), use_dropbox)
def load_data(input_file: str) -> Union[list, dict]:
def load_data(input_file: str, use_dropbox: bool = True) -> Union[list, dict]:
"""
Load the data in the path input_file
:return: the data as python types
"""
f = open(input_file, "r")
content = ""
for piece in read_in_chunks(f):
content += piece
if not use_dropbox:
f = open(input_file, "r")
content = ""
for piece in read_in_chunks(f):
content += piece
else:
if input_file[0] != "/":
input_file = "/" + input_file
md, response = dbx.files_download(input_file)
content = response.content
return json.loads(content)
......@@ -82,3 +121,9 @@ def recreate_frame(frame: List[List], L: float, grid_step: float) -> Sky:
sky.add_bird(Bird(np.array(pos), vel, ang_vel, angle))
return sky
# save_data({"a": 5, "b": 22, "plouf":275}, "/test/test.json")
# print(dbx.files_list_folder("/test/"))
# print(load_data("/test/test.json"))
......@@ -37,7 +37,7 @@ def launch_two_groups(output_file: str, L: float, n_birds_1: int, n_birds_2: int
physics = Physics(sky, interaction_radius, eta)
Life.simulate(physics, dt, total_time, verbose_prop=.1, output_file=output_file)
launch_simulation_random("simulation_data/test-machine.json", L=1000, n_birds=100000, eta=.2, total_time=10) # , evolve=evolve)
launch_simulation_random("simulation_data/test-machine.json", L=1000, n_birds=10000, eta=.2, total_time=10) # , evolve=evolve)
#
# # launch_two_groups("simulation_data/test.json", L=100, n_birds_1=100, n_birds_2=0, radius_1=5, radius_2=5,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment