Replace file logging with progress bars

pokeapi_ditto/commands/analyze.py

@@ -6,11 +6,12 @@
 from typing import List
 
 from genson import SchemaBuilder
+from tqdm import tqdm
 
 from pokeapi_ditto.common import from_dir
 
 
-def do_analyze(api_dir: str, schema_dir: str, log: bool):
+def do_analyze(api_dir: str, schema_dir: str):
     if not Path(schema_dir).exists():
         Path(schema_dir).mkdir(parents=True)
 
@@ -30,16 +31,14 @@ def gen_single_schema(path: Path) -> SchemaBuilder:
         )
         file_names = glob.iglob(glob_exp, recursive=True)
         schema = SchemaBuilder()
-        for file_name in file_names:
+        for file_name in tqdm(file_names, desc=str(path.parent)):
             with open(file_name) as f:
                 schema.add_object(json.load(f))
         return schema
 
     @from_dir(schema_dir)
     def gen_schemas(paths: List[Path]):
-        for path in paths:
-            if log:
-                print(Path(schema_dir).joinpath(path))
+        for path in tqdm(paths):
             if not path.parent.exists():
                 os.makedirs(path.parent)
             schema = gen_single_schema(path)

pokeapi_ditto/commands/clone.py

@@ -3,11 +3,12 @@
 import os.path
 
 import requests
+from tqdm import tqdm
 
 from pokeapi_ditto.common import BASE_URL_PLACEHOLDER
 
 
-def do_clone(src_url: str, dest_dir: str, log: bool):
+def do_clone(src_url: str, dest_dir: str):
     if not src_url.endswith("/"):
         src_url += "/"
 
@@ -29,13 +30,11 @@ def print_json(data, file_name):
     endpoints = requests.get(url)
 
     path = dest_dir + url.replace(src_url, "") + "index.json"
-    if log:
-        print(path)
     print_json(endpoints.json(), path)
 
     # Endpoints
 
-    for endpoint in endpoints.json().values():
+    for endpoint in tqdm(endpoints.json().values()):
         # Zero index
         url = endpoint + "?limit=0"
         resource_list = requests.get(url)
@@ -44,27 +43,22 @@ def print_json(data, file_name):
         # Full index
         url = endpoint + "?limit=" + count
         resource_list = requests.get(url)
-        path = dest_dir + endpoint.replace(src_url, "") + "index.json"
-        if log:
-            print(path)
+        endpoint_path = endpoint.replace(src_url, "")
+        path = dest_dir + endpoint_path + "index.json"
         print_json(resource_list.json(), path)
 
         # All resources
-        for resourceSummary in resource_list.json()["results"]:
+        desc = list(filter(None, endpoint_path.split("/")))[-1]
+        for resourceSummary in tqdm(resource_list.json()["results"], desc=desc):
             resource_url = resourceSummary["url"]
             path = dest_dir + resource_url.replace(src_url, "") + "index.json"
 
-            if not os.path.isfile(path):
-                if log:
-                    print(path)
-                resource = requests.get(resource_url)
-                print_json(resource.json(), path)
+            resource = requests.get(resource_url)
+            print_json(resource.json(), path)
 
             if endpoint.endswith("/pokemon/"):
                 resource_url += "encounters/"
                 path = dest_dir + resource_url.replace(src_url, "") + "index.json"
                 if not os.path.isfile(path):
-                    if log:
-                        print(path)
                     resource = requests.get(resource_url)
                     print_json(resource.json(), path)

pokeapi_ditto/commands/transform.py

@@ -1,10 +1,12 @@
 from pathlib import Path
-from typing import List
+from typing import Iterable
+
+from tqdm import tqdm
 
 from pokeapi_ditto.common import apply_base_url
 
 
-def do_transform(src_dir: str, dest_dir: str, base_url: str, log: bool):
+def do_transform(src_dir: str, dest_dir: str, base_url: str):
     src_dir: Path = Path(src_dir)
     dest_dir: Path = Path(dest_dir)
 
@@ -14,12 +16,10 @@ def do_transform(src_dir: str, dest_dir: str, base_url: str, log: bool):
     if not dest_dir.exists():
         dest_dir.mkdir(parents=True)
 
-    orig_paths: List[Path] = src_dir.glob("api/**/*.json")
+    orig_paths: Iterable[Path] = src_dir.glob("api/**/*.json")
 
-    for orig in orig_paths:
+    for orig in tqdm(list(orig_paths)):
         new = dest_dir.joinpath(orig.relative_to(src_dir))
-        if log:
-            print(new)
 
         if not new.parent.exists():
             new.parent.mkdir(parents=True)

pokeapi_ditto/main.py

@@ -11,13 +11,6 @@ class Ditto(object):
     def __init__(self):
         parser = argparse.ArgumentParser()
         parser.add_argument("--version", action="version", version=__version__)
-        parser.add_argument(
-            "--log",
-            action="store_const",
-            const=True,
-            default=False,
-            help="turn on logging of files saved",
-        )
         subparsers = parser.add_subparsers(dest="command")
 
         clone_args = subparsers.add_parser("clone")

pyproject.lock

@@ -342,6 +342,15 @@ platform = "*"
 python-versions = "*"
 version = "0.9.4"
 
+[[package]]
+category = "main"
+description = "Fast, Extensible Progress Meter"
+name = "tqdm"
+optional = false
+platform = "any"
+python-versions = ">=2.6, !=3.0.*, !=3.1.*"
+version = "4.25.0"
+
 [[package]]
 category = "main"
 description = "HTTP library with thread-safe connection pooling, file post, and more."
@@ -361,7 +370,7 @@ python-versions = "*"
 version = "0.14.1"
 
 [metadata]
-content-hash = "2084bde1ecc9f116f89986fc0a738d8269e3eaba20333963f7199aa0af1aa616"
+content-hash = "bd8d8fb1022f9d3d21cdfaba69d1b7574c307f71346ed689ea1fb770acace180"
 platform = "*"
 python-versions = "^3.6"
 
@@ -397,5 +406,6 @@ pytest = ["453cbbbe5ce6db38717d282b758b917de84802af4288910c12442984bde7b823", "a
 requests = ["63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", "ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a"]
 six = ["70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", "832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb"]
 toml = ["8e86bd6ce8cc11b9620cb637466453d94f5d57ad86f17e98a98d1f73e3baab2d"]
+tqdm = ["5ef526702c0d265d5a960a3b27f3971fac13c26cf0fb819294bfa71fc6026c88", "a3364bd83ce4777320b862e3c8a93d7da91e20a95f06ef79bed7dd71c654cafa"]
 urllib3 = ["a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", "b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5"]
 werkzeug = ["c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c", "d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b"]

pyproject.toml

@@ -29,6 +29,7 @@ flask-cors = "^3.0"
 requests = "^2.19"
 gevent = "^1.3"
 genson = "^1.0"
+tqdm = "^4.25"
 
 [tool.poetry.dev-dependencies]
 pytest = "^3.0"