|
|
@ -1,10 +1,15 @@ |
|
|
|
#!/bin/python |
|
|
|
|
|
|
|
import sys |
|
|
|
# import urllib library |
|
|
|
from urllib.request import urlopen |
|
|
|
|
|
|
|
|
|
|
|
# import json |
|
|
|
import json |
|
|
|
import naya |
|
|
|
|
|
|
|
# import sqlite utils library |
|
|
|
from sqlite_utils import Database |
|
|
|
|
|
|
|
# store the URL in url as |
|
|
|
# parameter for urlopen |
|
|
|
url = "https://api.scryfall.com/bulk-data" |
|
|
@ -28,10 +33,14 @@ for file in data_json["data"]: |
|
|
|
|
|
|
|
CHUNK = 256 * 1024 |
|
|
|
|
|
|
|
db = Database("cards.db", recreate=True) |
|
|
|
db.enable_wal() |
|
|
|
cards = db["cards"] |
|
|
|
|
|
|
|
for file in bulk_files: |
|
|
|
uri = file["download_uri"] |
|
|
|
filename = file["type"]+"_"+file["id"]+".json" |
|
|
|
|
|
|
|
print("Downloading {} ...".format(uri)) |
|
|
|
rep = urlopen(uri) |
|
|
|
with open(filename, 'wb') as f: |
|
|
|
while True: |
|
|
@ -39,4 +48,16 @@ for file in bulk_files: |
|
|
|
if not chunk: |
|
|
|
break |
|
|
|
f.write(chunk) |
|
|
|
|
|
|
|
print("Updating cards.db ...\n") |
|
|
|
numcard = 0 |
|
|
|
with open(filename, "r") as f: |
|
|
|
rawjson = naya.stream_array(naya.tokenize(f)) |
|
|
|
batch = [] |
|
|
|
for card in rawjson: |
|
|
|
batch.append(card) |
|
|
|
if len(batch) == 512: |
|
|
|
cards.insert_all(batch,alter=True,pk="id",batch_size=512) |
|
|
|
batch = [] |
|
|
|
sys.stdout.write("Adding card {} \r".format(numcard)) |
|
|
|
numcard+=1 |
|
|
|
print("\n") |