|
|
@ -21,7 +21,7 @@ response = urlopen(url) |
|
|
|
# from url in data |
|
|
|
data_json = json.loads(response.read()) |
|
|
|
|
|
|
|
bulk_files = []; |
|
|
|
bulk_files = [] |
|
|
|
|
|
|
|
# print the json response |
|
|
|
for file in data_json["data"]: |
|
|
@ -51,15 +51,42 @@ for file in bulk_files: |
|
|
|
#json downloaded, moving to db |
|
|
|
print("Updating cards.db ...\n") |
|
|
|
numcard = 0 |
|
|
|
numlayout = 0 |
|
|
|
with open(filename, "r") as f: |
|
|
|
rawjson = naya.stream_array(naya.tokenize(f)) |
|
|
|
batch = [] |
|
|
|
for card in rawjson: |
|
|
|
if card["lang"] in ['fr', 'en']: |
|
|
|
if card["layout"] in ["token","art_series", "double_faced_token", "planar", "scheme", "vanguard", "emblem"]: |
|
|
|
numlayout+=1 |
|
|
|
elif ((card["lang"] in ['fr', 'en']) and card["digital"] == 0): |
|
|
|
#print(card.keys()) |
|
|
|
card.pop("oversized") |
|
|
|
card.pop("digital") |
|
|
|
card.pop("object") |
|
|
|
if "tcgplayer_id" in card.keys(): |
|
|
|
card.pop("tcgplayer_id") |
|
|
|
if "tcgplayer_etched_id" in card.keys(): |
|
|
|
card.pop("tcgplayer_etched_id") |
|
|
|
if "arena_id" in card.keys(): |
|
|
|
card.pop("arena_id") |
|
|
|
if "mtgo_id" in card.keys(): |
|
|
|
card.pop("mtgo_id") |
|
|
|
if "mtgo_foil_id" in card.keys(): |
|
|
|
card.pop("mtgo_foil_id") |
|
|
|
if "cardmarket_id" in card.keys(): |
|
|
|
card.pop("cardmarket_id") |
|
|
|
if "multiverse_id" in card.keys(): |
|
|
|
card.pop("multiverse_id") |
|
|
|
if "security_stamp" in card.keys(): |
|
|
|
card.pop("security_stamp") |
|
|
|
card.pop("booster") |
|
|
|
card.pop("games") |
|
|
|
card.pop("set_type") |
|
|
|
batch.append(card) |
|
|
|
if len(batch) == 512: |
|
|
|
if len(batch) == 512: |
|
|
|
cards.insert_all(batch,alter=True,pk="id",batch_size=512) |
|
|
|
batch = [] |
|
|
|
print("Dropped {} cards because of layout \r".format(numlayout)) |
|
|
|
sys.stdout.write("Adding card {} \r".format(numcard)) |
|
|
|
numcard+=1 |
|
|
|
print("\n") |